From 270ac88b5182de0a7318fab41ecc14e41142c558 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 27 Mar 2021 15:45:15 +0000 Subject: [PATCH 001/505] Added GetBlocksMessage and BlocksMessage, which allow multiple blocks to be transferred between peers in a single request. When communicating with a peer that is running at least version 1.5.0, it will now sync multiple blocks at once in Synchronizer.syncToPeerChain(). This allows us to bypass the single block syncing (and retry mechanism), which has proven to be unviable when there are multiple active forks with several blocks in each chain. For peers below v1.5.0, the logic should remain unaffected and it will continue to sync blocks individually. --- src/main/java/org/qortal/block/Block.java | 14 +- .../org/qortal/controller/Controller.java | 80 ++++++ .../org/qortal/controller/Synchronizer.java | 239 ++++++++++++------ src/main/java/org/qortal/network/Network.java | 1 + .../qortal/network/message/BlocksMessage.java | 90 +++++++ .../network/message/GetBlocksMessage.java | 65 +++++ .../org/qortal/network/message/Message.java | 5 +- .../org/qortal/transaction/Transaction.java | 4 + .../transform/block/BlockTransformer.java | 22 +- 9 files changed, 436 insertions(+), 84 deletions(-) create mode 100644 src/main/java/org/qortal/network/message/BlocksMessage.java create mode 100644 src/main/java/org/qortal/network/message/GetBlocksMessage.java diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 8551e4e7..e2560ac2 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -469,6 +469,16 @@ public class Block { return this.minter; } + + public void setRepository(Repository repository) throws DataException { + this.repository = repository; + + for (Transaction transaction : this.getTransactions()) { + transaction.setRepository(repository); + } + } + + // More information /** @@ -517,8 +527,10 @@ public class Block { long nonAtTransactionCount = transactionsData.stream().filter(transactionData -> transactionData.getType() != TransactionType.AT).count(); // The number of non-AT transactions fetched from repository should correspond with Block's transactionCount - if (nonAtTransactionCount != this.blockData.getTransactionCount()) + if (nonAtTransactionCount != this.blockData.getTransactionCount()) { + LOGGER.error(() -> String.format("Block's transactions from repository (%d) do not match block's transaction count (%d)", nonAtTransactionCount, this.blockData.getTransactionCount())); throw new IllegalStateException("Block's transactions from repository do not match block's transaction count"); + } this.transactions = new ArrayList<>(); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index a0ca1d05..40c25d84 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -68,9 +68,11 @@ import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.message.ArbitraryDataMessage; import org.qortal.network.message.BlockSummariesMessage; +import org.qortal.network.message.BlocksMessage; import org.qortal.network.message.CachedBlockMessage; import org.qortal.network.message.GetArbitraryDataMessage; import org.qortal.network.message.GetBlockMessage; +import org.qortal.network.message.GetBlocksMessage; import org.qortal.network.message.GetBlockSummariesMessage; import org.qortal.network.message.GetOnlineAccountsMessage; import org.qortal.network.message.GetPeersMessage; @@ -216,6 +218,18 @@ public class Controller extends Thread { } public GetBlockMessageStats getBlockMessageStats = new GetBlockMessageStats(); + public static class GetBlocksMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong cacheHits = new AtomicLong(); + public AtomicLong unknownBlocks = new AtomicLong(); + public AtomicLong cacheFills = new AtomicLong(); + public AtomicLong fullyFromCache = new AtomicLong(); + + public GetBlocksMessageStats() { + } + } + public GetBlocksMessageStats getBlocksMessageStats = new GetBlocksMessageStats(); + public static class GetBlockSummariesStats { public AtomicLong requests = new AtomicLong(); public AtomicLong cacheHits = new AtomicLong(); @@ -1094,6 +1108,10 @@ public class Controller extends Thread { onNetworkGetBlockMessage(peer, message); break; + case GET_BLOCKS: + onNetworkGetBlocksMessage(peer, message); + break; + case TRANSACTION: onNetworkTransactionMessage(peer, message); break; @@ -1208,6 +1226,68 @@ public class Controller extends Thread { } } + private void onNetworkGetBlocksMessage(Peer peer, Message message) { + GetBlocksMessage getBlocksMessage = (GetBlocksMessage) message; + byte[] parentSignature = getBlocksMessage.getParentSignature(); + this.stats.getBlocksMessageStats.requests.incrementAndGet(); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // If peer's parent signature matches our latest block signature + // then we can short-circuit with an empty response + BlockData chainTip = getChainTip(); + if (chainTip != null && Arrays.equals(parentSignature, chainTip.getSignature())) { + Message blocksMessage = new BlocksMessage(Collections.emptyList()); + blocksMessage.setId(message.getId()); + if (!peer.sendMessage(blocksMessage)) + peer.disconnect("failed to send blocks"); + + return; + } + + List blockDataList = new ArrayList<>(); + + // Attempt to serve from our cache of latest blocks + synchronized (this.latestBlocks) { + blockDataList = this.latestBlocks.stream() + .dropWhile(cachedBlockData -> !Arrays.equals(cachedBlockData.getReference(), parentSignature)) + .map(BlockData::new) + .collect(Collectors.toList()); + } + + if (blockDataList.isEmpty()) { + int numberRequested = Math.min(Network.MAX_BLOCKS_PER_REPLY, getBlocksMessage.getNumberRequested()); + + BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); + + while (blockData != null && blockDataList.size() < numberRequested) { + blockDataList.add(blockData); + + blockData = repository.getBlockRepository().fromReference(blockData.getSignature()); + } + } else { + this.stats.getBlocksMessageStats.cacheHits.incrementAndGet(); + + if (blockDataList.size() >= getBlocksMessage.getNumberRequested()) + this.stats.getBlocksMessageStats.fullyFromCache.incrementAndGet(); + } + + List blocks = new ArrayList<>(); + for (BlockData blockData : blockDataList) { + Block block = new Block(repository, blockData); + blocks.add(block); + } + + Message blocksMessage = new BlocksMessage(blocks); + blocksMessage.setId(message.getId()); + if (!peer.sendMessage(blocksMessage)) + peer.disconnect("failed to send blocks"); + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while sending blocks after %s to peer %s", Base58.encode(parentSignature), peer), e); + } + } + private void onNetworkTransactionMessage(Peer peer, Message message) { TransactionMessage transactionMessage = (TransactionMessage) message; TransactionData transactionData = transactionMessage.getTransactionData(); diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 0804c2df..5b81622a 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -22,8 +22,10 @@ import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Peer; import org.qortal.network.message.BlockMessage; +import org.qortal.network.message.BlocksMessage; import org.qortal.network.message.BlockSummariesMessage; import org.qortal.network.message.GetBlockMessage; +import org.qortal.network.message.GetBlocksMessage; import org.qortal.network.message.GetBlockSummariesMessage; import org.qortal.network.message.GetSignaturesV2Message; import org.qortal.network.message.Message; @@ -56,6 +58,9 @@ public class Synchronizer { /** Number of retry attempts if a peer fails to respond with the requested data */ private static final int MAXIMUM_RETRIES = 3; // XXX move to Settings? + /* Minimum peer version that supports syncing multiple blocks at once via GetBlocksMessage */ + private static final long PEER_VERSION_150 = 0x0100050000L; + private static Synchronizer instance; @@ -360,97 +365,161 @@ public class Synchronizer { // Overall plan: fetch peer's blocks first, then orphan, then apply - // Convert any leftover (post-common) block summaries into signatures to request from peer - List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); // Calculate the total number of additional blocks this peer has beyond the common block int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight; - // Subtract the number of signatures that we already have, as we don't need to request them again - int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); - // Fetch remaining block signatures, if needed - int retryCount = 0; - while (numberSignaturesRequired > 0) { - byte[] latestPeerSignature = peerBlockSignatures.isEmpty() ? commonBlockSig : peerBlockSignatures.get(peerBlockSignatures.size() - 1); - int lastPeerHeight = commonBlockHeight + peerBlockSignatures.size(); - int numberOfSignaturesToRequest = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE); - LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", - numberOfSignaturesToRequest, (numberOfSignaturesToRequest != 1 ? "s": ""), lastPeerHeight, Base58.encode(latestPeerSignature))); - - List moreBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberOfSignaturesToRequest); - - if (moreBlockSignatures == null || moreBlockSignatures.isEmpty()) { - LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, - lastPeerHeight, Base58.encode(latestPeerSignature))); - - if (retryCount >= MAXIMUM_RETRIES) { - // Give up with this peer - return SynchronizationResult.NO_REPLY; - } - else { - // Retry until retryCount reaches MAXIMUM_RETRIES - retryCount++; - int triesRemaining = MAXIMUM_RETRIES - retryCount; - LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s": ""))); - continue; - } - } - - // Reset retryCount because the last request succeeded - retryCount = 0; - - LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); - - peerBlockSignatures.addAll(moreBlockSignatures); - numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); - } - - // Fetch blocks using signatures - LOGGER.debug(String.format("Fetching new blocks from peer %s after height %d", peer, commonBlockHeight)); + // Firstly, attempt to retrieve the blocks themselves, rather than signatures. This is supported by newer peers. + // We could optionally check for a version here if we didn't want to make unnecessary requests List peerBlocks = new ArrayList<>(); - retryCount = 0; - while (peerBlocks.size() < peerBlockSignatures.size()) { - byte[] blockSignature = peerBlockSignatures.get(peerBlocks.size()); + if (peer.getPeersVersion() >= PEER_VERSION_150) { + // This peer supports syncing multiple blocks at once via GetBlocksMessage + int numberBlocksRequired = additionalPeerBlocksAfterCommonBlock - peerBlocks.size(); + while (numberBlocksRequired > 0) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; - LOGGER.debug(String.format("Fetching block with signature %.8s", Base58.encode(blockSignature))); - int blockHeightToRequest = commonBlockHeight + peerBlocks.size() + 1; // +1 because we are requesting the next block, beyond what we already have in the peerBlocks array - Block newBlock = this.fetchBlock(repository, peer, blockSignature); + byte[] latestPeerSignature = peerBlocks.isEmpty() ? commonBlockSig : peerBlocks.get(peerBlocks.size() - 1).getSignature(); + int lastPeerHeight = commonBlockHeight + peerBlocks.size(); + int numberOfBlocksToRequest = Math.min(numberBlocksRequired, MAXIMUM_REQUEST_SIZE); - if (newBlock == null) { - LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, blockHeightToRequest, Base58.encode(blockSignature))); + LOGGER.trace(String.format("Requesting %d block%s after height %d, sig %.8s", + numberOfBlocksToRequest, (numberOfBlocksToRequest != 1 ? "s" : ""), lastPeerHeight, Base58.encode(latestPeerSignature))); - if (retryCount >= MAXIMUM_RETRIES) { - // Give up with this peer - return SynchronizationResult.NO_REPLY; + List blocks = this.fetchBlocks(repository, peer, latestPeerSignature, numberOfBlocksToRequest); + if (blocks == null || blocks.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with more blocks after height %d, sig %.8s", peer, + lastPeerHeight, Base58.encode(latestPeerSignature))); + + if (peerBlocks.isEmpty()) { + return SynchronizationResult.NO_REPLY; + } } - else { - // Retry until retryCount reaches MAXIMUM_RETRIES - retryCount++; - int triesRemaining = MAXIMUM_RETRIES - retryCount; - LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s": ""))); - continue; + + LOGGER.debug(String.format("Received %d blocks from peer %s", blocks.size(), peer)); + + try { + for (Block block : blocks) { + + // Set the repository, because we couldn't do that when originally constructing the Block + block.setRepository(repository); + + // Transactions are transmitted without approval status so determine that now + for (Transaction transaction : block.getTransactions()) { + transaction.setInitialApprovalStatus(); + } + + peerBlocks.add(block); + } + } catch (IllegalStateException e) { + LOGGER.error("Error processing transactions in block", e); + return SynchronizationResult.REPOSITORY_ISSUE; } + + numberBlocksRequired = additionalPeerBlocksAfterCommonBlock - peerBlocks.size(); } - - if (!newBlock.isSignatureValid()) { - LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, - blockHeightToRequest, Base58.encode(blockSignature))); - return SynchronizationResult.INVALID_DATA; - } - - // Reset retryCount because the last request succeeded - retryCount = 0; - - LOGGER.debug(String.format("Received block with height %d, sig: %.8s", newBlock.getBlockData().getHeight(), Base58.encode(blockSignature))); - - // Transactions are transmitted without approval status so determine that now - for (Transaction transaction : newBlock.getTransactions()) - transaction.setInitialApprovalStatus(); - - peerBlocks.add(newBlock); } + else { + // Older peer version - use slow sync + + // Convert any leftover (post-common) block summaries into signatures to request from peer + List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); + + // Subtract the number of signatures that we already have, as we don't need to request them again + int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); + + + // Fetch remaining block signatures, if needed + int retryCount = 0; + while (numberSignaturesRequired > 0) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + byte[] latestPeerSignature = peerBlockSignatures.isEmpty() ? commonBlockSig : peerBlockSignatures.get(peerBlockSignatures.size() - 1); + int lastPeerHeight = commonBlockHeight + peerBlockSignatures.size(); + int numberOfSignaturesToRequest = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE); + + LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", + numberOfSignaturesToRequest, (numberOfSignaturesToRequest != 1 ? "s" : ""), lastPeerHeight, Base58.encode(latestPeerSignature))); + + List moreBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberOfSignaturesToRequest); + + if (moreBlockSignatures == null || moreBlockSignatures.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, + lastPeerHeight, Base58.encode(latestPeerSignature))); + + if (retryCount >= MAXIMUM_RETRIES) { + // Give up with this peer + return SynchronizationResult.NO_REPLY; + } else { + // Retry until retryCount reaches MAXIMUM_RETRIES + retryCount++; + int triesRemaining = MAXIMUM_RETRIES - retryCount; + LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : ""))); + continue; + } + } + + // Reset retryCount because the last request succeeded + retryCount = 0; + + LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); + + peerBlockSignatures.addAll(moreBlockSignatures); + numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); + } + + // Fetch blocks using signatures + LOGGER.debug(String.format("Fetching new blocks from peer %s after height %d", peer, commonBlockHeight)); + + retryCount = 0; + while (peerBlocks.size() < peerBlockSignatures.size()) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + byte[] blockSignature = peerBlockSignatures.get(peerBlocks.size()); + + LOGGER.debug(String.format("Fetching block with signature %.8s", Base58.encode(blockSignature))); + int blockHeightToRequest = commonBlockHeight + peerBlocks.size() + 1; // +1 because we are requesting the next block, beyond what we already have in the peerBlocks array + Block newBlock = this.fetchBlock(repository, peer, blockSignature); + + if (newBlock == null) { + LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, blockHeightToRequest, Base58.encode(blockSignature))); + + if (retryCount >= MAXIMUM_RETRIES) { + // Give up with this peer + return SynchronizationResult.NO_REPLY; + } else { + // Retry until retryCount reaches MAXIMUM_RETRIES + retryCount++; + int triesRemaining = MAXIMUM_RETRIES - retryCount; + LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : ""))); + continue; + } + } + + if (!newBlock.isSignatureValid()) { + LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, + blockHeightToRequest, Base58.encode(blockSignature))); + return SynchronizationResult.INVALID_DATA; + } + + // Reset retryCount because the last request succeeded + retryCount = 0; + + LOGGER.debug(String.format("Received block with height %d, sig: %.8s", newBlock.getBlockData().getHeight(), Base58.encode(blockSignature))); + + // Transactions are transmitted without approval status so determine that now + for (Transaction transaction : newBlock.getTransactions()) + transaction.setInitialApprovalStatus(); + + peerBlocks.add(newBlock); + } + + } + // Unwind to common block (unless common block is our latest block) LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s", commonBlockHeight, commonBlockSig58)); @@ -625,6 +694,22 @@ public class Synchronizer { return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates()); } + private List fetchBlocks(Repository repository, Peer peer, byte[] parentSignature, int numberRequested) throws InterruptedException { + Message getBlocksMessage = new GetBlocksMessage(parentSignature, numberRequested); + + Message message = peer.getResponse(getBlocksMessage); + if (message == null || message.getType() != MessageType.BLOCKS) { + return null; + } + + BlocksMessage blocksMessage = (BlocksMessage) message; + if (blocksMessage == null || blocksMessage.getBlocks() == null) { + return null; + } + + return blocksMessage.getBlocks(); + } + private void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { final int firstBlockHeight = blockSummaries.get(0).getHeight(); diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 0e9ac32b..7a234c7a 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -89,6 +89,7 @@ public class Network { public static final int MAX_SIGNATURES_PER_REPLY = 500; public static final int MAX_BLOCK_SUMMARIES_PER_REPLY = 500; + public static final int MAX_BLOCKS_PER_REPLY = 500; // Generate our node keys / ID private final Ed25519PrivateKeyParameters edPrivateKeyParams = new Ed25519PrivateKeyParameters(new SecureRandom()); diff --git a/src/main/java/org/qortal/network/message/BlocksMessage.java b/src/main/java/org/qortal/network/message/BlocksMessage.java new file mode 100644 index 00000000..f53de301 --- /dev/null +++ b/src/main/java/org/qortal/network/message/BlocksMessage.java @@ -0,0 +1,90 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.Block; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.Triple; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class BlocksMessage extends Message { + + private static final Logger LOGGER = LogManager.getLogger(BlocksMessage.class); + + private List blocks; + + public BlocksMessage(List blocks) { + this(-1, blocks); + } + + private BlocksMessage(int id, List blocks) { + super(id, MessageType.BLOCKS); + + this.blocks = blocks; + } + + public List getBlocks() { + return this.blocks; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + + int count = bytes.getInt(); + List blocks = new ArrayList<>(); + + for (int i = 0; i < count; ++i) { + int height = bytes.getInt(); + + try { + boolean finalBlockInBuffer = (i == count-1); + + Triple, List> blockInfo = null; + blockInfo = BlockTransformer.fromByteBuffer(bytes, finalBlockInBuffer); + BlockData blockData = blockInfo.getA(); + blockData.setHeight(height); + + // We are unable to obtain a valid Repository instance here, so set it to null and we will attach it later + Block block = new Block(null, blockData, blockInfo.getB(), blockInfo.getC()); + blocks.add(block); + + } catch (TransformationException e) { + return null; + } + + } + + return new BlocksMessage(id, blocks); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(Ints.toByteArray(this.blocks.size())); + + for (Block block : this.blocks) { + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + bytes.write(BlockTransformer.toBytes(block)); + } + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } catch (TransformationException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/GetBlocksMessage.java b/src/main/java/org/qortal/network/message/GetBlocksMessage.java new file mode 100644 index 00000000..ae5a78c4 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetBlocksMessage.java @@ -0,0 +1,65 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.qortal.transform.Transformer; +import org.qortal.transform.block.BlockTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +public class GetBlocksMessage extends Message { + + private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; + + private byte[] parentSignature; + private int numberRequested; + + public GetBlocksMessage(byte[] parentSignature, int numberRequested) { + this(-1, parentSignature, numberRequested); + } + + private GetBlocksMessage(int id, byte[] parentSignature, int numberRequested) { + super(id, MessageType.GET_BLOCKS); + + this.parentSignature = parentSignature; + this.numberRequested = numberRequested; + } + + public byte[] getParentSignature() { + return this.parentSignature; + } + + public int getNumberRequested() { + return this.numberRequested; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + if (bytes.remaining() != BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH) + return null; + + byte[] parentSignature = new byte[BLOCK_SIGNATURE_LENGTH]; + bytes.get(parentSignature); + + int numberRequested = bytes.getInt(); + + return new GetBlocksMessage(id, parentSignature, numberRequested); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.parentSignature); + + bytes.write(Ints.toByteArray(this.numberRequested)); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index cc90fe81..d1546dce 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -80,7 +80,10 @@ public abstract class Message { GET_ONLINE_ACCOUNTS(81), ARBITRARY_DATA(90), - GET_ARBITRARY_DATA(91); + GET_ARBITRARY_DATA(91), + + BLOCKS(100), + GET_BLOCKS(101); public final int value; public final Method fromByteBufferMethod; diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index d7dd1455..2a57649c 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -315,6 +315,10 @@ public abstract class Transaction { return this.transactionData; } + public void setRepository(Repository repository) { + this.repository = repository; + } + // More information public static long getDeadline(TransactionData transactionData) { diff --git a/src/main/java/org/qortal/transform/block/BlockTransformer.java b/src/main/java/org/qortal/transform/block/BlockTransformer.java index 8b91fd11..cce3e7d7 100644 --- a/src/main/java/org/qortal/transform/block/BlockTransformer.java +++ b/src/main/java/org/qortal/transform/block/BlockTransformer.java @@ -74,19 +74,30 @@ public class BlockTransformer extends Transformer { } /** - * Extract block data and transaction data from serialized bytes. - * + * Extract block data and transaction data from serialized bytes containing a single block. + * * @param bytes * @return BlockData and a List of transactions. * @throws TransformationException */ public static Triple, List> fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { + return BlockTransformer.fromByteBuffer(byteBuffer, true); + } + + /** + * Extract block data and transaction data from serialized bytes containing one or more blocks. + * + * @param bytes + * @return the next block's BlockData and a List of transactions. + * @throws TransformationException + */ + public static Triple, List> fromByteBuffer(ByteBuffer byteBuffer, boolean finalBlockInBuffer) throws TransformationException { int version = byteBuffer.getInt(); - if (byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH) + if (finalBlockInBuffer && byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH) throw new TransformationException("Byte data too short for Block"); - if (byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize()) + if (finalBlockInBuffer && byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize()) throw new TransformationException("Byte data too long for Block"); long timestamp = byteBuffer.getLong(); @@ -210,7 +221,8 @@ public class BlockTransformer extends Transformer { byteBuffer.get(onlineAccountsSignatures); } - if (byteBuffer.hasRemaining()) + // We should only complain about excess byte data if we aren't expecting more blocks in this ByteBuffer + if (finalBlockInBuffer && byteBuffer.hasRemaining()) throw new TransformationException("Excess byte data found after parsing Block"); // We don't have a height! From a5308995b73f931962e00c5278d712609108aa5f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 27 Mar 2021 15:46:30 +0000 Subject: [PATCH 002/505] Bump version to 1.5.0, to allow nodes to start using the new syncing method. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3eda336f..6697cc81 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 1.4.5 + 1.5.0 jar true From dbcf6de2d50dac2e9bb1c7ab51c02235bb0bb7cd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 27 Mar 2021 17:59:23 +0000 Subject: [PATCH 003/505] Added new settings "fastSyncEnabled" (default: false) and "fastSyncEnabledWhenResolvingFork" (default: true). --- src/main/java/org/qortal/settings/Settings.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index fb068b8d..d15a4ab9 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -123,6 +123,11 @@ public class Settings { /** Maximum number of threads for network proof-of-work compute, used during handshaking. */ private int networkPoWComputePoolSize = 2; + /** Whether to sync multiple blocks at once in normal operation */ + private boolean fastSyncEnabled = false; + /** Whether to sync multiple blocks at once when the peer has a different chain */ + private boolean fastSyncEnabledWhenResolvingFork = true; + // Which blockchains this node is running private String blockchainConfig = null; // use default from resources private BitcoinNet bitcoinNet = BitcoinNet.MAIN; @@ -436,6 +441,14 @@ public class Settings { return this.repositoryConnectionPoolSize; } + public boolean isFastSyncEnabled() { + return this.fastSyncEnabled; + } + + public boolean isFastSyncEnabledWhenResolvingFork() { + return this.fastSyncEnabledWhenResolvingFork; + } + public boolean isAutoUpdateEnabled() { return this.autoUpdateEnabled; } From 8c3753326f7e38d0b8da9ec8788971c6fedad2f2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 27 Mar 2021 18:00:39 +0000 Subject: [PATCH 004/505] Check "isFastSyncEnabledWhenResolvingFork" setting before requesting multiple blocks from a peer. This allows users to opt out of this functionality, by setting it to false in their settings. --- src/main/java/org/qortal/controller/Synchronizer.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 5b81622a..0c32a777 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -34,6 +34,7 @@ import org.qortal.network.message.Message.MessageType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; @@ -370,12 +371,11 @@ public class Synchronizer { int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight; - // Firstly, attempt to retrieve the blocks themselves, rather than signatures. This is supported by newer peers. - // We could optionally check for a version here if we didn't want to make unnecessary requests + // Firstly, attempt to retrieve the blocks themselves, rather than signatures. This is supported by newer peers (version 1.5.0 and above). List peerBlocks = new ArrayList<>(); - if (peer.getPeersVersion() >= PEER_VERSION_150) { - // This peer supports syncing multiple blocks at once via GetBlocksMessage + if (Settings.getInstance().isFastSyncEnabledWhenResolvingFork() && peer.getPeersVersion() >= PEER_VERSION_150) { + // This peer supports syncing multiple blocks at once via GetBlocksMessage, and it is enabled in the settings int numberBlocksRequired = additionalPeerBlocksAfterCommonBlock - peerBlocks.size(); while (numberBlocksRequired > 0) { if (Controller.isStopping()) @@ -422,7 +422,7 @@ public class Synchronizer { } } else { - // Older peer version - use slow sync + // Older peer version, or fast sync is disabled in the settings - use slow sync // Convert any leftover (post-common) block summaries into signatures to request from peer List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); From 3e0ff7f43fbe5d67ad6d0f80c8d175d59c42091f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 27 Mar 2021 18:04:47 +0000 Subject: [PATCH 005/505] Split Synchronizer.applyNewBlocks() into two methods. If fast syncing is enabled in the settings (by default it's disabled) AND the peer is running at least v1.5.0, then it will route through to a new method which fetches multiple blocks at a time, in a very similar way to Synchronizer.syncToPeerChain(). If fast syncing is disabled in the settings, or we are communicating with a peer on an older version, it will continue to sync blocks individually. --- .../org/qortal/controller/Synchronizer.java | 97 ++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 0c32a777..2aa2fb8a 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -572,8 +572,103 @@ public class Synchronizer { } private SynchronizationResult applyNewBlocks(Repository repository, BlockData commonBlockData, int ourInitialHeight, + Peer peer, int peerHeight, List peerBlockSummaries) throws InterruptedException, DataException { + + if (Settings.getInstance().isFastSyncEnabled() && peer.getPeersVersion() >= PEER_VERSION_150) + // This peer supports syncing multiple blocks at once via GetBlocksMessage, and it is enabled in the settings + return this.applyNewBlocksUsingFastSync(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries); + else + // Older peer version, or fast sync is disabled in the settings - use slow sync + return this.applyNewBlocksUsingSlowSync(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries); + + } + + private SynchronizationResult applyNewBlocksUsingFastSync(Repository repository, BlockData commonBlockData, int ourInitialHeight, + Peer peer, int peerHeight, List peerBlockSummaries) throws InterruptedException, DataException { + LOGGER.debug(String.format("Fetching new blocks from peer %s using fast sync", peer)); + + final int commonBlockHeight = commonBlockData.getHeight(); + final byte[] commonBlockSig = commonBlockData.getSignature(); + byte[] latestPeerSignature = commonBlockSig; + + int ourHeight = ourInitialHeight; + + // Fetch, and apply, blocks from peer + int maxBatchHeight = commonBlockHeight + SYNC_BATCH_SIZE; + + while (ourHeight < peerHeight && ourHeight < maxBatchHeight) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_REQUEST_SIZE); + + LOGGER.trace(String.format("Fetching %d blocks after height %d, sig %.8s from %s", numberRequested, ourHeight, Base58.encode(latestPeerSignature), peer)); + List blocks = this.fetchBlocks(repository, peer, latestPeerSignature, numberRequested); + if (blocks == null || blocks.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with more blocks after height %d, sig %.8s", peer, + ourHeight, Base58.encode(latestPeerSignature))); + } + LOGGER.trace(String.format("Received %d blocks after height %d, sig %.8s from %s", blocks.size(), ourHeight, Base58.encode(latestPeerSignature), peer)); + + for (Block newBlock : blocks) { + ++ourHeight; + + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + if (newBlock == null) { + LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, + ourHeight, Base58.encode(latestPeerSignature))); + return SynchronizationResult.NO_REPLY; + } + + if (!newBlock.isSignatureValid()) { + LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, + ourHeight, Base58.encode(latestPeerSignature))); + return SynchronizationResult.INVALID_DATA; + } + + // Set the repository, because we couldn't do that when originally constructing the Block + newBlock.setRepository(repository); + + // Transactions are transmitted without approval status so determine that now + for (Transaction transaction : newBlock.getTransactions()) { + transaction.setInitialApprovalStatus(); + } + + ValidationResult blockResult = newBlock.isValid(); + if (blockResult != ValidationResult.OK) { + LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer, + ourHeight, Base58.encode(latestPeerSignature), blockResult.name())); + return SynchronizationResult.INVALID_DATA; + } + + // Save transactions attached to this block + for (Transaction transaction : newBlock.getTransactions()) { + TransactionData transactionData = transaction.getTransactionData(); + repository.getTransactionRepository().save(transactionData); + } + + newBlock.process(); + + LOGGER.trace(String.format("Processed block height %d, sig %.8s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()))); + + repository.saveChanges(); + + Controller.getInstance().onNewBlock(newBlock.getBlockData()); + + // Update latestPeerSignature so that subsequent batches start requesting from the correct block + latestPeerSignature = newBlock.getSignature(); + } + + } + + return SynchronizationResult.OK; + } + + private SynchronizationResult applyNewBlocksUsingSlowSync(Repository repository, BlockData commonBlockData, int ourInitialHeight, Peer peer, int peerHeight, List peerBlockSummaries) throws InterruptedException, DataException { - LOGGER.debug(String.format("Fetching new blocks from peer %s", peer)); + LOGGER.debug(String.format("Fetching new blocks from peer %s using slow sync", peer)); final int commonBlockHeight = commonBlockData.getHeight(); final byte[] commonBlockSig = commonBlockData.getSignature(); From 365662a2afd4c913cd8a0f180bb65cd18b9f9a6c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 27 Mar 2021 19:25:27 +0000 Subject: [PATCH 006/505] MAXIMUM_RETRIES reduced from 3 to 1. It will now only retry once, which should save around 6 seconds of wasted synchronization time if a node is unable to respond with the requested block (due to a re-org, etc). --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 2aa2fb8a..c747dd6e 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -57,7 +57,7 @@ public class Synchronizer { private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? /** Number of retry attempts if a peer fails to respond with the requested data */ - private static final int MAXIMUM_RETRIES = 3; // XXX move to Settings? + private static final int MAXIMUM_RETRIES = 1; // XXX move to Settings? /* Minimum peer version that supports syncing multiple blocks at once via GetBlocksMessage */ private static final long PEER_VERSION_150 = 0x0100050000L; From 2556855bd77d20dbfad6850a8b97c9c500b56cb2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 28 Mar 2021 14:19:53 +0100 Subject: [PATCH 007/505] Added missing return statement if a peer fails to respond with blocks when fast syncing. --- src/main/java/org/qortal/controller/Synchronizer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index c747dd6e..ff96c5b7 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -607,6 +607,7 @@ public class Synchronizer { if (blocks == null || blocks.isEmpty()) { LOGGER.info(String.format("Peer %s failed to respond with more blocks after height %d, sig %.8s", peer, ourHeight, Base58.encode(latestPeerSignature))); + return SynchronizationResult.NO_REPLY; } LOGGER.trace(String.format("Received %d blocks after height %d, sig %.8s from %s", blocks.size(), ourHeight, Base58.encode(latestPeerSignature), peer)); From f22f954ae3c708088e8521ba5d2717d5b72d35de Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 28 Mar 2021 14:35:47 +0100 Subject: [PATCH 008/505] Use MAXIMUM_BLOCKS_REQUEST_SIZE for GetBlocksMessage and BlockMessage, instead of MAXIMUM_REQUEST_SIZE. Currently set to 1, as serialization of the BlocksMessage data on mainnet is too slow to use this for any significant number of blocks right now. Hopefully we can find a way to optimise this process, which will allow us to use this for multiple block syncing. Until then, sticking with single blocks should still be enough to help solve the network congestion and re-orgs we are seeing, because it gives us the ability to request the next block based on the previous block's signature, which was unavailable using GET_BLOCK. This removes the requirement to fetch all block signatures upfront, and therefore it shouldn't matter if the peer does a partial re-org whilst a node is syncing to it. --- src/main/java/org/qortal/controller/Synchronizer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index ff96c5b7..0e953db1 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -56,6 +56,9 @@ public class Synchronizer { /** Maximum number of block signatures we ask from peer in one go */ private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? + /** Maximum number of blocks we ask from peer in one go */ + private static final int MAXIMUM_BLOCKS_REQUEST_SIZE = 1; // XXX move to Settings? + /** Number of retry attempts if a peer fails to respond with the requested data */ private static final int MAXIMUM_RETRIES = 1; // XXX move to Settings? @@ -383,7 +386,7 @@ public class Synchronizer { byte[] latestPeerSignature = peerBlocks.isEmpty() ? commonBlockSig : peerBlocks.get(peerBlocks.size() - 1).getSignature(); int lastPeerHeight = commonBlockHeight + peerBlocks.size(); - int numberOfBlocksToRequest = Math.min(numberBlocksRequired, MAXIMUM_REQUEST_SIZE); + int numberOfBlocksToRequest = Math.min(numberBlocksRequired, MAXIMUM_BLOCKS_REQUEST_SIZE); LOGGER.trace(String.format("Requesting %d block%s after height %d, sig %.8s", numberOfBlocksToRequest, (numberOfBlocksToRequest != 1 ? "s" : ""), lastPeerHeight, Base58.encode(latestPeerSignature))); @@ -600,7 +603,7 @@ public class Synchronizer { if (Controller.isStopping()) return SynchronizationResult.SHUTTING_DOWN; - int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_REQUEST_SIZE); + int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_BLOCKS_REQUEST_SIZE); LOGGER.trace(String.format("Fetching %d blocks after height %d, sig %.8s from %s", numberRequested, ourHeight, Base58.encode(latestPeerSignature), peer)); List blocks = this.fetchBlocks(repository, peer, latestPeerSignature, numberRequested); From cb80280eaf68a2cc94dd8f2dbe98652b714c188c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 28 Mar 2021 14:47:57 +0100 Subject: [PATCH 009/505] Bump Peer response timeout from 3s to 4s --- src/main/java/org/qortal/network/Peer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index f619111a..9212fb56 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -46,7 +46,7 @@ public class Peer { private static final int CONNECT_TIMEOUT = 2000; // ms /** Maximum time to wait for a message reply to arrive from peer. (ms) */ - private static final int RESPONSE_TIMEOUT = 3000; // ms + private static final int RESPONSE_TIMEOUT = 4000; // ms /** * Interval between PING messages to a peer. (ms) From f2bbafe6c295a967bcf49d5dc5ea0d8759a2a10a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 28 Mar 2021 16:52:15 +0100 Subject: [PATCH 010/505] Added missing break statement if a peer fails to respond with blocks when resolving a fork via fast sync. --- src/main/java/org/qortal/controller/Synchronizer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 0e953db1..ade6fb88 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -399,6 +399,7 @@ public class Synchronizer { if (peerBlocks.isEmpty()) { return SynchronizationResult.NO_REPLY; } + break; } LOGGER.debug(String.format("Received %d blocks from peer %s", blocks.size(), peer)); From 08f3d653ccbdce11f77a28311f32f5dca39207bb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 28 Mar 2021 17:27:04 +0100 Subject: [PATCH 011/505] Added new settings "maxBlocksPerRequest" and "maxBlocksPerResponse", to control the number of blocks requested and returned by nodes when using GetBlocksMessage and BlocksMessage. --- .../java/org/qortal/controller/Controller.java | 12 +++++++++--- .../java/org/qortal/controller/Synchronizer.java | 14 +++++++++----- src/main/java/org/qortal/network/Network.java | 1 - .../org/qortal/network/message/BlocksMessage.java | 1 + src/main/java/org/qortal/settings/Settings.java | 8 ++++++++ 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 40c25d84..d988f24d 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1245,6 +1245,10 @@ public class Controller extends Thread { return; } + // Ensure that we don't serve more blocks than the amount specified in the settings + // Serializing multiple blocks is very slow, so by default we are using a low limit + int blockLimitPerRequest = Settings.getInstance().getMaxBlocksPerResponse(); + List blockDataList = new ArrayList<>(); // Attempt to serve from our cache of latest blocks @@ -1256,7 +1260,7 @@ public class Controller extends Thread { } if (blockDataList.isEmpty()) { - int numberRequested = Math.min(Network.MAX_BLOCKS_PER_REPLY, getBlocksMessage.getNumberRequested()); + int numberRequested = Math.min(blockLimitPerRequest, getBlocksMessage.getNumberRequested()); BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); @@ -1274,8 +1278,10 @@ public class Controller extends Thread { List blocks = new ArrayList<>(); for (BlockData blockData : blockDataList) { - Block block = new Block(repository, blockData); - blocks.add(block); + if (blocks.size() < blockLimitPerRequest) { + Block block = new Block(repository, blockData); + blocks.add(block); + } } Message blocksMessage = new BlocksMessage(blocks); diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index ade6fb88..c72d18da 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -56,9 +56,6 @@ public class Synchronizer { /** Maximum number of block signatures we ask from peer in one go */ private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? - /** Maximum number of blocks we ask from peer in one go */ - private static final int MAXIMUM_BLOCKS_REQUEST_SIZE = 1; // XXX move to Settings? - /** Number of retry attempts if a peer fails to respond with the requested data */ private static final int MAXIMUM_RETRIES = 1; // XXX move to Settings? @@ -380,13 +377,17 @@ public class Synchronizer { if (Settings.getInstance().isFastSyncEnabledWhenResolvingFork() && peer.getPeersVersion() >= PEER_VERSION_150) { // This peer supports syncing multiple blocks at once via GetBlocksMessage, and it is enabled in the settings int numberBlocksRequired = additionalPeerBlocksAfterCommonBlock - peerBlocks.size(); + + // Ensure that we don't request more blocks than specified in the settings + int maxBlocksPerRequest = Settings.getInstance().getMaxBlocksPerRequest(); + while (numberBlocksRequired > 0) { if (Controller.isStopping()) return SynchronizationResult.SHUTTING_DOWN; byte[] latestPeerSignature = peerBlocks.isEmpty() ? commonBlockSig : peerBlocks.get(peerBlocks.size() - 1).getSignature(); int lastPeerHeight = commonBlockHeight + peerBlocks.size(); - int numberOfBlocksToRequest = Math.min(numberBlocksRequired, MAXIMUM_BLOCKS_REQUEST_SIZE); + int numberOfBlocksToRequest = Math.min(numberBlocksRequired, maxBlocksPerRequest); LOGGER.trace(String.format("Requesting %d block%s after height %d, sig %.8s", numberOfBlocksToRequest, (numberOfBlocksToRequest != 1 ? "s" : ""), lastPeerHeight, Base58.encode(latestPeerSignature))); @@ -600,11 +601,14 @@ public class Synchronizer { // Fetch, and apply, blocks from peer int maxBatchHeight = commonBlockHeight + SYNC_BATCH_SIZE; + // Ensure that we don't request more blocks than specified in the settings + int maxBlocksPerRequest = Settings.getInstance().getMaxBlocksPerRequest(); + while (ourHeight < peerHeight && ourHeight < maxBatchHeight) { if (Controller.isStopping()) return SynchronizationResult.SHUTTING_DOWN; - int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_BLOCKS_REQUEST_SIZE); + int numberRequested = Math.min(maxBatchHeight - ourHeight, maxBlocksPerRequest); LOGGER.trace(String.format("Fetching %d blocks after height %d, sig %.8s from %s", numberRequested, ourHeight, Base58.encode(latestPeerSignature), peer)); List blocks = this.fetchBlocks(repository, peer, latestPeerSignature, numberRequested); diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 7a234c7a..0e9ac32b 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -89,7 +89,6 @@ public class Network { public static final int MAX_SIGNATURES_PER_REPLY = 500; public static final int MAX_BLOCK_SUMMARIES_PER_REPLY = 500; - public static final int MAX_BLOCKS_PER_REPLY = 500; // Generate our node keys / ID private final Ed25519PrivateKeyParameters edPrivateKeyParams = new Ed25519PrivateKeyParameters(new SecureRandom()); diff --git a/src/main/java/org/qortal/network/message/BlocksMessage.java b/src/main/java/org/qortal/network/message/BlocksMessage.java index f53de301..b997ead5 100644 --- a/src/main/java/org/qortal/network/message/BlocksMessage.java +++ b/src/main/java/org/qortal/network/message/BlocksMessage.java @@ -78,6 +78,7 @@ public class BlocksMessage extends Message { bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); bytes.write(BlockTransformer.toBytes(block)); } + LOGGER.trace(String.format("Total length of %d blocks is %d bytes", this.blocks.size(), bytes.size())); return bytes.toByteArray(); } catch (IOException e) { diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index d15a4ab9..3070bf96 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -127,6 +127,10 @@ public class Settings { private boolean fastSyncEnabled = false; /** Whether to sync multiple blocks at once when the peer has a different chain */ private boolean fastSyncEnabledWhenResolvingFork = true; + /** Maximum number of blocks to request at once */ + private int maxBlocksPerRequest = 1; + /** Maximum number of blocks this node will serve in a single response */ + private int maxBlocksPerResponse = 5; // Which blockchains this node is running private String blockchainConfig = null; // use default from resources @@ -449,6 +453,10 @@ public class Settings { return this.fastSyncEnabledWhenResolvingFork; } + public int getMaxBlocksPerRequest() { return this.maxBlocksPerRequest; } + + public int getMaxBlocksPerResponse() { return this.maxBlocksPerResponse; } + public boolean isAutoUpdateEnabled() { return this.autoUpdateEnabled; } From 6c5dbf7bd053b6eb0ffa4e77bc0d32597482387f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 8 May 2021 16:36:42 +0100 Subject: [PATCH 012/505] Preliminary version for fast sync set to 1.6.0, because 1.5.x is already released. --- src/main/java/org/qortal/controller/Synchronizer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 4f3833b3..a160ea62 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -61,7 +61,7 @@ public class Synchronizer { private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? /* Minimum peer version that supports syncing multiple blocks at once via GetBlocksMessage */ - private static final long PEER_VERSION_150 = 0x0100050000L; + private static final long PEER_VERSION_160 = 0x0100060000L; private static Synchronizer instance; @@ -967,7 +967,7 @@ public class Synchronizer { private SynchronizationResult applyNewBlocks(Repository repository, BlockData commonBlockData, int ourInitialHeight, Peer peer, int peerHeight, List peerBlockSummaries) throws InterruptedException, DataException { - if (Settings.getInstance().isFastSyncEnabled() && peer.getPeersVersion() >= PEER_VERSION_150) + if (Settings.getInstance().isFastSyncEnabled() && peer.getPeersVersion() >= PEER_VERSION_160) // This peer supports syncing multiple blocks at once via GetBlocksMessage, and it is enabled in the settings return this.applyNewBlocksUsingFastSync(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries); else From 3aa9b5f0b6f3eee4bf2cb00ce4e49954472f19ed Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 8 May 2021 22:36:41 +0100 Subject: [PATCH 013/505] Increased timeout when syncing multiple blocks from 4s to 10s. --- .../org/qortal/controller/Synchronizer.java | 5 +++- src/main/java/org/qortal/network/Peer.java | 23 ++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a160ea62..cba5d3c0 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -63,6 +63,9 @@ public class Synchronizer { /* Minimum peer version that supports syncing multiple blocks at once via GetBlocksMessage */ private static final long PEER_VERSION_160 = 0x0100060000L; + /** Maximum time to wait for a peer to respond with blocks (ms) */ + private static final int FETCH_BLOCKS_TIMEOUT = 10000; + private static Synchronizer instance; @@ -1189,7 +1192,7 @@ public class Synchronizer { private List fetchBlocks(Repository repository, Peer peer, byte[] parentSignature, int numberRequested) throws InterruptedException { Message getBlocksMessage = new GetBlocksMessage(parentSignature, numberRequested); - Message message = peer.getResponse(getBlocksMessage); + Message message = peer.getResponseWithTimeout(getBlocksMessage, FETCH_BLOCKS_TIMEOUT); if (message == null || message.getType() != MessageType.BLOCKS) { return null; } diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index cc4ad918..e5bd369d 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -538,7 +538,23 @@ public class Peer { } /** - * Send message to peer and await response. + * Send message to peer and await response, using default RESPONSE_TIMEOUT. + *

+ * Message is assigned a random ID and sent. If a response with matching ID is received then it is returned to caller. + *

+ * If no response with matching ID within timeout, or some other error/exception occurs, then return null.
+ * (Assume peer will be rapidly disconnected after this). + * + * @param message + * @return Message if valid response received; null if not or error/exception occurs + * @throws InterruptedException + */ + public Message getResponse(Message message) throws InterruptedException { + return getResponseWithTimeout(message, RESPONSE_TIMEOUT); + } + + /** + * Send message to peer and await response, using custom timeout. *

* Message is assigned a random ID and sent. If a response with matching ID is received then it is returned to caller. *

@@ -546,10 +562,11 @@ public class Peer { * (Assume peer will be rapidly disconnected after this). * * @param message + * @param timeout * @return Message if valid response received; null if not or error/exception occurs * @throws InterruptedException */ - public Message getResponse(Message message) throws InterruptedException { + public Message getResponseWithTimeout(Message message, int timeout) throws InterruptedException { BlockingQueue blockingQueue = new ArrayBlockingQueue<>(1); // Assign random ID to this message @@ -570,7 +587,7 @@ public class Peer { } try { - return blockingQueue.poll(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); + return blockingQueue.poll(timeout, TimeUnit.MILLISECONDS); } finally { this.replyQueues.remove(id); } From d2ea5633fb82269d7c3e683b453b7b342ef6d589 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 08:25:24 +0100 Subject: [PATCH 014/505] Fixed divide by zero exception. Block.calcKeyDistance() cannot be called on some trimmed blocks, because the minter level is unable to be inferred in some cases. This generally hasn't been an issue, but the new Block.logDebugInfo() method is invoking it for all blocks. For now I am adding defensiveness to the debug method, but longer term we might want to add defensiveness to Block.calcKeyDistance() itself, if we ever encounter this issue again. I will leave it alone for now, to reduce risk. --- src/main/java/org/qortal/block/Block.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 384ca193..41adbd78 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -2021,7 +2021,7 @@ public class Block { LOGGER.debug(String.format("Online accounts: %d", this.getBlockData().getOnlineAccountsCount())); BlockSummaryData blockSummaryData = new BlockSummaryData(this.getBlockData()); - if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null) + if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null || minterLevel == 0) return; blockSummaryData.setMinterLevel(minterLevel); From 68544715bf5f98df2e2baae190dfaa30c895e50a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 09:00:53 +0100 Subject: [PATCH 015/505] Skip Block.logDebugInfo() altogether if the log level is more specific than DEBUG, to avoid wasting resources. --- src/main/java/org/qortal/block/Block.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 41adbd78..1f00afc2 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -2010,6 +2010,10 @@ public class Block { private void logDebugInfo() { try { + // Avoid calculations if possible. We have to check against INFO here, since Level.isMoreSpecificThan() confusingly uses <= rather than just < + if (LOGGER.getLevel().isMoreSpecificThan(Level.INFO)) + return; + if (this.repository == null || this.getMinter() == null || this.getBlockData() == null) return; From 428af3c0e8da7d9165d14f05a9e4a666eed1199c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 May 2021 13:57:47 +0100 Subject: [PATCH 016/505] Only use fast sync on trimmed blocks, as others are too large. This could probably be improved to make sure that all blocks in the next request are within the trimmed time range, but it's enough for now. --- src/main/java/org/qortal/controller/Synchronizer.java | 3 ++- src/main/java/org/qortal/data/block/BlockData.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index cba5d3c0..6ceee7f3 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -970,7 +970,8 @@ public class Synchronizer { private SynchronizationResult applyNewBlocks(Repository repository, BlockData commonBlockData, int ourInitialHeight, Peer peer, int peerHeight, List peerBlockSummaries) throws InterruptedException, DataException { - if (Settings.getInstance().isFastSyncEnabled() && peer.getPeersVersion() >= PEER_VERSION_160) + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (Settings.getInstance().isFastSyncEnabled() && peer.getPeersVersion() >= PEER_VERSION_160 && ourLatestBlockData.isTrimmed()) // This peer supports syncing multiple blocks at once via GetBlocksMessage, and it is enabled in the settings return this.applyNewBlocksUsingFastSync(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries); else diff --git a/src/main/java/org/qortal/data/block/BlockData.java b/src/main/java/org/qortal/data/block/BlockData.java index 3567d0f8..e2d6bad1 100644 --- a/src/main/java/org/qortal/data/block/BlockData.java +++ b/src/main/java/org/qortal/data/block/BlockData.java @@ -9,7 +9,10 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import org.qortal.block.BlockChain; +import org.qortal.settings.Settings; import org.qortal.crypto.Crypto; +import org.qortal.utils.NTP; // All properties to be converted to JSON via JAX-RS @XmlAccessorType(XmlAccessType.FIELD) @@ -204,6 +207,13 @@ public class BlockData implements Serializable { return this.onlineAccountsSignatures; } + public boolean isTrimmed() { + long onlineAccountSignaturesTrimmedTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime(); + long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); + long blockTimestamp = this.getTimestamp(); + return blockTimestamp < onlineAccountSignaturesTrimmedTimestamp && blockTimestamp < currentTrimmableTimestamp; + } + // JAXB special @XmlElement(name = "minterAddress") From 255233fe38fc048f04e0e87cef8224fc906c5aa8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 10 May 2021 09:10:14 +0100 Subject: [PATCH 017/505] Reduced log spam. --- src/main/java/org/qortal/network/Handshake.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index 8bee63a2..78b181ce 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -75,7 +75,7 @@ public enum Handshake { // Ensure the peer is running at least the minimum version allowed for connections final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); if (peer.isAtLeastVersion(minPeerVersion) == false) { - LOGGER.info(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); + LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); return null; } } From 688404011b3dd80e4c6de52b7e96457139f5be78 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 May 2021 10:08:50 +0100 Subject: [PATCH 018/505] Relocate FETCH_BLOCKS_TIMEOUT to Peer.java and use a static import. --- src/main/java/org/qortal/controller/Synchronizer.java | 5 +++-- src/main/java/org/qortal/network/Peer.java | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a0ffaff9..185e77a0 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -42,6 +42,8 @@ import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; import org.qortal.utils.NTP; +import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT; + public class Synchronizer { private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class); @@ -63,8 +65,7 @@ public class Synchronizer { /* Minimum peer version that supports syncing multiple blocks at once via GetBlocksMessage */ private static final long PEER_VERSION_160 = 0x0100060000L; - /** Maximum time to wait for a peer to respond with blocks (ms) */ - private static final int FETCH_BLOCKS_TIMEOUT = 10000; + private static Synchronizer instance; diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index e50c18d6..8c364dc7 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -47,6 +47,11 @@ public class Peer { */ private static final int RESPONSE_TIMEOUT = 3000; // ms + /** + * Maximum time to wait for a peer to respond with blocks (ms) + */ + public static final int FETCH_BLOCKS_TIMEOUT = 10000; + /** * Interval between PING messages to a peer. (ms) *

From f58a52eaa4ba6b68b9dcae0a04d2bda56906930b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 May 2021 13:10:38 +0100 Subject: [PATCH 019/505] Further work to increase the response timeout when requesting multiple blocks. --- .../java/org/qortal/controller/Controller.java | 4 +++- src/main/java/org/qortal/network/Peer.java | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 428b9043..96d325f4 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -103,6 +103,8 @@ import org.qortal.utils.Triple; import com.google.common.primitives.Longs; +import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT; + public class Controller extends Thread { static { @@ -1374,7 +1376,7 @@ public class Controller extends Thread { Message blocksMessage = new BlocksMessage(blocks); blocksMessage.setId(message.getId()); - if (!peer.sendMessage(blocksMessage)) + if (!peer.sendMessageWithTimeout(blocksMessage, FETCH_BLOCKS_TIMEOUT)) peer.disconnect("failed to send blocks"); } catch (DataException e) { diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 8c364dc7..c2535118 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -524,12 +524,22 @@ public class Peer { } /** - * Attempt to send Message to peer. + * Attempt to send Message to peer, using default RESPONSE_TIMEOUT. * * @param message message to be sent * @return true if message successfully sent; false otherwise */ public boolean sendMessage(Message message) { + return this.sendMessageWithTimeout(message, RESPONSE_TIMEOUT); + } + + /** + * Attempt to send Message to peer, using custom timeout. + * + * @param message message to be sent + * @return true if message successfully sent; false otherwise + */ + public boolean sendMessageWithTimeout(Message message, int timeout) { if (!this.socketChannel.isOpen()) { return false; } @@ -563,7 +573,7 @@ public class Peer { */ Thread.sleep(1L); //NOSONAR squid:S2276 - if (System.currentTimeMillis() - sendStart > RESPONSE_TIMEOUT) { + if (System.currentTimeMillis() - sendStart > timeout) { // We've taken too long to send this message return false; } @@ -630,7 +640,7 @@ public class Peer { message.setId(id); // Try to send message - if (!this.sendMessage(message)) { + if (!this.sendMessageWithTimeout(message, timeout)) { this.replyQueues.remove(id); return null; } From ed423ed0418a44b2d7842794d37e6513222fb6b4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 May 2021 14:54:13 +0100 Subject: [PATCH 020/505] Increased MAX_DATA_SIZE and SYNC_BATCH_SIZE, to increase the effectiveness of the batch sync. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- src/main/java/org/qortal/network/message/Message.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 185e77a0..c393232d 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -49,7 +49,7 @@ public class Synchronizer { private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class); /** Max number of new blocks we aim to add to chain tip in each sync round */ - private static final int SYNC_BATCH_SIZE = 200; // XXX move to Settings? + private static final int SYNC_BATCH_SIZE = 1000; // XXX move to Settings? /** Initial jump back of block height when searching for common block with peer */ private static final int INITIAL_BLOCK_STEP = 8; diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index d1546dce..07c44c7b 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -25,7 +25,7 @@ public abstract class Message { private static final int MAGIC_LENGTH = 4; private static final int CHECKSUM_LENGTH = 4; - private static final int MAX_DATA_SIZE = 1024 * 1024; // 1MB + private static final int MAX_DATA_SIZE = 10 * 1024 * 1024; // 10MB @SuppressWarnings("serial") public static class MessageException extends Exception { From 2ceba45782724b6b67196ebba21bdfb6bbe65150 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 May 2021 14:57:58 +0100 Subject: [PATCH 021/505] Fast sync default blocks per request increased to 100. --- src/main/java/org/qortal/settings/Settings.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index a9898a87..32308db1 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -138,9 +138,9 @@ public class Settings { /** Whether to sync multiple blocks at once when the peer has a different chain */ private boolean fastSyncEnabledWhenResolvingFork = true; /** Maximum number of blocks to request at once */ - private int maxBlocksPerRequest = 1; + private int maxBlocksPerRequest = 100; /** Maximum number of blocks this node will serve in a single response */ - private int maxBlocksPerResponse = 5; + private int maxBlocksPerResponse = 100; // Which blockchains this node is running private String blockchainConfig = null; // use default from resources From cffbd41f2614644f73750437c28f63a098654868 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Jun 2021 09:09:06 +0100 Subject: [PATCH 022/505] Reduce memory allocations in onNetworkGetBlocksMessage --- .../org/qortal/controller/Controller.java | 40 ++++--------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 96d325f4..2f8cb56d 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1338,40 +1338,16 @@ public class Controller extends Thread { // Ensure that we don't serve more blocks than the amount specified in the settings // Serializing multiple blocks is very slow, so by default we are using a low limit int blockLimitPerRequest = Settings.getInstance().getMaxBlocksPerResponse(); - - List blockDataList = new ArrayList<>(); - - // Attempt to serve from our cache of latest blocks - synchronized (this.latestBlocks) { - blockDataList = this.latestBlocks.stream() - .dropWhile(cachedBlockData -> !Arrays.equals(cachedBlockData.getReference(), parentSignature)) - .map(BlockData::new) - .collect(Collectors.toList()); - } - - if (blockDataList.isEmpty()) { - int numberRequested = Math.min(blockLimitPerRequest, getBlocksMessage.getNumberRequested()); - - BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); - - while (blockData != null && blockDataList.size() < numberRequested) { - blockDataList.add(blockData); - - blockData = repository.getBlockRepository().fromReference(blockData.getSignature()); - } - } else { - this.stats.getBlocksMessageStats.cacheHits.incrementAndGet(); - - if (blockDataList.size() >= getBlocksMessage.getNumberRequested()) - this.stats.getBlocksMessageStats.fullyFromCache.incrementAndGet(); - } + int untrimmedBlockLimitPerRequest = Settings.getInstance().getMaxBlocksPerResponse(); + int numberRequested = Math.min(blockLimitPerRequest, getBlocksMessage.getNumberRequested()); List blocks = new ArrayList<>(); - for (BlockData blockData : blockDataList) { - if (blocks.size() < blockLimitPerRequest) { - Block block = new Block(repository, blockData); - blocks.add(block); - } + BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); + + while (blockData != null && blocks.size() < numberRequested) { + Block block = new Block(repository, blockData); + blocks.add(block); + blockData = repository.getBlockRepository().fromReference(blockData.getSignature()); } Message blocksMessage = new BlocksMessage(blocks); From c63a7884cbb15bab3f2da8adaa947868d6d0e768 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Jun 2021 09:10:25 +0100 Subject: [PATCH 023/505] Limit to 10 untrimmed blocks per response, as they are larger than the trimmed ones. --- src/main/java/org/qortal/controller/Controller.java | 4 ++++ src/main/java/org/qortal/settings/Settings.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 2f8cb56d..8bf8e955 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1345,6 +1345,10 @@ public class Controller extends Thread { BlockData blockData = repository.getBlockRepository().fromReference(parentSignature); while (blockData != null && blocks.size() < numberRequested) { + // If we're dealing with untrimmed blocks, ensure we don't go above the untrimmedBlockLimitPerRequest + if (blockData.isTrimmed() == false && blocks.size() >= untrimmedBlockLimitPerRequest) { + break; + } Block block = new Block(repository, blockData); blocks.add(block); blockData = repository.getBlockRepository().fromReference(blockData.getSignature()); diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 32308db1..184cb0cd 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -141,6 +141,8 @@ public class Settings { private int maxBlocksPerRequest = 100; /** Maximum number of blocks this node will serve in a single response */ private int maxBlocksPerResponse = 100; + /** Maximum number of untrimmed blocks this node will serve in a single response */ + private int maxUntrimmedBlocksPerResponse = 10; // Which blockchains this node is running private String blockchainConfig = null; // use default from resources @@ -474,6 +476,8 @@ public class Settings { public int getMaxBlocksPerResponse() { return this.maxBlocksPerResponse; } + public int getMaxUntrimmedBlocksPerResponse() { return this.maxUntrimmedBlocksPerResponse; } + public boolean isAutoUpdateEnabled() { return this.autoUpdateEnabled; } From bc6b3fb5f4248beaeef615be6516b06f6a0bee9f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Jun 2021 13:04:49 +0100 Subject: [PATCH 024/505] Include timestamps in block-timings.sh --- tools/block-timings.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 5324209b..88d8d643 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -69,11 +69,13 @@ function fetch_and_process_blocks { online_accounts_count=$(echo "${block_minting_info}" | jq -r .onlineAccountsCount) key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio) time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta) + timestamp=$(echo "${block_minting_info}" | jq -r .timestamp) time_offset=$(calculate_time_offset "${key_distance_ratio}") block_time=$((target-deviation+time_offset)) echo "=== BLOCK ${height} ===" + echo "Timestamp: ${timestamp}" echo "Minter level: ${minter_level}" echo "Online accounts: ${online_accounts_count}" echo "Key distance ratio: ${key_distance_ratio}" From 904be3005fea9ab377ff26c02de5f8ecace6fcc0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 12 Jun 2021 13:07:25 +0100 Subject: [PATCH 025/505] Enable fast sync by default. --- src/main/java/org/qortal/settings/Settings.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 184cb0cd..55f421af 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -134,13 +134,13 @@ public class Settings { private boolean allowConnectionsWithOlderPeerVersions = true; /** Whether to sync multiple blocks at once in normal operation */ - private boolean fastSyncEnabled = false; + private boolean fastSyncEnabled = true; /** Whether to sync multiple blocks at once when the peer has a different chain */ private boolean fastSyncEnabledWhenResolvingFork = true; /** Maximum number of blocks to request at once */ private int maxBlocksPerRequest = 100; /** Maximum number of blocks this node will serve in a single response */ - private int maxBlocksPerResponse = 100; + private int maxBlocksPerResponse = 200; /** Maximum number of untrimmed blocks this node will serve in a single response */ private int maxUntrimmedBlocksPerResponse = 10; From 86aab7023c5c8d4daf57724cfd10a340ee370b36 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 14 Jun 2021 19:40:37 +0100 Subject: [PATCH 026/505] Initial settings for data node development. Most to be decided later. --- pom.xml | 4 ++-- src/main/java/org/qortal/settings/Settings.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 8aaa13b5..aac866b7 100644 --- a/pom.xml +++ b/pom.xml @@ -2,8 +2,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 org.qortal - qortal - 1.5.4 + qortal-data + 0.1.0 jar true diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 55f421af..4743c9f4 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -29,11 +29,11 @@ import org.qortal.crosschain.Litecoin.LitecoinNet; @XmlAccessorType(XmlAccessType.FIELD) public class Settings { - private static final int MAINNET_LISTEN_PORT = 12392; - private static final int TESTNET_LISTEN_PORT = 62392; + private static final int MAINNET_LISTEN_PORT = 12394; + private static final int TESTNET_LISTEN_PORT = 62394; - private static final int MAINNET_API_PORT = 12391; - private static final int TESTNET_API_PORT = 62391; + private static final int MAINNET_API_PORT = 12393; + private static final int TESTNET_API_PORT = 62393; private static final Logger LOGGER = LogManager.getLogger(Settings.class); private static final String SETTINGS_FILENAME = "settings.json"; From cc59510cd05ffb3f4845ee7d5389ac197a8fd0fc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 15 Jun 2021 09:27:05 +0100 Subject: [PATCH 027/505] Removed all cross-chain code. It's not needed for data nodes. --- src/main/java/org/qortal/api/ApiService.java | 6 - .../model/CrossChainBitcoinRedeemRequest.java | 34 - .../model/CrossChainBitcoinRefundRequest.java | 31 - .../CrossChainBitcoinTemplateRequest.java | 23 - .../model/CrossChainBitcoinyHTLCStatus.java | 31 - .../api/model/CrossChainBuildRequest.java | 39 - .../api/model/CrossChainCancelRequest.java | 20 - .../model/CrossChainDualSecretRequest.java | 29 - .../api/model/CrossChainOfferSummary.java | 127 -- .../api/model/CrossChainSecretRequest.java | 26 - .../api/model/CrossChainTradeRequest.java | 23 - .../api/model/CrossChainTradeSummary.java | 54 - .../model/crosschain/BitcoinSendRequest.java | 29 - .../model/crosschain/LitecoinSendRequest.java | 29 - .../crosschain/TradeBotCreateRequest.java | 46 - .../crosschain/TradeBotRespondRequest.java | 29 - .../qortal/api/resource/ApiDefinition.java | 3 +- .../CrossChainBitcoinACCTv1Resource.java | 363 ----- .../resource/CrossChainBitcoinResource.java | 167 --- .../api/resource/CrossChainHtlcResource.java | 603 -------- .../CrossChainLitecoinACCTv1Resource.java | 145 -- .../resource/CrossChainLitecoinResource.java | 167 --- .../api/resource/CrossChainResource.java | 424 ------ .../resource/CrossChainTradeBotResource.java | 286 ---- .../api/websocket/PresenceWebSocket.java | 244 ---- .../api/websocket/TradeBotWebSocket.java | 157 -- .../api/websocket/TradeOffersWebSocket.java | 351 ----- .../org/qortal/at/QortalFunctionCode.java | 15 - .../org/qortal/controller/Controller.java | 4 - .../controller/tradebot/AcctTradeBot.java | 30 - .../tradebot/BitcoinACCTv1TradeBot.java | 1273 ----------------- .../tradebot/LitecoinACCTv1TradeBot.java | 894 ------------ .../qortal/controller/tradebot/TradeBot.java | 373 ----- src/main/java/org/qortal/crosschain/ACCT.java | 23 - .../java/org/qortal/crosschain/AcctMode.java | 21 - .../java/org/qortal/crosschain/Bitcoin.java | 190 --- .../org/qortal/crosschain/BitcoinACCTv1.java | 921 ------------ .../java/org/qortal/crosschain/Bitcoiny.java | 740 ---------- .../BitcoinyBlockchainProvider.java | 40 - .../org/qortal/crosschain/BitcoinyHTLC.java | 438 ------ .../crosschain/BitcoinyTransaction.java | 146 -- .../java/org/qortal/crosschain/ElectrumX.java | 688 --------- .../qortal/crosschain/ForeignBlockchain.java | 9 - .../ForeignBlockchainException.java | 77 - .../java/org/qortal/crosschain/Litecoin.java | 175 --- .../org/qortal/crosschain/LitecoinACCTv1.java | 853 ----------- .../qortal/crosschain/SimpleTransaction.java | 32 - .../crosschain/SupportedBlockchain.java | 113 -- .../qortal/crosschain/TransactionHash.java | 31 - .../org/qortal/crosschain/UnspentOutput.java | 16 - .../data/crosschain/CrossChainTradeData.java | 109 -- .../qortal/data/crosschain/TradeBotData.java | 268 ---- .../transaction/PresenceTransactionData.java | 73 - .../data/transaction/TransactionData.java | 2 +- .../repository/CrossChainRepository.java | 21 - .../org/qortal/repository/Repository.java | 2 - .../hsqldb/HSQLDBCrossChainRepository.java | 202 --- .../hsqldb/HSQLDBDatabaseUpdates.java | 51 - .../repository/hsqldb/HSQLDBRepository.java | 70 +- .../HSQLDBPresenceTransactionRepository.java | 57 - .../java/org/qortal/settings/Settings.java | 12 - .../transaction/PresenceTransaction.java | 256 ---- .../PresenceTransactionTransformer.java | 108 -- .../java/org/qortal/test/PresenceTests.java | 133 -- .../java/org/qortal/test/RepositoryTests.java | 20 - .../qortal/test/api/CrossChainApiTests.java | 42 - .../qortal/test/crosschain/BitcoinTests.java | 115 -- .../test/crosschain/ElectrumXTests.java | 201 --- .../org/qortal/test/crosschain/HtlcTests.java | 128 -- .../qortal/test/crosschain/LitecoinTests.java | 114 -- .../test/crosschain/apps/BuildHTLC.java | 114 -- .../test/crosschain/apps/CheckHTLC.java | 135 -- .../qortal/test/crosschain/apps/Common.java | 158 -- .../apps/GetNextReceiveAddress.java | 78 - .../test/crosschain/apps/GetTransaction.java | 84 -- .../apps/GetWalletTransactions.java | 82 -- .../org/qortal/test/crosschain/apps/Pay.java | 80 -- .../test/crosschain/apps/RedeemHTLC.java | 166 --- .../test/crosschain/apps/RefundHTLC.java | 163 --- .../bitcoinv1/BitcoinACCTv1Tests.java | 795 ---------- .../test/crosschain/bitcoinv1/DeployAT.java | 169 --- .../test/crosschain/litecoinv1/DeployAT.java | 150 -- .../litecoinv1/LitecoinACCTv1Tests.java | 770 ---------- .../litecoinv1/SendCancelMessage.java | 90 -- .../litecoinv1/SendRedeemMessage.java | 101 -- .../litecoinv1/SendTradeMessage.java | 118 -- 86 files changed, 5 insertions(+), 15820 deletions(-) delete mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainBuildRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainCancelRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainOfferSummary.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainSecretRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainTradeRequest.java delete mode 100644 src/main/java/org/qortal/api/model/CrossChainTradeSummary.java delete mode 100644 src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java delete mode 100644 src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java delete mode 100644 src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java delete mode 100644 src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java delete mode 100644 src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java delete mode 100644 src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java delete mode 100644 src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java delete mode 100644 src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java delete mode 100644 src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java delete mode 100644 src/main/java/org/qortal/api/resource/CrossChainResource.java delete mode 100644 src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java delete mode 100644 src/main/java/org/qortal/api/websocket/PresenceWebSocket.java delete mode 100644 src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java delete mode 100644 src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java delete mode 100644 src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java delete mode 100644 src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java delete mode 100644 src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java delete mode 100644 src/main/java/org/qortal/controller/tradebot/TradeBot.java delete mode 100644 src/main/java/org/qortal/crosschain/ACCT.java delete mode 100644 src/main/java/org/qortal/crosschain/AcctMode.java delete mode 100644 src/main/java/org/qortal/crosschain/Bitcoin.java delete mode 100644 src/main/java/org/qortal/crosschain/BitcoinACCTv1.java delete mode 100644 src/main/java/org/qortal/crosschain/Bitcoiny.java delete mode 100644 src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java delete mode 100644 src/main/java/org/qortal/crosschain/BitcoinyHTLC.java delete mode 100644 src/main/java/org/qortal/crosschain/BitcoinyTransaction.java delete mode 100644 src/main/java/org/qortal/crosschain/ElectrumX.java delete mode 100644 src/main/java/org/qortal/crosschain/ForeignBlockchain.java delete mode 100644 src/main/java/org/qortal/crosschain/ForeignBlockchainException.java delete mode 100644 src/main/java/org/qortal/crosschain/Litecoin.java delete mode 100644 src/main/java/org/qortal/crosschain/LitecoinACCTv1.java delete mode 100644 src/main/java/org/qortal/crosschain/SimpleTransaction.java delete mode 100644 src/main/java/org/qortal/crosschain/SupportedBlockchain.java delete mode 100644 src/main/java/org/qortal/crosschain/TransactionHash.java delete mode 100644 src/main/java/org/qortal/crosschain/UnspentOutput.java delete mode 100644 src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java delete mode 100644 src/main/java/org/qortal/data/crosschain/TradeBotData.java delete mode 100644 src/main/java/org/qortal/data/transaction/PresenceTransactionData.java delete mode 100644 src/main/java/org/qortal/repository/CrossChainRepository.java delete mode 100644 src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java delete mode 100644 src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java delete mode 100644 src/main/java/org/qortal/transaction/PresenceTransaction.java delete mode 100644 src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java delete mode 100644 src/test/java/org/qortal/test/PresenceTests.java delete mode 100644 src/test/java/org/qortal/test/api/CrossChainApiTests.java delete mode 100644 src/test/java/org/qortal/test/crosschain/BitcoinTests.java delete mode 100644 src/test/java/org/qortal/test/crosschain/ElectrumXTests.java delete mode 100644 src/test/java/org/qortal/test/crosschain/HtlcTests.java delete mode 100644 src/test/java/org/qortal/test/crosschain/LitecoinTests.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/Common.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/Pay.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java delete mode 100644 src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java delete mode 100644 src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java delete mode 100644 src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java delete mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java delete mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java delete mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java delete mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java delete mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index 5baf2c5d..b88edb5a 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -43,9 +43,6 @@ import org.qortal.api.websocket.ActiveChatsWebSocket; import org.qortal.api.websocket.AdminStatusWebSocket; import org.qortal.api.websocket.BlocksWebSocket; import org.qortal.api.websocket.ChatMessagesWebSocket; -import org.qortal.api.websocket.PresenceWebSocket; -import org.qortal.api.websocket.TradeBotWebSocket; -import org.qortal.api.websocket.TradeOffersWebSocket; import org.qortal.settings.Settings; public class ApiService { @@ -199,9 +196,6 @@ public class ApiService { context.addServlet(BlocksWebSocket.class, "/websockets/blocks"); context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*"); context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages"); - context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers"); - context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot"); - context.addServlet(PresenceWebSocket.class, "/websockets/presence"); // Start server this.server.start(); diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java deleted file mode 100644 index 074fd24d..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.qortal.api.model; - -import java.math.BigDecimal; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainBitcoinRedeemRequest { - - @Schema(description = "Bitcoin HASH160(public key) for refund", example = "2nGDBPPPFS1c9w1h33YwFk4KUJU2") - public byte[] refundPublicKeyHash; - - @Schema(description = "Bitcoin PRIVATE KEY for redeem", example = "cUvGNSnu14q6Hr1X7TESjYVTqBpFjj8GGLGjGdpJwD9NhSQKeYUk") - public byte[] redeemPrivateKey; - - @Schema(description = "Qortal AT address") - public String atAddress; - - @Schema(description = "Bitcoin miner fee", example = "0.00001000") - public BigDecimal bitcoinMinerFee; - - @Schema(description = "32-byte secret", example = "6gVbAXCVzJXAWwtAVGAfgAkkXpeXvPUwSciPmCfSfXJG") - public byte[] secret; - - @Schema(description = "Bitcoin HASH160(public key) for receiving funds, or omit to derive from private key", example = "u17kBVKkKSp12oUzaxFwNnq1JZf") - public byte[] receivingAccountInfo; - - public CrossChainBitcoinRedeemRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java deleted file mode 100644 index f2485389..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.qortal.api.model; - -import java.math.BigDecimal; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainBitcoinRefundRequest { - - @Schema(description = "Bitcoin PRIVATE KEY for refund", example = "cSP3zTb6bfm8GATtAcEJ8LqYtNQmzZ9jE2wQUVnZGiBzojDdrwKV") - public byte[] refundPrivateKey; - - @Schema(description = "Bitcoin HASH160(public key) for redeem", example = "2daMveGc5pdjRyFacbxBzMksCbyC") - public byte[] redeemPublicKeyHash; - - @Schema(description = "Qortal AT address") - public String atAddress; - - @Schema(description = "Bitcoin miner fee", example = "0.00001000") - public BigDecimal bitcoinMinerFee; - - @Schema(description = "Bitcoin HASH160(public key) for receiving funds, or omit to derive from private key", example = "u17kBVKkKSp12oUzaxFwNnq1JZf") - public byte[] receivingAccountInfo; - - public CrossChainBitcoinRefundRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java deleted file mode 100644 index b7510eaa..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.qortal.api.model; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainBitcoinTemplateRequest { - - @Schema(description = "Bitcoin HASH160(public key) for refund", example = "2nGDBPPPFS1c9w1h33YwFk4KUJU2") - public byte[] refundPublicKeyHash; - - @Schema(description = "Bitcoin HASH160(public key) for redeem", example = "2daMveGc5pdjRyFacbxBzMksCbyC") - public byte[] redeemPublicKeyHash; - - @Schema(description = "Qortal AT address") - public String atAddress; - - public CrossChainBitcoinTemplateRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java deleted file mode 100644 index 2772eae1..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.qortal.api.model; - -import java.math.BigDecimal; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainBitcoinyHTLCStatus { - - @Schema(description = "P2SH address", example = "3CdH27kTpV8dcFHVRYjQ8EEV5FJg9X8pSJ (mainnet), 2fMiRRXVsxhZeyfum9ifybZvaMHbQTmwdZw (testnet)") - public String bitcoinP2shAddress; - - @Schema(description = "P2SH balance") - public BigDecimal bitcoinP2shBalance; - - @Schema(description = "Can HTLC redeem yet?") - public boolean canRedeem; - - @Schema(description = "Can HTLC refund yet?") - public boolean canRefund; - - @Schema(description = "Secret used by HTLC redeemer") - public byte[] secret; - - public CrossChainBitcoinyHTLCStatus() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java b/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java deleted file mode 100644 index e8d38703..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.qortal.api.model; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainBuildRequest { - - @Schema(description = "AT creator's public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") - public byte[] creatorPublicKey; - - @Schema(description = "Final QORT amount paid out on successful trade", example = "80.40200000") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long qortAmount; - - @Schema(description = "QORT amount funding AT, including covering AT execution fees", example = "123.45670000") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long fundingQortAmount; - - @Schema(description = "HASH160 of creator's Bitcoin public key", example = "2daMveGc5pdjRyFacbxBzMksCbyC") - public byte[] bitcoinPublicKeyHash; - - @Schema(description = "HASH160 of secret", example = "43vnftqkjxrhb5kJdkU1ZFQLEnWV") - public byte[] hashOfSecretB; - - @Schema(description = "Bitcoin P2SH BTC balance for release of secret", example = "0.00864200") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long bitcoinAmount; - - @Schema(description = "Trade time window (minutes) from trade agreement to automatic refund", example = "10080") - public Integer tradeTimeout; - - public CrossChainBuildRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java b/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java deleted file mode 100644 index 25a18952..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.qortal.api.model; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainCancelRequest { - - @Schema(description = "AT creator's public key", example = "K6wuddsBV3HzRrXFFezE7P5MoRXp5m3mEDokRDGZB6ry") - public byte[] creatorPublicKey; - - @Schema(description = "Qortal trade AT address") - public String atAddress; - - public CrossChainCancelRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java b/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java deleted file mode 100644 index b6705d5d..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.qortal.api.model; - -import io.swagger.v3.oas.annotations.media.Schema; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainDualSecretRequest { - - @Schema(description = "Public key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") - public byte[] partnerPublicKey; - - @Schema(description = "Qortal AT address") - public String atAddress; - - @Schema(description = "secret-A (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1") - public byte[] secretA; - - @Schema(description = "secret-B (32 bytes)", example = "EN2Bgx3BcEMtxFCewmCVSMkfZjVKYhx3KEXC5A21KBGx") - public byte[] secretB; - - @Schema(description = "Qortal address for receiving QORT from AT") - public String receivingAddress; - - public CrossChainDualSecretRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java b/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java deleted file mode 100644 index bf71c2d2..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.qortal.api.model; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import org.qortal.crosschain.AcctMode; -import org.qortal.data.crosschain.CrossChainTradeData; - -import io.swagger.v3.oas.annotations.media.Schema; - -// All properties to be converted to JSON via JAXB -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainOfferSummary { - - // Properties - - @Schema(description = "AT's Qortal address") - private String qortalAtAddress; - - @Schema(description = "AT creator's Qortal address") - private String qortalCreator; - - @Schema(description = "AT creator's ephemeral trading key-pair represented as Qortal address") - private String qortalCreatorTradeAddress; - - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long qortAmount; - - @Schema(description = "Bitcoin amount - DEPRECATED: use foreignAmount") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - @Deprecated - private long btcAmount; - - @Schema(description = "Foreign blockchain amount") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long foreignAmount; - - @Schema(description = "Suggested trade timeout (minutes)", example = "10080") - private int tradeTimeout; - - @Schema(description = "Current AT execution mode") - private AcctMode mode; - - private long timestamp; - - @Schema(description = "Trade partner's Qortal receiving address") - private String partnerQortalReceivingAddress; - - private String foreignBlockchain; - - private String acctName; - - protected CrossChainOfferSummary() { - /* For JAXB */ - } - - public CrossChainOfferSummary(CrossChainTradeData crossChainTradeData, long timestamp) { - this.qortalAtAddress = crossChainTradeData.qortalAtAddress; - this.qortalCreator = crossChainTradeData.qortalCreator; - this.qortalCreatorTradeAddress = crossChainTradeData.qortalCreatorTradeAddress; - this.qortAmount = crossChainTradeData.qortAmount; - this.foreignAmount = crossChainTradeData.expectedForeignAmount; - this.btcAmount = this.foreignAmount; // Duplicate for deprecated field - this.tradeTimeout = crossChainTradeData.tradeTimeout; - this.mode = crossChainTradeData.mode; - this.timestamp = timestamp; - this.partnerQortalReceivingAddress = crossChainTradeData.qortalPartnerReceivingAddress; - this.foreignBlockchain = crossChainTradeData.foreignBlockchain; - this.acctName = crossChainTradeData.acctName; - } - - public String getQortalAtAddress() { - return this.qortalAtAddress; - } - - public String getQortalCreator() { - return this.qortalCreator; - } - - public String getQortalCreatorTradeAddress() { - return this.qortalCreatorTradeAddress; - } - - public long getQortAmount() { - return this.qortAmount; - } - - public long getBtcAmount() { - return this.btcAmount; - } - - public long getForeignAmount() { - return this.foreignAmount; - } - - public int getTradeTimeout() { - return this.tradeTimeout; - } - - public AcctMode getMode() { - return this.mode; - } - - public long getTimestamp() { - return this.timestamp; - } - - public String getPartnerQortalReceivingAddress() { - return this.partnerQortalReceivingAddress; - } - - public String getForeignBlockchain() { - return this.foreignBlockchain; - } - - public String getAcctName() { - return this.acctName; - } - - // For debugging mostly - - public String toString() { - return String.format("%s: %s", this.qortalAtAddress, this.mode); - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java b/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java deleted file mode 100644 index 2db475e5..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.qortal.api.model; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainSecretRequest { - - @Schema(description = "Private key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") - public byte[] partnerPrivateKey; - - @Schema(description = "Qortal AT address") - public String atAddress; - - @Schema(description = "Secret (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1") - public byte[] secret; - - @Schema(description = "Qortal address for receiving QORT from AT") - public String receivingAddress; - - public CrossChainSecretRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java b/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java deleted file mode 100644 index 1afd7290..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.qortal.api.model; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainTradeRequest { - - @Schema(description = "AT creator's 'trade' public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") - public byte[] tradePublicKey; - - @Schema(description = "Qortal AT address") - public String atAddress; - - @Schema(description = "Signature of trading partner's 'offer' MESSAGE transaction") - public byte[] messageTransactionSignature; - - public CrossChainTradeRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java b/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java deleted file mode 100644 index 274dd818..00000000 --- a/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.qortal.api.model; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import org.qortal.data.crosschain.CrossChainTradeData; - -import io.swagger.v3.oas.annotations.media.Schema; - -// All properties to be converted to JSON via JAXB -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainTradeSummary { - - private long tradeTimestamp; - - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long qortAmount; - - @Deprecated - @Schema(description = "DEPRECATED: use foreignAmount instead") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long btcAmount; - - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long foreignAmount; - - protected CrossChainTradeSummary() { - /* For JAXB */ - } - - public CrossChainTradeSummary(CrossChainTradeData crossChainTradeData, long timestamp) { - this.tradeTimestamp = timestamp; - this.qortAmount = crossChainTradeData.qortAmount; - this.foreignAmount = crossChainTradeData.expectedForeignAmount; - this.btcAmount = this.foreignAmount; - } - - public long getTradeTimestamp() { - return this.tradeTimestamp; - } - - public long getQortAmount() { - return this.qortAmount; - } - - public long getBtcAmount() { - return this.btcAmount; - } - - public long getForeignAmount() { - return this.foreignAmount; - } -} diff --git a/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java b/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java deleted file mode 100644 index 86d3d7c8..00000000 --- a/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.qortal.api.model.crosschain; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class BitcoinSendRequest { - - @Schema(description = "Bitcoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________") - public String xprv58; - - @Schema(description = "Recipient's Bitcoin address ('legacy' P2PKH only)", example = "1BitcoinEaterAddressDontSendf59kuE") - public String receivingAddress; - - @Schema(description = "Amount of BTC to send", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long bitcoinAmount; - - @Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 BTC (100 sats) per byte", example = "0.00000100", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public Long feePerByte; - - public BitcoinSendRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java b/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java deleted file mode 100644 index 5f215740..00000000 --- a/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.qortal.api.model.crosschain; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class LitecoinSendRequest { - - @Schema(description = "Litecoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________") - public String xprv58; - - @Schema(description = "Recipient's Litecoin address ('legacy' P2PKH only)", example = "LiTecoinEaterAddressDontSendhLfzKD") - public String receivingAddress; - - @Schema(description = "Amount of LTC to send", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long litecoinAmount; - - @Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 LTC (100 sats) per byte", example = "0.00000100", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public Long feePerByte; - - public LitecoinSendRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java b/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java deleted file mode 100644 index 1f96488e..00000000 --- a/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.qortal.api.model.crosschain; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import org.qortal.crosschain.SupportedBlockchain; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class TradeBotCreateRequest { - - @Schema(description = "Trade creator's public key", example = "2zR1WFsbM7akHghqSCYKBPk6LDP8aKiQSRS1FrwoLvoB") - public byte[] creatorPublicKey; - - @Schema(description = "QORT amount paid out on successful trade", example = "80.40000000", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long qortAmount; - - @Schema(description = "QORT amount funding AT, including covering AT execution fees", example = "80.50000000", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long fundingQortAmount; - - @Deprecated - @Schema(description = "Bitcoin amount wanted in return. DEPRECATED: use foreignAmount instead", example = "0.00864200", type = "number", hidden = true) - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public Long bitcoinAmount; - - @Schema(description = "Foreign blockchain. Note: default (BITCOIN) to be removed in the future", example = "BITCOIN", implementation = SupportedBlockchain.class) - public SupportedBlockchain foreignBlockchain; - - @Schema(description = "Foreign blockchain amount wanted in return", example = "0.00864200", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public Long foreignAmount; - - @Schema(description = "Suggested trade timeout (minutes)", example = "10080") - public int tradeTimeout; - - @Schema(description = "Foreign blockchain address for receiving", example = "1BitcoinEaterAddressDontSendf59kuE") - public String receivingAddress; - - public TradeBotCreateRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java b/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java deleted file mode 100644 index ecc8ed6f..00000000 --- a/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.qortal.api.model.crosschain; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import io.swagger.v3.oas.annotations.media.Schema; - -@XmlAccessorType(XmlAccessType.FIELD) -public class TradeBotRespondRequest { - - @Schema(description = "Qortal AT address", example = "Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - public String atAddress; - - @Deprecated - @Schema(description = "Bitcoin BIP32 extended private key. DEPRECATED: use foreignKey instead", hidden = true, - example = "xprv___________________________________________________________________________________________________________") - public String xprv58; - - @Schema(description = "Foreign blockchain private key, e.g. BIP32 'm' key for Bitcoin/Litecoin starting with 'xprv'", - example = "xprv___________________________________________________________________________________________________________") - public String foreignKey; - - @Schema(description = "Qortal address for receiving QORT from AT", example = "Qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq") - public String receivingAddress; - - public TradeBotRespondRequest() { - } - -} diff --git a/src/main/java/org/qortal/api/resource/ApiDefinition.java b/src/main/java/org/qortal/api/resource/ApiDefinition.java index f9ec7459..fa27bfbb 100644 --- a/src/main/java/org/qortal/api/resource/ApiDefinition.java +++ b/src/main/java/org/qortal/api/resource/ApiDefinition.java @@ -22,7 +22,6 @@ import org.qortal.api.Security; @Tag(name = "Automated Transactions"), @Tag(name = "Blocks"), @Tag(name = "Chat"), - @Tag(name = "Cross-Chain"), @Tag(name = "Groups"), @Tag(name = "Names"), @Tag(name = "Payments"), @@ -41,4 +40,4 @@ import org.qortal.api.Security; @SecurityScheme(name = "apiKey", type = SecuritySchemeType.APIKEY, in = SecuritySchemeIn.HEADER, paramName = Security.API_KEY_HEADER) }) public class ApiDefinition { -} \ No newline at end of file +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java deleted file mode 100644 index 20a27241..00000000 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java +++ /dev/null @@ -1,363 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; - -import java.util.Arrays; -import java.util.Random; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; - -import org.qortal.account.PublicKeyAccount; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; -import org.qortal.api.model.CrossChainBuildRequest; -import org.qortal.api.model.CrossChainDualSecretRequest; -import org.qortal.api.model.CrossChainTradeRequest; -import org.qortal.asset.Asset; -import org.qortal.crosschain.BitcoinACCTv1; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.AcctMode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction; -import org.qortal.transaction.Transaction.TransactionType; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.TransformationException; -import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.DeployAtTransactionTransformer; -import org.qortal.transform.transaction.MessageTransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -@Path("/crosschain/BitcoinACCTv1") -@Tag(name = "Cross-Chain (BitcoinACCTv1)") -public class CrossChainBitcoinACCTv1Resource { - - @Context - HttpServletRequest request; - - @POST - @Path("/build") - @Operation( - summary = "Build Bitcoin cross-chain trading AT", - description = "Returns raw, unsigned DEPLOY_AT transaction", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainBuildRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_DATA, ApiError.INVALID_REFERENCE, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) - public String buildTrade(CrossChainBuildRequest tradeRequest) { - Security.checkApiCallAllowed(request); - - byte[] creatorPublicKey = tradeRequest.creatorPublicKey; - - if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (tradeRequest.hashOfSecretB == null || tradeRequest.hashOfSecretB.length != Bitcoiny.HASH160_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.tradeTimeout == null) - tradeRequest.tradeTimeout = 7 * 24 * 60; // 7 days - else - if (tradeRequest.tradeTimeout < 10 || tradeRequest.tradeTimeout > 50000) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.qortAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.fundingQortAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - // funding amount must exceed initial + final - if (tradeRequest.fundingQortAmount <= tradeRequest.qortAmount) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (tradeRequest.bitcoinAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - try (final Repository repository = RepositoryManager.getRepository()) { - PublicKeyAccount creatorAccount = new PublicKeyAccount(repository, creatorPublicKey); - - byte[] creationBytes = BitcoinACCTv1.buildQortalAT(creatorAccount.getAddress(), tradeRequest.bitcoinPublicKeyHash, tradeRequest.hashOfSecretB, - tradeRequest.qortAmount, tradeRequest.bitcoinAmount, tradeRequest.tradeTimeout); - - long txTimestamp = NTP.getTime(); - byte[] lastReference = creatorAccount.getLastReference(); - if (lastReference == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_REFERENCE); - - long fee = 0; - String name = "QORT-BTC cross-chain trade"; - String description = "Qortal-Bitcoin cross-chain trade"; - String atType = "ACCT"; - String tags = "QORT-BTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, creatorAccount.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, tradeRequest.fundingQortAmount, Asset.QORT); - - Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - ValidationResult result = deployAtTransaction.isValidUnconfirmed(); - if (result != ValidationResult.OK) - throw TransactionsResource.createTransactionInvalidException(request, result); - - byte[] bytes = DeployAtTransactionTransformer.toBytes(deployAtTransactionData); - return Base58.encode(bytes); - } catch (TransformationException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/trademessage") - @Operation( - summary = "Builds raw, unsigned 'trade' MESSAGE transaction that sends cross-chain trade recipient address, triggering 'trade' mode", - description = "Specify address of cross-chain AT that needs to be messaged, and signature of 'offer' MESSAGE from trade partner.
" - + "AT needs to be in 'offer' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
" - + "You need to sign output with trade private key otherwise the MESSAGE transaction will be invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainTradeRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content( - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public String buildTradeMessage(CrossChainTradeRequest tradeRequest) { - Security.checkApiCallAllowed(request); - - byte[] tradePublicKey = tradeRequest.tradePublicKey; - - if (tradePublicKey == null || tradePublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (tradeRequest.atAddress == null || !Crypto.isValidAtAddress(tradeRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (tradeRequest.messageTransactionSignature == null || !Crypto.isValidAddress(tradeRequest.messageTransactionSignature)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, tradeRequest.atAddress); - CrossChainTradeData crossChainTradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - - if (crossChainTradeData.mode != AcctMode.OFFERING) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Does supplied public key match trade public key? - if (!Crypto.toAddress(tradePublicKey).equals(crossChainTradeData.qortalCreatorTradeAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - TransactionData transactionData = repository.getTransactionRepository().fromSignature(tradeRequest.messageTransactionSignature); - if (transactionData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_UNKNOWN); - - if (transactionData.getType() != TransactionType.MESSAGE) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); - - MessageTransactionData messageTransactionData = (MessageTransactionData) transactionData; - byte[] messageData = messageTransactionData.getData(); - BitcoinACCTv1.OfferMessageData offerMessageData = BitcoinACCTv1.extractOfferMessageData(messageData); - if (offerMessageData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); - - // Good to make MESSAGE - - byte[] aliceForeignPublicKeyHash = offerMessageData.partnerBitcoinPKH; - byte[] hashOfSecretA = offerMessageData.hashOfSecretA; - int lockTimeA = (int) offerMessageData.lockTimeA; - - String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(messageTransactionData.getTimestamp(), lockTimeA); - - byte[] outgoingMessageData = BitcoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - byte[] messageTransactionBytes = buildAtMessage(repository, tradePublicKey, tradeRequest.atAddress, outgoingMessageData); - - return Base58.encode(messageTransactionBytes); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/redeemmessage") - @Operation( - summary = "Builds raw, unsigned 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner", - description = "Specify address of cross-chain AT that needs to be messaged, both 32-byte secrets and an address for receiving QORT from AT.
" - + "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
" - + "You need to sign output with account the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainDualSecretRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content( - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public String buildRedeemMessage(CrossChainDualSecretRequest secretRequest) { - Security.checkApiCallAllowed(request); - - byte[] partnerPublicKey = secretRequest.partnerPublicKey; - - if (partnerPublicKey == null || partnerPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (secretRequest.secretA == null || secretRequest.secretA.length != BitcoinACCTv1.SECRET_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (secretRequest.secretB == null || secretRequest.secretB.length != BitcoinACCTv1.SECRET_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress); - CrossChainTradeData crossChainTradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - - if (crossChainTradeData.mode != AcctMode.TRADING) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - String partnerAddress = Crypto.toAddress(partnerPublicKey); - - // MESSAGE must come from address that AT considers trade partner - if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - // Good to make MESSAGE - - byte[] messageData = BitcoinACCTv1.buildRedeemMessage(secretRequest.secretA, secretRequest.secretB, secretRequest.receivingAddress); - byte[] messageTransactionBytes = buildAtMessage(repository, partnerPublicKey, secretRequest.atAddress, messageData); - - return Base58.encode(messageTransactionBytes); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - // Must be correct AT - check functionality using code hash - if (!Arrays.equals(atData.getCodeHash(), BitcoinACCTv1.CODE_BYTES_HASH)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // No point sending message to AT that's finished - if (atData.getIsFinished()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - return atData; - } - - private byte[] buildAtMessage(Repository repository, byte[] senderPublicKey, String atAddress, byte[] messageData) throws DataException { - long txTimestamp = NTP.getTime(); - - // senderPublicKey could be ephemeral trade public key where there is no corresponding account and hence no reference - String senderAddress = Crypto.toAddress(senderPublicKey); - byte[] lastReference = repository.getAccountRepository().getLastReference(senderAddress); - final boolean requiresPoW = lastReference == null; - - if (requiresPoW) { - Random random = new Random(); - lastReference = new byte[Transformer.SIGNATURE_LENGTH]; - random.nextBytes(lastReference); - } - - int version = 4; - int nonce = 0; - long amount = 0L; - Long assetId = null; // no assetId as amount is zero - Long fee = 0L; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, senderPublicKey, fee, null); - TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, atAddress, amount, assetId, messageData, false, false); - - MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); - - if (requiresPoW) { - messageTransaction.computeNonce(); - } else { - fee = messageTransaction.calcRecommendedFee(); - messageTransactionData.setFee(fee); - } - - ValidationResult result = messageTransaction.isValidUnconfirmed(); - if (result != ValidationResult.OK) - throw TransactionsResource.createTransactionInvalidException(request, result); - - try { - return MessageTransactionTransformer.toBytes(messageTransactionData); - } catch (TransformationException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java deleted file mode 100644 index 2c1c6991..00000000 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.media.ArraySchema; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; - -import java.util.List; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; - -import org.bitcoinj.core.Transaction; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; -import org.qortal.api.model.crosschain.BitcoinSendRequest; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.SimpleTransaction; - -@Path("/crosschain/btc") -@Tag(name = "Cross-Chain (Bitcoin)") -public class CrossChainBitcoinResource { - - @Context - HttpServletRequest request; - - @POST - @Path("/walletbalance") - @Operation( - summary = "Returns BTC balance for hierarchical, deterministic BIP32 wallet", - description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", - description = "BIP32 'm' private/public key in base58", - example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) - public String getBitcoinWalletBalance(String key58) { - Security.checkApiCallAllowed(request); - - Bitcoin bitcoin = Bitcoin.getInstance(); - - if (!bitcoin.isValidDeterministicKey(key58)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - Long balance = bitcoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - - return balance.toString(); - } - - @POST - @Path("/wallettransactions") - @Operation( - summary = "Returns transactions for hierarchical, deterministic BIP32 wallet", - description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", - description = "BIP32 'm' private/public key in base58", - example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) ) - ) - } - ) - @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) - public List getBitcoinWalletTransactions(String key58) { - Security.checkApiCallAllowed(request); - - Bitcoin bitcoin = Bitcoin.getInstance(); - - if (!bitcoin.isValidDeterministicKey(key58)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - try { - return bitcoin.getWalletTransactions(key58); - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } - } - - @POST - @Path("/send") - @Operation( - summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address", - description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = BitcoinSendRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) - public String sendBitcoin(BitcoinSendRequest bitcoinSendRequest) { - Security.checkApiCallAllowed(request); - - if (bitcoinSendRequest.bitcoinAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - if (bitcoinSendRequest.feePerByte != null && bitcoinSendRequest.feePerByte <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - Bitcoin bitcoin = Bitcoin.getInstance(); - - if (!bitcoin.isValidAddress(bitcoinSendRequest.receivingAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (!bitcoin.isValidDeterministicKey(bitcoinSendRequest.xprv58)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - Transaction spendTransaction = bitcoin.buildSpend(bitcoinSendRequest.xprv58, - bitcoinSendRequest.receivingAddress, - bitcoinSendRequest.bitcoinAmount, - bitcoinSendRequest.feePerByte); - - if (spendTransaction == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); - - try { - bitcoin.broadcastTransaction(spendTransaction); - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } - - return spendTransaction.getTxId().toString(); - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java deleted file mode 100644 index 98e9b01d..00000000 --- a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java +++ /dev/null @@ -1,603 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; - -import java.math.BigDecimal; -import java.util.List; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bitcoinj.core.*; -import org.bitcoinj.script.Script; -import org.qortal.api.*; -import org.qortal.api.model.CrossChainBitcoinyHTLCStatus; -import org.qortal.crosschain.*; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -@Path("/crosschain/htlc") -@Tag(name = "Cross-Chain (Hash time-locked contracts)") -public class CrossChainHtlcResource { - - private static final Logger LOGGER = LogManager.getLogger(CrossChainHtlcResource.class); - - @Context - HttpServletRequest request; - - @GET - @Path("/address/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}") - @Operation( - summary = "Returns HTLC address based on trade info", - description = "Blockchain can be BITCOIN or LITECOIN. Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_CRITERIA}) - public String deriveHtlcAddress(@PathParam("blockchain") String blockchainName, - @PathParam("refundPKH") String refundPKH, - @PathParam("locktime") int lockTime, - @PathParam("redeemPKH") String redeemPKH, - @PathParam("hashOfSecret") String hashOfSecret) { - SupportedBlockchain blockchain = SupportedBlockchain.valueOf(blockchainName); - if (blockchain == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - byte[] refunderPubKeyHash; - byte[] redeemerPubKeyHash; - byte[] decodedHashOfSecret; - - try { - refunderPubKeyHash = Base58.decode(refundPKH); - redeemerPubKeyHash = Base58.decode(redeemPKH); - - if (refunderPubKeyHash.length != 20 || redeemerPubKeyHash.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - } catch (IllegalArgumentException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - } - - try { - decodedHashOfSecret = Base58.decode(hashOfSecret); - if (decodedHashOfSecret.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } catch (IllegalArgumentException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - byte[] redeemScript = BitcoinyHTLC.buildScript(refunderPubKeyHash, lockTime, redeemerPubKeyHash, decodedHashOfSecret); - - Bitcoiny bitcoiny = (Bitcoiny) blockchain.getInstance(); - - return bitcoiny.deriveP2shAddress(redeemScript); - } - - @GET - @Path("/status/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}") - @Operation( - summary = "Checks HTLC status", - description = "Blockchain can be BITCOIN or LITECOIN. Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = CrossChainBitcoinyHTLCStatus.class)) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) - public CrossChainBitcoinyHTLCStatus checkHtlcStatus(@PathParam("blockchain") String blockchainName, - @PathParam("refundPKH") String refundPKH, - @PathParam("locktime") int lockTime, - @PathParam("redeemPKH") String redeemPKH, - @PathParam("hashOfSecret") String hashOfSecret) { - Security.checkApiCallAllowed(request); - - SupportedBlockchain blockchain = SupportedBlockchain.valueOf(blockchainName); - if (blockchain == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - byte[] refunderPubKeyHash; - byte[] redeemerPubKeyHash; - byte[] decodedHashOfSecret; - - try { - refunderPubKeyHash = Base58.decode(refundPKH); - redeemerPubKeyHash = Base58.decode(redeemPKH); - - if (refunderPubKeyHash.length != 20 || redeemerPubKeyHash.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - } catch (IllegalArgumentException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - } - - try { - decodedHashOfSecret = Base58.decode(hashOfSecret); - if (decodedHashOfSecret.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } catch (IllegalArgumentException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - byte[] redeemScript = BitcoinyHTLC.buildScript(refunderPubKeyHash, lockTime, redeemerPubKeyHash, decodedHashOfSecret); - - Bitcoiny bitcoiny = (Bitcoiny) blockchain.getInstance(); - - String p2shAddress = bitcoiny.deriveP2shAddress(redeemScript); - - long now = NTP.getTime(); - - try { - int medianBlockTime = bitcoiny.getMedianBlockTime(); - - // Check P2SH is funded - long p2shBalance = bitcoiny.getConfirmedBalance(p2shAddress.toString()); - - CrossChainBitcoinyHTLCStatus htlcStatus = new CrossChainBitcoinyHTLCStatus(); - htlcStatus.bitcoinP2shAddress = p2shAddress; - htlcStatus.bitcoinP2shBalance = BigDecimal.valueOf(p2shBalance, 8); - - List fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddress.toString()); - - if (p2shBalance > 0L && !fundingOutputs.isEmpty()) { - htlcStatus.canRedeem = now >= medianBlockTime * 1000L; - htlcStatus.canRefund = now >= lockTime * 1000L; - } - - if (now >= medianBlockTime * 1000L) { - // See if we can extract secret - htlcStatus.secret = BitcoinyHTLC.findHtlcSecret(bitcoiny, htlcStatus.bitcoinP2shAddress); - } - - return htlcStatus; - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } - } - - @GET - @Path("/redeem/LITECOIN/{ataddress}/{tradePrivateKey}/{secret}/{receivingAddress}") - @Operation( - summary = "Redeems HTLC associated with supplied AT, using private key, secret, and receiving address", - description = "Secret and private key should be 32 bytes (base58 encoded). Receiving address must be a valid LTC P2PKH address.
" + - "The secret can be found in Alice's trade bot data or in the message to Bob's AT.
" + - "The trade private key and receiving address can be found in Bob's trade bot data.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) - public boolean redeemHtlc(@PathParam("ataddress") String atAddress, - @PathParam("tradePrivateKey") String tradePrivateKey, - @PathParam("secret") String secret, - @PathParam("receivingAddress") String receivingAddress) { - Security.checkApiCallAllowed(request); - - // base58 decode the trade private key - byte[] decodedTradePrivateKey = null; - if (tradePrivateKey != null) - decodedTradePrivateKey = Base58.decode(tradePrivateKey); - - // base58 decode the secret - byte[] decodedSecret = null; - if (secret != null) - decodedSecret = Base58.decode(secret); - - // Convert supplied Litecoin receiving address into public key hash (we only support P2PKH at this time) - Address litecoinReceivingAddress; - try { - litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), receivingAddress); - } catch (AddressFormatException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - if (litecoinReceivingAddress.getOutputScriptType() != Script.ScriptType.P2PKH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); - - return this.doRedeemHtlc(atAddress, decodedTradePrivateKey, decodedSecret, litecoinReceivingAccountInfo); - } - - @GET - @Path("/redeem/LITECOIN/{ataddress}") - @Operation( - summary = "Redeems HTLC associated with supplied AT", - description = "To be used by a QORT seller (Bob) who needs to redeem LTC proceeds that are stuck in a P2SH.
" + - "This requires Bob's trade bot data to be present in the database for this AT.
" + - "It will fail if the buyer has yet to redeem the QORT held in the AT.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) - public boolean redeemHtlc(@PathParam("ataddress") String atAddress) { - Security.checkApiCallAllowed(request); - - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); - if (acct == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - if (crossChainTradeData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Attempt to find secret from the buyer's message to AT - byte[] decodedSecret = LitecoinACCTv1.findSecretA(repository, crossChainTradeData); - if (decodedSecret == null) { - LOGGER.info(() -> String.format("Unable to find secret-A from redeem message to AT %s", atAddress)); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); - TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); - - // Search for the tradePrivateKey in the tradebot data - byte[] decodedPrivateKey = null; - if (tradeBotData != null) - decodedPrivateKey = tradeBotData.getTradePrivateKey(); - - // Search for the litecoin receiving address in the tradebot data - byte[] litecoinReceivingAccountInfo = null; - if (tradeBotData != null) - // Use receiving address PKH from tradebot data - litecoinReceivingAccountInfo = tradeBotData.getReceivingAccountInfo(); - - return this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, litecoinReceivingAccountInfo); - - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @GET - @Path("/redeemAll/LITECOIN") - @Operation( - summary = "Redeems HTLC for all applicable ATs in tradebot data", - description = "To be used by a QORT seller (Bob) who needs to redeem LTC proceeds that are stuck in P2SH transactions.
" + - "This requires Bob's trade bot data to be present in the database for any ATs that need redeeming.
" + - "Returns true if at least one trade is redeemed. More detail is available in the log.txt.* file.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) - public boolean redeemAllHtlc() { - Security.checkApiCallAllowed(request); - boolean success = false; - - try (final Repository repository = RepositoryManager.getRepository()) { - List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); - - for (TradeBotData tradeBotData : allTradeBotData) { - String atAddress = tradeBotData.getAtAddress(); - if (atAddress == null) { - LOGGER.info("Missing AT address in tradebot data", atAddress); - continue; - } - - String tradeState = tradeBotData.getState(); - if (tradeState == null) { - LOGGER.info("Missing trade state for AT {}", atAddress); - continue; - } - - if (tradeState.startsWith("ALICE")) { - LOGGER.info("AT {} isn't redeemable because it is a buy order", atAddress); - continue; - } - - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) { - LOGGER.info("Couldn't find AT with address {}", atAddress); - continue; - } - - ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); - if (acct == null) { - continue; - } - - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - if (crossChainTradeData == null) { - LOGGER.info("Couldn't find crosschain trade data for AT {}", atAddress); - continue; - } - - // Attempt to find secret from the buyer's message to AT - byte[] decodedSecret = LitecoinACCTv1.findSecretA(repository, crossChainTradeData); - if (decodedSecret == null) { - LOGGER.info("Unable to find secret-A from redeem message to AT {}", atAddress); - continue; - } - - // Search for the tradePrivateKey in the tradebot data - byte[] decodedPrivateKey = tradeBotData.getTradePrivateKey(); - - // Search for the litecoin receiving address PKH in the tradebot data - byte[] litecoinReceivingAccountInfo = tradeBotData.getReceivingAccountInfo(); - - try { - LOGGER.info("Attempting to redeem P2SH balance associated with AT {}...", atAddress); - boolean redeemed = this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, litecoinReceivingAccountInfo); - if (redeemed) { - LOGGER.info("Redeemed P2SH balance associated with AT {}", atAddress); - success = true; - } - else { - LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Already redeemed?", atAddress); - } - } catch (ApiException e) { - LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Missing data?", atAddress); - } - } - - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - - return success; - } - - private boolean doRedeemHtlc(String atAddress, byte[] decodedTradePrivateKey, byte[] decodedSecret, byte[] litecoinReceivingAccountInfo) { - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); - if (acct == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - if (crossChainTradeData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Validate trade private key - if (decodedTradePrivateKey == null || decodedTradePrivateKey.length != 32) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Validate secret - if (decodedSecret == null || decodedSecret.length != 32) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Validate receiving address - if (litecoinReceivingAccountInfo == null || litecoinReceivingAccountInfo.length != 20) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Make sure the receiving address isn't a QORT address, given that we can share the same field for both QORT and LTC - if (Crypto.isValidAddress(litecoinReceivingAccountInfo)) - if (Base58.encode(litecoinReceivingAccountInfo).startsWith("Q")) - // This is likely a QORT address, not an LTC - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - - // Use secret-A to redeem P2SH-A - - Litecoin litecoin = Litecoin.getInstance(); - - int lockTime = crossChainTradeData.lockTimeA; - byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTime, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - LOGGER.info(String.format("Redeeming P2SH address: %s", p2shAddressA)); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund - return false; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Double-check that we have redeemed P2SH-A... - return false; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // Wait for AT to auto-refund - return false; - - case FUNDED: { - Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey redeemKey = ECKey.fromPrivate(decodedTradePrivateKey); - List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); - - Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, - fundingOutputs, redeemScriptA, decodedSecret, litecoinReceivingAccountInfo); - - litecoin.broadcastTransaction(p2shRedeemTransaction); - return true; // TODO: validate? - } - } - - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); - } - - return false; - } - - @GET - @Path("/refund/LITECOIN/{ataddress}") - @Operation( - summary = "Refunds HTLC associated with supplied AT", - description = "To be used by a QORT buyer (Alice) who needs to refund their LTC that is stuck in a P2SH.
" + - "This requires Alice's trade bot data to be present in the database for this AT.
" + - "It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) - public boolean refundHtlc(@PathParam("ataddress") String atAddress) { - Security.checkApiCallAllowed(request); - - try (final Repository repository = RepositoryManager.getRepository()) { - List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); - TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); - if (tradeBotData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - if (tradeBotData.getForeignKey() == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Determine LTC receive address for refund - Litecoin litecoin = Litecoin.getInstance(); - String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); - - return this.doRefundHtlc(atAddress, receiveAddress); - - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); - } - } - - @GET - @Path("/refund/LITECOIN/{ataddress}/{receivingAddress}") - @Operation( - summary = "Refunds HTLC associated with supplied AT, to the specified LTC receiving address", - description = "To be used by a QORT buyer (Alice) who needs to refund their LTC that is stuck in a P2SH.
" + - "This requires Alice's trade bot data to be present in the database for this AT.
" + - "It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) - public boolean refundHtlc(@PathParam("ataddress") String atAddress, - @PathParam("receivingAddress") String receivingAddress) { - Security.checkApiCallAllowed(request); - return this.doRefundHtlc(atAddress, receivingAddress); - } - - - private boolean doRefundHtlc(String atAddress, String receiveAddress) { - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); - if (acct == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - if (crossChainTradeData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); - TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); - if (tradeBotData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - - int lockTime = tradeBotData.getLockTimeA(); - - // We can't refund P2SH-A until lockTime-A has passed - if (NTP.getTime() <= lockTime * 1000L) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); - - Litecoin litecoin = Litecoin.getInstance(); - - // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) - int medianBlockTime = litecoin.getMedianBlockTime(); - if (medianBlockTime <= lockTime) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); - - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTime, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - LOGGER.info(String.format("Refunding P2SH address: %s", p2shAddressA)); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // Still waiting for P2SH-A to be funded... - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); - - case REDEEM_IN_PROGRESS: - case REDEEMED: - case REFUND_IN_PROGRESS: - case REFUNDED: - // Too late! - return false; - - case FUNDED:{ - Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); - - // Validate the destination LTC address - Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); - if (receiving.getOutputScriptType() != Script.ScriptType.P2PKH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, - fundingOutputs, redeemScriptA, lockTime, receiving.getHash()); - - litecoin.broadcastTransaction(p2shRefundTransaction); - return true; // TODO: validate? - } - } - - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); - } - - return false; - } - - private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { - return (lockTimeA - tradeTimeout * 60) * 1000L; - } - -} diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java deleted file mode 100644 index 04923133..00000000 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; -import org.qortal.api.model.CrossChainSecretRequest; -import org.qortal.crosschain.AcctMode; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.TransformationException; -import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.MessageTransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import java.util.Arrays; -import java.util.Random; - -@Path("/crosschain/LitecoinACCTv1") -@Tag(name = "Cross-Chain (LitecoinACCTv1)") -public class CrossChainLitecoinACCTv1Resource { - - @Context - HttpServletRequest request; - - @POST - @Path("/redeemmessage") - @Operation( - summary = "Signs and broadcasts a 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner", - description = "Specify address of cross-chain AT that needs to be messaged, Alice's trade private key, the 32-byte secret,
" - + "and an address for receiving QORT from AT. All of these can be found in Alice's trade bot data.
" - + "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
" - + "You need to use the private key that the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainSecretRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content( - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public boolean buildRedeemMessage(CrossChainSecretRequest secretRequest) { - Security.checkApiCallAllowed(request); - - byte[] partnerPrivateKey = secretRequest.partnerPrivateKey; - - if (partnerPrivateKey == null || partnerPrivateKey.length != Transformer.PRIVATE_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (secretRequest.secret == null || secretRequest.secret.length != LitecoinACCTv1.SECRET_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - - if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress); - CrossChainTradeData crossChainTradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - - if (crossChainTradeData.mode != AcctMode.TRADING) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - byte[] partnerPublicKey = new PrivateKeyAccount(null, partnerPrivateKey).getPublicKey(); - String partnerAddress = Crypto.toAddress(partnerPublicKey); - - // MESSAGE must come from address that AT considers trade partner - if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - // Good to make MESSAGE - - byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secretRequest.secret, secretRequest.receivingAddress); - - PrivateKeyAccount sender = new PrivateKeyAccount(repository, partnerPrivateKey); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, secretRequest.atAddress, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); - - return true; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - // Must be correct AT - check functionality using code hash - if (!Arrays.equals(atData.getCodeHash(), LitecoinACCTv1.CODE_BYTES_HASH)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // No point sending message to AT that's finished - if (atData.getIsFinished()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - return atData; - } - -} diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java deleted file mode 100644 index 8883f964..00000000 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.media.ArraySchema; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; - -import java.util.List; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; - -import org.bitcoinj.core.Transaction; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; -import org.qortal.api.model.crosschain.LitecoinSendRequest; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.Litecoin; -import org.qortal.crosschain.SimpleTransaction; - -@Path("/crosschain/ltc") -@Tag(name = "Cross-Chain (Litecoin)") -public class CrossChainLitecoinResource { - - @Context - HttpServletRequest request; - - @POST - @Path("/walletbalance") - @Operation( - summary = "Returns LTC balance for hierarchical, deterministic BIP32 wallet", - description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", - description = "BIP32 'm' private/public key in base58", - example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) - public String getLitecoinWalletBalance(String key58) { - Security.checkApiCallAllowed(request); - - Litecoin litecoin = Litecoin.getInstance(); - - if (!litecoin.isValidDeterministicKey(key58)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - Long balance = litecoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - - return balance.toString(); - } - - @POST - @Path("/wallettransactions") - @Operation( - summary = "Returns transactions for hierarchical, deterministic BIP32 wallet", - description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", - description = "BIP32 'm' private/public key in base58", - example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) ) - ) - } - ) - @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) - public List getLitecoinWalletTransactions(String key58) { - Security.checkApiCallAllowed(request); - - Litecoin litecoin = Litecoin.getInstance(); - - if (!litecoin.isValidDeterministicKey(key58)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - try { - return litecoin.getWalletTransactions(key58); - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } - } - - @POST - @Path("/send") - @Operation( - summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address", - description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = LitecoinSendRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) - public String sendBitcoin(LitecoinSendRequest litecoinSendRequest) { - Security.checkApiCallAllowed(request); - - if (litecoinSendRequest.litecoinAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - if (litecoinSendRequest.feePerByte != null && litecoinSendRequest.feePerByte <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - Litecoin litecoin = Litecoin.getInstance(); - - if (!litecoin.isValidAddress(litecoinSendRequest.receivingAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (!litecoin.isValidDeterministicKey(litecoinSendRequest.xprv58)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - Transaction spendTransaction = litecoin.buildSpend(litecoinSendRequest.xprv58, - litecoinSendRequest.receivingAddress, - litecoinSendRequest.litecoinAmount, - litecoinSendRequest.feePerByte); - - if (spendTransaction == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); - - try { - litecoin.broadcastTransaction(spendTransaction); - } catch (ForeignBlockchainException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } - - return spendTransaction.getTxId().toString(); - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java deleted file mode 100644 index fdd74b7d..00000000 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ /dev/null @@ -1,424 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.ArraySchema; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.security.SecurityRequirement; -import io.swagger.v3.oas.annotations.tags.Tag; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.function.Supplier; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; - -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; -import org.qortal.api.model.CrossChainCancelRequest; -import org.qortal.api.model.CrossChainTradeSummary; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.AcctMode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.TransformationException; -import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.MessageTransactionTransformer; -import org.qortal.utils.Amounts; -import org.qortal.utils.Base58; -import org.qortal.utils.ByteArray; -import org.qortal.utils.NTP; - -@Path("/crosschain") -@Tag(name = "Cross-Chain") -public class CrossChainResource { - - @Context - HttpServletRequest request; - - @GET - @Path("/tradeoffers") - @Operation( - summary = "Find cross-chain trade offers", - responses = { - @ApiResponse( - content = @Content( - array = @ArraySchema( - schema = @Schema( - implementation = CrossChainTradeData.class - ) - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public List getTradeOffers( - @Parameter( - description = "Limit to specific blockchain", - example = "LITECOIN", - schema = @Schema(implementation = SupportedBlockchain.class) - ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain, - @Parameter( ref = "limit") @QueryParam("limit") Integer limit, - @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, - @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { - // Impose a limit on 'limit' - if (limit != null && limit > 100) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - final boolean isExecutable = true; - List crossChainTradesData = new ArrayList<>(); - - try (final Repository repository = RepositoryManager.getRepository()) { - Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); - - for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { - byte[] codeHash = acctInfo.getKey().value; - ACCT acct = acctInfo.getValue().get(); - - List atsData = repository.getATRepository().getATsByFunctionality(codeHash, isExecutable, limit, offset, reverse); - - for (ATData atData : atsData) { - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - crossChainTradesData.add(crossChainTradeData); - } - } - - return crossChainTradesData; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @GET - @Path("/trade/{ataddress}") - @Operation( - summary = "Show detailed trade info", - responses = { - @ApiResponse( - content = @Content( - schema = @Schema( - implementation = CrossChainTradeData.class - ) - ) - ) - } - ) - @ApiErrors({ApiError.ADDRESS_UNKNOWN, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public CrossChainTradeData getTrade(@PathParam("ataddress") String atAddress) { - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); - if (acct == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - return acct.populateTradeData(repository, atData); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @GET - @Path("/trades") - @Operation( - summary = "Find completed cross-chain trades", - description = "Returns summary info about successfully completed cross-chain trades", - responses = { - @ApiResponse( - content = @Content( - array = @ArraySchema( - schema = @Schema( - implementation = CrossChainTradeSummary.class - ) - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public List getCompletedTrades( - @Parameter( - description = "Limit to specific blockchain", - example = "LITECOIN", - schema = @Schema(implementation = SupportedBlockchain.class) - ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain, - @Parameter( - description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)", - example = "1597310000000" - ) @QueryParam("minimumTimestamp") Long minimumTimestamp, - @Parameter( ref = "limit") @QueryParam("limit") Integer limit, - @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, - @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { - // Impose a limit on 'limit' - if (limit != null && limit > 100) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // minimumTimestamp (if given) needs to be positive - if (minimumTimestamp != null && minimumTimestamp <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - final Boolean isFinished = Boolean.TRUE; - - try (final Repository repository = RepositoryManager.getRepository()) { - Integer minimumFinalHeight = null; - - if (minimumTimestamp != null) { - minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp); - - if (minimumFinalHeight == 0) - // We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return - return Collections.emptyList(); - - // height returned from repository is for block BEFORE timestamp - // but we want trades AFTER timestamp so bump height accordingly - minimumFinalHeight++; - } - - List crossChainTrades = new ArrayList<>(); - - Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); - - for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { - byte[] codeHash = acctInfo.getKey().value; - ACCT acct = acctInfo.getValue().get(); - - List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, - isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight, - limit, offset, reverse); - - for (ATStateData atState : atStates) { - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); - - // We also need block timestamp for use as trade timestamp - long timestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); - - CrossChainTradeSummary crossChainTradeSummary = new CrossChainTradeSummary(crossChainTradeData, timestamp); - crossChainTrades.add(crossChainTradeSummary); - } - } - - return crossChainTrades; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @GET - @Path("/price/{blockchain}") - @Operation( - summary = "Request current estimated trading price", - description = "Returns price based on most recent completed trades. Price is expressed in terms of QORT per unit foreign currency.", - responses = { - @ApiResponse( - content = @Content( - schema = @Schema( - type = "number" - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - public long getTradePriceEstimate( - @Parameter( - description = "foreign blockchain", - example = "LITECOIN", - schema = @Schema(implementation = SupportedBlockchain.class) - ) @PathParam("blockchain") SupportedBlockchain foreignBlockchain, - @Parameter( - description = "Maximum number of trades to include in price calculation", - example = "10", - schema = @Schema(type = "integer", defaultValue = "10") - ) @QueryParam("maxtrades") Integer maxtrades) { - // foreignBlockchain is required - if (foreignBlockchain == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // We want both a minimum of 5 trades and enough trades to span at least 4 hours - int minimumCount = 5; - int maximumCount = maxtrades != null ? maxtrades : 10; - long minimumPeriod = 4 * 60 * 60 * 1000L; // ms - Boolean isFinished = Boolean.TRUE; - - try (final Repository repository = RepositoryManager.getRepository()) { - Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); - - long totalForeign = 0; - long totalQort = 0; - - for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { - byte[] codeHash = acctInfo.getKey().value; - ACCT acct = acctInfo.getValue().get(); - - List atStates = repository.getATRepository().getMatchingFinalATStatesQuorum(codeHash, - isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, maximumCount, minimumPeriod); - - for (ATStateData atState : atStates) { - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); - totalForeign += crossChainTradeData.expectedForeignAmount; - totalQort += crossChainTradeData.qortAmount; - } - } - - return Amounts.scaledDivide(totalQort, totalForeign); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @DELETE - @Path("/tradeoffer") - @Operation( - summary = "Builds raw, unsigned 'cancel' MESSAGE transaction that cancels cross-chain trade offer", - description = "Specify address of cross-chain AT that needs to be cancelled.
" - + "AT needs to be in 'offer' mode. Messages sent to an AT in 'trade' mode will be ignored.
" - + "Performs MESSAGE proof-of-work.
" - + "You need to sign output with AT creator's private key otherwise the MESSAGE transaction will be invalid.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = CrossChainCancelRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content( - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) - @SecurityRequirement(name = "apiKey") - public String cancelTrade(CrossChainCancelRequest cancelRequest) { - Security.checkApiCallAllowed(request); - - byte[] creatorPublicKey = cancelRequest.creatorPublicKey; - - if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - if (cancelRequest.atAddress == null || !Crypto.isValidAtAddress(cancelRequest.atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, cancelRequest.atAddress); - - ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); - if (acct == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - - if (crossChainTradeData.mode != AcctMode.OFFERING) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - // Does supplied public key match AT creator's public key? - if (!Arrays.equals(creatorPublicKey, atData.getCreatorPublicKey())) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); - - // Good to make MESSAGE - - String atCreatorAddress = Crypto.toAddress(creatorPublicKey); - byte[] messageData = acct.buildCancelMessage(atCreatorAddress); - - byte[] messageTransactionBytes = buildAtMessage(repository, creatorPublicKey, cancelRequest.atAddress, messageData); - - return Base58.encode(messageTransactionBytes); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - // No point sending message to AT that's finished - if (atData.getIsFinished()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - return atData; - } - - private byte[] buildAtMessage(Repository repository, byte[] senderPublicKey, String atAddress, byte[] messageData) throws DataException { - long txTimestamp = NTP.getTime(); - - // senderPublicKey could be ephemeral trade public key where there is no corresponding account and hence no reference - String senderAddress = Crypto.toAddress(senderPublicKey); - byte[] lastReference = repository.getAccountRepository().getLastReference(senderAddress); - final boolean requiresPoW = lastReference == null; - - if (requiresPoW) { - Random random = new Random(); - lastReference = new byte[Transformer.SIGNATURE_LENGTH]; - random.nextBytes(lastReference); - } - - int version = 4; - int nonce = 0; - long amount = 0L; - Long assetId = null; // no assetId as amount is zero - Long fee = 0L; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, senderPublicKey, fee, null); - TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, atAddress, amount, assetId, messageData, false, false); - - MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); - - if (requiresPoW) { - messageTransaction.computeNonce(); - } else { - fee = messageTransaction.calcRecommendedFee(); - messageTransactionData.setFee(fee); - } - - ValidationResult result = messageTransaction.isValidUnconfirmed(); - if (result != ValidationResult.OK) - throw TransactionsResource.createTransactionInvalidException(request, result); - - try { - return MessageTransactionTransformer.toBytes(messageTransactionData); - } catch (TransformationException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java b/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java deleted file mode 100644 index cd8766ca..00000000 --- a/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java +++ /dev/null @@ -1,286 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.ArraySchema; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; - -import java.util.List; -import java.util.stream.Collectors; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; - -import org.qortal.account.Account; -import org.qortal.account.PublicKeyAccount; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; -import org.qortal.api.model.crosschain.TradeBotCreateRequest; -import org.qortal.api.model.crosschain.TradeBotRespondRequest; -import org.qortal.asset.Asset; -import org.qortal.controller.tradebot.AcctTradeBot; -import org.qortal.controller.tradebot.TradeBot; -import org.qortal.crosschain.ForeignBlockchain; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.AcctMode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.utils.Base58; - -@Path("/crosschain/tradebot") -@Tag(name = "Cross-Chain (Trade-Bot)") -public class CrossChainTradeBotResource { - - @Context - HttpServletRequest request; - - @GET - @Operation( - summary = "List current trade-bot states", - responses = { - @ApiResponse( - content = @Content( - array = @ArraySchema( - schema = @Schema( - implementation = TradeBotData.class - ) - ) - ) - ) - } - ) - @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public List getTradeBotStates( - @Parameter( - description = "Limit to specific blockchain", - example = "LITECOIN", - schema = @Schema(implementation = SupportedBlockchain.class) - ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain) { - Security.checkApiCallAllowed(request); - - try (final Repository repository = RepositoryManager.getRepository()) { - List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); - - if (foreignBlockchain == null) - return allTradeBotData; - - return allTradeBotData.stream().filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain.name())).collect(Collectors.toList()); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/create") - @Operation( - summary = "Create a trade offer (trade-bot entry)", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = TradeBotCreateRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.INSUFFICIENT_BALANCE, ApiError.REPOSITORY_ISSUE}) - @SuppressWarnings("deprecation") - public String tradeBotCreator(TradeBotCreateRequest tradeBotCreateRequest) { - Security.checkApiCallAllowed(request); - - if (tradeBotCreateRequest.foreignBlockchain == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - ForeignBlockchain foreignBlockchain = tradeBotCreateRequest.foreignBlockchain.getInstance(); - - // We prefer foreignAmount to deprecated bitcoinAmount - if (tradeBotCreateRequest.foreignAmount == null) - tradeBotCreateRequest.foreignAmount = tradeBotCreateRequest.bitcoinAmount; - - if (!foreignBlockchain.isValidAddress(tradeBotCreateRequest.receivingAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (tradeBotCreateRequest.tradeTimeout < 60) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - if (tradeBotCreateRequest.foreignAmount == null || tradeBotCreateRequest.foreignAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - if (tradeBotCreateRequest.qortAmount <= 0 || tradeBotCreateRequest.fundingQortAmount <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - try (final Repository repository = RepositoryManager.getRepository()) { - // Do some simple checking first - Account creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); - - if (creator.getConfirmedBalance(Asset.QORT) < tradeBotCreateRequest.fundingQortAmount) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INSUFFICIENT_BALANCE); - - byte[] unsignedBytes = TradeBot.getInstance().createTrade(repository, tradeBotCreateRequest); - if (unsignedBytes == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - return Base58.encode(unsignedBytes); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/respond") - @Operation( - summary = "Respond to a trade offer. NOTE: WILL SPEND FUNDS!)", - description = "Start a new trade-bot entry to respond to chosen trade offer.", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.APPLICATION_JSON, - schema = @Schema( - implementation = TradeBotRespondRequest.class - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE, ApiError.REPOSITORY_ISSUE}) - @SuppressWarnings("deprecation") - public String tradeBotResponder(TradeBotRespondRequest tradeBotRespondRequest) { - Security.checkApiCallAllowed(request); - - final String atAddress = tradeBotRespondRequest.atAddress; - - // We prefer foreignKey to deprecated xprv58 - if (tradeBotRespondRequest.foreignKey == null) - tradeBotRespondRequest.foreignKey = tradeBotRespondRequest.xprv58; - - if (tradeBotRespondRequest.foreignKey == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - if (atAddress == null || !Crypto.isValidAtAddress(atAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (tradeBotRespondRequest.receivingAddress == null || !Crypto.isValidAddress(tradeBotRespondRequest.receivingAddress)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - // Extract data from cross-chain trading AT - try (final Repository repository = RepositoryManager.getRepository()) { - ATData atData = fetchAtDataWithChecking(repository, atAddress); - - // TradeBot uses AT's code hash to map to ACCT - ACCT acct = TradeBot.getInstance().getAcctUsingAtData(atData); - if (acct == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (!acct.getBlockchain().isValidWalletKey(tradeBotRespondRequest.foreignKey)) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); - if (crossChainTradeData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - - if (crossChainTradeData.mode != AcctMode.OFFERING) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - AcctTradeBot.ResponseResult result = TradeBot.getInstance().startResponse(repository, atData, acct, crossChainTradeData, - tradeBotRespondRequest.foreignKey, tradeBotRespondRequest.receivingAddress); - - switch (result) { - case OK: - return "true"; - - case BALANCE_ISSUE: - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); - - case NETWORK_ISSUE: - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - - default: - return "false"; - } - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @DELETE - @Operation( - summary = "Delete completed trade", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", - example = "93MB2qRDNVLxbmmPuYpLdAqn3u2x9ZhaVZK5wELHueP8" - ) - ) - ), - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) - ) - } - ) - @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE}) - public String tradeBotDelete(String tradePrivateKey58) { - Security.checkApiCallAllowed(request); - - final byte[] tradePrivateKey; - try { - tradePrivateKey = Base58.decode(tradePrivateKey58); - - if (tradePrivateKey.length != 32) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - } catch (NumberFormatException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - // Handed off to TradeBot - return TradeBot.getInstance().deleteEntry(repository, tradePrivateKey) ? "true" : "false"; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - if (atData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - - // No point sending message to AT that's finished - if (atData.getIsFinished()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - return atData; - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java deleted file mode 100644 index 26d131c4..00000000 --- a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java +++ /dev/null @@ -1,244 +0,0 @@ -package org.qortal.api.websocket; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import org.eclipse.jetty.websocket.api.Session; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; -import org.eclipse.jetty.websocket.api.annotations.WebSocket; -import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; -import org.qortal.controller.Controller; -import org.qortal.crypto.Crypto; -import org.qortal.data.transaction.PresenceTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.event.Event; -import org.qortal.event.EventBus; -import org.qortal.event.Listener; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.transaction.PresenceTransaction.PresenceType; -import org.qortal.transaction.Transaction.TransactionType; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -@WebSocket -@SuppressWarnings("serial") -public class PresenceWebSocket extends ApiWebSocket implements Listener { - - @XmlAccessorType(XmlAccessType.FIELD) - @SuppressWarnings("unused") - private static class PresenceInfo { - private final PresenceType presenceType; - private final String publicKey; - private final long timestamp; - private final String address; - - protected PresenceInfo() { - this.presenceType = null; - this.publicKey = null; - this.timestamp = 0L; - this.address = null; - } - - public PresenceInfo(PresenceType presenceType, String pubKey58, long timestamp) { - this.presenceType = presenceType; - this.publicKey = pubKey58; - this.timestamp = timestamp; - this.address = Crypto.toAddress(Base58.decode(this.publicKey)); - } - - public PresenceType getPresenceType() { - return this.presenceType; - } - - public String getPublicKey() { - return this.publicKey; - } - - public long getTimestamp() { - return this.timestamp; - } - - public String getAddress() { - return this.address; - } - } - - /** Outer map key is PresenceType (enum), inner map key is public key in base58, inner map value is timestamp */ - private static final Map> currentEntries = Collections.synchronizedMap(new EnumMap<>(PresenceType.class)); - - /** (Optional) PresenceType used for filtering by that Session. */ - private static final Map sessionPresenceTypes = Collections.synchronizedMap(new HashMap<>()); - - @Override - public void configure(WebSocketServletFactory factory) { - factory.register(PresenceWebSocket.class); - - try (final Repository repository = RepositoryManager.getRepository()) { - populateCurrentInfo(repository); - } catch (DataException e) { - // How to fail properly? - return; - } - - EventBus.INSTANCE.addListener(this::listen); - } - - @Override - public void listen(Event event) { - // We use NewBlockEvent as a proxy for 1-minute timer - if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent)) - return; - - removeOldEntries(); - - if (event instanceof Controller.NewBlockEvent) - // We only wanted a chance to cull old entries - return; - - TransactionData transactionData = ((Controller.NewTransactionEvent) event).getTransactionData(); - - if (transactionData.getType() != TransactionType.PRESENCE) - return; - - PresenceTransactionData presenceData = (PresenceTransactionData) transactionData; - PresenceType presenceType = presenceData.getPresenceType(); - - // Put/replace for this publickey making sure we keep newest timestamp - String pubKey58 = Base58.encode(presenceData.getCreatorPublicKey()); - long ourTimestamp = presenceData.getTimestamp(); - long computedTimestamp = mergePresence(presenceType, pubKey58, ourTimestamp); - - if (computedTimestamp != ourTimestamp) - // nothing changed - return; - - List presenceInfo = Collections.singletonList(new PresenceInfo(presenceType, pubKey58, computedTimestamp)); - - // Notify sessions - for (Session session : getSessions()) { - PresenceType sessionPresenceType = sessionPresenceTypes.get(session); - - if (sessionPresenceType == null || sessionPresenceType == presenceType) - sendPresenceInfo(session, presenceInfo); - } - } - - @OnWebSocketConnect - @Override - public void onWebSocketConnect(Session session) { - Map> queryParams = session.getUpgradeRequest().getParameterMap(); - List presenceTypes = queryParams.get("presenceType"); - - // We only support ONE presenceType - String presenceTypeName = presenceTypes == null || presenceTypes.isEmpty() ? null : presenceTypes.get(0); - - PresenceType presenceType = presenceTypeName == null ? null : PresenceType.fromString(presenceTypeName); - - // Make sure that if caller does give a presenceType, that it is a valid/known one. - if (presenceTypeName != null && presenceType == null) { - session.close(4003, "unknown presenceType: " + presenceTypeName); - return; - } - - // Save session's requested PresenceType, if given - if (presenceType != null) - sessionPresenceTypes.put(session, presenceType); - - List presenceInfo; - - synchronized (currentEntries) { - presenceInfo = currentEntries.entrySet().stream() - .filter(entry -> presenceType == null ? true : entry.getKey() == presenceType) - .flatMap(entry -> entry.getValue().entrySet().stream().map(innerEntry -> new PresenceInfo(entry.getKey(), innerEntry.getKey(), innerEntry.getValue()))) - .collect(Collectors.toList()); - } - - if (!sendPresenceInfo(session, presenceInfo)) { - session.close(4002, "websocket issue"); - return; - } - - super.onWebSocketConnect(session); - } - - @OnWebSocketClose - @Override - public void onWebSocketClose(Session session, int statusCode, String reason) { - // clean up - sessionPresenceTypes.remove(session); - - super.onWebSocketClose(session, statusCode, reason); - } - - @OnWebSocketError - public void onWebSocketError(Session session, Throwable throwable) { - /* ignored */ - } - - @OnWebSocketMessage - public void onWebSocketMessage(Session session, String message) { - /* ignored */ - } - - private boolean sendPresenceInfo(Session session, List presenceInfo) { - try { - StringWriter stringWriter = new StringWriter(); - marshall(stringWriter, presenceInfo); - - String output = stringWriter.toString(); - session.getRemote().sendStringByFuture(output); - } catch (IOException e) { - // No output this time? - return false; - } - - return true; - } - - private static void populateCurrentInfo(Repository repository) throws DataException { - // We want ALL PRESENCE transactions - - List presenceTransactionsData = repository.getTransactionRepository().getUnconfirmedTransactions(TransactionType.PRESENCE, null); - - for (TransactionData transactionData : presenceTransactionsData) { - PresenceTransactionData presenceData = (PresenceTransactionData) transactionData; - - PresenceType presenceType = presenceData.getPresenceType(); - - // Put/replace for this publickey making sure we keep newest timestamp - String pubKey58 = Base58.encode(presenceData.getCreatorPublicKey()); - long ourTimestamp = presenceData.getTimestamp(); - - mergePresence(presenceType, pubKey58, ourTimestamp); - } - } - - private static long mergePresence(PresenceType presenceType, String pubKey58, long ourTimestamp) { - Map typedPubkeyTimestamps = currentEntries.computeIfAbsent(presenceType, someType -> Collections.synchronizedMap(new HashMap<>())); - return typedPubkeyTimestamps.compute(pubKey58, (somePubKey58, currentTimestamp) -> (currentTimestamp == null || currentTimestamp < ourTimestamp) ? ourTimestamp : currentTimestamp); - } - - private static void removeOldEntries() { - long now = NTP.getTime(); - - currentEntries.entrySet().forEach(entry -> { - long expiryThreshold = now - entry.getKey().getLifetime(); - entry.getValue().entrySet().removeIf(pubkeyTimestamp -> pubkeyTimestamp.getValue() < expiryThreshold); - }); - } - -} diff --git a/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java deleted file mode 100644 index 55969c6b..00000000 --- a/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java +++ /dev/null @@ -1,157 +0,0 @@ -package org.qortal.api.websocket; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.eclipse.jetty.websocket.api.Session; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; -import org.eclipse.jetty.websocket.api.annotations.WebSocket; -import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; -import org.qortal.controller.tradebot.TradeBot; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.event.Event; -import org.qortal.event.EventBus; -import org.qortal.event.Listener; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.utils.Base58; - -@WebSocket -@SuppressWarnings("serial") -public class TradeBotWebSocket extends ApiWebSocket implements Listener { - - /** Cache of trade-bot entry states, keyed by trade-bot entry's "trade private key" (base58) */ - private static final Map PREVIOUS_STATES = new HashMap<>(); - - private static final Map sessionBlockchain = Collections.synchronizedMap(new HashMap<>()); - - @Override - public void configure(WebSocketServletFactory factory) { - factory.register(TradeBotWebSocket.class); - - try (final Repository repository = RepositoryManager.getRepository()) { - List tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); - if (tradeBotEntries == null) - // How do we properly fail here? - return; - - PREVIOUS_STATES.putAll(tradeBotEntries.stream().collect(Collectors.toMap(entry -> Base58.encode(entry.getTradePrivateKey()), TradeBotData::getStateValue))); - } catch (DataException e) { - // No output this time - } - - EventBus.INSTANCE.addListener(this::listen); - } - - @Override - public void listen(Event event) { - if (!(event instanceof TradeBot.StateChangeEvent)) - return; - - TradeBotData tradeBotData = ((TradeBot.StateChangeEvent) event).getTradeBotData(); - String tradePrivateKey58 = Base58.encode(tradeBotData.getTradePrivateKey()); - - synchronized (PREVIOUS_STATES) { - Integer previousStateValue = PREVIOUS_STATES.get(tradePrivateKey58); - if (previousStateValue != null && previousStateValue == tradeBotData.getStateValue()) - // Not changed - return; - - PREVIOUS_STATES.put(tradePrivateKey58, tradeBotData.getStateValue()); - } - - List tradeBotEntries = Collections.singletonList(tradeBotData); - - for (Session session : getSessions()) { - // Only send if this session has this/no preferred blockchain - String preferredBlockchain = sessionBlockchain.get(session); - - if (preferredBlockchain == null || preferredBlockchain.equals(tradeBotData.getForeignBlockchain())) - sendEntries(session, tradeBotEntries); - } - } - - @OnWebSocketConnect - @Override - public void onWebSocketConnect(Session session) { - Map> queryParams = session.getUpgradeRequest().getParameterMap(); - - List foreignBlockchains = queryParams.get("foreignBlockchain"); - final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); - - // Make sure blockchain (if any) is valid - if (foreignBlockchain != null && SupportedBlockchain.fromString(foreignBlockchain) == null) { - session.close(4003, "unknown blockchain: " + foreignBlockchain); - return; - } - - // save session's preferred blockchain (if any) - sessionBlockchain.put(session, foreignBlockchain); - - // Send all known trade-bot entries - try (final Repository repository = RepositoryManager.getRepository()) { - List tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); - - // Optional filtering - if (foreignBlockchain != null) - tradeBotEntries = tradeBotEntries.stream() - .filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain)) - .collect(Collectors.toList()); - - if (!sendEntries(session, tradeBotEntries)) { - session.close(4002, "websocket issue"); - return; - } - } catch (DataException e) { - session.close(4001, "repository issue fetching trade-bot entries"); - return; - } - - super.onWebSocketConnect(session); - } - - @OnWebSocketClose - @Override - public void onWebSocketClose(Session session, int statusCode, String reason) { - // clean up - sessionBlockchain.remove(session); - - super.onWebSocketClose(session, statusCode, reason); - } - - @OnWebSocketError - public void onWebSocketError(Session session, Throwable throwable) { - /* ignored */ - } - - @OnWebSocketMessage - public void onWebSocketMessage(Session session, String message) { - /* ignored */ - } - - private boolean sendEntries(Session session, List tradeBotEntries) { - try { - StringWriter stringWriter = new StringWriter(); - marshall(stringWriter, tradeBotEntries); - - String output = stringWriter.toString(); - session.getRemote().sendStringByFuture(output); - } catch (IOException e) { - // No output this time? - return false; - } - - return true; - } - -} diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java deleted file mode 100644 index 186f79e3..00000000 --- a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java +++ /dev/null @@ -1,351 +0,0 @@ -package org.qortal.api.websocket; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.eclipse.jetty.websocket.api.Session; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; -import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; -import org.eclipse.jetty.websocket.api.annotations.WebSocket; -import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; -import org.qortal.api.model.CrossChainOfferSummary; -import org.qortal.controller.Controller; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.AcctMode; -import org.qortal.data.at.ATStateData; -import org.qortal.data.block.BlockData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.event.Event; -import org.qortal.event.EventBus; -import org.qortal.event.Listener; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.utils.ByteArray; -import org.qortal.utils.NTP; - -@WebSocket -@SuppressWarnings("serial") -public class TradeOffersWebSocket extends ApiWebSocket implements Listener { - - private static final Logger LOGGER = LogManager.getLogger(TradeOffersWebSocket.class); - - private static class CachedOfferInfo { - public final Map previousAtModes = new HashMap<>(); - - // OFFERING - public final Map currentSummaries = new HashMap<>(); - // REDEEMED/REFUNDED/CANCELLED - public final Map historicSummaries = new HashMap<>(); - } - // Manual synchronization - private static final Map cachedInfoByBlockchain = new HashMap<>(); - - private static final Predicate isHistoric = offerSummary - -> offerSummary.getMode() == AcctMode.REDEEMED - || offerSummary.getMode() == AcctMode.REFUNDED - || offerSummary.getMode() == AcctMode.CANCELLED; - - private static final Map sessionBlockchain = Collections.synchronizedMap(new HashMap<>()); - - @Override - public void configure(WebSocketServletFactory factory) { - factory.register(TradeOffersWebSocket.class); - - try (final Repository repository = RepositoryManager.getRepository()) { - populateCurrentSummaries(repository); - - populateHistoricSummaries(repository); - } catch (DataException e) { - // How to fail properly? - return; - } - - EventBus.INSTANCE.addListener(this::listen); - } - - @Override - public void listen(Event event) { - if (!(event instanceof Controller.NewBlockEvent)) - return; - - BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData(); - - // Process any new info - - try (final Repository repository = RepositoryManager.getRepository()) { - // Find any new/changed trade ATs since this block - final Boolean isFinished = null; - final Integer dataByteOffset = null; - final Long expectedValue = null; - final Integer minimumFinalHeight = blockData.getHeight(); - - for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { - Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); - - List crossChainOfferSummaries = new ArrayList<>(); - - synchronized (cachedInfoByBlockchain) { - CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); - - for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { - byte[] codeHash = acctInfo.getKey().value; - ACCT acct = acctInfo.getValue().get(); - - List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, - isFinished, dataByteOffset, expectedValue, minimumFinalHeight, - null, null, null); - - crossChainOfferSummaries.addAll(produceSummaries(repository, acct, atStates, blockData.getTimestamp())); - } - - // Remove any entries unchanged from last time - crossChainOfferSummaries.removeIf(offerSummary -> cachedInfo.previousAtModes.get(offerSummary.getQortalAtAddress()) == offerSummary.getMode()); - - // Skip to next blockchain if nothing has changed (for this blockchain) - if (crossChainOfferSummaries.isEmpty()) - continue; - - // Update - for (CrossChainOfferSummary offerSummary : crossChainOfferSummaries) { - String offerAtAddress = offerSummary.getQortalAtAddress(); - - cachedInfo.previousAtModes.put(offerAtAddress, offerSummary.getMode()); - LOGGER.trace(() -> String.format("Block height: %d, AT: %s, mode: %s", blockData.getHeight(), offerAtAddress, offerSummary.getMode().name())); - - switch (offerSummary.getMode()) { - case OFFERING: - cachedInfo.currentSummaries.put(offerAtAddress, offerSummary); - cachedInfo.historicSummaries.remove(offerAtAddress); - break; - - case REDEEMED: - case REFUNDED: - case CANCELLED: - cachedInfo.currentSummaries.remove(offerAtAddress); - cachedInfo.historicSummaries.put(offerAtAddress, offerSummary); - break; - - case TRADING: - cachedInfo.currentSummaries.remove(offerAtAddress); - cachedInfo.historicSummaries.remove(offerAtAddress); - break; - } - } - - // Remove any historic offers that are over 24 hours old - final long tooOldTimestamp = NTP.getTime() - 24 * 60 * 60 * 1000L; - cachedInfo.historicSummaries.values().removeIf(historicSummary -> historicSummary.getTimestamp() < tooOldTimestamp); - } - - // Notify sessions - for (Session session : getSessions()) { - // Only send if this session has this/no preferred blockchain - String preferredBlockchain = sessionBlockchain.get(session); - - if (preferredBlockchain == null || preferredBlockchain.equals(blockchain.name())) - sendOfferSummaries(session, crossChainOfferSummaries); - } - - } - } catch (DataException e) { - // No output this time - } - } - - @OnWebSocketConnect - @Override - public void onWebSocketConnect(Session session) { - Map> queryParams = session.getUpgradeRequest().getParameterMap(); - final boolean includeHistoric = queryParams.get("includeHistoric") != null; - - List foreignBlockchains = queryParams.get("foreignBlockchain"); - final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); - - // Make sure blockchain (if any) is valid - if (foreignBlockchain != null && SupportedBlockchain.fromString(foreignBlockchain) == null) { - session.close(4003, "unknown blockchain: " + foreignBlockchain); - return; - } - - // Save session's preferred blockchain, if given - if (foreignBlockchain != null) - sessionBlockchain.put(session, foreignBlockchain); - - List crossChainOfferSummaries = new ArrayList<>(); - - synchronized (cachedInfoByBlockchain) { - Collection cachedInfos; - - if (foreignBlockchain == null) - // No preferred blockchain, so iterate through all of them - cachedInfos = cachedInfoByBlockchain.values(); - else - cachedInfos = Collections.singleton(cachedInfoByBlockchain.computeIfAbsent(foreignBlockchain, k -> new CachedOfferInfo())); - - for (CachedOfferInfo cachedInfo : cachedInfos) { - crossChainOfferSummaries.addAll(cachedInfo.currentSummaries.values()); - - if (includeHistoric) - crossChainOfferSummaries.addAll(cachedInfo.historicSummaries.values()); - } - } - - if (!sendOfferSummaries(session, crossChainOfferSummaries)) { - session.close(4002, "websocket issue"); - return; - } - - super.onWebSocketConnect(session); - } - - @OnWebSocketClose - @Override - public void onWebSocketClose(Session session, int statusCode, String reason) { - // clean up - sessionBlockchain.remove(session); - - super.onWebSocketClose(session, statusCode, reason); - } - - @OnWebSocketError - public void onWebSocketError(Session session, Throwable throwable) { - /* ignored */ - } - - @OnWebSocketMessage - public void onWebSocketMessage(Session session, String message) { - /* ignored */ - } - - private boolean sendOfferSummaries(Session session, List crossChainOfferSummaries) { - try { - StringWriter stringWriter = new StringWriter(); - marshall(stringWriter, crossChainOfferSummaries); - - String output = stringWriter.toString(); - session.getRemote().sendStringByFuture(output); - } catch (IOException e) { - // No output this time? - return false; - } - - return true; - } - - private static void populateCurrentSummaries(Repository repository) throws DataException { - // We want ALL OFFERING trades - Boolean isFinished = Boolean.FALSE; - Long expectedValue = (long) AcctMode.OFFERING.value; - Integer minimumFinalHeight = null; - - for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { - Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); - - CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); - - for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { - byte[] codeHash = acctInfo.getKey().value; - ACCT acct = acctInfo.getValue().get(); - - Integer dataByteOffset = acct.getModeByteOffset(); - List initialAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, - isFinished, dataByteOffset, expectedValue, minimumFinalHeight, - null, null, null); - - if (initialAtStates == null) - throw new DataException("Couldn't fetch current trades from repository"); - - // Save initial AT modes - cachedInfo.previousAtModes.putAll(initialAtStates.stream().collect(Collectors.toMap(ATStateData::getATAddress, atState -> AcctMode.OFFERING))); - - // Convert to offer summaries - cachedInfo.currentSummaries.putAll(produceSummaries(repository, acct, initialAtStates, null).stream() - .collect(Collectors.toMap(CrossChainOfferSummary::getQortalAtAddress, offerSummary -> offerSummary))); - } - } - } - - private static void populateHistoricSummaries(Repository repository) throws DataException { - // We want REDEEMED/REFUNDED/CANCELLED trades over the last 24 hours - long timestamp = System.currentTimeMillis() - 24 * 60 * 60 * 1000L; - int minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(timestamp); - - if (minimumFinalHeight == 0) - throw new DataException("Couldn't fetch block timestamp from repository"); - - Boolean isFinished = Boolean.TRUE; - Integer dataByteOffset = null; - Long expectedValue = null; - ++minimumFinalHeight; // because height is just *before* timestamp - - for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { - Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); - - CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); - - for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { - byte[] codeHash = acctInfo.getKey().value; - ACCT acct = acctInfo.getValue().get(); - - List historicAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, - isFinished, dataByteOffset, expectedValue, minimumFinalHeight, - null, null, null); - - if (historicAtStates == null) - throw new DataException("Couldn't fetch historic trades from repository"); - - for (ATStateData historicAtState : historicAtStates) { - CrossChainOfferSummary historicOfferSummary = produceSummary(repository, acct, historicAtState, null); - - if (!isHistoric.test(historicOfferSummary)) - continue; - - // Add summary to initial burst - cachedInfo.historicSummaries.put(historicOfferSummary.getQortalAtAddress(), historicOfferSummary); - - // Save initial AT mode - cachedInfo.previousAtModes.put(historicOfferSummary.getQortalAtAddress(), historicOfferSummary.getMode()); - } - } - } - } - - private static CrossChainOfferSummary produceSummary(Repository repository, ACCT acct, ATStateData atState, Long timestamp) throws DataException { - CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); - - long atStateTimestamp; - - if (crossChainTradeData.mode == AcctMode.OFFERING) - // We want when trade was created, not when it was last updated - atStateTimestamp = crossChainTradeData.creationTimestamp; - else - atStateTimestamp = timestamp != null ? timestamp : repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); - - return new CrossChainOfferSummary(crossChainTradeData, atStateTimestamp); - } - - private static List produceSummaries(Repository repository, ACCT acct, List atStates, Long timestamp) throws DataException { - List offerSummaries = new ArrayList<>(); - - for (ATStateData atState : atStates) - offerSummaries.add(produceSummary(repository, acct, atState, timestamp)); - - return offerSummaries; - } - -} diff --git a/src/main/java/org/qortal/at/QortalFunctionCode.java b/src/main/java/org/qortal/at/QortalFunctionCode.java index 0d11e488..8f989e19 100644 --- a/src/main/java/org/qortal/at/QortalFunctionCode.java +++ b/src/main/java/org/qortal/at/QortalFunctionCode.java @@ -10,10 +10,8 @@ import org.ciyam.at.ExecutionException; import org.ciyam.at.FunctionData; import org.ciyam.at.IllegalFunctionCodeException; import org.ciyam.at.MachineState; -import org.qortal.crosschain.Bitcoin; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.TransactionData; -import org.qortal.settings.Settings; /** * Qortal-specific CIYAM-AT Functions. @@ -100,19 +98,6 @@ public enum QortalFunctionCode { setB(state, pkh); } }, - /** - * Convert 20-byte value in LSB of B1, and all of B2 & B3 to P2SH.
- * 0x0511
- * P2SH stored in lower 25 bytes of B. - */ - CONVERT_B_TO_P2SH(0x0511, 0, false) { - @Override - protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException { - byte addressPrefix = Settings.getInstance().getBitcoinNet() == Bitcoin.BitcoinNet.MAIN ? 0x05 : (byte) 0xc4; - - convertAddressInB(addressPrefix, state); - } - }, /** * Convert 20-byte value in LSB of B1, and all of B2 & B3 to Qortal address.
* 0x0512
diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index c7bccb73..1e74f365 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -46,7 +46,6 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; -import org.qortal.controller.tradebot.TradeBot; import org.qortal.crypto.Crypto; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; @@ -483,9 +482,6 @@ public class Controller extends Thread { blockMinter = new BlockMinter(); blockMinter.start(); - LOGGER.info("Starting trade-bot"); - TradeBot.getInstance(); - // Arbitrary transaction data manager // LOGGER.info("Starting arbitrary-transaction data manager"); // ArbitraryDataManager.getInstance().start(); diff --git a/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java b/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java deleted file mode 100644 index 84a0d484..00000000 --- a/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.qortal.controller.tradebot; - -import java.util.List; - -import org.qortal.api.model.crosschain.TradeBotCreateRequest; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; - -public interface AcctTradeBot { - - public enum ResponseResult { OK, BALANCE_ISSUE, NETWORK_ISSUE, TRADE_ALREADY_EXISTS } - - /** Returns list of state names for trade-bot entries that have ended, e.g. redeemed, refunded or cancelled. */ - public List getEndStates(); - - public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException; - - public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, - CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException; - - public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException; - - public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException; - -} diff --git a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java deleted file mode 100644 index ca2e2518..00000000 --- a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java +++ /dev/null @@ -1,1273 +0,0 @@ -package org.qortal.controller.tradebot; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bitcoinj.core.Address; -import org.bitcoinj.core.AddressFormatException; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.account.PublicKeyAccount; -import org.qortal.api.model.crosschain.TradeBotCreateRequest; -import org.qortal.asset.Asset; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.AcctMode; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.BitcoinACCTv1; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.DeployAtTransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -/** - * Performing cross-chain trading steps on behalf of user. - *

- * We deal with three different independent state-spaces here: - *

    - *
  • Qortal blockchain
    • - *
  • Foreign blockchain
    • - *
  • Trade-bot entries
    • - *
- */ -public class BitcoinACCTv1TradeBot implements AcctTradeBot { - - private static final Logger LOGGER = LogManager.getLogger(BitcoinACCTv1TradeBot.class); - - public enum State implements TradeBot.StateNameAndValueSupplier { - BOB_WAITING_FOR_AT_CONFIRM(10, false, false), - BOB_WAITING_FOR_MESSAGE(15, true, true), - BOB_WAITING_FOR_P2SH_B(20, true, true), - BOB_WAITING_FOR_AT_REDEEM(25, true, true), - BOB_DONE(30, false, false), - BOB_REFUNDED(35, false, false), - - ALICE_WAITING_FOR_P2SH_A(80, true, true), - ALICE_WAITING_FOR_AT_LOCK(85, true, true), - ALICE_WATCH_P2SH_B(90, true, true), - ALICE_DONE(95, false, false), - ALICE_REFUNDING_B(100, true, true), - ALICE_REFUNDING_A(105, true, true), - ALICE_REFUNDED(110, false, false); - - private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); - - public final int value; - public final boolean requiresAtData; - public final boolean requiresTradeData; - - State(int value, boolean requiresAtData, boolean requiresTradeData) { - this.value = value; - this.requiresAtData = requiresAtData; - this.requiresTradeData = requiresTradeData; - } - - public static State valueOf(int value) { - return map.get(value); - } - - @Override - public String getState() { - return this.name(); - } - - @Override - public int getStateValue() { - return this.value; - } - } - - /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ - private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms - - /** P2SH-B output amount to avoid dust threshold (3000 sats/kB). */ - private static final long P2SH_B_OUTPUT_AMOUNT = 1000L; - - private static BitcoinACCTv1TradeBot instance; - - private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDING_B, State.ALICE_REFUNDED).stream() - .map(State::name) - .collect(Collectors.toUnmodifiableList()); - - private BitcoinACCTv1TradeBot() { - } - - public static synchronized BitcoinACCTv1TradeBot getInstance() { - if (instance == null) - instance = new BitcoinACCTv1TradeBot(); - - return instance; - } - - @Override - public List getEndStates() { - return this.endStates; - } - - /** - * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for BTC. - *

- * Generates: - *

    - *
  • new 'trade' private key
    • - *
  • secret-B
    • - *
- * Derives: - *
    - *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • - *
  • 'foreign' (as in Bitcoin) public key, public key hash
    • - *
  • HASH160 of secret-B
    • - *
- * A Qortal AT is then constructed including the following as constants in the 'data segment': - *
    - *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • - *
  • 'foreign'/Bitcoin public key hash - used by Alice's P2SH scripts to allow redeem
    • - *
  • HASH160 of secret-B - used by AT and P2SH to validate a potential secret-B
    • - *
  • QORT amount on offer by Bob
    • - *
  • BTC amount expected in return by Bob (from Alice)
    • - *
  • trading timeout, in case things go wrong and everyone needs to refund
    • - *
- * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. - *

- * Trade-bot will wait for Bob's AT to be deployed before taking next step. - *

- * @param repository - * @param tradeBotCreateRequest - * @return raw, unsigned DEPLOY_AT transaction - * @throws DataException - */ - public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - byte[] secretB = TradeBot.generateSecret(); - byte[] hashOfSecretB = Crypto.hash160(secretB); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - - // Convert Bitcoin receiving address into public key hash (we only support P2PKH at this time) - Address bitcoinReceivingAddress; - try { - bitcoinReceivingAddress = Address.fromString(Bitcoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); - } catch (AddressFormatException e) { - throw new DataException("Unsupported Bitcoin receiving address: " + tradeBotCreateRequest.receivingAddress); - } - if (bitcoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) - throw new DataException("Unsupported Bitcoin receiving address: " + tradeBotCreateRequest.receivingAddress); - - byte[] bitcoinReceivingAccountInfo = bitcoinReceivingAddress.getHash(); - - PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); - - // Deploy AT - long timestamp = NTP.getTime(); - byte[] reference = creator.getLastReference(); - long fee = 0L; - byte[] signature = null; - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); - - String name = "QORT/BTC ACCT"; - String description = "QORT/BTC cross-chain trade"; - String aTType = "ACCT"; - String tags = "ACCT QORT BTC"; - byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, hashOfSecretB, tradeBotCreateRequest.qortAmount, - tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); - long amount = tradeBotCreateRequest.fundingQortAmount; - - DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - DeployAtTransaction.ensureATAddress(deployAtTransactionData); - String atAddress = deployAtTransactionData.getAtAddress(); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, BitcoinACCTv1.NAME, - State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, - creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - secretB, hashOfSecretB, - SupportedBlockchain.BITCOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - tradeBotCreateRequest.foreignAmount, null, null, null, bitcoinReceivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); - - // Return to user for signing and broadcast as we don't have their Qortal private key - try { - return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); - } catch (TransformationException e) { - throw new DataException("Failed to transform DEPLOY_AT transaction?", e); - } - } - - /** - * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching BTC to an existing offer. - *

- * Requires a chosen trade offer from Bob, passed by crossChainTradeData - * and access to a Bitcoin wallet via xprv58. - *

- * The crossChainTradeData contains the current trade offer state - * as extracted from the AT's data segment. - *

- * Access to a funded wallet is via a Bitcoin BIP32 hierarchical deterministic key, - * passed via xprv58. - * This key will be stored in your node's database - * to allow trade-bot to create/fund the necessary P2SH transactions! - * However, due to the nature of BIP32 keys, it is possible to give the trade-bot - * only a subset of wallet access (see BIP32 for more details). - *

- * As an example, the xprv58 can be extract from a legacy, password-less - * Electrum wallet by going to the console tab and entering:
- * wallet.keystore.xprv
- * which should result in a base58 string starting with either 'xprv' (for Bitcoin main-net) - * or 'tprv' for (Bitcoin test-net). - *

- * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. - *

- * If sufficient funds are available, this method will actually fund the P2SH-A - * with the Bitcoin amount expected by 'Bob'. - *

- * If the Bitcoin transaction is successfully broadcast to the network then the trade-bot entry - * is saved to the repository and the cross-chain trading process commences. - *

- * Trade-bot will wait for P2SH-A to confirm before taking next step. - *

- * @param repository - * @param crossChainTradeData chosen trade OFFER that Alice wants to match - * @param xprv58 funded wallet xprv in base58 - * @return true if P2SH-A funding transaction successfully broadcast to Bitcoin network, false otherwise - * @throws DataException - */ - public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - byte[] secretA = TradeBot.generateSecret(); - byte[] hashOfSecretA = Crypto.hash160(secretA); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH - - // We need to generate lockTime-A: add tradeTimeout to now - long now = NTP.getTime(); - int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, BitcoinACCTv1.NAME, - State.ALICE_WAITING_FOR_P2SH_A.name(), State.ALICE_WAITING_FOR_P2SH_A.value, - receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - secretA, hashOfSecretA, - SupportedBlockchain.BITCOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); - - // Check we have enough funds via xprv58 to fund both P2SHs to cover expectedBitcoin - String tradeForeignAddress = Bitcoin.getInstance().pkhToAddress(tradeForeignPublicKeyHash); - - long p2shFee; - try { - p2shFee = Bitcoin.getInstance().getP2shFee(now); - } catch (ForeignBlockchainException e) { - LOGGER.debug("Couldn't estimate Bitcoin fees?"); - return ResponseResult.NETWORK_ISSUE; - } - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long fundsRequiredForP2shA = p2shFee /*funding P2SH-A*/ + crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-A*/; - long fundsRequiredForP2shB = p2shFee /*funding P2SH-B*/ + P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-B*/; - long totalFundsRequired = fundsRequiredForP2shA + fundsRequiredForP2shB; - - // As buildSpend also adds a fee, this is more pessimistic than required - Transaction fundingCheckTransaction = Bitcoin.getInstance().buildSpend(xprv58, tradeForeignAddress, totalFundsRequired); - if (fundingCheckTransaction == null) - return ResponseResult.BALANCE_ISSUE; - - // P2SH-A to be funded - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); - String p2shAddress = Bitcoin.getInstance().deriveP2shAddress(redeemScriptBytes); - - // Fund P2SH-A - - // Do not include fee for funding transaction as this is covered by buildSpend() - long amountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-A*/; - - Transaction p2shFundingTransaction = Bitcoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); - if (p2shFundingTransaction == null) { - LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); - return ResponseResult.BALANCE_ISSUE; - } - - try { - Bitcoin.getInstance().broadcastTransaction(p2shFundingTransaction); - } catch (ForeignBlockchainException e) { - // We couldn't fund P2SH-A at this time - LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); - return ResponseResult.NETWORK_ISSUE; - } - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Waiting for confirmation", p2shAddress)); - - return ResponseResult.OK; - } - - @Override - public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) - return true; - - // If the AT doesn't exist then we might as well let the user tidy up - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) - return true; - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - case ALICE_DONE: - case BOB_DONE: - case ALICE_REFUNDED: - case BOB_REFUNDED: - return true; - - default: - return false; - } - } - - @Override - public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) { - LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); - return; - } - - ATData atData = null; - CrossChainTradeData tradeData = null; - - if (tradeBotState.requiresAtData) { - // Attempt to fetch AT data - atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); - if (atData == null) { - LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); - return; - } - - if (tradeBotState.requiresTradeData) { - tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - if (tradeData == null) { - LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); - return; - } - } - } - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - handleBobWaitingForAtConfirm(repository, tradeBotData); - break; - - case ALICE_WAITING_FOR_P2SH_A: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceWaitingForP2shA(repository, tradeBotData, atData, tradeData); - break; - - case BOB_WAITING_FOR_MESSAGE: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_WAITING_FOR_AT_LOCK: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); - break; - - case BOB_WAITING_FOR_P2SH_B: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForP2shB(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_WATCH_P2SH_B: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceWatchingP2shB(repository, tradeBotData, atData, tradeData); - break; - - case BOB_WAITING_FOR_AT_REDEEM: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_DONE: - case BOB_DONE: - break; - - case ALICE_REFUNDING_B: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceRefundingP2shB(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_REFUNDING_A: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_REFUNDED: - case BOB_REFUNDED: - break; - } - } - - /** - * Trade-bot is waiting for Bob's AT to deploy. - *

- * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. - */ - private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { - if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) - return; - - // We've waited ages for AT to be confirmed into a block but something has gone awry. - // After this long we assume transaction loss so give up with trade-bot entry too. - tradeBotData.setState(State.BOB_REFUNDED.name()); - tradeBotData.setStateValue(State.BOB_REFUNDED.value); - tradeBotData.setTimestamp(NTP.getTime()); - // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() - repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); - repository.saveChanges(); - - LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); - TradeBot.notifyStateChange(tradeBotData); - return; - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, - () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); - } - - /** - * Trade-bot is waiting for Alice's P2SH-A to confirm. - *

- * If P2SH-A is confirmed, then trade-bot's next step is to MESSAGE Bob's trade address with Alice's trade info. - *

- * It is possible between broadcast and confirmation of P2SH-A funding transaction, that Bob has cancelled his trade offer. - * If this is detected then trade-bot's next step is to wait until P2SH-A can refund back to Alice. - *

- * In normal operation, trade-bot send a zero-fee, PoW MESSAGE on Alice's behalf containing: - *

    - *
  • Alice's 'foreign'/Bitcoin public key hash - so Bob's trade-bot can derive P2SH-A address and check balance
    • - *
  • HASH160 of Alice's secret-A - also used to derive P2SH-A address
    • - *
  • lockTime of P2SH-A - also used to derive P2SH-A address, but also for other use later in the trading process
    • - *
- * If MESSAGE transaction is successfully broadcast, trade-bot's next step is to wait until Bob's AT has locked trade to Alice only. - * @throws ForeignBlockchainException - */ - private void handleAliceWaitingForP2shA(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) - return; - - Bitcoin bitcoin = Bitcoin.getInstance(); - - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getLockTimeA(), crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestampA = calcP2shAFeeTimestamp(tradeBotData.getLockTimeA(), crossChainTradeData.tradeTimeout); - long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); - long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // This shouldn't occur, but defensively check P2SH-B in case we haven't redeemed the AT - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, - () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); - return; - - case FUNDED: - // Fall-through out of switch... - break; - } - - // P2SH-A funding confirmed - - // Attempt to send MESSAGE to Bob's Qortal trade address - byte[] messageData = BitcoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); - String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WAITING_FOR_AT_LOCK, - () -> String.format("P2SH-A %s funding confirmed. Messaged %s. Waiting for AT %s to lock to us", - p2shAddressA, messageRecipient, tradeBotData.getAtAddress())); - } - - /** - * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. - *

- * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, - * in which case trade-bot is done with this specific trade and finalizes on refunded state. - *

- * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. - *

- * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. - *

- * Assuming P2SH-A has at least expected Bitcoin balance, - * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. - *

- * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. - *

- * Trade-bot's next step is to wait for P2SH-B, which will allow Bob to reveal his secret-B, - * needed by Alice to progress her side of the trade. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // If AT has finished then Bob likely cancelled his trade offer - if (atData.getIsFinished()) { - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); - return; - } - - Bitcoin bitcoin = Bitcoin.getInstance(); - - String address = tradeBotData.getTradeNativeAddress(); - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); - - final byte[] originalLastTransactionSignature = tradeBotData.getLastTransactionSignature(); - - // Skip past previously processed messages - if (originalLastTransactionSignature != null) - for (int i = 0; i < messageTransactionsData.size(); ++i) - if (Arrays.equals(messageTransactionsData.get(i).getSignature(), originalLastTransactionSignature)) { - messageTransactionsData.subList(0, i + 1).clear(); - break; - } - - while (!messageTransactionsData.isEmpty()) { - MessageTransactionData messageTransactionData = messageTransactionsData.remove(0); - tradeBotData.setLastTransactionSignature(messageTransactionData.getSignature()); - - if (messageTransactionData.isText()) - continue; - - // We're expecting: HASH160(secret-A), Alice's Bitcoin pubkeyhash and lockTime-A - byte[] messageData = messageTransactionData.getData(); - BitcoinACCTv1.OfferMessageData offerMessageData = BitcoinACCTv1.extractOfferMessageData(messageData); - if (offerMessageData == null) - continue; - - byte[] aliceForeignPublicKeyHash = offerMessageData.partnerBitcoinPKH; - byte[] hashOfSecretA = offerMessageData.hashOfSecretA; - int lockTimeA = (int) offerMessageData.lockTimeA; - - // Determine P2SH-A address and confirm funded - byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); - String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); - - long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); - final long minimumAmountA = tradeBotData.getForeignAmount() - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; - - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // There might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // This shouldn't occur, but defensively bump to next state - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_P2SH_B, - () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case FUNDED: - // Fall-through out of switch... - break; - } - - // Good to go - send MESSAGE to AT - - String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(messageTransactionData.getTimestamp(), lockTimeA); - - // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume - byte[] outgoingMessageData = BitcoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); - - outgoingMessageTransaction.computeNonce(); - outgoingMessageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - byte[] redeemScriptB = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeB, tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret()); - String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_P2SH_B, - () -> String.format("Locked AT %s to %s. Waiting for P2SH-B %s", tradeBotData.getAtAddress(), aliceNativeAddress, p2shAddressB)); - - return; - } - - // Don't resave/notify if we don't need to - if (tradeBotData.getLastTransactionSignature() != originalLastTransactionSignature) - TradeBot.updateTradeBotState(repository, tradeBotData, null); - } - - /** - * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. - *

- * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow - * this process has taken so long that we've reached P2SH-A's locktime, or that someone else - * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. - *

- * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. - *

- * If all is well, trade-bot then uses Bitcoin wallet to (token) fund P2SH-B. - *

- * If P2SH-B funding transaction is successfully broadcast to the Bitcoin network, trade-bot's next - * step is to watch for Bob revealing secret-B by redeeming P2SH-B. - * @throws ForeignBlockchainException - */ - private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) - return; - - Bitcoin bitcoin = Bitcoin.getInstance(); - int lockTimeA = tradeBotData.getLockTimeA(); - - // Refund P2SH-A if we've passed lockTime-A - if (NTP.getTime() >= tradeBotData.getLockTimeA() * 1000L) { - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); - - long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); - long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // This shouldn't occur, but defensively revert back to waiting for P2SH-A - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WAITING_FOR_P2SH_A, - () -> String.format("P2SH-A %s no longer funded? Defensively checking P2SH-A next", p2shAddressA)); - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // This shouldn't occur, but defensively bump to next state - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, - () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); - return; - - case FUNDED: - // Fall-through out of switch... - break; - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> atData.getIsFinished() - ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) - : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); - - return; - } - - // We're waiting for AT to be in TRADE mode - if (crossChainTradeData.mode != AcctMode.TRADING) - return; - - // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above - - // Alice needs to fund P2SH-B here - - // Find our MESSAGE to AT from previous state - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), - crossChainTradeData.qortalCreatorTradeAddress, null, null, null); - if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { - LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); - return; - } - - long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(recipientMessageTimestamp, lockTimeA); - - // Our calculated lockTime-B should match AT's calculated lockTime-B - if (lockTimeB != crossChainTradeData.lockTimeB) { - LOGGER.debug(() -> String.format("Trade AT lockTime-B '%d' doesn't match our lockTime-B '%d'", crossChainTradeData.lockTimeB, lockTimeB)); - // We'll eventually refund - return; - } - - byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); - String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); - - long feeTimestampB = calcP2shBFeeTimestamp(lockTimeA, lockTimeB); - long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); - - // Have we funded P2SH-B already? - final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; - - BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); - - switch (htlcStatusB) { - case UNFUNDED: { - // Do not include fee for funding transaction as this is covered by buildSpend() - long amountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB /*redeeming/refunding P2SH-B*/; - - Transaction p2shFundingTransaction = bitcoin.buildSpend(tradeBotData.getForeignKey(), p2shAddressB, amountB); - if (p2shFundingTransaction == null) { - LOGGER.debug("Unable to build P2SH-B funding transaction - lack of funds?"); - return; - } - - bitcoin.broadcastTransaction(p2shFundingTransaction); - break; - } - - case FUNDING_IN_PROGRESS: - case FUNDED: - break; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // This shouldn't occur, but defensively bump to next state - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, - () -> String.format("P2SH-B %s already spent? Defensively checking P2SH-B next", p2shAddressB)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("P2SH-B %s already refunded. Refunding P2SH-A next", p2shAddressB)); - return; - } - - // P2SH-B funded, now we wait for Bob to redeem it - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, - () -> String.format("AT %s locked to us (%s). P2SH-B %s funded. Watching P2SH-B for secret-B", - tradeBotData.getAtAddress(), tradeBotData.getTradeNativeAddress(), p2shAddressB)); - } - - /** - * Trade-bot is waiting for P2SH-B to funded. - *

- * It's possible than Bob's AT has reached it's trading timeout and automatically refunded QORT back to Bob. - * In which case, trade-bot is done with this specific trade and finalizes on refunded state. - *

- * Assuming P2SH-B is funded, trade-bot 'redeems' this P2SH using secret-B, thus revealing it to Alice. - *

- * Trade-bot's next step is to wait for Alice to use secret-B, and her secret-A, to redeem Bob's AT. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForP2shB(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // If we've passed AT refund timestamp then AT will have finished after auto-refunding - if (atData.getIsFinished()) { - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); - - return; - } - - // It's possible AT hasn't processed our previous MESSAGE yet and so lockTimeB won't be set - if (crossChainTradeData.lockTimeB == null) - // AT yet to process MESSAGE - return; - - Bitcoin bitcoin = Bitcoin.getInstance(); - - byte[] redeemScriptB = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, crossChainTradeData.lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); - String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); - - long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, crossChainTradeData.lockTimeB); - long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); - - final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; - - BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); - - switch (htlcStatusB) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // Still waiting for P2SH-B to be funded... - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // This shouldn't occur, but defensively bump to next state - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, - () -> String.format("P2SH-B %s already spent (exposing secret-B)? Checking AT %s for secret-A", p2shAddressB, tradeBotData.getAtAddress())); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // AT should auto-refund - we don't need to do anything here - return; - - case FUNDED: - break; - } - - // Redeem P2SH-B using secret-B - Coin redeemAmount = Coin.valueOf(P2SH_B_OUTPUT_AMOUNT); // An actual amount to avoid dust filter, remaining used as fees. The real funds are in P2SH-A. - ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressB); - byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); - - Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoin.getNetworkParameters(), redeemAmount, redeemKey, - fundingOutputs, redeemScriptB, tradeBotData.getSecret(), receivingAccountInfo); - - bitcoin.broadcastTransaction(p2shRedeemTransaction); - - // P2SH-B redeemed, now we wait for Alice to use secret-A to redeem AT - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, - () -> String.format("P2SH-B %s redeemed (exposing secret-B). Watching AT %s for secret-A", p2shAddressB, tradeBotData.getAtAddress())); - } - - /** - * Trade-bot is waiting for Bob to redeem P2SH-B thus revealing secret-B to Alice. - *

- * It's possible that this process has taken so long that we've reached P2SH-B's locktime. - * In which case, trade-bot switches to begin the refund process. - *

- * If trade-bot can extract a valid secret-B from the spend of P2SH-B, then it creates a - * zero-fee, PoW MESSAGE to send to Bob's AT, including both secret-B and also Alice's secret-A. - *

- * Both secrets are needed to release the QORT funds from Bob's AT to Alice's 'native'/Qortal - * trade address. - *

- * In revealing a valid secret-A, Bob can then redeem the BTC funds from P2SH-A. - *

- * If trade-bot successfully broadcasts the MESSAGE transaction, then this specific trade is done. - * @throws ForeignBlockchainException - */ - private void handleAliceWatchingP2shB(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) - return; - - Bitcoin bitcoin = Bitcoin.getInstance(); - - byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), crossChainTradeData.lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); - String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); - - long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, crossChainTradeData.lockTimeB); - long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); - final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; - - BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); - - switch (htlcStatusB) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - case FUNDED: - case REDEEM_IN_PROGRESS: - // Still waiting for P2SH-B to be funded/redeemed... - return; - - case REDEEMED: - // Bob has redeemed P2SH-B, so double-check that we have redeemed AT... - break; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // We've refunded P2SH-B? Bump to refunding P2SH-A then - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("P2SH-B %s already refunded. Refunding P2SH-A next", p2shAddressB)); - return; - } - - byte[] secretB = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddressB); - if (secretB == null) - // Secret not revealed at this time - return; - - // Send 'redeem' MESSAGE to AT using both secrets - byte[] secretA = tradeBotData.getSecret(); - String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH - byte[] messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, qortalReceivingAddress); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // Reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-B %s redeemed, using secrets to redeem AT %s. Funds should arrive at %s", - p2shAddressB, tradeBotData.getAtAddress(), qortalReceivingAddress)); - } - - /** - * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the BTC funds from P2SH-A. - *

- * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, - * trade-bot is done with this specific trade and finalizes in refunded state. - *

- * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the BTC funds from P2SH-A - * to Bob's 'foreign'/Bitcoin trade legacy-format address, as derived from trade private key. - *

- * (This could potentially be 'improved' to send BTC to any address of Bob's choosing by changing the transaction output). - *

- * If trade-bot successfully broadcasts the transaction, then this specific trade is done. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // AT should be 'finished' once Alice has redeemed QORT funds - if (!atData.getIsFinished()) - // Not finished yet - return; - - // If AT is not REDEEMED then something has gone wrong - if (crossChainTradeData.mode != AcctMode.REDEEMED) { - // Not redeemed so must be refunded/cancelled - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); - - return; - } - - byte[] secretA = BitcoinACCTv1.findSecretA(repository, crossChainTradeData); - if (secretA == null) { - LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); - return; - } - - // Use secret-A to redeem P2SH-A - - Bitcoin bitcoin = Bitcoin.getInstance(); - int lockTimeA = crossChainTradeData.lockTimeA; - - byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); - byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); - String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); - long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Double-check that we have redeemed P2SH-A... - break; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // Wait for AT to auto-refund - return; - - case FUNDED: { - Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT); - ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressA); - - Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoin.getNetworkParameters(), redeemAmount, redeemKey, - fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); - - bitcoin.broadcastTransaction(p2shRedeemTransaction); - break; - } - } - - String receivingAddress = bitcoin.pkhToAddress(receivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, - () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); - } - - /** - * Trade-bot is attempting to refund P2SH-B. - *

- * We could potentially skip this step as P2SH-B is only funded with a token amount to cover the mining fee should Bob redeem P2SH-B. - *

- * Upon successful broadcast of P2SH-B refunding transaction, trade-bot's next step is to begin refunding of P2SH-A. - * @throws ForeignBlockchainException - */ - private void handleAliceRefundingP2shB(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - int lockTimeB = crossChainTradeData.lockTimeB; - - // We can't refund P2SH-B until lockTime-B has passed - if (NTP.getTime() <= lockTimeB * 1000L) - return; - - Bitcoin bitcoin = Bitcoin.getInstance(); - - // We can't refund P2SH-B until median block time has passed lockTime-B (see BIP113) - int medianBlockTime = bitcoin.getMedianBlockTime(); - if (medianBlockTime <= lockTimeB) - return; - - byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); - String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); - - long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, lockTimeB); - long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); - final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; - - BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); - - switch (htlcStatusB) { - case UNFUNDED: - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("P2SH-B %s never funded?. Refunding P2SH-A next", p2shAddressB)); - return; - - case FUNDING_IN_PROGRESS: - // Still waiting for P2SH-B to be funded... - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // We must be very close to trade timeout. Defensively try to refund P2SH-A - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("P2SH-B %s already spent?. Refunding P2SH-A next", p2shAddressB)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - break; - - case FUNDED:{ - Coin refundAmount = Coin.valueOf(P2SH_B_OUTPUT_AMOUNT); // An actual amount to avoid dust filter, remaining used as fees. - ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressB); - - // Determine receive address for refund - String receiveAddress = bitcoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); - Address receiving = Address.fromString(bitcoin.getNetworkParameters(), receiveAddress); - - Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoin.getNetworkParameters(), refundAmount, refundKey, - fundingOutputs, redeemScriptB, lockTimeB, receiving.getHash()); - - bitcoin.broadcastTransaction(p2shRefundTransaction); - break; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("Refunded P2SH-B %s. Waiting for LockTime-A", p2shAddressB)); - } - - /** - * Trade-bot is attempting to refund P2SH-A. - * @throws ForeignBlockchainException - */ - private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - int lockTimeA = tradeBotData.getLockTimeA(); - - // We can't refund P2SH-A until lockTime-A has passed - if (NTP.getTime() <= lockTimeA * 1000L) - return; - - Bitcoin bitcoin = Bitcoin.getInstance(); - - // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) - int medianBlockTime = bitcoin.getMedianBlockTime(); - if (medianBlockTime <= lockTimeA) - return; - - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); - long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // Still waiting for P2SH-A to be funded... - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Too late! - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-A %s already spent!", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - break; - - case FUNDED:{ - Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT); - ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressA); - - // Determine receive address for refund - String receiveAddress = bitcoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); - Address receiving = Address.fromString(bitcoin.getNetworkParameters(), receiveAddress); - - Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoin.getNetworkParameters(), refundAmount, refundKey, - fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); - - bitcoin.broadcastTransaction(p2shRefundTransaction); - break; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); - } - - /** - * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. - *

- * Will automatically update trade-bot state to ALICE_REFUNDING_B or ALICE_DONE as necessary. - * - * @throws DataException - * @throws ForeignBlockchainException - */ - private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // This is OK - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) - return false; - - boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); - - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING && isAtLockedToUs) - return false; - - if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { - // We've redeemed already? - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); - } else { - // Any other state is not good, so start defensive refund - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_B, - () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); - } - - return true; - } - - private long calcP2shAFeeTimestamp(int lockTimeA, int tradeTimeout) { - return (lockTimeA - tradeTimeout * 60) * 1000L; - } - - private long calcP2shBFeeTimestamp(int lockTimeA, int lockTimeB) { - // lockTimeB is halfway between offerMessageTimestamp and lockTimeA - return (lockTimeA - (lockTimeA - lockTimeB) * 2) * 1000L; - } - -} diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java deleted file mode 100644 index 0bd2972b..00000000 --- a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java +++ /dev/null @@ -1,894 +0,0 @@ -package org.qortal.controller.tradebot; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bitcoinj.core.Address; -import org.bitcoinj.core.AddressFormatException; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.account.PublicKeyAccount; -import org.qortal.api.model.crosschain.TradeBotCreateRequest; -import org.qortal.asset.Asset; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.AcctMode; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.Litecoin; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.DeployAtTransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -/** - * Performing cross-chain trading steps on behalf of user. - *

- * We deal with three different independent state-spaces here: - *

    - *
  • Qortal blockchain
    • - *
  • Foreign blockchain
    • - *
  • Trade-bot entries
    • - *
- */ -public class LitecoinACCTv1TradeBot implements AcctTradeBot { - - private static final Logger LOGGER = LogManager.getLogger(LitecoinACCTv1TradeBot.class); - - public enum State implements TradeBot.StateNameAndValueSupplier { - BOB_WAITING_FOR_AT_CONFIRM(10, false, false), - BOB_WAITING_FOR_MESSAGE(15, true, true), - BOB_WAITING_FOR_AT_REDEEM(25, true, true), - BOB_DONE(30, false, false), - BOB_REFUNDED(35, false, false), - - ALICE_WAITING_FOR_AT_LOCK(85, true, true), - ALICE_DONE(95, false, false), - ALICE_REFUNDING_A(105, true, true), - ALICE_REFUNDED(110, false, false); - - private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); - - public final int value; - public final boolean requiresAtData; - public final boolean requiresTradeData; - - State(int value, boolean requiresAtData, boolean requiresTradeData) { - this.value = value; - this.requiresAtData = requiresAtData; - this.requiresTradeData = requiresTradeData; - } - - public static State valueOf(int value) { - return map.get(value); - } - - @Override - public String getState() { - return this.name(); - } - - @Override - public int getStateValue() { - return this.value; - } - } - - /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ - private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms - - private static LitecoinACCTv1TradeBot instance; - - private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() - .map(State::name) - .collect(Collectors.toUnmodifiableList()); - - private LitecoinACCTv1TradeBot() { - } - - public static synchronized LitecoinACCTv1TradeBot getInstance() { - if (instance == null) - instance = new LitecoinACCTv1TradeBot(); - - return instance; - } - - @Override - public List getEndStates() { - return this.endStates; - } - - /** - * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for LTC. - *

- * Generates: - *

    - *
  • new 'trade' private key
    • - *
- * Derives: - *
    - *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • - *
  • 'foreign' (as in Litecoin) public key, public key hash
    • - *
- * A Qortal AT is then constructed including the following as constants in the 'data segment': - *
    - *
  • 'native'/Qortal 'trade' address - used as a MESSAGE contact
    • - *
  • 'foreign'/Litecoin public key hash - used by Alice's P2SH scripts to allow redeem
    • - *
  • QORT amount on offer by Bob
    • - *
  • LTC amount expected in return by Bob (from Alice)
    • - *
  • trading timeout, in case things go wrong and everyone needs to refund
    • - *
- * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. - *

- * Trade-bot will wait for Bob's AT to be deployed before taking next step. - *

- * @param repository - * @param tradeBotCreateRequest - * @return raw, unsigned DEPLOY_AT transaction - * @throws DataException - */ - public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - - // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) - Address litecoinReceivingAddress; - try { - litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); - } catch (AddressFormatException e) { - throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); - } - if (litecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) - throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); - - byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); - - PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); - - // Deploy AT - long timestamp = NTP.getTime(); - byte[] reference = creator.getLastReference(); - long fee = 0L; - byte[] signature = null; - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); - - String name = "QORT/LTC ACCT"; - String description = "QORT/LTC cross-chain trade"; - String aTType = "ACCT"; - String tags = "ACCT QORT LTC"; - byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, - tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); - long amount = tradeBotCreateRequest.fundingQortAmount; - - DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - DeployAtTransaction.ensureATAddress(deployAtTransactionData); - String atAddress = deployAtTransactionData.getAtAddress(); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, - State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, - creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - null, null, - SupportedBlockchain.LITECOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - tradeBotCreateRequest.foreignAmount, null, null, null, litecoinReceivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); - - // Attempt to backup the trade bot data - TradeBot.backupTradeBotData(repository); - - // Return to user for signing and broadcast as we don't have their Qortal private key - try { - return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); - } catch (TransformationException e) { - throw new DataException("Failed to transform DEPLOY_AT transaction?", e); - } - } - - /** - * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching LTC to an existing offer. - *

- * Requires a chosen trade offer from Bob, passed by crossChainTradeData - * and access to a Litecoin wallet via xprv58. - *

- * The crossChainTradeData contains the current trade offer state - * as extracted from the AT's data segment. - *

- * Access to a funded wallet is via a Litecoin BIP32 hierarchical deterministic key, - * passed via xprv58. - * This key will be stored in your node's database - * to allow trade-bot to create/fund the necessary P2SH transactions! - * However, due to the nature of BIP32 keys, it is possible to give the trade-bot - * only a subset of wallet access (see BIP32 for more details). - *

- * As an example, the xprv58 can be extract from a legacy, password-less - * Electrum wallet by going to the console tab and entering:
- * wallet.keystore.xprv
- * which should result in a base58 string starting with either 'xprv' (for Litecoin main-net) - * or 'tprv' for (Litecoin test-net). - *

- * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. - *

- * If sufficient funds are available, this method will actually fund the P2SH-A - * with the Litecoin amount expected by 'Bob'. - *

- * If the Litecoin transaction is successfully broadcast to the network then - * we also send a MESSAGE to Bob's trade-bot to let them know. - *

- * The trade-bot entry is saved to the repository and the cross-chain trading process commences. - *

- * @param repository - * @param crossChainTradeData chosen trade OFFER that Alice wants to match - * @param xprv58 funded wallet xprv in base58 - * @return true if P2SH-A funding transaction successfully broadcast to Litecoin network, false otherwise - * @throws DataException - */ - public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { - byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); - byte[] secretA = TradeBot.generateSecret(); - byte[] hashOfSecretA = Crypto.hash160(secretA); - - byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); - byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); - String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); - - byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); - byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); - byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH - - // We need to generate lockTime-A: add tradeTimeout to now - long now = NTP.getTime(); - int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, - State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, - receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - secretA, hashOfSecretA, - SupportedBlockchain.LITECOIN.name(), - tradeForeignPublicKey, tradeForeignPublicKeyHash, - crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); - - // Attempt to backup the trade bot data - TradeBot.backupTradeBotData(repository); - - // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount - long p2shFee; - try { - p2shFee = Litecoin.getInstance().getP2shFee(now); - } catch (ForeignBlockchainException e) { - LOGGER.debug("Couldn't estimate Litecoin fees?"); - return ResponseResult.NETWORK_ISSUE; - } - - // Fee for redeem/refund is subtracted from P2SH-A balance. - // Do not include fee for funding transaction as this is covered by buildSpend() - long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; - - // P2SH-A to be funded - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); - String p2shAddress = Litecoin.getInstance().deriveP2shAddress(redeemScriptBytes); - - // Build transaction for funding P2SH-A - Transaction p2shFundingTransaction = Litecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); - if (p2shFundingTransaction == null) { - LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); - return ResponseResult.BALANCE_ISSUE; - } - - try { - Litecoin.getInstance().broadcastTransaction(p2shFundingTransaction); - } catch (ForeignBlockchainException e) { - // We couldn't fund P2SH-A at this time - LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); - return ResponseResult.NETWORK_ISSUE; - } - - // Attempt to send MESSAGE to Bob's Qortal trade address - byte[] messageData = LitecoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); - String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); - return ResponseResult.NETWORK_ISSUE; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); - - return ResponseResult.OK; - } - - @Override - public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) - return true; - - // If the AT doesn't exist then we might as well let the user tidy up - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) - return true; - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - case ALICE_DONE: - case BOB_DONE: - case ALICE_REFUNDED: - case BOB_REFUNDED: - return true; - - default: - return false; - } - } - - @Override - public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { - State tradeBotState = State.valueOf(tradeBotData.getStateValue()); - if (tradeBotState == null) { - LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); - return; - } - - ATData atData = null; - CrossChainTradeData tradeData = null; - - if (tradeBotState.requiresAtData) { - // Attempt to fetch AT data - atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); - if (atData == null) { - LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); - return; - } - - if (tradeBotState.requiresTradeData) { - tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - if (tradeData == null) { - LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); - return; - } - } - } - - switch (tradeBotState) { - case BOB_WAITING_FOR_AT_CONFIRM: - handleBobWaitingForAtConfirm(repository, tradeBotData); - break; - - case BOB_WAITING_FOR_MESSAGE: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_WAITING_FOR_AT_LOCK: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); - break; - - case BOB_WAITING_FOR_AT_REDEEM: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_DONE: - case BOB_DONE: - break; - - case ALICE_REFUNDING_A: - TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); - handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); - break; - - case ALICE_REFUNDED: - case BOB_REFUNDED: - break; - } - } - - /** - * Trade-bot is waiting for Bob's AT to deploy. - *

- * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. - */ - private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { - if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { - if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) - return; - - // We've waited ages for AT to be confirmed into a block but something has gone awry. - // After this long we assume transaction loss so give up with trade-bot entry too. - tradeBotData.setState(State.BOB_REFUNDED.name()); - tradeBotData.setStateValue(State.BOB_REFUNDED.value); - tradeBotData.setTimestamp(NTP.getTime()); - // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() - repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); - repository.saveChanges(); - - LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); - TradeBot.notifyStateChange(tradeBotData); - return; - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, - () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); - } - - /** - * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. - *

- * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, - * in which case trade-bot is done with this specific trade and finalizes on refunded state. - *

- * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. - *

- * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. - *

- * Assuming P2SH-A has at least expected Litecoin balance, - * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. - *

- * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. - *

- * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to - * extract secret-A needed to redeem Alice's P2SH. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // If AT has finished then Bob likely cancelled his trade offer - if (atData.getIsFinished()) { - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); - return; - } - - Litecoin litecoin = Litecoin.getInstance(); - - String address = tradeBotData.getTradeNativeAddress(); - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); - - for (MessageTransactionData messageTransactionData : messageTransactionsData) { - if (messageTransactionData.isText()) - continue; - - // We're expecting: HASH160(secret-A), Alice's Litecoin pubkeyhash and lockTime-A - byte[] messageData = messageTransactionData.getData(); - LitecoinACCTv1.OfferMessageData offerMessageData = LitecoinACCTv1.extractOfferMessageData(messageData); - if (offerMessageData == null) - continue; - - byte[] aliceForeignPublicKeyHash = offerMessageData.partnerLitecoinPKH; - byte[] hashOfSecretA = offerMessageData.hashOfSecretA; - int lockTimeA = (int) offerMessageData.lockTimeA; - long messageTimestamp = messageTransactionData.getTimestamp(); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(messageTimestamp, lockTimeA); - - // Determine P2SH-A address and confirm funded - byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; - - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // There might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // We've already redeemed this? - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, - () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... - continue; - - case FUNDED: - // Fall-through out of switch... - break; - } - - // Good to go - send MESSAGE to AT - - String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); - - // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume - byte[] outgoingMessageData = LitecoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); - - outgoingMessageTransaction.computeNonce(); - outgoingMessageTransaction.sign(sender); - - // reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, - () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); - - return; - } - } - - /** - * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. - *

- * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow - * this process has taken so long that we've reached P2SH-A's locktime, or that someone else - * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. - *

- * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. - *

- * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. - *

- * In revealing a valid secret-A, Bob can then redeem the LTC funds from P2SH-A. - *

- * @throws ForeignBlockchainException - */ - private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) - return; - - Litecoin litecoin = Litecoin.getInstance(); - int lockTimeA = tradeBotData.getLockTimeA(); - - // Refund P2SH-A if we've passed lockTime-A - if (NTP.getTime() >= lockTimeA * 1000L) { - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - case FUNDED: - break; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Already redeemed? - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); - return; - - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> atData.getIsFinished() - ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) - : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); - - return; - } - - // We're waiting for AT to be in TRADE mode - if (crossChainTradeData.mode != AcctMode.TRADING) - return; - - // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above - - // Find our MESSAGE to AT from previous state - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), - crossChainTradeData.qortalCreatorTradeAddress, null, null, null); - if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { - LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); - return; - } - - long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); - - // Our calculated refundTimeout should match AT's refundTimeout - if (refundTimeout != crossChainTradeData.refundTimeout) { - LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); - // We'll eventually refund - return; - } - - // We're good to redeem AT - - // Send 'redeem' MESSAGE to AT using both secret - byte[] secretA = tradeBotData.getSecret(); - String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH - byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secretA, qortalReceivingAddress); - String messageRecipient = tradeBotData.getAtAddress(); - - boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); - if (!isMessageAlreadySent) { - PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); - - messageTransaction.computeNonce(); - messageTransaction.sign(sender); - - // Reset repository state to prevent deadlock - repository.discardChanges(); - ValidationResult result = messageTransaction.importAsUnconfirmed(); - - if (result != ValidationResult.OK) { - LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); - return; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("Redeeming AT %s. Funds should arrive at %s", - tradeBotData.getAtAddress(), qortalReceivingAddress)); - } - - /** - * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the LTC funds from P2SH-A. - *

- * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, - * trade-bot is done with this specific trade and finalizes in refunded state. - *

- * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the LTC funds from P2SH-A - * to Bob's 'foreign'/Litecoin trade legacy-format address, as derived from trade private key. - *

- * (This could potentially be 'improved' to send LTC to any address of Bob's choosing by changing the transaction output). - *

- * If trade-bot successfully broadcasts the transaction, then this specific trade is done. - * @throws ForeignBlockchainException - */ - private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // AT should be 'finished' once Alice has redeemed QORT funds - if (!atData.getIsFinished()) - // Not finished yet - return; - - // If AT is REFUNDED or CANCELLED then something has gone wrong - if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { - // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the LTC - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, - () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); - - return; - } - - byte[] secretA = LitecoinACCTv1.findSecretA(repository, crossChainTradeData); - if (secretA == null) { - LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); - return; - } - - // Use secret-A to redeem P2SH-A - - Litecoin litecoin = Litecoin.getInstance(); - - byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); - int lockTimeA = crossChainTradeData.lockTimeA; - byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Double-check that we have redeemed P2SH-A... - break; - - case REFUND_IN_PROGRESS: - case REFUNDED: - // Wait for AT to auto-refund - return; - - case FUNDED: { - Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); - - Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, - fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); - - litecoin.broadcastTransaction(p2shRedeemTransaction); - break; - } - } - - String receivingAddress = litecoin.pkhToAddress(receivingAccountInfo); - - TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, - () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); - } - - /** - * Trade-bot is attempting to refund P2SH-A. - * @throws ForeignBlockchainException - */ - private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - int lockTimeA = tradeBotData.getLockTimeA(); - - // We can't refund P2SH-A until lockTime-A has passed - if (NTP.getTime() <= lockTimeA * 1000L) - return; - - Litecoin litecoin = Litecoin.getInstance(); - - // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) - int medianBlockTime = litecoin.getMedianBlockTime(); - if (medianBlockTime <= lockTimeA) - return; - - byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); - String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); - - // Fee for redeem/refund is subtracted from P2SH-A balance. - long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); - long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); - long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; - BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); - - switch (htlcStatusA) { - case UNFUNDED: - case FUNDING_IN_PROGRESS: - // Still waiting for P2SH-A to be funded... - return; - - case REDEEM_IN_PROGRESS: - case REDEEMED: - // Too late! - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("P2SH-A %s already spent!", p2shAddressA)); - return; - - case REFUND_IN_PROGRESS: - case REFUNDED: - break; - - case FUNDED:{ - Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); - ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); - List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); - - // Determine receive address for refund - String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); - Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); - - Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, - fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); - - litecoin.broadcastTransaction(p2shRefundTransaction); - break; - } - } - - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, - () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); - } - - /** - * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. - *

- * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. - * - * @throws DataException - * @throws ForeignBlockchainException - */ - private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, - ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { - // This is OK - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) - return false; - - boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); - - if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) - if (isAtLockedToUs) { - // AT is trading with us - OK - return false; - } else { - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); - - return true; - } - - if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { - // We've redeemed already? - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, - () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); - } else { - // Any other state is not good, so start defensive refund - TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, - () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); - } - - return true; - } - - private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { - return (lockTimeA - tradeTimeout * 60) * 1000L; - } - -} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java deleted file mode 100644 index fa3b599e..00000000 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ /dev/null @@ -1,373 +0,0 @@ -package org.qortal.controller.tradebot; - -import java.awt.TrayIcon.MessageType; -import java.security.SecureRandom; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.util.Supplier; -import org.bitcoinj.core.ECKey; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.api.model.crosschain.TradeBotCreateRequest; -import org.qortal.controller.Controller; -import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.BitcoinACCTv1; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.PresenceTransactionData; -import org.qortal.event.Event; -import org.qortal.event.EventBus; -import org.qortal.event.Listener; -import org.qortal.group.Group; -import org.qortal.gui.SysTray; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.settings.Settings; -import org.qortal.transaction.PresenceTransaction; -import org.qortal.transaction.PresenceTransaction.PresenceType; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.transform.transaction.TransactionTransformer; -import org.qortal.utils.NTP; - -import com.google.common.primitives.Longs; - -/** - * Performing cross-chain trading steps on behalf of user. - *

- * We deal with three different independent state-spaces here: - *

    - *
  • Qortal blockchain
    • - *
  • Foreign blockchain
    • - *
  • Trade-bot entries
    • - *
- */ -public class TradeBot implements Listener { - - private static final Logger LOGGER = LogManager.getLogger(TradeBot.class); - private static final Random RANDOM = new SecureRandom(); - - public interface StateNameAndValueSupplier { - public String getState(); - public int getStateValue(); - } - - public static class StateChangeEvent implements Event { - private final TradeBotData tradeBotData; - - public StateChangeEvent(TradeBotData tradeBotData) { - this.tradeBotData = tradeBotData; - } - - public TradeBotData getTradeBotData() { - return this.tradeBotData; - } - } - - private static final Map, Supplier> acctTradeBotSuppliers = new HashMap<>(); - static { - acctTradeBotSuppliers.put(BitcoinACCTv1.class, BitcoinACCTv1TradeBot::getInstance); - acctTradeBotSuppliers.put(LitecoinACCTv1.class, LitecoinACCTv1TradeBot::getInstance); - } - - private static TradeBot instance; - - private final Map presenceTimestampsByAtAddress = Collections.synchronizedMap(new HashMap<>()); - - private TradeBot() { - EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event)); - } - - public static synchronized TradeBot getInstance() { - if (instance == null) - instance = new TradeBot(); - - return instance; - } - - public ACCT getAcctUsingAtData(ATData atData) { - byte[] codeHash = atData.getCodeHash(); - if (codeHash == null) - return null; - - return SupportedBlockchain.getAcctByCodeHash(codeHash); - } - - public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { - ACCT acct = this.getAcctUsingAtData(atData); - if (acct == null) - return null; - - return acct.populateTradeData(repository, atData); - } - - /** - * Creates a new trade-bot entry from the "Bob" viewpoint, - * i.e. OFFERing QORT in exchange for foreign blockchain currency. - *

- * Generates: - *

    - *
  • new 'trade' private key
    • - *
  • secret(s)
    • - *
- * Derives: - *
    - *
  • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
    • - *
  • 'foreign' public key, public key hash
    • - *
  • hash(es) of secret(s)
    • - *
- * A Qortal AT is then constructed including the following as constants in the 'data segment': - *
    - *
  • 'native' (Qortal) 'trade' address - used to MESSAGE AT
    • - *
  • 'foreign' public key hash - used by Alice's to allow redeem of currency on foreign blockchain
    • - *
  • hash(es) of secret(s) - used by AT (optional) and foreign blockchain as needed
    • - *
  • QORT amount on offer by Bob
    • - *
  • foreign currency amount expected in return by Bob (from Alice)
    • - *
  • trading timeout, in case things go wrong and everyone needs to refund
    • - *
- * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. - *

- * Trade-bot will wait for Bob's AT to be deployed before taking next step. - *

- * @param repository - * @param tradeBotCreateRequest - * @return raw, unsigned DEPLOY_AT transaction - * @throws DataException - */ - public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { - // Fetch latest ACCT version for requested foreign blockchain - ACCT acct = tradeBotCreateRequest.foreignBlockchain.getLatestAcct(); - - AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); - if (acctTradeBot == null) - return null; - - return acctTradeBot.createTrade(repository, tradeBotCreateRequest); - } - - /** - * Creates a trade-bot entry from the 'Alice' viewpoint, - * i.e. matching foreign blockchain currency to an existing QORT offer. - *

- * Requires a chosen trade offer from Bob, passed by crossChainTradeData - * and access to a foreign blockchain wallet via foreignKey. - *

- * @param repository - * @param crossChainTradeData chosen trade OFFER that Alice wants to match - * @param foreignKey foreign blockchain wallet key - * @throws DataException - */ - public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, - CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException { - AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); - if (acctTradeBot == null) { - LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot for AT %s", atData.getATAddress())); - return ResponseResult.NETWORK_ISSUE; - } - - // Check Alice doesn't already have an existing, on-going trade-bot entry for this AT. - if (repository.getCrossChainRepository().existsTradeWithAtExcludingStates(atData.getATAddress(), acctTradeBot.getEndStates())) - return ResponseResult.TRADE_ALREADY_EXISTS; - - return acctTradeBot.startResponse(repository, atData, acct, crossChainTradeData, foreignKey, receivingAddress); - } - - public boolean deleteEntry(Repository repository, byte[] tradePrivateKey) throws DataException { - TradeBotData tradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); - if (tradeBotData == null) - // Can't delete what we don't have! - return false; - - boolean canDelete = false; - - ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName()); - if (acct == null) - // We can't/no longer support this ACCT - canDelete = true; - else { - AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); - canDelete = acctTradeBot == null || acctTradeBot.canDelete(repository, tradeBotData); - } - - if (canDelete) { - repository.getCrossChainRepository().delete(tradePrivateKey); - repository.saveChanges(); - } - - return canDelete; - } - - @Override - public void listen(Event event) { - if (!(event instanceof Controller.NewBlockEvent)) - return; - - synchronized (this) { - List allTradeBotData; - - try (final Repository repository = RepositoryManager.getRepository()) { - allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); - } catch (DataException e) { - LOGGER.error("Couldn't run trade bot due to repository issue", e); - return; - } - - for (TradeBotData tradeBotData : allTradeBotData) - try (final Repository repository = RepositoryManager.getRepository()) { - // Find ACCT-specific trade-bot for this entry - ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName()); - if (acct == null) { - LOGGER.debug(() -> String.format("Couldn't find ACCT matching name %s", tradeBotData.getAcctName())); - continue; - } - - AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); - if (acctTradeBot == null) { - LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot matching name %s", tradeBotData.getAcctName())); - continue; - } - - acctTradeBot.progress(repository, tradeBotData); - } catch (DataException e) { - LOGGER.error("Couldn't run trade bot due to repository issue", e); - } catch (ForeignBlockchainException e) { - LOGGER.warn(() -> String.format("Foreign blockchain issue processing trade-bot entry for AT %s: %s", tradeBotData.getAtAddress(), e.getMessage())); - } - } - } - - /*package*/ static byte[] generateTradePrivateKey() { - // The private key is used for both Curve25519 and secp256k1 so needs to be valid for both. - // Curve25519 accepts any seed, so generate a valid secp256k1 key and use that. - return new ECKey().getPrivKeyBytes(); - } - - /*package*/ static byte[] deriveTradeNativePublicKey(byte[] privateKey) { - return PrivateKeyAccount.toPublicKey(privateKey); - } - - /*package*/ static byte[] deriveTradeForeignPublicKey(byte[] privateKey) { - return ECKey.fromPrivate(privateKey).getPubKey(); - } - - /*package*/ static byte[] generateSecret() { - byte[] secret = new byte[32]; - RANDOM.nextBytes(secret); - return secret; - } - - /*package*/ static void backupTradeBotData(Repository repository) { - // Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure - try { - LOGGER.info("About to backup trade bot data..."); - repository.exportNodeLocalData(); - } catch (DataException e) { - LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage())); - } - } - - /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ - /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, - String newState, int newStateValue, Supplier logMessageSupplier) throws DataException { - tradeBotData.setState(newState); - tradeBotData.setStateValue(newStateValue); - tradeBotData.setTimestamp(NTP.getTime()); - repository.getCrossChainRepository().save(tradeBotData); - repository.saveChanges(); - - if (Settings.getInstance().isTradebotSystrayEnabled()) - SysTray.getInstance().showMessage("Trade-Bot", String.format("%s: %s", tradeBotData.getAtAddress(), newState), MessageType.INFO); - - if (logMessageSupplier != null) - LOGGER.info(logMessageSupplier); - - LOGGER.debug(() -> String.format("new state for trade-bot entry based on AT %s: %s", tradeBotData.getAtAddress(), newState)); - - notifyStateChange(tradeBotData); - } - - /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ - /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, StateNameAndValueSupplier newStateSupplier, Supplier logMessageSupplier) throws DataException { - updateTradeBotState(repository, tradeBotData, newStateSupplier.getState(), newStateSupplier.getStateValue(), logMessageSupplier); - } - - /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ - /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, Supplier logMessageSupplier) throws DataException { - updateTradeBotState(repository, tradeBotData, tradeBotData.getState(), tradeBotData.getStateValue(), logMessageSupplier); - } - - /*package*/ static void notifyStateChange(TradeBotData tradeBotData) { - StateChangeEvent stateChangeEvent = new StateChangeEvent(tradeBotData); - EventBus.INSTANCE.notify(stateChangeEvent); - } - - /*package*/ static AcctTradeBot findTradeBotForAcct(ACCT acct) { - Supplier acctTradeBotSupplier = acctTradeBotSuppliers.get(acct.getClass()); - if (acctTradeBotSupplier == null) - return null; - - return acctTradeBotSupplier.get(); - } - - // PRESENCE-related - /*package*/ void updatePresence(Repository repository, TradeBotData tradeBotData, CrossChainTradeData tradeData) - throws DataException { - String atAddress = tradeBotData.getAtAddress(); - - PrivateKeyAccount tradeNativeAccount = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); - String signerAddress = tradeNativeAccount.getAddress(); - - /* - * There's no point in Alice trying to build a PRESENCE transaction - * for an AT that isn't locked to her, as other peers won't be able - * to validate the PRESENCE transaction as signing public key won't - * be visible. - */ - if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) - // Signer is neither Bob, nor Alice, or trade not yet locked to Alice - return; - - long now = NTP.getTime(); - long threshold = now - PresenceType.TRADE_BOT.getLifetime(); - - long timestamp = presenceTimestampsByAtAddress.compute(atAddress, (k, v) -> (v == null || v < threshold) ? now : v); - - // If timestamp hasn't been updated then nothing to do - if (timestamp != now) - return; - - int txGroupId = Group.NO_GROUP; - byte[] reference = new byte[TransactionTransformer.SIGNATURE_LENGTH]; - byte[] creatorPublicKey = tradeNativeAccount.getPublicKey(); - long fee = 0L; - - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, null); - - int nonce = 0; - byte[] timestampSignature = tradeNativeAccount.sign(Longs.toByteArray(timestamp)); - - PresenceTransactionData transactionData = new PresenceTransactionData(baseTransactionData, nonce, PresenceType.TRADE_BOT, timestampSignature); - - PresenceTransaction presenceTransaction = new PresenceTransaction(repository, transactionData); - presenceTransaction.computeNonce(); - - presenceTransaction.sign(tradeNativeAccount); - - ValidationResult result = presenceTransaction.importAsUnconfirmed(); - if (result != ValidationResult.OK) - LOGGER.debug(() -> String.format("Unable to build trade-bot PRESENCE transaction for %s: %s", tradeBotData.getAtAddress(), result.name())); - } - -} diff --git a/src/main/java/org/qortal/crosschain/ACCT.java b/src/main/java/org/qortal/crosschain/ACCT.java deleted file mode 100644 index e557a3e2..00000000 --- a/src/main/java/org/qortal/crosschain/ACCT.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.qortal.crosschain; - -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; - -public interface ACCT { - - public byte[] getCodeBytesHash(); - - public int getModeByteOffset(); - - public ForeignBlockchain getBlockchain(); - - public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException; - - public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException; - - public byte[] buildCancelMessage(String creatorQortalAddress); - -} diff --git a/src/main/java/org/qortal/crosschain/AcctMode.java b/src/main/java/org/qortal/crosschain/AcctMode.java deleted file mode 100644 index 21496032..00000000 --- a/src/main/java/org/qortal/crosschain/AcctMode.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.qortal.crosschain; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - -import java.util.Map; - -public enum AcctMode { - OFFERING(0), TRADING(1), CANCELLED(2), REFUNDED(3), REDEEMED(4); - - public final int value; - private static final Map map = stream(AcctMode.values()).collect(toMap(mode -> mode.value, mode -> mode)); - - AcctMode(int value) { - this.value = value; - } - - public static AcctMode valueOf(int value) { - return map.get(value); - } -} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/Bitcoin.java b/src/main/java/org/qortal/crosschain/Bitcoin.java deleted file mode 100644 index 28275d6a..00000000 --- a/src/main/java/org/qortal/crosschain/Bitcoin.java +++ /dev/null @@ -1,190 +0,0 @@ -package org.qortal.crosschain; - -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumMap; -import java.util.Map; - -import org.bitcoinj.core.Context; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.params.MainNetParams; -import org.bitcoinj.params.RegTestParams; -import org.bitcoinj.params.TestNet3Params; -import org.qortal.crosschain.ElectrumX.Server; -import org.qortal.crosschain.ElectrumX.Server.ConnectionType; -import org.qortal.settings.Settings; - -public class Bitcoin extends Bitcoiny { - - public static final String CURRENCY_CODE = "BTC"; - - // Temporary values until a dynamic fee system is written. - private static final long OLD_FEE_AMOUNT = 4_000L; // Not 5000 so that existing P2SH-B can output 1000, avoiding dust issue, leaving 4000 for fees. - private static final long NEW_FEE_TIMESTAMP = 1598280000000L; // milliseconds since epoch - private static final long NEW_FEE_AMOUNT = 10_000L; - - private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST - - private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); - static { - DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); - DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); - } - - public enum BitcoinNet { - MAIN { - @Override - public NetworkParameters getParams() { - return MainNetParams.get(); - } - - @Override - public Collection getServers() { - return Arrays.asList( - // Servers chosen on NO BASIS WHATSOEVER from various sources! - new Server("128.0.190.26", Server.ConnectionType.SSL, 50002), - new Server("hodlers.beer", Server.ConnectionType.SSL, 50002), - new Server("electrumx.erbium.eu", Server.ConnectionType.TCP, 50001), - new Server("electrumx.erbium.eu", Server.ConnectionType.SSL, 50002), - new Server("btc.lastingcoin.net", Server.ConnectionType.SSL, 50002), - new Server("electrum.bitaroo.net", Server.ConnectionType.SSL, 50002), - new Server("bitcoin.grey.pw", Server.ConnectionType.SSL, 50002), - new Server("2electrumx.hopto.me", Server.ConnectionType.SSL, 56022), - new Server("185.64.116.15", Server.ConnectionType.SSL, 50002), - new Server("kirsche.emzy.de", Server.ConnectionType.SSL, 50002), - new Server("alviss.coinjoined.com", Server.ConnectionType.SSL, 50002), - new Server("electrum.emzy.de", Server.ConnectionType.SSL, 50002), - new Server("electrum.emzy.de", Server.ConnectionType.TCP, 50001), - new Server("vmd71287.contaboserver.net", Server.ConnectionType.SSL, 50002), - new Server("btc.litepay.ch", Server.ConnectionType.SSL, 50002), - new Server("electrum.stippy.com", Server.ConnectionType.SSL, 50002), - new Server("xtrum.com", Server.ConnectionType.SSL, 50002), - new Server("electrum.acinq.co", Server.ConnectionType.SSL, 50002), - new Server("electrum2.taborsky.cz", Server.ConnectionType.SSL, 50002), - new Server("vmd63185.contaboserver.net", Server.ConnectionType.SSL, 50002), - new Server("electrum2.privateservers.network", Server.ConnectionType.SSL, 50002), - new Server("electrumx.alexridevski.net", Server.ConnectionType.SSL, 50002), - new Server("192.166.219.200", Server.ConnectionType.SSL, 50002), - new Server("2ex.digitaleveryware.com", Server.ConnectionType.SSL, 50002), - new Server("dxm.no-ip.biz", Server.ConnectionType.SSL, 50002), - new Server("caleb.vegas", Server.ConnectionType.SSL, 50002)); - } - - @Override - public String getGenesisHash() { - return "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"; - } - - @Override - public long getP2shFee(Long timestamp) { - // TODO: This will need to be replaced with something better in the near future! - if (timestamp != null && timestamp < NEW_FEE_TIMESTAMP) - return OLD_FEE_AMOUNT; - - return NEW_FEE_AMOUNT; - } - }, - TEST3 { - @Override - public NetworkParameters getParams() { - return TestNet3Params.get(); - } - - @Override - public Collection getServers() { - return Arrays.asList( - new Server("tn.not.fyi", Server.ConnectionType.SSL, 55002), - new Server("electrumx-test.1209k.com", Server.ConnectionType.SSL, 50002), - new Server("testnet.qtornado.com", Server.ConnectionType.SSL, 51002), - new Server("testnet.aranguren.org", Server.ConnectionType.TCP, 51001), - new Server("testnet.aranguren.org", Server.ConnectionType.SSL, 51002), - new Server("testnet.hsmiths.com", Server.ConnectionType.SSL, 53012)); - } - - @Override - public String getGenesisHash() { - return "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"; - } - - @Override - public long getP2shFee(Long timestamp) { - return NON_MAINNET_FEE; - } - }, - REGTEST { - @Override - public NetworkParameters getParams() { - return RegTestParams.get(); - } - - @Override - public Collection getServers() { - return Arrays.asList( - new Server("localhost", Server.ConnectionType.TCP, 50001), - new Server("localhost", Server.ConnectionType.SSL, 50002)); - } - - @Override - public String getGenesisHash() { - // This is unique to each regtest instance - return null; - } - - @Override - public long getP2shFee(Long timestamp) { - return NON_MAINNET_FEE; - } - }; - - public abstract NetworkParameters getParams(); - public abstract Collection getServers(); - public abstract String getGenesisHash(); - public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; - } - - private static Bitcoin instance; - - private final BitcoinNet bitcoinNet; - - // Constructors and instance - - private Bitcoin(BitcoinNet bitcoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { - super(blockchain, bitcoinjContext, currencyCode); - this.bitcoinNet = bitcoinNet; - - LOGGER.info(() -> String.format("Starting Bitcoin support using %s", this.bitcoinNet.name())); - } - - public static synchronized Bitcoin getInstance() { - if (instance == null) { - BitcoinNet bitcoinNet = Settings.getInstance().getBitcoinNet(); - - BitcoinyBlockchainProvider electrumX = new ElectrumX("Bitcoin-" + bitcoinNet.name(), bitcoinNet.getGenesisHash(), bitcoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); - Context bitcoinjContext = new Context(bitcoinNet.getParams()); - - instance = new Bitcoin(bitcoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); - } - - return instance; - } - - // Getters & setters - - public static synchronized void resetForTesting() { - instance = null; - } - - // Actual useful methods for use by other classes - - /** - * Returns estimated BTC fee, in sats per 1000bytes, optionally for historic timestamp. - * - * @param timestamp optional milliseconds since epoch, or null for 'now' - * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong - */ - @Override - public long getP2shFee(Long timestamp) throws ForeignBlockchainException { - return this.bitcoinNet.getP2shFee(timestamp); - } - -} diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java deleted file mode 100644 index 5118e103..00000000 --- a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java +++ /dev/null @@ -1,921 +0,0 @@ -package org.qortal.crosschain; - -import static org.ciyam.at.OpCode.calcOffset; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; - -import org.ciyam.at.API; -import org.ciyam.at.CompilationException; -import org.ciyam.at.FunctionCode; -import org.ciyam.at.MachineState; -import org.ciyam.at.OpCode; -import org.ciyam.at.Timestamp; -import org.qortal.account.Account; -import org.qortal.asset.Asset; -import org.qortal.at.QortalFunctionCode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.utils.Base58; -import org.qortal.utils.BitTwiddling; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -/** - * Cross-chain trade AT - * - *

- *

    - *
  • Bob generates Bitcoin & Qortal 'trade' keys, and secret-b - *
      - *
    • private key required to sign P2SH redeem tx
      • - *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • - *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • - *
    - *
    • - *
  • Bob deploys Qortal AT - *
      - *
    - *
    • - *
  • Alice finds Qortal AT and wants to trade - *
      - *
    • Alice generates Bitcoin & Qortal 'trade' keys
      • - *
    • Alice funds Bitcoin P2SH-A
      • - *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: - *
        - *
      • hash-of-secret-A
        • - *
      • her 'trade' Bitcoin PKH
        • - *
      - *
      • - *
    - *
    • - *
  • Bob receives "offer" MESSAGE - *
      - *
    • Checks Alice's P2SH-A
      • - *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: - *
        - *
      • Alice's trade Qortal address
        • - *
      • Alice's trade Bitcoin PKH
        • - *
      • hash-of-secret-A
        • - *
      - *
      • - *
    - *
    • - *
  • Alice checks Qortal AT to confirm it's locked to her - *
      - *
    • Alice creates/funds Bitcoin P2SH-B
      • - *
    - *
    • - *
  • Bob checks P2SH-B is funded - *
      - *
    • Bob redeems P2SH-B using his Bitcoin trade key and secret-B
      • - *
    - *
    • - *
  • Alice scans P2SH-B redeem transaction to extract secret-B - *
      - *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: - *
        - *
      • secret-A
        • - *
      • secret-B
        • - *
      • Qortal receiving address of her chosing
        • - *
      - *
      • - *
    • AT's QORT funds are sent to Qortal receiving address
      • - *
    - *
    • - *
  • Bob checks AT, extracts secret-A - *
      - *
    • Bob redeems P2SH-A using his Bitcoin trade key and secret-A
      • - *
    • P2SH-A BTC funds end up at Bitcoin address determined by redeem transaction output(s)
      • - *
    - *
    • - *
- */ -public class BitcoinACCTv1 implements ACCT { - - public static final String NAME = BitcoinACCTv1.class.getSimpleName(); - public static final byte[] CODE_BYTES_HASH = HashCode.fromString("f7f419522a9aaa3c671149878f8c1374dfc59d4fd86ca43ff2a4d913cfbc9e89").asBytes(); // SHA256 of AT code bytes - - public static final int SECRET_LENGTH = 32; - - /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ - private static final int MODE_VALUE_OFFSET = 68; - /** Byte offset into AT state data where 'mode' variable (long) is stored. */ - public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); - - public static class OfferMessageData { - public byte[] partnerBitcoinPKH; - public byte[] hashOfSecretA; - public long lockTimeA; - } - public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerBitcoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; - public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ - + 24 /*partner's Bitcoin PKH (padded from 20 to 24)*/ - + 8 /*lockTimeB*/ - + 24 /*hash of secret-A (padded from 20 to 24)*/ - + 8 /*lockTimeA*/; - public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret*/ + 32 /*secret*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; - public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; - - private static BitcoinACCTv1 instance; - - private BitcoinACCTv1() { - } - - public static synchronized BitcoinACCTv1 getInstance() { - if (instance == null) - instance = new BitcoinACCTv1(); - - return instance; - } - - @Override - public byte[] getCodeBytesHash() { - return CODE_BYTES_HASH; - } - - @Override - public int getModeByteOffset() { - return MODE_BYTE_OFFSET; - } - - @Override - public ForeignBlockchain getBlockchain() { - return Bitcoin.getInstance(); - } - - /** - * Returns Qortal AT creation bytes for cross-chain trading AT. - *

- * tradeTimeout (minutes) is the time window for the trade partner to send the - * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. - * - * @param creatorTradeAddress AT creator's trade Qortal address, also used for refunds - * @param bitcoinPublicKeyHash 20-byte HASH160 of creator's trade Bitcoin public key - * @param hashOfSecretB 20-byte HASH160 of 32-byte secret-B - * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT - * @param bitcoinAmount how much BTC the AT creator is expecting to trade - * @param tradeTimeout suggested timeout for entire trade - */ - public static byte[] buildQortalAT(String creatorTradeAddress, byte[] bitcoinPublicKeyHash, byte[] hashOfSecretB, long qortAmount, long bitcoinAmount, int tradeTimeout) { - // Labels for data segment addresses - int addrCounter = 0; - - // Constants (with corresponding dataByteBuffer.put*() calls below) - - final int addrCreatorTradeAddress1 = addrCounter++; - final int addrCreatorTradeAddress2 = addrCounter++; - final int addrCreatorTradeAddress3 = addrCounter++; - final int addrCreatorTradeAddress4 = addrCounter++; - - final int addrBitcoinPublicKeyHash = addrCounter; - addrCounter += 4; - - final int addrHashOfSecretB = addrCounter; - addrCounter += 4; - - final int addrQortAmount = addrCounter++; - final int addrBitcoinAmount = addrCounter++; - final int addrTradeTimeout = addrCounter++; - - final int addrMessageTxnType = addrCounter++; - final int addrExpectedTradeMessageLength = addrCounter++; - final int addrExpectedRedeemMessageLength = addrCounter++; - - final int addrCreatorAddressPointer = addrCounter++; - final int addrHashOfSecretBPointer = addrCounter++; - final int addrQortalPartnerAddressPointer = addrCounter++; - final int addrMessageSenderPointer = addrCounter++; - - final int addrTradeMessagePartnerBitcoinPKHOffset = addrCounter++; - final int addrPartnerBitcoinPKHPointer = addrCounter++; - final int addrTradeMessageHashOfSecretAOffset = addrCounter++; - final int addrHashOfSecretAPointer = addrCounter++; - - final int addrRedeemMessageSecretBOffset = addrCounter++; - final int addrRedeemMessageReceivingAddressOffset = addrCounter++; - - final int addrMessageDataPointer = addrCounter++; - final int addrMessageDataLength = addrCounter++; - - final int addrPartnerReceivingAddressPointer = addrCounter++; - - final int addrEndOfConstants = addrCounter; - - // Variables - - final int addrCreatorAddress1 = addrCounter++; - final int addrCreatorAddress2 = addrCounter++; - final int addrCreatorAddress3 = addrCounter++; - final int addrCreatorAddress4 = addrCounter++; - - final int addrQortalPartnerAddress1 = addrCounter++; - final int addrQortalPartnerAddress2 = addrCounter++; - final int addrQortalPartnerAddress3 = addrCounter++; - final int addrQortalPartnerAddress4 = addrCounter++; - - final int addrLockTimeA = addrCounter++; - final int addrLockTimeB = addrCounter++; - final int addrRefundTimeout = addrCounter++; - final int addrRefundTimestamp = addrCounter++; - final int addrLastTxnTimestamp = addrCounter++; - final int addrBlockTimestamp = addrCounter++; - final int addrTxnType = addrCounter++; - final int addrResult = addrCounter++; - - final int addrMessageSender1 = addrCounter++; - final int addrMessageSender2 = addrCounter++; - final int addrMessageSender3 = addrCounter++; - final int addrMessageSender4 = addrCounter++; - - final int addrMessageLength = addrCounter++; - - final int addrMessageData = addrCounter; - addrCounter += 4; - - final int addrHashOfSecretA = addrCounter; - addrCounter += 4; - - final int addrPartnerBitcoinPKH = addrCounter; - addrCounter += 4; - - final int addrPartnerReceivingAddress = addrCounter; - addrCounter += 4; - - final int addrMode = addrCounter++; - assert addrMode == MODE_VALUE_OFFSET : "MODE_VALUE_OFFSET does not match addrMode"; - - // Data segment - ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); - - // AT creator's trade Qortal address, decoded from Base58 - assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; - byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); - dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); - - // Bitcoin public key hash - assert dataByteBuffer.position() == addrBitcoinPublicKeyHash * MachineState.VALUE_SIZE : "addrBitcoinPublicKeyHash incorrect"; - dataByteBuffer.put(Bytes.ensureCapacity(bitcoinPublicKeyHash, 32, 0)); - - // Hash of secret-B - assert dataByteBuffer.position() == addrHashOfSecretB * MachineState.VALUE_SIZE : "addrHashOfSecretB incorrect"; - dataByteBuffer.put(Bytes.ensureCapacity(hashOfSecretB, 32, 0)); - - // Redeem Qort amount - assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; - dataByteBuffer.putLong(qortAmount); - - // Expected Bitcoin amount - assert dataByteBuffer.position() == addrBitcoinAmount * MachineState.VALUE_SIZE : "addrBitcoinAmount incorrect"; - dataByteBuffer.putLong(bitcoinAmount); - - // Suggested trade timeout (minutes) - assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; - dataByteBuffer.putLong(tradeTimeout); - - // We're only interested in MESSAGE transactions - assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; - dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); - - // Expected length of 'trade' MESSAGE data from AT creator - assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; - dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); - - // Expected length of 'redeem' MESSAGE data from trade partner - assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; - dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); - - // Index into data segment of AT creator's address, used by GET_B_IND - assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; - dataByteBuffer.putLong(addrCreatorAddress1); - - // Index into data segment of hash of secret B, used by GET_B_IND - assert dataByteBuffer.position() == addrHashOfSecretBPointer * MachineState.VALUE_SIZE : "addrHashOfSecretBPointer incorrect"; - dataByteBuffer.putLong(addrHashOfSecretB); - - // Index into data segment of partner's Qortal address, used by SET_B_IND - assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; - dataByteBuffer.putLong(addrQortalPartnerAddress1); - - // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND - assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; - dataByteBuffer.putLong(addrMessageSender1); - - // Offset into 'trade' MESSAGE data payload for extracting partner's Bitcoin PKH - assert dataByteBuffer.position() == addrTradeMessagePartnerBitcoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerBitcoinPKHOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Index into data segment of partner's Bitcoin PKH, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerBitcoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerBitcoinPKHPointer incorrect"; - dataByteBuffer.putLong(addrPartnerBitcoinPKH); - - // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A - assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; - dataByteBuffer.putLong(64L); - - // Index into data segment to hash of secret A, used by GET_B_IND - assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; - dataByteBuffer.putLong(addrHashOfSecretA); - - // Offset into 'redeem' MESSAGE data payload for extracting secret-B - assert dataByteBuffer.position() == addrRedeemMessageSecretBOffset * MachineState.VALUE_SIZE : "addrRedeemMessageSecretBOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address - assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; - dataByteBuffer.putLong(64L); - - // Source location and length for hashing any passed secret - assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; - dataByteBuffer.putLong(addrMessageData); - assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; - dataByteBuffer.putLong(32L); - - // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; - dataByteBuffer.putLong(addrPartnerReceivingAddress); - - assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; - - // Code labels - Integer labelRefund = null; - - Integer labelTradeTxnLoop = null; - Integer labelCheckTradeTxn = null; - Integer labelCheckCancelTxn = null; - Integer labelNotTradeNorCancelTxn = null; - Integer labelCheckNonRefundTradeTxn = null; - Integer labelTradeTxnExtract = null; - Integer labelRedeemTxnLoop = null; - Integer labelCheckRedeemTxn = null; - Integer labelCheckRedeemTxnSender = null; - Integer labelCheckSecretB = null; - Integer labelPayout = null; - - ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); - - // Two-pass version - for (int pass = 0; pass < 2; ++pass) { - codeByteBuffer.clear(); - - try { - /* Initialization */ - - // Use AT creation 'timestamp' as starting point for finding transactions sent to AT - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); - - // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ - - /* Transaction processing loop */ - labelTradeTxnLoop = codeByteBuffer.position(); - - // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckTradeTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); - - /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - // Message sender's address matches AT creator's trade address so go process 'trade' message - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); - - /* Checking message sender for possible cancel message */ - labelCheckCancelTxn = codeByteBuffer.position(); - - // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - // Partner address is AT creator's address, so cancel offer and finish. - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - /* Not trade nor cancel message */ - labelNotTradeNorCancelTxn = codeByteBuffer.position(); - - // Loop to find another transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Possible switch-to-trade-mode message */ - labelCheckNonRefundTradeTxn = codeByteBuffer.position(); - - // Check 'trade' message we received has expected number of message bytes - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to info extraction code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); - // Message length didn't match - go back to finding another 'trade' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Extracting info from 'trade' MESSAGE transaction */ - labelTradeTxnExtract = codeByteBuffer.position(); - - // Extract message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); - - // Extract trade partner's Bitcoin public key hash (PKH) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerBitcoinPKHOffset)); - // Extract partner's Bitcoin PKH (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerBitcoinPKHPointer)); - // Also extract lockTimeB - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeB)); - - // Grab next 32 bytes - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); - - // Extract hash-of-secret-a (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); - // Extract lockTimeA (from B4) - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); - - // Calculate trade refund timeout: (lockTimeA - lockTimeB) / 2 / 60 - codeByteBuffer.put(OpCode.SET_DAT.compile(addrRefundTimeout, addrLockTimeA)); // refundTimeout = lockTimeA - codeByteBuffer.put(OpCode.SUB_DAT.compile(addrRefundTimeout, addrLockTimeB)); // refundTimeout -= lockTimeB - codeByteBuffer.put(OpCode.DIV_VAL.compile(addrRefundTimeout, 2L * 60L)); // refundTimeout /= 2 * 60 - // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); - - /* We are in 'trade mode' */ - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ - - // Fetch current block 'timestamp' - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); - // If we're not past refund 'timestamp' then look for next transaction - codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - // We're past refund 'timestamp' so go refund everything back to AT creator - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); - - /* Transaction processing loop */ - labelRedeemTxnLoop = codeByteBuffer.position(); - - // Find next transaction to this AT since the last one (if any) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckRedeemTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check message payload length */ - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to sender checking code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); - // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Check transaction's sender */ - labelCheckRedeemTxnSender = codeByteBuffer.position(); - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check 'secret-A' in transaction's message */ - - // Extract secret-A from first 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); - // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); - // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). - // Save the equality result (1 if they match, 0 otherwise) into addrResult. - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); - // If hashes don't match, addrResult will be zero so go find another transaction - codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckSecretB))); - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Check 'secret-B' in transaction's message */ - - labelCheckSecretB = codeByteBuffer.position(); - - // Extract secret-B from next 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageSecretBOffset)); - // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); - // Load B register with expected hash result (as pointed to by addrHashOfSecretBPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretBPointer)); - // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). - // Save the equality result (1 if they match, 0 otherwise) into addrResult. - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); - // If hashes don't match, addrResult will be zero so go find another transaction - codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Success! Pay arranged amount to receiving address */ - labelPayout = codeByteBuffer.position(); - - // Extract Qortal receiving address from next 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); - // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); - // Pay AT's balance to receiving address - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); - // Set redeemed mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - // Fall-through to refunding any remaining balance back to AT creator - - /* Refund balance back to AT creator */ - labelRefund = codeByteBuffer.position(); - - // Set refunded mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - } catch (CompilationException e) { - throw new IllegalStateException("Unable to compile BTC-QORT ACCT?", e); - } - } - - codeByteBuffer.flip(); - - byte[] codeBytes = new byte[codeByteBuffer.limit()]; - codeByteBuffer.get(codeBytes); - - assert Arrays.equals(Crypto.digest(codeBytes), BitcoinACCTv1.CODE_BYTES_HASH) - : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); - - final short ciyamAtVersion = 2; - final short numCallStackPages = 0; - final short numUserStackPages = 0; - final long minActivationAmount = 0L; - - return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { - ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { - byte[] addressBytes = new byte[25]; // for general use - String atAddress = atStateData.getATAddress(); - - CrossChainTradeData tradeData = new CrossChainTradeData(); - - tradeData.foreignBlockchain = SupportedBlockchain.BITCOIN.name(); - tradeData.acctName = NAME; - - tradeData.qortalAtAddress = atAddress; - tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); - tradeData.creationTimestamp = creationTimestamp; - - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); - - byte[] stateData = atStateData.getStateData(); - ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); - dataByteBuffer.position(MachineState.HEADER_LENGTH); - - /* Constants */ - - // Skip creator's trade address - dataByteBuffer.get(addressBytes); - tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Creator's Bitcoin/foreign public key hash - tradeData.creatorForeignPKH = new byte[20]; - dataByteBuffer.get(tradeData.creatorForeignPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes - - // Hash of secret B - tradeData.hashOfSecretB = new byte[20]; - dataByteBuffer.get(tradeData.hashOfSecretB); - dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.hashOfSecretB.length); // skip to 32 bytes - - // Redeem payout - tradeData.qortAmount = dataByteBuffer.getLong(); - - // Expected BTC amount - tradeData.expectedForeignAmount = dataByteBuffer.getLong(); - - // Trade timeout - tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); - - // Skip MESSAGE transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'trade' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'redeem' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to hash-of-secret-B - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's Qortal trade address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message sender - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for partner's bitcoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's bitcoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'redeem' message data offset for secret-B - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'redeem' message data offset for partner's Qortal receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message data - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip message data length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - /* End of constants / begin variables */ - - // Skip AT creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Partner's trade address (if present) - dataByteBuffer.get(addressBytes); - String qortalRecipient = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Potential lockTimeA (if in trade mode) - int lockTimeA = (int) dataByteBuffer.getLong(); - - // Potential lockTimeB (if in trade mode) - int lockTimeB = (int) dataByteBuffer.getLong(); - - // AT refund timeout (probably only useful for debugging) - int refundTimeout = (int) dataByteBuffer.getLong(); - - // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) - long tradeRefundTimestamp = dataByteBuffer.getLong(); - - // Skip last transaction timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip block timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary result - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message sender - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Skip message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message data - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Potential hash160 of secret A - byte[] hashOfSecretA = new byte[20]; - dataByteBuffer.get(hashOfSecretA); - dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes - - // Potential partner's Bitcoin PKH - byte[] partnerBitcoinPKH = new byte[20]; - dataByteBuffer.get(partnerBitcoinPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerBitcoinPKH.length); // skip to 32 bytes - - // Partner's receiving address (if present) - byte[] partnerReceivingAddress = new byte[25]; - dataByteBuffer.get(partnerReceivingAddress); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes - - // Trade AT's 'mode' - long modeValue = dataByteBuffer.getLong(); - AcctMode acctMode = AcctMode.valueOf((int) (modeValue & 0xffL)); - - /* End of variables */ - - if (acctMode != null && acctMode != AcctMode.OFFERING) { - tradeData.mode = acctMode; - tradeData.refundTimeout = refundTimeout; - tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; - tradeData.qortalPartnerAddress = qortalRecipient; - tradeData.hashOfSecretA = hashOfSecretA; - tradeData.partnerForeignPKH = partnerBitcoinPKH; - tradeData.lockTimeA = lockTimeA; - tradeData.lockTimeB = lockTimeB; - - if (acctMode == AcctMode.REDEEMED) - tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); - } else { - tradeData.mode = AcctMode.OFFERING; - } - - tradeData.duplicateDeprecated(); - - return tradeData; - } - - /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ - public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); - } - - /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ - public static OfferMessageData extractOfferMessageData(byte[] messageData) { - if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) - return null; - - OfferMessageData offerMessageData = new OfferMessageData(); - offerMessageData.partnerBitcoinPKH = Arrays.copyOfRange(messageData, 0, 20); - offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); - offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); - - return offerMessageData; - } - - /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ - public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int lockTimeB) { - byte[] data = new byte[TRADE_MESSAGE_LENGTH]; - byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - byte[] lockTimeBBytes = BitTwiddling.toBEByteArray((long) lockTimeB); - - System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); - System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); - System.arraycopy(lockTimeBBytes, 0, data, 56, lockTimeBBytes.length); - System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); - System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); - - return data; - } - - /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ - @Override - public byte[] buildCancelMessage(String creatorQortalAddress) { - byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; - byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); - - System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); - - return data; - } - - /** Returns 'redeem' MESSAGE payload for trade partner/ to send to AT. */ - public static byte[] buildRedeemMessage(byte[] secretA, byte[] secretB, String qortalReceivingAddress) { - byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; - byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); - - System.arraycopy(secretA, 0, data, 0, secretA.length); - System.arraycopy(secretB, 0, data, 32, secretB.length); - System.arraycopy(qortalReceivingAddressBytes, 0, data, 64, qortalReceivingAddressBytes.length); - - return data; - } - - /** Returns P2SH-B lockTime (epoch seconds) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ - public static int calcLockTimeB(long offerMessageTimestamp, int lockTimeA) { - // lockTimeB is halfway between offerMessageTimestamp and lockTimeA - return (int) ((lockTimeA + (offerMessageTimestamp / 1000L)) / 2L); - } - - public static byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { - String atAddress = crossChainTradeData.qortalAtAddress; - String redeemerAddress = crossChainTradeData.qortalPartnerAddress; - - // We don't have partner's public key so we check every message to AT - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); - if (messageTransactionsData == null) - return null; - - // Find 'redeem' message - for (MessageTransactionData messageTransactionData : messageTransactionsData) { - // Check message payload type/encryption - if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) - continue; - - // Check message payload size - byte[] messageData = messageTransactionData.getData(); - if (messageData.length != REDEEM_MESSAGE_LENGTH) - // Wrong payload length - continue; - - // Check sender - if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) - // Wrong sender; - continue; - - // Extract both secretA & secretB - byte[] secretA = new byte[32]; - System.arraycopy(messageData, 0, secretA, 0, secretA.length); - byte[] secretB = new byte[32]; - System.arraycopy(messageData, 32, secretB, 0, secretB.length); - - byte[] hashOfSecretA = Crypto.hash160(secretA); - if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) - continue; - - byte[] hashOfSecretB = Crypto.hash160(secretB); - if (!Arrays.equals(hashOfSecretB, crossChainTradeData.hashOfSecretB)) - continue; - - return secretA; - } - - return null; - } - -} diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java deleted file mode 100644 index fc98f959..00000000 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ /dev/null @@ -1,740 +0,0 @@ -package org.qortal.crosschain; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bitcoinj.core.Address; -import org.bitcoinj.core.AddressFormatException; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.Context; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.InsufficientMoneyException; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Sha256Hash; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.core.UTXO; -import org.bitcoinj.core.UTXOProvider; -import org.bitcoinj.core.UTXOProviderException; -import org.bitcoinj.crypto.ChildNumber; -import org.bitcoinj.crypto.DeterministicHierarchy; -import org.bitcoinj.crypto.DeterministicKey; -import org.bitcoinj.script.Script.ScriptType; -import org.bitcoinj.script.ScriptBuilder; -import org.bitcoinj.wallet.DeterministicKeyChain; -import org.bitcoinj.wallet.SendRequest; -import org.bitcoinj.wallet.Wallet; -import org.qortal.api.model.SimpleForeignTransaction; -import org.qortal.crypto.Crypto; -import org.qortal.utils.Amounts; -import org.qortal.utils.BitTwiddling; - -import com.google.common.hash.HashCode; - -/** Bitcoin-like (Bitcoin, Litecoin, etc.) support */ -public abstract class Bitcoiny implements ForeignBlockchain { - - protected static final Logger LOGGER = LogManager.getLogger(Bitcoiny.class); - - public static final int HASH160_LENGTH = 20; - - protected final BitcoinyBlockchainProvider blockchain; - protected final Context bitcoinjContext; - protected final String currencyCode; - - protected final NetworkParameters params; - - /** Keys that have been previously marked as fully spent,
- * i.e. keys with transactions but with no unspent outputs. */ - protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); - - /** How many bitcoinj wallet keys to generate in each batch. */ - private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; - - /** Byte offset into raw block headers to block timestamp. */ - private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; - - // Constructors and instance - - protected Bitcoiny(BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { - this.blockchain = blockchain; - this.bitcoinjContext = bitcoinjContext; - this.currencyCode = currencyCode; - - this.params = this.bitcoinjContext.getParams(); - } - - // Getters & setters - - public BitcoinyBlockchainProvider getBlockchainProvider() { - return this.blockchain; - } - - public Context getBitcoinjContext() { - return this.bitcoinjContext; - } - - public String getCurrencyCode() { - return this.currencyCode; - } - - public NetworkParameters getNetworkParameters() { - return this.params; - } - - // Interface obligations - - @Override - public boolean isValidAddress(String address) { - try { - ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType(); - - return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH; - } catch (AddressFormatException e) { - return false; - } - } - - @Override - public boolean isValidWalletKey(String walletKey) { - return this.isValidDeterministicKey(walletKey); - } - - // Actual useful methods for use by other classes - - public String format(Coin amount) { - return this.format(amount.value); - } - - public String format(long amount) { - return Amounts.prettyAmount(amount) + " " + this.currencyCode; - } - - public boolean isValidDeterministicKey(String key58) { - try { - Context.propagate(this.bitcoinjContext); - DeterministicKey.deserializeB58(null, key58, this.params); - return true; - } catch (IllegalArgumentException e) { - return false; - } - } - - /** Returns P2PKH address using passed public key hash. */ - public String pkhToAddress(byte[] publicKeyHash) { - Context.propagate(this.bitcoinjContext); - return LegacyAddress.fromPubKeyHash(this.params, publicKeyHash).toString(); - } - - /** Returns P2SH address using passed redeem script. */ - public String deriveP2shAddress(byte[] redeemScriptBytes) { - Context.propagate(bitcoinjContext); - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - return LegacyAddress.fromScriptHash(this.params, redeemScriptHash).toString(); - } - - /** - * Returns median timestamp from latest 11 blocks, in seconds. - *

- * @throws ForeignBlockchainException if error occurs - */ - public int getMedianBlockTime() throws ForeignBlockchainException { - int height = this.blockchain.getCurrentHeight(); - - // Grab latest 11 blocks - List blockHeaders = this.blockchain.getRawBlockHeaders(height - 11, 11); - if (blockHeaders.size() < 11) - throw new ForeignBlockchainException("Not enough blocks to determine median block time"); - - List blockTimestamps = blockHeaders.stream().map(blockHeader -> BitTwiddling.intFromLEBytes(blockHeader, TIMESTAMP_OFFSET)).collect(Collectors.toList()); - - // Descending order - blockTimestamps.sort((a, b) -> Integer.compare(b, a)); - - // Pick median - return blockTimestamps.get(5); - } - - /** Returns fee per transaction KB. To be overridden for testnet/regtest. */ - public Coin getFeePerKb() { - return this.bitcoinjContext.getFeePerKb(); - } - - /** - * Returns fixed P2SH spending fee, in sats per 1000bytes, optionally for historic timestamp. - * - * @param timestamp optional milliseconds since epoch, or null for 'now' - * @return sats per 1000bytes - * @throws ForeignBlockchainException if something went wrong - */ - public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; - - /** - * Returns confirmed balance, based on passed payment script. - *

- * @return confirmed balance, or zero if script unknown - * @throws ForeignBlockchainException if there was an error - */ - public long getConfirmedBalance(String base58Address) throws ForeignBlockchainException { - return this.blockchain.getConfirmedBalance(addressToScriptPubKey(base58Address)); - } - - /** - * Returns list of unspent outputs pertaining to passed address. - *

- * @return list of unspent outputs, or empty list if address unknown - * @throws ForeignBlockchainException if there was an error. - */ - // TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead - public List getUnspentOutputs(String base58Address) throws ForeignBlockchainException { - List unspentOutputs = this.blockchain.getUnspentOutputs(addressToScriptPubKey(base58Address), false); - - List unspentTransactionOutputs = new ArrayList<>(); - for (UnspentOutput unspentOutput : unspentOutputs) { - List transactionOutputs = this.getOutputs(unspentOutput.hash); - - unspentTransactionOutputs.add(transactionOutputs.get(unspentOutput.index)); - } - - return unspentTransactionOutputs; - } - - /** - * Returns list of outputs pertaining to passed transaction hash. - *

- * @return list of outputs, or empty list if transaction unknown - * @throws ForeignBlockchainException if there was an error. - */ - // TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead - public List getOutputs(byte[] txHash) throws ForeignBlockchainException { - byte[] rawTransactionBytes = this.blockchain.getRawTransaction(txHash); - - Context.propagate(bitcoinjContext); - Transaction transaction = new Transaction(this.params, rawTransactionBytes); - return transaction.getOutputs(); - } - - /** - * Returns list of transaction hashes pertaining to passed address. - *

- * @return list of unspent outputs, or empty list if script unknown - * @throws ForeignBlockchainException if there was an error. - */ - public List getAddressTransactions(String base58Address, boolean includeUnconfirmed) throws ForeignBlockchainException { - return this.blockchain.getAddressTransactions(addressToScriptPubKey(base58Address), includeUnconfirmed); - } - - /** - * Returns list of raw, confirmed transactions involving given address. - *

- * @throws ForeignBlockchainException if there was an error - */ - public List getAddressTransactions(String base58Address) throws ForeignBlockchainException { - List transactionHashes = this.blockchain.getAddressTransactions(addressToScriptPubKey(base58Address), false); - - List rawTransactions = new ArrayList<>(); - for (TransactionHash transactionInfo : transactionHashes) { - byte[] rawTransaction = this.blockchain.getRawTransaction(HashCode.fromString(transactionInfo.txHash).asBytes()); - rawTransactions.add(rawTransaction); - } - - return rawTransactions; - } - - /** - * Returns transaction info for passed transaction hash. - *

- * @throws ForeignBlockchainException.NotFoundException if transaction unknown - * @throws ForeignBlockchainException if error occurs - */ - public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { - return this.blockchain.getTransaction(txHash); - } - - /** - * Broadcasts raw transaction to network. - *

- * @throws ForeignBlockchainException if error occurs - */ - public void broadcastTransaction(Transaction transaction) throws ForeignBlockchainException { - this.blockchain.broadcastTransaction(transaction.bitcoinSerialize()); - } - - /** - * Returns bitcoinj transaction sending amount to recipient. - * - * @param xprv58 BIP32 private key - * @param recipient P2PKH address - * @param amount unscaled amount - * @param feePerByte unscaled fee per byte, or null to use default fees - * @return transaction, or null if insufficient funds - */ - public Transaction buildSpend(String xprv58, String recipient, long amount, Long feePerByte) { - Context.propagate(bitcoinjContext); - - Wallet wallet = Wallet.fromSpendingKeyB58(this.params, xprv58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); - wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); - - Address destination = Address.fromString(this.params, recipient); - SendRequest sendRequest = SendRequest.to(destination, Coin.valueOf(amount)); - - if (feePerByte != null) - sendRequest.feePerKb = Coin.valueOf(feePerByte * 1000L); // Note: 1000 not 1024 - else - // Allow override of default for TestNet3, etc. - sendRequest.feePerKb = this.getFeePerKb(); - - try { - wallet.completeTx(sendRequest); - return sendRequest.tx; - } catch (InsufficientMoneyException e) { - return null; - } - } - - /** - * Returns bitcoinj transaction sending amount to recipient using default fees. - * - * @param xprv58 BIP32 private key - * @param recipient P2PKH address - * @param amount unscaled amount - * @return transaction, or null if insufficient funds - */ - public Transaction buildSpend(String xprv58, String recipient, long amount) { - return buildSpend(xprv58, recipient, amount, null); - } - - /** - * Returns unspent Bitcoin balance given 'm' BIP32 key. - * - * @param key58 BIP32/HD extended Bitcoin private/public key - * @return unspent BTC balance, or null if unable to determine balance - */ - public Long getWalletBalance(String key58) { - Context.propagate(bitcoinjContext); - - Wallet wallet = walletFromDeterministicKey58(key58); - wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); - - Coin balance = wallet.getBalance(); - if (balance == null) - return null; - - return balance.value; - } - - public List getWalletTransactions(String key58) throws ForeignBlockchainException { - Context.propagate(bitcoinjContext); - - Wallet wallet = walletFromDeterministicKey58(key58); - DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); - - keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); - keyChain.maybeLookAhead(); - - List keys = new ArrayList<>(keyChain.getLeafKeys()); - - Set walletTransactions = new HashSet<>(); - Set keySet = new HashSet<>(); - - int ki = 0; - do { - boolean areAllKeysUnused = true; - - for (; ki < keys.size(); ++ki) { - DeterministicKey dKey = keys.get(ki); - - // Check for transactions - Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); - keySet.add(address.toString()); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); - - if (!historicTransactionHashes.isEmpty()) { - areAllKeysUnused = false; - - for (TransactionHash transactionHash : historicTransactionHashes) - walletTransactions.add(this.getTransaction(transactionHash.txHash)); - } - } - - if (areAllKeysUnused) - // No transactions for this batch of keys so assume we're done searching. - break; - - // Generate some more keys - keys.addAll(generateMoreKeys(keyChain)); - - // Process new keys - } while (true); - - Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); - - return walletTransactions.stream().map(t -> convertToSimpleTransaction(t, keySet)).sorted(newestTimestampFirstComparator).collect(Collectors.toList()); - } - - protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) { - long amount = 0; - long total = 0L; - for (BitcoinyTransaction.Input input : t.inputs) { - try { - BitcoinyTransaction t2 = getTransaction(input.outputTxHash); - List senders = t2.outputs.get(input.outputVout).addresses; - for (String sender : senders) { - if (keySet.contains(sender)) { - total += t2.outputs.get(input.outputVout).value; - } - } - } catch (ForeignBlockchainException e) { - LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash); - } - } - if (t.outputs != null && !t.outputs.isEmpty()) { - for (BitcoinyTransaction.Output output : t.outputs) { - for (String address : output.addresses) { - if (keySet.contains(address)) { - if (total > 0L) { - amount -= (total - output.value); - } else { - amount += output.value; - } - } - } - } - } - return new SimpleTransaction(t.txHash, t.timestamp, amount); - } - - /** - * Returns first unused receive address given 'm' BIP32 key. - * - * @param key58 BIP32/HD extended Bitcoin private/public key - * @return P2PKH address - * @throws ForeignBlockchainException if something went wrong - */ - public String getUnusedReceiveAddress(String key58) throws ForeignBlockchainException { - Context.propagate(bitcoinjContext); - - Wallet wallet = walletFromDeterministicKey58(key58); - DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); - - keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); - keyChain.maybeLookAhead(); - - final int keyChainPathSize = keyChain.getAccountPath().size(); - List keys = new ArrayList<>(keyChain.getLeafKeys()); - - int ki = 0; - do { - for (; ki < keys.size(); ++ki) { - DeterministicKey dKey = keys.get(ki); - List dKeyPath = dKey.getPath(); - - // If keyChain is based on 'm', then make sure dKey is m/0/ki - i.e. a 'receive' address, not 'change' (m/1/ki) - if (dKeyPath.size() != keyChainPathSize + 2 || dKeyPath.get(dKeyPath.size() - 2) != ChildNumber.ZERO) - continue; - - // Check unspent - Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - - List unspentOutputs = this.blockchain.getUnspentOutputs(script, false); - - /* - * If there are no unspent outputs then either: - * a) all the outputs have been spent - * b) address has never been used - * - * For case (a) we want to remember not to check this address (key) again. - */ - - if (unspentOutputs.isEmpty()) { - // If this is a known key that has been spent before, then we can skip asking for transaction history - if (this.spentKeys.contains(dKey)) { - wallet.getActiveKeyChain().markKeyAsUsed(dKey); - continue; - } - - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); - - if (!historicTransactionHashes.isEmpty()) { - // Fully spent key - case (a) - this.spentKeys.add(dKey); - wallet.getActiveKeyChain().markKeyAsUsed(dKey); - continue; - } - - // Key never been used - case (b) - return address.toString(); - } - - // Key has unspent outputs, hence used, so no good to us - this.spentKeys.remove(dKey); - } - - // Generate some more keys - keys.addAll(generateMoreKeys(keyChain)); - - // Process new keys - } while (true); - } - - // UTXOProvider support - - static class WalletAwareUTXOProvider implements UTXOProvider { - private final Bitcoiny bitcoiny; - private final Wallet wallet; - - private final DeterministicKeyChain keyChain; - - public WalletAwareUTXOProvider(Bitcoiny bitcoiny, Wallet wallet) { - this.bitcoiny = bitcoiny; - this.wallet = wallet; - this.keyChain = this.wallet.getActiveKeyChain(); - - // Set up wallet's key chain - this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); - this.keyChain.maybeLookAhead(); - } - - @Override - public List getOpenTransactionOutputs(List keys) throws UTXOProviderException { - List allUnspentOutputs = new ArrayList<>(); - final boolean coinbase = false; - - int ki = 0; - do { - boolean areAllKeysUnspent = true; - - for (; ki < keys.size(); ++ki) { - ECKey key = keys.get(ki); - - Address address = Address.fromKey(this.bitcoiny.params, key, ScriptType.P2PKH); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - - List unspentOutputs; - try { - unspentOutputs = this.bitcoiny.blockchain.getUnspentOutputs(script, false); - } catch (ForeignBlockchainException e) { - throw new UTXOProviderException(String.format("Unable to fetch unspent outputs for %s", address)); - } - - /* - * If there are no unspent outputs then either: - * a) all the outputs have been spent - * b) address has never been used - * - * For case (a) we want to remember not to check this address (key) again. - */ - - if (unspentOutputs.isEmpty()) { - // If this is a known key that has been spent before, then we can skip asking for transaction history - if (this.bitcoiny.spentKeys.contains(key)) { - this.wallet.getActiveKeyChain().markKeyAsUsed((DeterministicKey) key); - areAllKeysUnspent = false; - continue; - } - - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes; - try { - historicTransactionHashes = this.bitcoiny.blockchain.getAddressTransactions(script, false); - } catch (ForeignBlockchainException e) { - throw new UTXOProviderException(String.format("Unable to fetch transaction history for %s", address)); - } - - if (!historicTransactionHashes.isEmpty()) { - // Fully spent key - case (a) - this.bitcoiny.spentKeys.add(key); - this.wallet.getActiveKeyChain().markKeyAsUsed((DeterministicKey) key); - areAllKeysUnspent = false; - } else { - // Key never been used - case (b) - } - - continue; - } - - // If we reach here, then there's definitely at least one unspent key - this.bitcoiny.spentKeys.remove(key); - - for (UnspentOutput unspentOutput : unspentOutputs) { - List transactionOutputs; - try { - transactionOutputs = this.bitcoiny.getOutputs(unspentOutput.hash); - } catch (ForeignBlockchainException e) { - throw new UTXOProviderException(String.format("Unable to fetch outputs for TX %s", - HashCode.fromBytes(unspentOutput.hash))); - } - - TransactionOutput transactionOutput = transactionOutputs.get(unspentOutput.index); - - UTXO utxo = new UTXO(Sha256Hash.wrap(unspentOutput.hash), unspentOutput.index, - Coin.valueOf(unspentOutput.value), unspentOutput.height, coinbase, - transactionOutput.getScriptPubKey()); - - allUnspentOutputs.add(utxo); - } - } - - if (areAllKeysUnspent) - // No transactions for this batch of keys so assume we're done searching. - return allUnspentOutputs; - - // Generate some more keys - keys.addAll(Bitcoiny.generateMoreKeys(this.keyChain)); - - // Process new keys - } while (true); - } - - @Override - public int getChainHeadHeight() throws UTXOProviderException { - try { - return this.bitcoiny.blockchain.getCurrentHeight(); - } catch (ForeignBlockchainException e) { - throw new UTXOProviderException("Unable to determine Bitcoiny chain height"); - } - } - - @Override - public NetworkParameters getParams() { - return this.bitcoiny.params; - } - } - - // Utility methods for others - - public static List simplifyWalletTransactions(List transactions) { - // Sort by oldest timestamp first - transactions.sort(Comparator.comparingInt(t -> t.timestamp)); - - // Manual 2nd-level sort same-timestamp transactions so that a transaction's input comes first - int fromIndex = 0; - do { - int timestamp = transactions.get(fromIndex).timestamp; - - int toIndex; - for (toIndex = fromIndex + 1; toIndex < transactions.size(); ++toIndex) - if (transactions.get(toIndex).timestamp != timestamp) - break; - - // Process same-timestamp sub-list - List subList = transactions.subList(fromIndex, toIndex); - - // Only if necessary - if (subList.size() > 1) { - // Quick index lookup - Map indexByTxHash = subList.stream().collect(Collectors.toMap(t -> t.txHash, t -> t.timestamp)); - - int restartIndex = 0; - boolean isSorted; - do { - isSorted = true; - - for (int ourIndex = restartIndex; ourIndex < subList.size(); ++ourIndex) { - BitcoinyTransaction ourTx = subList.get(ourIndex); - - for (BitcoinyTransaction.Input input : ourTx.inputs) { - Integer inputIndex = indexByTxHash.get(input.outputTxHash); - - if (inputIndex != null && inputIndex > ourIndex) { - // Input tx is currently after current tx, so swap - BitcoinyTransaction tmpTx = subList.get(inputIndex); - subList.set(inputIndex, ourTx); - subList.set(ourIndex, tmpTx); - - // Update index lookup too - indexByTxHash.put(ourTx.txHash, inputIndex); - indexByTxHash.put(tmpTx.txHash, ourIndex); - - if (isSorted) - restartIndex = Math.max(restartIndex, ourIndex); - - isSorted = false; - break; - } - } - } - } while (!isSorted); - } - - fromIndex = toIndex; - } while (fromIndex < transactions.size()); - - // Simplify - List simpleTransactions = new ArrayList<>(); - - // Quick lookup of txs in our wallet - Set walletTxHashes = transactions.stream().map(t -> t.txHash).collect(Collectors.toSet()); - - for (BitcoinyTransaction transaction : transactions) { - SimpleForeignTransaction.Builder builder = new SimpleForeignTransaction.Builder(); - builder.txHash(transaction.txHash); - builder.timestamp(transaction.timestamp); - - builder.isSentNotReceived(false); - - for (BitcoinyTransaction.Input input : transaction.inputs) { - // TODO: add input via builder - - if (walletTxHashes.contains(input.outputTxHash)) - builder.isSentNotReceived(true); - } - - for (BitcoinyTransaction.Output output : transaction.outputs) - builder.output(output.addresses, output.value); - - simpleTransactions.add(builder.build()); - } - - return simpleTransactions; - } - - // Utility methods for us - - protected static List generateMoreKeys(DeterministicKeyChain keyChain) { - int existingLeafKeyCount = keyChain.getLeafKeys().size(); - - // Increase lookahead size... - keyChain.setLookaheadSize(keyChain.getLookaheadSize() + Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); - // ...and lookahead threshold (minimum number of keys to generate)... - keyChain.setLookaheadThreshold(0); - // ...so that this call will generate more keys - keyChain.maybeLookAhead(); - - // This returns *all* keys - List allLeafKeys = keyChain.getLeafKeys(); - - // Only return newly generated keys - return allLeafKeys.subList(existingLeafKeyCount, allLeafKeys.size()); - } - - protected byte[] addressToScriptPubKey(String base58Address) { - Context.propagate(this.bitcoinjContext); - Address address = Address.fromString(this.params, base58Address); - return ScriptBuilder.createOutputScript(address).getProgram(); - } - - protected Wallet walletFromDeterministicKey58(String key58) { - DeterministicKey dKey = DeterministicKey.deserializeB58(null, key58, this.params); - - if (dKey.hasPrivKey()) - return Wallet.fromSpendingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); - else - return Wallet.fromWatchingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); - } - -} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java b/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java deleted file mode 100644 index 7691efb1..00000000 --- a/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.qortal.crosschain; - -import java.util.List; - -public abstract class BitcoinyBlockchainProvider { - - public static final boolean INCLUDE_UNCONFIRMED = true; - public static final boolean EXCLUDE_UNCONFIRMED = false; - - /** Returns ID unique to bitcoiny network (e.g. "Litecoin-TEST3") */ - public abstract String getNetId(); - - /** Returns current blockchain height. */ - public abstract int getCurrentHeight() throws ForeignBlockchainException; - - /** Returns a list of raw block headers, starting at startHeight (inclusive), up to count max. */ - public abstract List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException; - - /** Returns balance of address represented by scriptPubKey. */ - public abstract long getConfirmedBalance(byte[] scriptPubKey) throws ForeignBlockchainException; - - /** Returns raw, serialized, transaction bytes given txHash. */ - public abstract byte[] getRawTransaction(String txHash) throws ForeignBlockchainException; - - /** Returns raw, serialized, transaction bytes given txHash. */ - public abstract byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException; - - /** Returns unpacked transaction given txHash. */ - public abstract BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException; - - /** Returns list of transaction hashes (and heights) for address represented by scriptPubKey, optionally including unconfirmed transactions. */ - public abstract List getAddressTransactions(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException; - - /** Returns list of unspent transaction outputs for address represented by scriptPubKey, optionally including unconfirmed transactions. */ - public abstract List getUnspentOutputs(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException; - - /** Broadcasts raw, serialized, transaction bytes to network, returning success/failure. */ - public abstract void broadcastTransaction(byte[] rawTransaction) throws ForeignBlockchainException; - -} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java b/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java deleted file mode 100644 index 8ebfffa2..00000000 --- a/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java +++ /dev/null @@ -1,438 +0,0 @@ -package org.qortal.crosschain; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.Transaction.SigHash; -import org.bitcoinj.core.TransactionInput; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.crypto.TransactionSignature; -import org.bitcoinj.script.Script; -import org.bitcoinj.script.ScriptBuilder; -import org.bitcoinj.script.ScriptChunk; -import org.bitcoinj.script.ScriptOpCodes; -import org.qortal.crypto.Crypto; -import org.qortal.utils.Base58; -import org.qortal.utils.BitTwiddling; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -public class BitcoinyHTLC { - - public enum Status { - UNFUNDED, FUNDING_IN_PROGRESS, FUNDED, REDEEM_IN_PROGRESS, REDEEMED, REFUND_IN_PROGRESS, REFUNDED - } - - public static final int SECRET_LENGTH = 32; - public static final int MIN_LOCKTIME = 1500000000; - - public static final long NO_LOCKTIME_NO_RBF_SEQUENCE = 0xFFFFFFFFL; - public static final long LOCKTIME_NO_RBF_SEQUENCE = NO_LOCKTIME_NO_RBF_SEQUENCE - 1; - - // Assuming node's trade-bot has no more than 100 entries? - private static final int MAX_CACHE_ENTRIES = 100; - - // Max time-to-live for cache entries (milliseconds) - private static final long CACHE_TIMEOUT = 30_000L; - - @SuppressWarnings("serial") - private static final Map SECRET_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) { - // This method is called just after a new entry has been added - @Override - public boolean removeEldestEntry(Map.Entry eldest) { - return size() > MAX_CACHE_ENTRIES; - } - }; - private static final byte[] NO_SECRET_CACHE_ENTRY = new byte[0]; - - @SuppressWarnings("serial") - private static final Map STATUS_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) { - // This method is called just after a new entry has been added - @Override - public boolean removeEldestEntry(Map.Entry eldest) { - return size() > MAX_CACHE_ENTRIES; - } - }; - - /* - * OP_TUCK (to copy public key to before signature) - * OP_CHECKSIGVERIFY (sig & pubkey must verify or script fails) - * OP_HASH160 (convert public key to PKH) - * OP_DUP (duplicate PKH) - * OP_EQUAL (does PKH match refund PKH?) - * OP_IF - * OP_DROP (no need for duplicate PKH) - * - * OP_CHECKLOCKTIMEVERIFY (if this passes, leftover stack is so script passes) - * OP_ELSE - * OP_EQUALVERIFY (duplicate PKH must match redeem PKH or script fails) - * OP_HASH160 (hash secret) - * OP_EQUAL (do hashes of secrets match? if true, script passes else script fails) - * OP_ENDIF - */ - - private static final byte[] redeemScript1 = HashCode.fromString("7dada97614").asBytes(); // OP_TUCK OP_CHECKSIGVERIFY OP_HASH160 OP_DUP push(0x14 bytes) - private static final byte[] redeemScript2 = HashCode.fromString("87637504").asBytes(); // OP_EQUAL OP_IF OP_DROP push(0x4 bytes) - private static final byte[] redeemScript3 = HashCode.fromString("b16714").asBytes(); // OP_CHECKLOCKTIMEVERIFY OP_ELSE push(0x14 bytes) - private static final byte[] redeemScript4 = HashCode.fromString("88a914").asBytes(); // OP_EQUALVERIFY OP_HASH160 push(0x14 bytes) - private static final byte[] redeemScript5 = HashCode.fromString("8768").asBytes(); // OP_EQUAL OP_ENDIF - - /** - * Returns redeemScript used for cross-chain trading. - *

- * See comments in {@link BitcoinyHTLC} for more details. - * - * @param refunderPubKeyHash 20-byte HASH160 of P2SH funder's public key, for refunding purposes - * @param lockTime seconds-since-epoch threshold, after which P2SH funder can claim refund - * @param redeemerPubKeyHash 20-byte HASH160 of P2SH redeemer's public key - * @param hashOfSecret 20-byte HASH160 of secret, used by P2SH redeemer to claim funds - */ - public static byte[] buildScript(byte[] refunderPubKeyHash, int lockTime, byte[] redeemerPubKeyHash, byte[] hashOfSecret) { - return Bytes.concat(redeemScript1, refunderPubKeyHash, redeemScript2, BitTwiddling.toLEByteArray((int) (lockTime & 0xffffffffL)), - redeemScript3, redeemerPubKeyHash, redeemScript4, hashOfSecret, redeemScript5); - } - - /** - * Builds a custom transaction to spend HTLC P2SH. - * - * @param params blockchain network parameters - * @param amount output amount, should be total of input amounts, less miner fees - * @param spendKey key for signing transaction, and also where funds are 'sent' (output) - * @param fundingOutput output from transaction that funded P2SH address - * @param redeemScriptBytes the redeemScript itself, in byte[] form - * @param lockTime (optional) transaction nLockTime, used in refund scenario - * @param scriptSigBuilder function for building scriptSig using transaction input signature - * @param outputPublicKeyHash PKH used to create P2PKH output - * @return Signed transaction for spending P2SH - */ - public static Transaction buildP2shTransaction(NetworkParameters params, Coin amount, ECKey spendKey, - List fundingOutputs, byte[] redeemScriptBytes, - Long lockTime, Function scriptSigBuilder, byte[] outputPublicKeyHash) { - Transaction transaction = new Transaction(params); - transaction.setVersion(2); - - // Output is back to P2SH funder - transaction.addOutput(amount, ScriptBuilder.createP2PKHOutputScript(outputPublicKeyHash)); - - for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { - TransactionOutput fundingOutput = fundingOutputs.get(inputIndex); - - // Input (without scriptSig prior to signing) - TransactionInput input = new TransactionInput(params, null, redeemScriptBytes, fundingOutput.getOutPointFor()); - if (lockTime != null) - input.setSequenceNumber(LOCKTIME_NO_RBF_SEQUENCE); // Use max-value - 1, so lockTime can be used but not RBF - else - input.setSequenceNumber(NO_LOCKTIME_NO_RBF_SEQUENCE); // Use max-value, so no lockTime and no RBF - transaction.addInput(input); - } - - // Set locktime after inputs added but before input signatures are generated - if (lockTime != null) - transaction.setLockTime(lockTime); - - for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { - // Generate transaction signature for input - final boolean anyoneCanPay = false; - TransactionSignature txSig = transaction.calculateSignature(inputIndex, spendKey, redeemScriptBytes, SigHash.ALL, anyoneCanPay); - - // Calculate transaction signature - byte[] txSigBytes = txSig.encodeToBitcoin(); - - // Build scriptSig using lambda and tx signature - Script scriptSig = scriptSigBuilder.apply(txSigBytes); - - // Set input scriptSig - transaction.getInput(inputIndex).setScriptSig(scriptSig); - } - - return transaction; - } - - /** - * Returns signed transaction claiming refund from HTLC P2SH. - * - * @param params blockchain network parameters - * @param refundAmount refund amount, should be total of input amounts, less miner fees - * @param refundKey key for signing transaction - * @param fundingOutputs outputs from transaction that funded P2SH address - * @param redeemScriptBytes the redeemScript itself, in byte[] form - * @param lockTime transaction nLockTime - must be at least locktime used in redeemScript - * @param receivingAccountInfo public-key-hash used for P2PKH output - * @return Signed transaction for refunding P2SH - */ - public static Transaction buildRefundTransaction(NetworkParameters params, Coin refundAmount, ECKey refundKey, - List fundingOutputs, byte[] redeemScriptBytes, long lockTime, byte[] receivingAccountInfo) { - Function refundSigScriptBuilder = (txSigBytes) -> { - // Build scriptSig with... - ScriptBuilder scriptBuilder = new ScriptBuilder(); - - // transaction signature - scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); - - // redeem public key - byte[] refundPubKey = refundKey.getPubKey(); - scriptBuilder.addChunk(new ScriptChunk(refundPubKey.length, refundPubKey)); - - // redeem script - scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); - - return scriptBuilder.build(); - }; - - // Send funds back to funding address - return buildP2shTransaction(params, refundAmount, refundKey, fundingOutputs, redeemScriptBytes, lockTime, refundSigScriptBuilder, receivingAccountInfo); - } - - /** - * Returns signed transaction redeeming funds from P2SH address. - * - * @param params blockchain network parameters - * @param redeemAmount redeem amount, should be total of input amounts, less miner fees - * @param redeemKey key for signing transaction - * @param fundingOutputs outputs from transaction that funded P2SH address - * @param redeemScriptBytes the redeemScript itself, in byte[] form - * @param secret actual 32-byte secret used when building redeemScript - * @param receivingAccountInfo Bitcoin PKH used for output - * @return Signed transaction for redeeming P2SH - */ - public static Transaction buildRedeemTransaction(NetworkParameters params, Coin redeemAmount, ECKey redeemKey, - List fundingOutputs, byte[] redeemScriptBytes, byte[] secret, byte[] receivingAccountInfo) { - Function redeemSigScriptBuilder = (txSigBytes) -> { - // Build scriptSig with... - ScriptBuilder scriptBuilder = new ScriptBuilder(); - - // secret - scriptBuilder.addChunk(new ScriptChunk(secret.length, secret)); - - // transaction signature - scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); - - // redeem public key - byte[] redeemPubKey = redeemKey.getPubKey(); - scriptBuilder.addChunk(new ScriptChunk(redeemPubKey.length, redeemPubKey)); - - // redeem script - scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); - - return scriptBuilder.build(); - }; - - return buildP2shTransaction(params, redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, null, redeemSigScriptBuilder, receivingAccountInfo); - } - - /** - * Returns 'secret', if any, given HTLC's P2SH address. - *

- * @throws ForeignBlockchainException - */ - public static byte[] findHtlcSecret(Bitcoiny bitcoiny, String p2shAddress) throws ForeignBlockchainException { - NetworkParameters params = bitcoiny.getNetworkParameters(); - String compoundKey = String.format("%s-%s-%d", params.getId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT); - - byte[] secret = SECRET_CACHE.getOrDefault(compoundKey, NO_SECRET_CACHE_ENTRY); - if (secret != NO_SECRET_CACHE_ENTRY) - return secret; - - List rawTransactions = bitcoiny.getAddressTransactions(p2shAddress); - - for (byte[] rawTransaction : rawTransactions) { - Transaction transaction = new Transaction(params, rawTransaction); - - // Cycle through inputs, looking for one that spends our HTLC - for (TransactionInput input : transaction.getInputs()) { - Script scriptSig = input.getScriptSig(); - List scriptChunks = scriptSig.getChunks(); - - // Expected number of script chunks for redeem. Refund might not have the same number. - int expectedChunkCount = 1 /*secret*/ + 1 /*sig*/ + 1 /*pubkey*/ + 1 /*redeemScript*/; - if (scriptChunks.size() != expectedChunkCount) - continue; - - // We're expecting last chunk to contain the actual redeemScript - ScriptChunk lastChunk = scriptChunks.get(scriptChunks.size() - 1); - byte[] redeemScriptBytes = lastChunk.data; - - // If non-push scripts, redeemScript will be null - if (redeemScriptBytes == null) - continue; - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address inputAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!inputAddress.toString().equals(p2shAddress)) - // Input isn't spending our HTLC - continue; - - secret = scriptChunks.get(0).data; - if (secret.length != BitcoinyHTLC.SECRET_LENGTH) - continue; - - // Cache secret for a while - SECRET_CACHE.put(compoundKey, secret); - - return secret; - } - } - - // Cache negative result - SECRET_CACHE.put(compoundKey, null); - - return null; - } - - /** - * Returns HTLC status, given P2SH address and expected redeem/refund amount - *

- * @throws ForeignBlockchainException if error occurs - */ - public static Status determineHtlcStatus(BitcoinyBlockchainProvider blockchain, String p2shAddress, long minimumAmount) throws ForeignBlockchainException { - String compoundKey = String.format("%s-%s-%d", blockchain.getNetId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT); - - Status cachedStatus = STATUS_CACHE.getOrDefault(compoundKey, null); - if (cachedStatus != null) - return cachedStatus; - - byte[] ourScriptPubKey = addressToScriptPubKey(p2shAddress); - List transactionHashes = blockchain.getAddressTransactions(ourScriptPubKey, BitcoinyBlockchainProvider.INCLUDE_UNCONFIRMED); - - // Sort by confirmed first, followed by ascending height - transactionHashes.sort(TransactionHash.CONFIRMED_FIRST.thenComparing(TransactionHash::getHeight)); - - // Transaction cache - Map transactionsByHash = new HashMap<>(); - // HASH160(redeem script) for this p2shAddress - byte[] ourRedeemScriptHash = addressToRedeemScriptHash(p2shAddress); - - // Check for spends first, caching full transaction info as we progress just in case we don't return in this loop - for (TransactionHash transactionInfo : transactionHashes) { - BitcoinyTransaction bitcoinyTransaction = blockchain.getTransaction(transactionInfo.txHash); - - // Cache for possible later reuse - transactionsByHash.put(transactionInfo.txHash, bitcoinyTransaction); - - // Acceptable funding is one transaction output, so we're expecting only one input - if (bitcoinyTransaction.inputs.size() != 1) - // Wrong number of inputs - continue; - - String scriptSig = bitcoinyTransaction.inputs.get(0).scriptSig; - - List scriptSigChunks = extractScriptSigChunks(HashCode.fromString(scriptSig).asBytes()); - if (scriptSigChunks.size() < 3 || scriptSigChunks.size() > 4) - // Not valid chunks for our form of HTLC - continue; - - // Last chunk is redeem script - byte[] redeemScriptBytes = scriptSigChunks.get(scriptSigChunks.size() - 1); - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - if (!Arrays.equals(redeemScriptHash, ourRedeemScriptHash)) - // Not spending our specific HTLC redeem script - continue; - - if (scriptSigChunks.size() == 4) - // If we have 4 chunks, then secret is present, hence redeem - cachedStatus = transactionInfo.height == 0 ? Status.REDEEM_IN_PROGRESS : Status.REDEEMED; - else - cachedStatus = transactionInfo.height == 0 ? Status.REFUND_IN_PROGRESS : Status.REFUNDED; - - STATUS_CACHE.put(compoundKey, cachedStatus); - return cachedStatus; - } - - String ourScriptPubKeyHex = HashCode.fromBytes(ourScriptPubKey).toString(); - - // Check for funding - for (TransactionHash transactionInfo : transactionHashes) { - BitcoinyTransaction bitcoinyTransaction = transactionsByHash.get(transactionInfo.txHash); - if (bitcoinyTransaction == null) - // Should be present in map! - throw new ForeignBlockchainException("Cached Bitcoin transaction now missing?"); - - // Check outputs for our specific P2SH - for (BitcoinyTransaction.Output output : bitcoinyTransaction.outputs) { - // Check amount - if (output.value < minimumAmount) - // Output amount too small (not taking fees into account) - continue; - - String scriptPubKeyHex = output.scriptPubKey; - if (!scriptPubKeyHex.equals(ourScriptPubKeyHex)) - // Not funding our specific P2SH - continue; - - cachedStatus = transactionInfo.height == 0 ? Status.FUNDING_IN_PROGRESS : Status.FUNDED; - STATUS_CACHE.put(compoundKey, cachedStatus); - return cachedStatus; - } - } - - cachedStatus = Status.UNFUNDED; - STATUS_CACHE.put(compoundKey, cachedStatus); - return cachedStatus; - } - - private static List extractScriptSigChunks(byte[] scriptSigBytes) { - List chunks = new ArrayList<>(); - - int offset = 0; - int previousOffset = 0; - while (offset < scriptSigBytes.length) { - byte pushOp = scriptSigBytes[offset++]; - - if (pushOp < 0 || pushOp > 0x4c) - // Unacceptable OP - return Collections.emptyList(); - - // Special treatment for OP_PUSHDATA1 - if (pushOp == 0x4c) { - if (offset >= scriptSigBytes.length) - // Run out of scriptSig bytes? - return Collections.emptyList(); - - pushOp = scriptSigBytes[offset++]; - } - - previousOffset = offset; - offset += Byte.toUnsignedInt(pushOp); - - byte[] chunk = Arrays.copyOfRange(scriptSigBytes, previousOffset, offset); - chunks.add(chunk); - } - - return chunks; - } - - private static byte[] addressToScriptPubKey(String p2shAddress) { - // We want the HASH160 part of the P2SH address - byte[] p2shAddressBytes = Base58.decode(p2shAddress); - - byte[] scriptPubKey = new byte[1 + 1 + 20 + 1]; - scriptPubKey[0x00] = (byte) 0xa9; /* OP_HASH160 */ - scriptPubKey[0x01] = (byte) 0x14; /* PUSH 0x14 bytes */ - System.arraycopy(p2shAddressBytes, 1, scriptPubKey, 2, 0x14); - scriptPubKey[0x16] = (byte) 0x87; /* OP_EQUAL */ - - return scriptPubKey; - } - - private static byte[] addressToRedeemScriptHash(String p2shAddress) { - // We want the HASH160 part of the P2SH address - byte[] p2shAddressBytes = Base58.decode(p2shAddress); - - return Arrays.copyOfRange(p2shAddressBytes, 1, 1 + 20); - } - -} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java b/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java deleted file mode 100644 index caf0b36d..00000000 --- a/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java +++ /dev/null @@ -1,146 +0,0 @@ -package org.qortal.crosschain; - -import java.util.List; -import java.util.stream.Collectors; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlTransient; - -@XmlAccessorType(XmlAccessType.FIELD) -public class BitcoinyTransaction { - - public final String txHash; - - @XmlTransient - public final int size; - - @XmlTransient - public final int locktime; - - // Not present if transaction is unconfirmed - public final Integer timestamp; - - public static class Input { - @XmlTransient - public final String scriptSig; - - @XmlTransient - public final int sequence; - - public final String outputTxHash; - - public final int outputVout; - - // For JAXB - protected Input() { - this.scriptSig = null; - this.sequence = 0; - this.outputTxHash = null; - this.outputVout = 0; - } - - public Input(String scriptSig, int sequence, String outputTxHash, int outputVout) { - this.scriptSig = scriptSig; - this.sequence = sequence; - this.outputTxHash = outputTxHash; - this.outputVout = outputVout; - } - - public String toString() { - return String.format("{output %s:%d, sequence %d, scriptSig %s}", - this.outputTxHash, this.outputVout, this.sequence, this.scriptSig); - } - } - @XmlTransient - public final List inputs; - - public static class Output { - @XmlTransient - public final String scriptPubKey; - - public final long value; - - public final List addresses; - - // For JAXB - protected Output() { - this.scriptPubKey = null; - this.value = 0; - this.addresses = null; - } - - public Output(String scriptPubKey, long value) { - this.scriptPubKey = scriptPubKey; - this.value = value; - this.addresses = null; - } - - public Output(String scriptPubKey, long value, List addresses) { - this.scriptPubKey = scriptPubKey; - this.value = value; - this.addresses = addresses; - } - - public String toString() { - return String.format("{value %d, scriptPubKey %s}", this.value, this.scriptPubKey); - } - } - public final List outputs; - - public final long totalAmount; - - // For JAXB - protected BitcoinyTransaction() { - this.txHash = null; - this.size = 0; - this.locktime = 0; - this.timestamp = 0; - this.inputs = null; - this.outputs = null; - this.totalAmount = 0; - } - - public BitcoinyTransaction(String txHash, int size, int locktime, Integer timestamp, - List inputs, List outputs) { - this.txHash = txHash; - this.size = size; - this.locktime = locktime; - this.timestamp = timestamp; - this.inputs = inputs; - this.outputs = outputs; - - this.totalAmount = outputs.stream().map(output -> output.value).reduce(0L, Long::sum); - } - - public String toString() { - return String.format("txHash %s, size %d, locktime %d, timestamp %d\n" - + "\tinputs: [%s]\n" - + "\toutputs: [%s]\n", - this.txHash, - this.size, - this.locktime, - this.timestamp, - this.inputs.stream().map(Input::toString).collect(Collectors.joining(",\n\t\t")), - this.outputs.stream().map(Output::toString).collect(Collectors.joining(",\n\t\t"))); - } - - @Override - public boolean equals(Object other) { - if (other == this) - return true; - - if (!(other instanceof BitcoinyTransaction)) - return false; - - BitcoinyTransaction otherTransaction = (BitcoinyTransaction) other; - - return this.txHash.equals(otherTransaction.txHash); - } - - @Override - public int hashCode() { - return this.txHash.hashCode(); - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java deleted file mode 100644 index b34aa199..00000000 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ /dev/null @@ -1,688 +0,0 @@ -package org.qortal.crosschain; - -import java.io.IOException; -import java.math.BigDecimal; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Random; -import java.util.Scanner; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.net.ssl.SSLSocketFactory; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.JSONValue; -import org.qortal.crypto.Crypto; -import org.qortal.crypto.TrustlessSSLSocketFactory; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -/** ElectrumX network support for querying Bitcoiny-related info like block headers, transaction outputs, etc. */ -public class ElectrumX extends BitcoinyBlockchainProvider { - - private static final Logger LOGGER = LogManager.getLogger(ElectrumX.class); - private static final Random RANDOM = new Random(); - - private static final double MIN_PROTOCOL_VERSION = 1.2; - private static final int BLOCK_HEADER_LENGTH = 80; - - // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" - private static final Pattern DAEMON_ERROR_REGEX = Pattern.compile("DaemonError\\(\\{.*'code': ?(-?[0-9]+).*\\}\\)\\z"); // Capture 'code' inside curly-brace content - - /** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */ - private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported"; - - public static class Server { - String hostname; - - public enum ConnectionType { TCP, SSL } - ConnectionType connectionType; - - int port; - - public Server(String hostname, ConnectionType connectionType, int port) { - this.hostname = hostname; - this.connectionType = connectionType; - this.port = port; - } - - @Override - public boolean equals(Object other) { - if (other == this) - return true; - - if (!(other instanceof Server)) - return false; - - Server otherServer = (Server) other; - - return this.connectionType == otherServer.connectionType - && this.port == otherServer.port - && this.hostname.equals(otherServer.hostname); - } - - @Override - public int hashCode() { - return this.hostname.hashCode() ^ this.port; - } - - @Override - public String toString() { - return String.format("%s:%s:%d", this.connectionType.name(), this.hostname, this.port); - } - } - private Set servers = new HashSet<>(); - private List remainingServers = new ArrayList<>(); - private Set uselessServers = Collections.synchronizedSet(new HashSet<>()); - - private final String netId; - private final String expectedGenesisHash; - private final Map defaultPorts = new EnumMap<>(Server.ConnectionType.class); - - private final Object serverLock = new Object(); - private Server currentServer; - private Socket socket; - private Scanner scanner; - private int nextId = 1; - - private static final int TX_CACHE_SIZE = 200; - @SuppressWarnings("serial") - private final Map transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { - // This method is called just after a new entry has been added - @Override - public boolean removeEldestEntry(Map.Entry eldest) { - return size() > TX_CACHE_SIZE; - } - }); - - // Constructors - - public ElectrumX(String netId, String genesisHash, Collection initialServerList, Map defaultPorts) { - this.netId = netId; - this.expectedGenesisHash = genesisHash; - this.servers.addAll(initialServerList); - this.defaultPorts.putAll(defaultPorts); - } - - // Methods for use by other classes - - @Override - public String getNetId() { - return this.netId; - } - - /** - * Returns current blockchain height. - *

- * @throws ForeignBlockchainException if error occurs - */ - @Override - public int getCurrentHeight() throws ForeignBlockchainException { - Object blockObj = this.rpc("blockchain.headers.subscribe"); - if (!(blockObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.headers.subscribe RPC"); - - JSONObject blockJson = (JSONObject) blockObj; - - Object heightObj = blockJson.get("height"); - - if (!(heightObj instanceof Long)) - throw new ForeignBlockchainException.NetworkException("Missing/invalid 'height' in JSON from ElectrumX blockchain.headers.subscribe RPC"); - - return ((Long) heightObj).intValue(); - } - - /** - * Returns list of raw block headers, starting from startHeight inclusive. - *

- * @throws ForeignBlockchainException if error occurs - */ - @Override - public List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException { - Object blockObj = this.rpc("blockchain.block.headers", startHeight, count); - if (!(blockObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.block.headers RPC"); - - JSONObject blockJson = (JSONObject) blockObj; - - Object countObj = blockJson.get("count"); - Object hexObj = blockJson.get("hex"); - - if (!(countObj instanceof Long) || !(hexObj instanceof String)) - throw new ForeignBlockchainException.NetworkException("Missing/invalid 'count' or 'hex' entries in JSON from ElectrumX blockchain.block.headers RPC"); - - Long returnedCount = (Long) countObj; - String hex = (String) hexObj; - - byte[] raw = HashCode.fromString(hex).asBytes(); - if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) - throw new ForeignBlockchainException.NetworkException("Unexpected raw header length in JSON from ElectrumX blockchain.block.headers RPC"); - - List rawBlockHeaders = new ArrayList<>(returnedCount.intValue()); - for (int i = 0; i < returnedCount; ++i) - rawBlockHeaders.add(Arrays.copyOfRange(raw, i * BLOCK_HEADER_LENGTH, (i + 1) * BLOCK_HEADER_LENGTH)); - - return rawBlockHeaders; - } - - /** - * Returns confirmed balance, based on passed payment script. - *

- * @return confirmed balance, or zero if script unknown - * @throws ForeignBlockchainException if there was an error - */ - @Override - public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException { - byte[] scriptHash = Crypto.digest(script); - Bytes.reverse(scriptHash); - - Object balanceObj = this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()); - if (!(balanceObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.scripthash.get_balance RPC"); - - JSONObject balanceJson = (JSONObject) balanceObj; - - Object confirmedBalanceObj = balanceJson.get("confirmed"); - - if (!(confirmedBalanceObj instanceof Long)) - throw new ForeignBlockchainException.NetworkException("Missing confirmed balance from ElectrumX blockchain.scripthash.get_balance RPC"); - - return (Long) balanceJson.get("confirmed"); - } - - /** - * Returns list of unspent outputs pertaining to passed payment script. - *

- * @return list of unspent outputs, or empty list if script unknown - * @throws ForeignBlockchainException if there was an error. - */ - @Override - public List getUnspentOutputs(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { - byte[] scriptHash = Crypto.digest(script); - Bytes.reverse(scriptHash); - - Object unspentJson = this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()); - if (!(unspentJson instanceof JSONArray)) - throw new ForeignBlockchainException("Expected array output from ElectrumX blockchain.scripthash.listunspent RPC"); - - List unspentOutputs = new ArrayList<>(); - for (Object rawUnspent : (JSONArray) unspentJson) { - JSONObject unspent = (JSONObject) rawUnspent; - - int height = ((Long) unspent.get("height")).intValue(); - // We only want unspent outputs from confirmed transactions (and definitely not mempool duplicates with height 0) - if (!includeUnconfirmed && height <= 0) - continue; - - byte[] txHash = HashCode.fromString((String) unspent.get("tx_hash")).asBytes(); - int outputIndex = ((Long) unspent.get("tx_pos")).intValue(); - long value = (Long) unspent.get("value"); - - unspentOutputs.add(new UnspentOutput(txHash, outputIndex, height, value)); - } - - return unspentOutputs; - } - - /** - * Returns raw transaction for passed transaction hash. - *

- * NOTE: Do not mutate returned byte[]! - * - * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs - */ - @Override - public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException { - Object rawTransactionHex; - try { - rawTransactionHex = this.rpc("blockchain.transaction.get", txHash, false); - } catch (ForeignBlockchainException.NetworkException e) { - // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) - if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) - throw new ForeignBlockchainException.NotFoundException(e.getMessage()); - - throw e; - } - - if (!(rawTransactionHex instanceof String)) - throw new ForeignBlockchainException.NetworkException("Expected hex string as raw transaction from ElectrumX blockchain.transaction.get RPC"); - - return HashCode.fromString((String) rawTransactionHex).asBytes(); - } - - /** - * Returns raw transaction for passed transaction hash. - *

- * NOTE: Do not mutate returned byte[]! - * - * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs - */ - @Override - public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException { - return getRawTransaction(HashCode.fromBytes(txHash).toString()); - } - - /** - * Returns transaction info for passed transaction hash. - *

- * @throws ForeignBlockchainException.NotFoundException if transaction not found - * @throws ForeignBlockchainException if error occurs - */ - @Override - public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { - // Check cache first - BitcoinyTransaction transaction = transactionCache.get(txHash); - if (transaction != null) - return transaction; - - Object transactionObj = null; - - do { - try { - transactionObj = this.rpc("blockchain.transaction.get", txHash, true); - } catch (ForeignBlockchainException.NetworkException e) { - // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) - if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) - throw new ForeignBlockchainException.NotFoundException(e.getMessage()); - - // Some servers also return non-standard responses like this: - // {"error":"verbose transactions are currently unsupported","id":3,"jsonrpc":"2.0"} - // We should probably not use this server any more - if (e.getServer() != null && e.getMessage() != null && e.getMessage().contains(VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE)) { - Server uselessServer = (Server) e.getServer(); - LOGGER.trace(() -> String.format("Server %s doesn't support verbose transactions - barring use of that server", uselessServer)); - this.uselessServers.add(uselessServer); - this.closeServer(uselessServer); - continue; - } - - throw e; - } - } while (transactionObj == null); - - if (!(transactionObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException("Expected JSONObject as response from ElectrumX blockchain.transaction.get RPC"); - - JSONObject transactionJson = (JSONObject) transactionObj; - - Object inputsObj = transactionJson.get("vin"); - if (!(inputsObj instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vin' from ElectrumX blockchain.transaction.get RPC"); - - Object outputsObj = transactionJson.get("vout"); - if (!(outputsObj instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vout' from ElectrumX blockchain.transaction.get RPC"); - - try { - int size = ((Long) transactionJson.get("size")).intValue(); - int locktime = ((Long) transactionJson.get("locktime")).intValue(); - - // Timestamp might not be present, e.g. for unconfirmed transaction - Object timeObj = transactionJson.get("time"); - Integer timestamp = timeObj != null - ? ((Long) timeObj).intValue() - : null; - - List inputs = new ArrayList<>(); - for (Object inputObj : (JSONArray) inputsObj) { - JSONObject inputJson = (JSONObject) inputObj; - - String scriptSig = (String) ((JSONObject) inputJson.get("scriptSig")).get("hex"); - int sequence = ((Long) inputJson.get("sequence")).intValue(); - String outputTxHash = (String) inputJson.get("txid"); - int outputVout = ((Long) inputJson.get("vout")).intValue(); - - inputs.add(new BitcoinyTransaction.Input(scriptSig, sequence, outputTxHash, outputVout)); - } - - List outputs = new ArrayList<>(); - for (Object outputObj : (JSONArray) outputsObj) { - JSONObject outputJson = (JSONObject) outputObj; - - String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex"); - long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue(); - - // address too, if present - List addresses = null; - Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses"); - if (addressesObj instanceof JSONArray) { - addresses = new ArrayList<>(); - for (Object addressObj : (JSONArray) addressesObj) - addresses.add((String) addressObj); - } - - outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses)); - } - - transaction = new BitcoinyTransaction(txHash, size, locktime, timestamp, inputs, outputs); - - // Save into cache - transactionCache.put(txHash, transaction); - - return transaction; - } catch (NullPointerException | ClassCastException e) { - // Unexpected / invalid response from ElectrumX server - } - - throw new ForeignBlockchainException.NetworkException("Unexpected JSON format from ElectrumX blockchain.transaction.get RPC"); - } - - /** - * Returns list of transactions, relating to passed payment script. - *

- * @return list of related transactions, or empty list if script unknown - * @throws ForeignBlockchainException if error occurs - */ - @Override - public List getAddressTransactions(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { - byte[] scriptHash = Crypto.digest(script); - Bytes.reverse(scriptHash); - - Object transactionsJson = this.rpc("blockchain.scripthash.get_history", HashCode.fromBytes(scriptHash).toString()); - if (!(transactionsJson instanceof JSONArray)) - throw new ForeignBlockchainException.NetworkException("Expected array output from ElectrumX blockchain.scripthash.get_history RPC"); - - List transactionHashes = new ArrayList<>(); - - for (Object rawTransactionInfo : (JSONArray) transactionsJson) { - JSONObject transactionInfo = (JSONObject) rawTransactionInfo; - - Long height = (Long) transactionInfo.get("height"); - if (!includeUnconfirmed && (height == null || height == 0)) - // We only want confirmed transactions - continue; - - String txHash = (String) transactionInfo.get("tx_hash"); - - transactionHashes.add(new TransactionHash(height.intValue(), txHash)); - } - - return transactionHashes; - } - - /** - * Broadcasts raw transaction to network. - *

- * @throws ForeignBlockchainException if error occurs - */ - @Override - public void broadcastTransaction(byte[] transactionBytes) throws ForeignBlockchainException { - Object rawBroadcastResult = this.rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString()); - - // We're expecting a simple string that is the transaction hash - if (!(rawBroadcastResult instanceof String)) - throw new ForeignBlockchainException.NetworkException("Unexpected response from ElectrumX blockchain.transaction.broadcast RPC"); - } - - // Class-private utility methods - - /** - * Query current server for its list of peer servers, and return those we can parse. - *

- * @throws ForeignBlockchainException - * @throws ClassCastException to be handled by caller - */ - private Set serverPeersSubscribe() throws ForeignBlockchainException { - Set newServers = new HashSet<>(); - - Object peers = this.connectedRpc("server.peers.subscribe"); - - for (Object rawPeer : (JSONArray) peers) { - JSONArray peer = (JSONArray) rawPeer; - if (peer.size() < 3) - // We're expecting at least 3 fields for each peer entry: IP, hostname, features - continue; - - String hostname = (String) peer.get(1); - JSONArray features = (JSONArray) peer.get(2); - - for (Object rawFeature : features) { - String feature = (String) rawFeature; - Server.ConnectionType connectionType = null; - Integer port = null; - - switch (feature.charAt(0)) { - case 's': - connectionType = Server.ConnectionType.SSL; - port = this.defaultPorts.get(connectionType); - break; - - case 't': - connectionType = Server.ConnectionType.TCP; - port = this.defaultPorts.get(connectionType); - break; - - default: - // e.g. could be 'v' for protocol version, or 'p' for pruning limit - break; - } - - if (connectionType == null || port == null) - // We couldn't extract any peer connection info? - continue; - - // Possible non-default port? - if (feature.length() > 1) - try { - port = Integer.parseInt(feature.substring(1)); - } catch (NumberFormatException e) { - // no good - continue; // for-loop above - } - - Server newServer = new Server(hostname, connectionType, port); - newServers.add(newServer); - } - } - - return newServers; - } - - /** - * Performs RPC call, with automatic reconnection to different server if needed. - *

- * @return "result" object from within JSON output - * @throws ForeignBlockchainException if server returns error or something goes wrong - */ - private Object rpc(String method, Object...params) throws ForeignBlockchainException { - synchronized (this.serverLock) { - if (this.remainingServers.isEmpty()) - this.remainingServers.addAll(this.servers); - - while (haveConnection()) { - Object response = connectedRpc(method, params); - if (response != null) - return response; - - // Didn't work, try another server... - this.closeServer(); - } - - // Failed to perform RPC - maybe lack of servers? - throw new ForeignBlockchainException.NetworkException(String.format("Failed to perform ElectrumX RPC %s", method)); - } - } - - /** Returns true if we have, or create, a connection to an ElectrumX server. */ - private boolean haveConnection() throws ForeignBlockchainException { - if (this.currentServer != null) - return true; - - while (!this.remainingServers.isEmpty()) { - Server server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); - LOGGER.trace(() -> String.format("Connecting to %s", server)); - - try { - SocketAddress endpoint = new InetSocketAddress(server.hostname, server.port); - int timeout = 5000; // ms - - this.socket = new Socket(); - this.socket.connect(endpoint, timeout); - this.socket.setTcpNoDelay(true); - - if (server.connectionType == Server.ConnectionType.SSL) { - SSLSocketFactory factory = TrustlessSSLSocketFactory.getSocketFactory(); - this.socket = factory.createSocket(this.socket, server.hostname, server.port, true); - } - - this.scanner = new Scanner(this.socket.getInputStream()); - this.scanner.useDelimiter("\n"); - - // Check connection is suitable by asking for server features, including genesis block hash - JSONObject featuresJson = (JSONObject) this.connectedRpc("server.features"); - - if (featuresJson == null || Double.valueOf((String) featuresJson.get("protocol_min")) < MIN_PROTOCOL_VERSION) - continue; - - if (this.expectedGenesisHash != null && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) - continue; - - // Ask for more servers - Set moreServers = serverPeersSubscribe(); - // Discard duplicate servers we already know - moreServers.removeAll(this.servers); - // Add to both lists - this.remainingServers.addAll(moreServers); - this.servers.addAll(moreServers); - - LOGGER.debug(() -> String.format("Connected to %s", server)); - this.currentServer = server; - return true; - } catch (IOException | ForeignBlockchainException | ClassCastException | NullPointerException e) { - // Didn't work, try another server... - closeServer(); - } - } - - return false; - } - - /** - * Perform RPC using currently connected server. - *

- * @param method - * @param params - * @return response Object, or null if server fails to respond - * @throws ForeignBlockchainException if server returns error - */ - @SuppressWarnings("unchecked") - private Object connectedRpc(String method, Object...params) throws ForeignBlockchainException { - JSONObject requestJson = new JSONObject(); - requestJson.put("id", this.nextId++); - requestJson.put("method", method); - requestJson.put("jsonrpc", "2.0"); - - JSONArray requestParams = new JSONArray(); - requestParams.addAll(Arrays.asList(params)); - requestJson.put("params", requestParams); - - String request = requestJson.toJSONString() + "\n"; - LOGGER.trace(() -> String.format("Request: %s", request)); - - final String response; - - try { - this.socket.getOutputStream().write(request.getBytes()); - response = scanner.next(); - } catch (IOException | NoSuchElementException e) { - // Unable to send, or receive -- try another server? - return null; - } - - LOGGER.trace(() -> String.format("Response: %s", response)); - - if (response.isEmpty()) - // Empty response - try another server? - return null; - - Object responseObj = JSONValue.parse(response); - if (!(responseObj instanceof JSONObject)) - // Unexpected response - try another server? - return null; - - JSONObject responseJson = (JSONObject) responseObj; - - Object errorObj = responseJson.get("error"); - if (errorObj != null) { - if (errorObj instanceof String) - throw new ForeignBlockchainException.NetworkException(String.format("Unexpected error message from ElectrumX RPC %s: %s", method, (String) errorObj), this.currentServer); - - if (!(errorObj instanceof JSONObject)) - throw new ForeignBlockchainException.NetworkException(String.format("Unexpected error response from ElectrumX RPC %s", method), this.currentServer); - - JSONObject errorJson = (JSONObject) errorObj; - - Object messageObj = errorJson.get("message"); - - if (!(messageObj instanceof String)) - throw new ForeignBlockchainException.NetworkException(String.format("Missing/invalid message in error response from ElectrumX RPC %s", method), this.currentServer); - - String message = (String) messageObj; - - // Some error 'messages' are actually wrapped upstream bitcoind errors: - // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" - // We want to detect these and extract the upstream error code for caller's use - Matcher messageMatcher = DAEMON_ERROR_REGEX.matcher(message); - if (messageMatcher.find()) - try { - int daemonErrorCode = Integer.parseInt(messageMatcher.group(1)); - throw new ForeignBlockchainException.NetworkException(daemonErrorCode, message, this.currentServer); - } catch (NumberFormatException e) { - // We couldn't parse the error code integer? Fall-through to generic exception... - } - - throw new ForeignBlockchainException.NetworkException(message, this.currentServer); - } - - return responseJson.get("result"); - } - - /** - * Closes connection to server if it is currently connected server. - * @param server - */ - private void closeServer(Server server) { - synchronized (this.serverLock) { - if (this.currentServer == null || !this.currentServer.equals(server)) - return; - - if (this.socket != null && !this.socket.isClosed()) - try { - this.socket.close(); - } catch (IOException e) { - // We did try... - } - - this.socket = null; - this.scanner = null; - this.currentServer = null; - } - } - - /** Closes connection to currently connected server (if any). */ - private void closeServer() { - synchronized (this.serverLock) { - this.closeServer(this.currentServer); - } - } - -} diff --git a/src/main/java/org/qortal/crosschain/ForeignBlockchain.java b/src/main/java/org/qortal/crosschain/ForeignBlockchain.java deleted file mode 100644 index 0a71e9d9..00000000 --- a/src/main/java/org/qortal/crosschain/ForeignBlockchain.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.qortal.crosschain; - -public interface ForeignBlockchain { - - public boolean isValidAddress(String address); - - public boolean isValidWalletKey(String walletKey); - -} diff --git a/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java b/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java deleted file mode 100644 index 1e658621..00000000 --- a/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.qortal.crosschain; - -@SuppressWarnings("serial") -public class ForeignBlockchainException extends Exception { - - public ForeignBlockchainException() { - super(); - } - - public ForeignBlockchainException(String message) { - super(message); - } - - public static class NetworkException extends ForeignBlockchainException { - private final Integer daemonErrorCode; - private final transient Object server; - - public NetworkException() { - super(); - this.daemonErrorCode = null; - this.server = null; - } - - public NetworkException(String message) { - super(message); - this.daemonErrorCode = null; - this.server = null; - } - - public NetworkException(int errorCode, String message) { - super(message); - this.daemonErrorCode = errorCode; - this.server = null; - } - - public NetworkException(String message, Object server) { - super(message); - this.daemonErrorCode = null; - this.server = server; - } - - public NetworkException(int errorCode, String message, Object server) { - super(message); - this.daemonErrorCode = errorCode; - this.server = server; - } - - public Integer getDaemonErrorCode() { - return this.daemonErrorCode; - } - - public Object getServer() { - return this.server; - } - } - - public static class NotFoundException extends ForeignBlockchainException { - public NotFoundException() { - super(); - } - - public NotFoundException(String message) { - super(message); - } - } - - public static class InsufficientFundsException extends ForeignBlockchainException { - public InsufficientFundsException() { - super(); - } - - public InsufficientFundsException(String message) { - super(message); - } - } - -} diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java deleted file mode 100644 index 5cbe4044..00000000 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.qortal.crosschain; - -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumMap; -import java.util.Map; - -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.Context; -import org.bitcoinj.core.NetworkParameters; -import org.libdohj.params.LitecoinMainNetParams; -import org.libdohj.params.LitecoinRegTestParams; -import org.libdohj.params.LitecoinTestNet3Params; -import org.qortal.crosschain.ElectrumX.Server; -import org.qortal.crosschain.ElectrumX.Server.ConnectionType; -import org.qortal.settings.Settings; - -public class Litecoin extends Bitcoiny { - - public static final String CURRENCY_CODE = "LTC"; - - private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(10000); // 0.0001 LTC per 1000 bytes - - // Temporary values until a dynamic fee system is written. - private static final long MAINNET_FEE = 1000L; - private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST - - private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); - static { - DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); - DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); - } - - public enum LitecoinNet { - MAIN { - @Override - public NetworkParameters getParams() { - return LitecoinMainNetParams.get(); - } - - @Override - public Collection getServers() { - return Arrays.asList( - // Servers chosen on NO BASIS WHATSOEVER from various sources! - new Server("electrum-ltc.someguy123.net", Server.ConnectionType.SSL, 50002), - new Server("backup.electrum-ltc.org", Server.ConnectionType.TCP, 50001), - new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 50001), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), - new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002), - new Server("ltc.rentonisk.com", Server.ConnectionType.TCP, 50001), - new Server("ltc.rentonisk.com", Server.ConnectionType.SSL, 50002), - new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), - new Server("ltc.litepay.ch", Server.ConnectionType.SSL, 50022)); - } - - @Override - public String getGenesisHash() { - return "12a765e31ffd4059bada1e25190f6e98c99d9714d334efa41a195a7e7e04bfe2"; - } - - @Override - public long getP2shFee(Long timestamp) { - // TODO: This will need to be replaced with something better in the near future! - return MAINNET_FEE; - } - }, - TEST3 { - @Override - public NetworkParameters getParams() { - return LitecoinTestNet3Params.get(); - } - - @Override - public Collection getServers() { - return Arrays.asList( - new Server("electrum-ltc.bysh.me", Server.ConnectionType.TCP, 51001), - new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 51002), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 51001), - new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 51002)); - } - - @Override - public String getGenesisHash() { - return "4966625a4b2851d9fdee139e56211a0d88575f59ed816ff5e6a63deb4e3e29a0"; - } - - @Override - public long getP2shFee(Long timestamp) { - return NON_MAINNET_FEE; - } - }, - REGTEST { - @Override - public NetworkParameters getParams() { - return LitecoinRegTestParams.get(); - } - - @Override - public Collection getServers() { - return Arrays.asList( - new Server("localhost", Server.ConnectionType.TCP, 50001), - new Server("localhost", Server.ConnectionType.SSL, 50002)); - } - - @Override - public String getGenesisHash() { - // This is unique to each regtest instance - return null; - } - - @Override - public long getP2shFee(Long timestamp) { - return NON_MAINNET_FEE; - } - }; - - public abstract NetworkParameters getParams(); - public abstract Collection getServers(); - public abstract String getGenesisHash(); - public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; - } - - private static Litecoin instance; - - private final LitecoinNet litecoinNet; - - // Constructors and instance - - private Litecoin(LitecoinNet litecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { - super(blockchain, bitcoinjContext, currencyCode); - this.litecoinNet = litecoinNet; - - LOGGER.info(() -> String.format("Starting Litecoin support using %s", this.litecoinNet.name())); - } - - public static synchronized Litecoin getInstance() { - if (instance == null) { - LitecoinNet litecoinNet = Settings.getInstance().getLitecoinNet(); - - BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); - Context bitcoinjContext = new Context(litecoinNet.getParams()); - - instance = new Litecoin(litecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); - } - - return instance; - } - - // Getters & setters - - public static synchronized void resetForTesting() { - instance = null; - } - - // Actual useful methods for use by other classes - - /** Default Litecoin fee is lower than Bitcoin: only 10sats/byte. */ - @Override - public Coin getFeePerKb() { - return DEFAULT_FEE_PER_KB; - } - - /** - * Returns estimated LTC fee, in sats per 1000bytes, optionally for historic timestamp. - * - * @param timestamp optional milliseconds since epoch, or null for 'now' - * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong - */ - @Override - public long getP2shFee(Long timestamp) throws ForeignBlockchainException { - return this.litecoinNet.getP2shFee(timestamp); - } - -} diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java deleted file mode 100644 index 454e80c2..00000000 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java +++ /dev/null @@ -1,853 +0,0 @@ -package org.qortal.crosschain; - -import static org.ciyam.at.OpCode.calcOffset; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; - -import org.ciyam.at.API; -import org.ciyam.at.CompilationException; -import org.ciyam.at.FunctionCode; -import org.ciyam.at.MachineState; -import org.ciyam.at.OpCode; -import org.ciyam.at.Timestamp; -import org.qortal.account.Account; -import org.qortal.asset.Asset; -import org.qortal.at.QortalFunctionCode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.utils.Base58; -import org.qortal.utils.BitTwiddling; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -/** - * Cross-chain trade AT - * - *

- *

    - *
  • Bob generates Litecoin & Qortal 'trade' keys - *
      - *
    • private key required to sign P2SH redeem tx
      • - *
    • private key could be used to create 'secret' (e.g. double-SHA256)
      • - *
    • encrypted private key could be stored in Qortal AT for access by Bob from any node
      • - *
    - *
    • - *
  • Bob deploys Qortal AT - *
      - *
    - *
    • - *
  • Alice finds Qortal AT and wants to trade - *
      - *
    • Alice generates Litecoin & Qortal 'trade' keys
      • - *
    • Alice funds Litecoin P2SH-A
      • - *
    • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: - *
        - *
      • hash-of-secret-A
        • - *
      • her 'trade' Litecoin PKH
        • - *
      - *
      • - *
    - *
    • - *
  • Bob receives "offer" MESSAGE - *
      - *
    • Checks Alice's P2SH-A
      • - *
    • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: - *
        - *
      • Alice's trade Qortal address
        • - *
      • Alice's trade Litecoin PKH
        • - *
      • hash-of-secret-A
        • - *
      - *
      • - *
    - *
    • - *
  • Alice checks Qortal AT to confirm it's locked to her - *
      - *
    • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: - *
        - *
      • secret-A
        • - *
      • Qortal receiving address of her chosing
        • - *
      - *
      • - *
    • AT's QORT funds are sent to Qortal receiving address
      • - *
    - *
    • - *
  • Bob checks AT, extracts secret-A - *
      - *
    • Bob redeems P2SH-A using his Litecoin trade key and secret-A
      • - *
    • P2SH-A LTC funds end up at Litecoin address determined by redeem transaction output(s)
      • - *
    - *
    • - *
- */ -public class LitecoinACCTv1 implements ACCT { - - public static final String NAME = LitecoinACCTv1.class.getSimpleName(); - public static final byte[] CODE_BYTES_HASH = HashCode.fromString("0fb15ad9ad1867dfbcafa51155481aa15d984ff9506f2b428eca4e2a2feac2b3").asBytes(); // SHA256 of AT code bytes - - public static final int SECRET_LENGTH = 32; - - /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ - private static final int MODE_VALUE_OFFSET = 61; - /** Byte offset into AT state data where 'mode' variable (long) is stored. */ - public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); - - public static class OfferMessageData { - public byte[] partnerLitecoinPKH; - public byte[] hashOfSecretA; - public long lockTimeA; - } - public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerLitecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; - public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ - + 24 /*partner's Litecoin PKH (padded from 20 to 24)*/ - + 8 /*AT trade timeout (minutes)*/ - + 24 /*hash of secret-A (padded from 20 to 24)*/ - + 8 /*lockTimeA*/; - public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; - public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; - - private static LitecoinACCTv1 instance; - - private LitecoinACCTv1() { - } - - public static synchronized LitecoinACCTv1 getInstance() { - if (instance == null) - instance = new LitecoinACCTv1(); - - return instance; - } - - @Override - public byte[] getCodeBytesHash() { - return CODE_BYTES_HASH; - } - - @Override - public int getModeByteOffset() { - return MODE_BYTE_OFFSET; - } - - @Override - public ForeignBlockchain getBlockchain() { - return Litecoin.getInstance(); - } - - /** - * Returns Qortal AT creation bytes for cross-chain trading AT. - *

- * tradeTimeout (minutes) is the time window for the trade partner to send the - * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. - * - * @param creatorTradeAddress AT creator's trade Qortal address - * @param litecoinPublicKeyHash 20-byte HASH160 of creator's trade Litecoin public key - * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT - * @param litecoinAmount how much LTC the AT creator is expecting to trade - * @param tradeTimeout suggested timeout for entire trade - */ - public static byte[] buildQortalAT(String creatorTradeAddress, byte[] litecoinPublicKeyHash, long qortAmount, long litecoinAmount, int tradeTimeout) { - if (litecoinPublicKeyHash.length != 20) - throw new IllegalArgumentException("Litecoin public key hash should be 20 bytes"); - - // Labels for data segment addresses - int addrCounter = 0; - - // Constants (with corresponding dataByteBuffer.put*() calls below) - - final int addrCreatorTradeAddress1 = addrCounter++; - final int addrCreatorTradeAddress2 = addrCounter++; - final int addrCreatorTradeAddress3 = addrCounter++; - final int addrCreatorTradeAddress4 = addrCounter++; - - final int addrLitecoinPublicKeyHash = addrCounter; - addrCounter += 4; - - final int addrQortAmount = addrCounter++; - final int addrLitecoinAmount = addrCounter++; - final int addrTradeTimeout = addrCounter++; - - final int addrMessageTxnType = addrCounter++; - final int addrExpectedTradeMessageLength = addrCounter++; - final int addrExpectedRedeemMessageLength = addrCounter++; - - final int addrCreatorAddressPointer = addrCounter++; - final int addrQortalPartnerAddressPointer = addrCounter++; - final int addrMessageSenderPointer = addrCounter++; - - final int addrTradeMessagePartnerLitecoinPKHOffset = addrCounter++; - final int addrPartnerLitecoinPKHPointer = addrCounter++; - final int addrTradeMessageHashOfSecretAOffset = addrCounter++; - final int addrHashOfSecretAPointer = addrCounter++; - - final int addrRedeemMessageReceivingAddressOffset = addrCounter++; - - final int addrMessageDataPointer = addrCounter++; - final int addrMessageDataLength = addrCounter++; - - final int addrPartnerReceivingAddressPointer = addrCounter++; - - final int addrEndOfConstants = addrCounter; - - // Variables - - final int addrCreatorAddress1 = addrCounter++; - final int addrCreatorAddress2 = addrCounter++; - final int addrCreatorAddress3 = addrCounter++; - final int addrCreatorAddress4 = addrCounter++; - - final int addrQortalPartnerAddress1 = addrCounter++; - final int addrQortalPartnerAddress2 = addrCounter++; - final int addrQortalPartnerAddress3 = addrCounter++; - final int addrQortalPartnerAddress4 = addrCounter++; - - final int addrLockTimeA = addrCounter++; - final int addrRefundTimeout = addrCounter++; - final int addrRefundTimestamp = addrCounter++; - final int addrLastTxnTimestamp = addrCounter++; - final int addrBlockTimestamp = addrCounter++; - final int addrTxnType = addrCounter++; - final int addrResult = addrCounter++; - - final int addrMessageSender1 = addrCounter++; - final int addrMessageSender2 = addrCounter++; - final int addrMessageSender3 = addrCounter++; - final int addrMessageSender4 = addrCounter++; - - final int addrMessageLength = addrCounter++; - - final int addrMessageData = addrCounter; - addrCounter += 4; - - final int addrHashOfSecretA = addrCounter; - addrCounter += 4; - - final int addrPartnerLitecoinPKH = addrCounter; - addrCounter += 4; - - final int addrPartnerReceivingAddress = addrCounter; - addrCounter += 4; - - final int addrMode = addrCounter++; - assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); - - // Data segment - ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); - - // AT creator's trade Qortal address, decoded from Base58 - assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; - byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); - dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); - - // Litecoin public key hash - assert dataByteBuffer.position() == addrLitecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrLitecoinPublicKeyHash incorrect"; - dataByteBuffer.put(Bytes.ensureCapacity(litecoinPublicKeyHash, 32, 0)); - - // Redeem Qort amount - assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; - dataByteBuffer.putLong(qortAmount); - - // Expected Litecoin amount - assert dataByteBuffer.position() == addrLitecoinAmount * MachineState.VALUE_SIZE : "addrLitecoinAmount incorrect"; - dataByteBuffer.putLong(litecoinAmount); - - // Suggested trade timeout (minutes) - assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; - dataByteBuffer.putLong(tradeTimeout); - - // We're only interested in MESSAGE transactions - assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; - dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); - - // Expected length of 'trade' MESSAGE data from AT creator - assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; - dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); - - // Expected length of 'redeem' MESSAGE data from trade partner - assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; - dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); - - // Index into data segment of AT creator's address, used by GET_B_IND - assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; - dataByteBuffer.putLong(addrCreatorAddress1); - - // Index into data segment of partner's Qortal address, used by SET_B_IND - assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; - dataByteBuffer.putLong(addrQortalPartnerAddress1); - - // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND - assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; - dataByteBuffer.putLong(addrMessageSender1); - - // Offset into 'trade' MESSAGE data payload for extracting partner's Litecoin PKH - assert dataByteBuffer.position() == addrTradeMessagePartnerLitecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerLitecoinPKHOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Index into data segment of partner's Litecoin PKH, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerLitecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerLitecoinPKHPointer incorrect"; - dataByteBuffer.putLong(addrPartnerLitecoinPKH); - - // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A - assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; - dataByteBuffer.putLong(64L); - - // Index into data segment to hash of secret A, used by GET_B_IND - assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; - dataByteBuffer.putLong(addrHashOfSecretA); - - // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address - assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; - dataByteBuffer.putLong(32L); - - // Source location and length for hashing any passed secret - assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; - dataByteBuffer.putLong(addrMessageData); - assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; - dataByteBuffer.putLong(32L); - - // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND - assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; - dataByteBuffer.putLong(addrPartnerReceivingAddress); - - assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; - - // Code labels - Integer labelRefund = null; - - Integer labelTradeTxnLoop = null; - Integer labelCheckTradeTxn = null; - Integer labelCheckCancelTxn = null; - Integer labelNotTradeNorCancelTxn = null; - Integer labelCheckNonRefundTradeTxn = null; - Integer labelTradeTxnExtract = null; - Integer labelRedeemTxnLoop = null; - Integer labelCheckRedeemTxn = null; - Integer labelCheckRedeemTxnSender = null; - Integer labelPayout = null; - - ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); - - // Two-pass version - for (int pass = 0; pass < 2; ++pass) { - codeByteBuffer.clear(); - - try { - /* Initialization */ - - // Use AT creation 'timestamp' as starting point for finding transactions sent to AT - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); - - // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ - - /* Transaction processing loop */ - labelTradeTxnLoop = codeByteBuffer.position(); - - // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckTradeTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); - - /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); - // Message sender's address matches AT creator's trade address so go process 'trade' message - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); - - /* Checking message sender for possible cancel message */ - labelCheckCancelTxn = codeByteBuffer.position(); - - // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); - // Partner address is AT creator's address, so cancel offer and finish. - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - /* Not trade nor cancel message */ - labelNotTradeNorCancelTxn = codeByteBuffer.position(); - - // Loop to find another transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Possible switch-to-trade-mode message */ - labelCheckNonRefundTradeTxn = codeByteBuffer.position(); - - // Check 'trade' message we received has expected number of message bytes - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to info extraction code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); - // Message length didn't match - go back to finding another 'trade' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); - - /* Extracting info from 'trade' MESSAGE transaction */ - labelTradeTxnExtract = codeByteBuffer.position(); - - // Extract message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); - - // Extract trade partner's Litecoin public key hash (PKH) from message into B - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerLitecoinPKHOffset)); - // Store partner's Litecoin PKH (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerLitecoinPKHPointer)); - // Extract AT trade timeout (minutes) (from B4) - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); - - // Grab next 32 bytes - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); - - // Extract hash-of-secret-A (we only really use values from B1-B3) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); - // Extract lockTime-A (from B4) - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); - - // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); - - /* We are in 'trade mode' */ - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); - - // Set restart position to after this opcode - codeByteBuffer.put(OpCode.SET_PCS.compile()); - - /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ - - // Fetch current block 'timestamp' - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); - // If we're not past refund 'timestamp' then look for next transaction - codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - // We're past refund 'timestamp' so go refund everything back to AT creator - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); - - /* Transaction processing loop */ - labelRedeemTxnLoop = codeByteBuffer.position(); - - // Find next transaction to this AT since the last one (if any) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); - // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); - // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction - codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - - /* Check transaction */ - labelCheckRedeemTxn = codeByteBuffer.position(); - - // Update our 'last found transaction's timestamp' using 'timestamp' from transaction - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); - // Extract transaction type (message/payment) from transaction and save type in addrTxnType - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); - // If transaction type is not MESSAGE type then go look for another transaction - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check message payload length */ - codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); - // If message length matches, branch to sender checking code - codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); - // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Check transaction's sender */ - labelCheckRedeemTxnSender = codeByteBuffer.position(); - - // Extract sender address from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); - // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); - - /* Check 'secret-A' in transaction's message */ - - // Extract secret-A from first 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); - // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); - // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); - // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). - // Save the equality result (1 if they match, 0 otherwise) into addrResult. - codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); - // If hashes don't match, addrResult will be zero so go find another transaction - codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); - codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); - - /* Success! Pay arranged amount to receiving address */ - labelPayout = codeByteBuffer.position(); - - // Extract Qortal receiving address from next 32 bytes of message from transaction into B register - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); - // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); - // Pay AT's balance to receiving address - codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); - // Set redeemed mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - - // Fall-through to refunding any remaining balance back to AT creator - - /* Refund balance back to AT creator */ - labelRefund = codeByteBuffer.position(); - - // Set refunded mode - codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); - // We're finished forever (finishing auto-refunds remaining balance to AT creator) - codeByteBuffer.put(OpCode.FIN_IMD.compile()); - } catch (CompilationException e) { - throw new IllegalStateException("Unable to compile LTC-QORT ACCT?", e); - } - } - - codeByteBuffer.flip(); - - byte[] codeBytes = new byte[codeByteBuffer.limit()]; - codeByteBuffer.get(codeBytes); - - assert Arrays.equals(Crypto.digest(codeBytes), LitecoinACCTv1.CODE_BYTES_HASH) - : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); - - final short ciyamAtVersion = 2; - final short numCallStackPages = 0; - final short numUserStackPages = 0; - final long minActivationAmount = 0L; - - return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { - ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - @Override - public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); - } - - /** - * Returns CrossChainTradeData with useful info extracted from AT. - */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { - byte[] addressBytes = new byte[25]; // for general use - String atAddress = atStateData.getATAddress(); - - CrossChainTradeData tradeData = new CrossChainTradeData(); - - tradeData.foreignBlockchain = SupportedBlockchain.LITECOIN.name(); - tradeData.acctName = NAME; - - tradeData.qortalAtAddress = atAddress; - tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); - tradeData.creationTimestamp = creationTimestamp; - - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); - - byte[] stateData = atStateData.getStateData(); - ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); - dataByteBuffer.position(MachineState.HEADER_LENGTH); - - /* Constants */ - - // Skip creator's trade address - dataByteBuffer.get(addressBytes); - tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Creator's Litecoin/foreign public key hash - tradeData.creatorForeignPKH = new byte[20]; - dataByteBuffer.get(tradeData.creatorForeignPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes - - // We don't use secret-B - tradeData.hashOfSecretB = null; - - // Redeem payout - tradeData.qortAmount = dataByteBuffer.getLong(); - - // Expected LTC amount - tradeData.expectedForeignAmount = dataByteBuffer.getLong(); - - // Trade timeout - tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); - - // Skip MESSAGE transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'trade' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip expected 'redeem' message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's Qortal trade address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message sender - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for partner's Litecoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's Litecoin PKH - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'trade' message data offset for hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to hash-of-secret-A - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip 'redeem' message data offset for partner's Qortal receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to message data - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip message data length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip pointer to partner's receiving address - dataByteBuffer.position(dataByteBuffer.position() + 8); - - /* End of constants / begin variables */ - - // Skip AT creator's address - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Partner's trade address (if present) - dataByteBuffer.get(addressBytes); - String qortalRecipient = Base58.encode(addressBytes); - dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); - - // Potential lockTimeA (if in trade mode) - int lockTimeA = (int) dataByteBuffer.getLong(); - - // AT refund timeout (probably only useful for debugging) - int refundTimeout = (int) dataByteBuffer.getLong(); - - // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) - long tradeRefundTimestamp = dataByteBuffer.getLong(); - - // Skip last transaction timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip block timestamp - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip transaction type - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary result - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message sender - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Skip message length - dataByteBuffer.position(dataByteBuffer.position() + 8); - - // Skip temporary message data - dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); - - // Potential hash160 of secret A - byte[] hashOfSecretA = new byte[20]; - dataByteBuffer.get(hashOfSecretA); - dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes - - // Potential partner's Litecoin PKH - byte[] partnerLitecoinPKH = new byte[20]; - dataByteBuffer.get(partnerLitecoinPKH); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerLitecoinPKH.length); // skip to 32 bytes - - // Partner's receiving address (if present) - byte[] partnerReceivingAddress = new byte[25]; - dataByteBuffer.get(partnerReceivingAddress); - dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes - - // Trade AT's 'mode' - long modeValue = dataByteBuffer.getLong(); - AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); - - /* End of variables */ - - if (mode != null && mode != AcctMode.OFFERING) { - tradeData.mode = mode; - tradeData.refundTimeout = refundTimeout; - tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; - tradeData.qortalPartnerAddress = qortalRecipient; - tradeData.hashOfSecretA = hashOfSecretA; - tradeData.partnerForeignPKH = partnerLitecoinPKH; - tradeData.lockTimeA = lockTimeA; - - if (mode == AcctMode.REDEEMED) - tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); - } else { - tradeData.mode = AcctMode.OFFERING; - } - - tradeData.duplicateDeprecated(); - - return tradeData; - } - - /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ - public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); - } - - /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ - public static OfferMessageData extractOfferMessageData(byte[] messageData) { - if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) - return null; - - OfferMessageData offerMessageData = new OfferMessageData(); - offerMessageData.partnerLitecoinPKH = Arrays.copyOfRange(messageData, 0, 20); - offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); - offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); - - return offerMessageData; - } - - /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ - public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { - byte[] data = new byte[TRADE_MESSAGE_LENGTH]; - byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); - byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); - byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); - - System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); - System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); - System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); - System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); - System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); - - return data; - } - - /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ - @Override - public byte[] buildCancelMessage(String creatorQortalAddress) { - byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; - byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); - - System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); - - return data; - } - - /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ - public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { - byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; - byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); - - System.arraycopy(secretA, 0, data, 0, secretA.length); - System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); - - return data; - } - - /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ - public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { - // refund should be triggered halfway between offerMessageTimestamp and lockTimeA - return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); - } - - public static byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { - String atAddress = crossChainTradeData.qortalAtAddress; - String redeemerAddress = crossChainTradeData.qortalPartnerAddress; - - // We don't have partner's public key so we check every message to AT - List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); - if (messageTransactionsData == null) - return null; - - // Find 'redeem' message - for (MessageTransactionData messageTransactionData : messageTransactionsData) { - // Check message payload type/encryption - if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) - continue; - - // Check message payload size - byte[] messageData = messageTransactionData.getData(); - if (messageData.length != REDEEM_MESSAGE_LENGTH) - // Wrong payload length - continue; - - // Check sender - if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) - // Wrong sender; - continue; - - // Extract secretA - byte[] secretA = new byte[32]; - System.arraycopy(messageData, 0, secretA, 0, secretA.length); - - byte[] hashOfSecretA = Crypto.hash160(secretA); - if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) - continue; - - return secretA; - } - - return null; - } - -} diff --git a/src/main/java/org/qortal/crosschain/SimpleTransaction.java b/src/main/java/org/qortal/crosschain/SimpleTransaction.java deleted file mode 100644 index 0fae20a5..00000000 --- a/src/main/java/org/qortal/crosschain/SimpleTransaction.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.qortal.crosschain; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -@XmlAccessorType(XmlAccessType.FIELD) -public class SimpleTransaction { - private String txHash; - private Integer timestamp; - private long totalAmount; - - public SimpleTransaction() { - } - - public SimpleTransaction(String txHash, Integer timestamp, long totalAmount) { - this.txHash = txHash; - this.timestamp = timestamp; - this.totalAmount = totalAmount; - } - - public String getTxHash() { - return txHash; - } - - public Integer getTimestamp() { - return timestamp; - } - - public long getTotalAmount() { - return totalAmount; - } -} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java deleted file mode 100644 index 7b6f91f5..00000000 --- a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.qortal.crosschain; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import org.qortal.utils.ByteArray; -import org.qortal.utils.Triple; - -public enum SupportedBlockchain { - - BITCOIN(Arrays.asList( - Triple.valueOf(BitcoinACCTv1.NAME, BitcoinACCTv1.CODE_BYTES_HASH, BitcoinACCTv1::getInstance) - // Could add improved BitcoinACCTv2 here in the future - )) { - @Override - public ForeignBlockchain getInstance() { - return Bitcoin.getInstance(); - } - - @Override - public ACCT getLatestAcct() { - return BitcoinACCTv1.getInstance(); - } - }, - - LITECOIN(Arrays.asList( - Triple.valueOf(LitecoinACCTv1.NAME, LitecoinACCTv1.CODE_BYTES_HASH, LitecoinACCTv1::getInstance) - )) { - @Override - public ForeignBlockchain getInstance() { - return Litecoin.getInstance(); - } - - @Override - public ACCT getLatestAcct() { - return LitecoinACCTv1.getInstance(); - } - }; - - private static final Map> supportedAcctsByCodeHash = Arrays.stream(SupportedBlockchain.values()) - .map(supportedBlockchain -> supportedBlockchain.supportedAccts) - .flatMap(List::stream) - .collect(Collectors.toUnmodifiableMap(triple -> new ByteArray(triple.getB()), Triple::getC)); - - private static final Map> supportedAcctsByName = Arrays.stream(SupportedBlockchain.values()) - .map(supportedBlockchain -> supportedBlockchain.supportedAccts) - .flatMap(List::stream) - .collect(Collectors.toUnmodifiableMap(Triple::getA, Triple::getC)); - - private static final Map blockchainsByName = Arrays.stream(SupportedBlockchain.values()) - .collect(Collectors.toUnmodifiableMap(Enum::name, blockchain -> blockchain)); - - private final List>> supportedAccts; - - SupportedBlockchain(List>> supportedAccts) { - this.supportedAccts = supportedAccts; - } - - public abstract ForeignBlockchain getInstance(); - public abstract ACCT getLatestAcct(); - - public static Map> getAcctMap() { - return supportedAcctsByCodeHash; - } - - public static SupportedBlockchain fromString(String name) { - return blockchainsByName.get(name); - } - - public static Map> getFilteredAcctMap(SupportedBlockchain blockchain) { - if (blockchain == null) - return getAcctMap(); - - return blockchain.supportedAccts.stream() - .collect(Collectors.toUnmodifiableMap(triple -> new ByteArray(triple.getB()), Triple::getC)); - } - - public static Map> getFilteredAcctMap(String specificBlockchain) { - if (specificBlockchain == null) - return getAcctMap(); - - SupportedBlockchain blockchain = blockchainsByName.get(specificBlockchain); - if (blockchain == null) - return Collections.emptyMap(); - - return getFilteredAcctMap(blockchain); - } - - public static ACCT getAcctByCodeHash(byte[] codeHash) { - ByteArray wrappedCodeHash = new ByteArray(codeHash); - - Supplier acctInstanceSupplier = supportedAcctsByCodeHash.get(wrappedCodeHash); - - if (acctInstanceSupplier == null) - return null; - - return acctInstanceSupplier.get(); - } - - public static ACCT getAcctByName(String acctName) { - Supplier acctInstanceSupplier = supportedAcctsByName.get(acctName); - - if (acctInstanceSupplier == null) - return null; - - return acctInstanceSupplier.get(); - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/TransactionHash.java b/src/main/java/org/qortal/crosschain/TransactionHash.java deleted file mode 100644 index c002ae80..00000000 --- a/src/main/java/org/qortal/crosschain/TransactionHash.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.qortal.crosschain; - -import java.util.Comparator; - -public class TransactionHash { - - public static final Comparator CONFIRMED_FIRST = (a, b) -> Boolean.compare(a.height != 0, b.height != 0); - - public final int height; - public final String txHash; - - public TransactionHash(int height, String txHash) { - this.height = height; - this.txHash = txHash; - } - - public int getHeight() { - return this.height; - } - - public String getTxHash() { - return this.txHash; - } - - public String toString() { - return this.height == 0 - ? String.format("txHash %s (unconfirmed)", this.txHash) - : String.format("txHash %s (height %d)", this.txHash, this.height); - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/UnspentOutput.java b/src/main/java/org/qortal/crosschain/UnspentOutput.java deleted file mode 100644 index 86aa533d..00000000 --- a/src/main/java/org/qortal/crosschain/UnspentOutput.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.qortal.crosschain; - -/** Unspent output info as returned by ElectrumX network. */ -public class UnspentOutput { - public final byte[] hash; - public final int index; - public final int height; - public final long value; - - public UnspentOutput(byte[] hash, int index, int height, long value) { - this.hash = hash; - this.index = index; - this.height = height; - this.value = value; - } -} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java b/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java deleted file mode 100644 index 69250e54..00000000 --- a/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java +++ /dev/null @@ -1,109 +0,0 @@ -package org.qortal.data.crosschain; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import org.qortal.crosschain.AcctMode; - -import io.swagger.v3.oas.annotations.media.Schema; - -// All properties to be converted to JSON via JAXB -@XmlAccessorType(XmlAccessType.FIELD) -public class CrossChainTradeData { - - // Properties - - @Schema(description = "AT's Qortal address") - public String qortalAtAddress; - - @Schema(description = "AT creator's Qortal address") - public String qortalCreator; - - @Schema(description = "AT creator's ephemeral trading key-pair represented as Qortal address") - public String qortalCreatorTradeAddress; - - @Deprecated - @Schema(description = "DEPRECATED: use creatorForeignPKH instead") - public byte[] creatorBitcoinPKH; - - @Schema(description = "AT creator's foreign blockchain trade public-key-hash (PKH)") - public byte[] creatorForeignPKH; - - @Schema(description = "Timestamp when AT was created (milliseconds since epoch)") - public long creationTimestamp; - - @Schema(description = "Suggested trade timeout (minutes)", example = "10080") - public int tradeTimeout; - - @Schema(description = "AT's current QORT balance") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long qortBalance; - - @Schema(description = "HASH160 of 32-byte secret-A") - public byte[] hashOfSecretA; - - @Schema(description = "HASH160 of 32-byte secret-B") - public byte[] hashOfSecretB; - - @Schema(description = "Final QORT payment that will be sent to Qortal trade partner") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long qortAmount; - - @Schema(description = "Trade partner's Qortal address (trade begins when this is set)") - public String qortalPartnerAddress; - - @Schema(description = "Timestamp when AT switched to trade mode") - public Long tradeModeTimestamp; - - @Schema(description = "How long from AT creation until AT triggers automatic refund to AT creator (minutes)") - public Integer refundTimeout; - - @Schema(description = "Actual Qortal block height when AT will automatically refund to AT creator (after trade begins)") - public Integer tradeRefundHeight; - - @Deprecated - @Schema(description = "DEPRECATED: use expectedForeignAmount instread") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long expectedBitcoin; - - @Schema(description = "Amount, in foreign blockchain currency, that AT creator expects trade partner to pay out (excluding miner fees)") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - public long expectedForeignAmount; - - @Schema(description = "Current AT execution mode") - public AcctMode mode; - - @Schema(description = "Suggested P2SH-A nLockTime based on trade timeout") - public Integer lockTimeA; - - @Schema(description = "Suggested P2SH-B nLockTime based on trade timeout") - public Integer lockTimeB; - - @Deprecated - @Schema(description = "DEPRECATED: use partnerForeignPKH instead") - public byte[] partnerBitcoinPKH; - - @Schema(description = "Trade partner's foreign blockchain public-key-hash (PKH)") - public byte[] partnerForeignPKH; - - @Schema(description = "Trade partner's Qortal receiving address") - public String qortalPartnerReceivingAddress; - - public String foreignBlockchain; - - public String acctName; - - // Constructors - - // Necessary for JAXB - public CrossChainTradeData() { - } - - public void duplicateDeprecated() { - this.creatorBitcoinPKH = this.creatorForeignPKH; - this.expectedBitcoin = this.expectedForeignAmount; - this.partnerBitcoinPKH = this.partnerForeignPKH; - } - -} diff --git a/src/main/java/org/qortal/data/crosschain/TradeBotData.java b/src/main/java/org/qortal/data/crosschain/TradeBotData.java deleted file mode 100644 index 19481466..00000000 --- a/src/main/java/org/qortal/data/crosschain/TradeBotData.java +++ /dev/null @@ -1,268 +0,0 @@ -package org.qortal.data.crosschain; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; - -import io.swagger.v3.oas.annotations.media.Schema; -import org.json.JSONObject; - -import org.qortal.utils.Base58; - -// All properties to be converted to JSON via JAXB -@XmlAccessorType(XmlAccessType.FIELD) -public class TradeBotData { - - private byte[] tradePrivateKey; - - private String acctName; - private String tradeState; - - // Internal use - not shown via API - @XmlTransient - @Schema(hidden = true) - private int tradeStateValue; - - private String creatorAddress; - private String atAddress; - - private long timestamp; - - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long qortAmount; - - private byte[] tradeNativePublicKey; - private byte[] tradeNativePublicKeyHash; - String tradeNativeAddress; - - private byte[] secret; - private byte[] hashOfSecret; - - private String foreignBlockchain; - private byte[] tradeForeignPublicKey; - private byte[] tradeForeignPublicKeyHash; - - @Deprecated - @Schema(description = "DEPRECATED: use foreignAmount instead", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long bitcoinAmount; - - @Schema(description = "amount in foreign blockchain currency", type = "number") - @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) - private long foreignAmount; - - // Never expose this via API - @XmlTransient - @Schema(hidden = true) - private String foreignKey; - - private byte[] lastTransactionSignature; - private Integer lockTimeA; - - // Could be Bitcoin or Qortal... - private byte[] receivingAccountInfo; - - protected TradeBotData() { - /* JAXB */ - } - - public TradeBotData(byte[] tradePrivateKey, String acctName, String tradeState, int tradeStateValue, - String creatorAddress, String atAddress, - long timestamp, long qortAmount, - byte[] tradeNativePublicKey, byte[] tradeNativePublicKeyHash, String tradeNativeAddress, - byte[] secret, byte[] hashOfSecret, - String foreignBlockchain, byte[] tradeForeignPublicKey, byte[] tradeForeignPublicKeyHash, - long foreignAmount, String foreignKey, - byte[] lastTransactionSignature, Integer lockTimeA, byte[] receivingAccountInfo) { - this.tradePrivateKey = tradePrivateKey; - this.acctName = acctName; - this.tradeState = tradeState; - this.tradeStateValue = tradeStateValue; - this.creatorAddress = creatorAddress; - this.atAddress = atAddress; - this.timestamp = timestamp; - this.qortAmount = qortAmount; - this.tradeNativePublicKey = tradeNativePublicKey; - this.tradeNativePublicKeyHash = tradeNativePublicKeyHash; - this.tradeNativeAddress = tradeNativeAddress; - this.secret = secret; - this.hashOfSecret = hashOfSecret; - this.foreignBlockchain = foreignBlockchain; - this.tradeForeignPublicKey = tradeForeignPublicKey; - this.tradeForeignPublicKeyHash = tradeForeignPublicKeyHash; - // deprecated copy - this.bitcoinAmount = foreignAmount; - this.foreignAmount = foreignAmount; - this.foreignKey = foreignKey; - this.lastTransactionSignature = lastTransactionSignature; - this.lockTimeA = lockTimeA; - this.receivingAccountInfo = receivingAccountInfo; - } - - public byte[] getTradePrivateKey() { - return this.tradePrivateKey; - } - - public String getAcctName() { - return this.acctName; - } - - public String getState() { - return this.tradeState; - } - - public void setState(String state) { - this.tradeState = state; - } - - public int getStateValue() { - return this.tradeStateValue; - } - - public void setStateValue(int stateValue) { - this.tradeStateValue = stateValue; - } - - public String getCreatorAddress() { - return this.creatorAddress; - } - - public String getAtAddress() { - return this.atAddress; - } - - public void setAtAddress(String atAddress) { - this.atAddress = atAddress; - } - - public long getTimestamp() { - return this.timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public long getQortAmount() { - return this.qortAmount; - } - - public byte[] getTradeNativePublicKey() { - return this.tradeNativePublicKey; - } - - public byte[] getTradeNativePublicKeyHash() { - return this.tradeNativePublicKeyHash; - } - - public String getTradeNativeAddress() { - return this.tradeNativeAddress; - } - - public byte[] getSecret() { - return this.secret; - } - - public byte[] getHashOfSecret() { - return this.hashOfSecret; - } - - public String getForeignBlockchain() { - return this.foreignBlockchain; - } - - public byte[] getTradeForeignPublicKey() { - return this.tradeForeignPublicKey; - } - - public byte[] getTradeForeignPublicKeyHash() { - return this.tradeForeignPublicKeyHash; - } - - public long getForeignAmount() { - return this.foreignAmount; - } - - public String getForeignKey() { - return this.foreignKey; - } - - public byte[] getLastTransactionSignature() { - return this.lastTransactionSignature; - } - - public void setLastTransactionSignature(byte[] lastTransactionSignature) { - this.lastTransactionSignature = lastTransactionSignature; - } - - public Integer getLockTimeA() { - return this.lockTimeA; - } - - public void setLockTimeA(Integer lockTimeA) { - this.lockTimeA = lockTimeA; - } - - public byte[] getReceivingAccountInfo() { - return this.receivingAccountInfo; - } - - public JSONObject toJson() { - JSONObject jsonObject = new JSONObject(); - jsonObject.put("tradePrivateKey", Base58.encode(this.getTradePrivateKey())); - jsonObject.put("acctName", this.getAcctName()); - jsonObject.put("tradeState", this.getState()); - jsonObject.put("tradeStateValue", this.getStateValue()); - jsonObject.put("creatorAddress", this.getCreatorAddress()); - jsonObject.put("atAddress", this.getAtAddress()); - jsonObject.put("timestamp", this.getTimestamp()); - jsonObject.put("qortAmount", this.getQortAmount()); - if (this.getTradeNativePublicKey() != null) jsonObject.put("tradeNativePublicKey", Base58.encode(this.getTradeNativePublicKey())); - if (this.getTradeNativePublicKeyHash() != null) jsonObject.put("tradeNativePublicKeyHash", Base58.encode(this.getTradeNativePublicKeyHash())); - jsonObject.put("tradeNativeAddress", this.getTradeNativeAddress()); - if (this.getSecret() != null) jsonObject.put("secret", Base58.encode(this.getSecret())); - if (this.getHashOfSecret() != null) jsonObject.put("hashOfSecret", Base58.encode(this.getHashOfSecret())); - jsonObject.put("foreignBlockchain", this.getForeignBlockchain()); - if (this.getTradeForeignPublicKey() != null) jsonObject.put("tradeForeignPublicKey", Base58.encode(this.getTradeForeignPublicKey())); - if (this.getTradeForeignPublicKeyHash() != null) jsonObject.put("tradeForeignPublicKeyHash", Base58.encode(this.getTradeForeignPublicKeyHash())); - jsonObject.put("foreignKey", this.getForeignKey()); - jsonObject.put("foreignAmount", this.getForeignAmount()); - if (this.getLastTransactionSignature() != null) jsonObject.put("lastTransactionSignature", Base58.encode(this.getLastTransactionSignature())); - jsonObject.put("lockTimeA", this.getLockTimeA()); - if (this.getReceivingAccountInfo() != null) jsonObject.put("receivingAccountInfo", Base58.encode(this.getReceivingAccountInfo())); - return jsonObject; - } - - public static TradeBotData fromJson(JSONObject json) { - return new TradeBotData( - json.isNull("tradePrivateKey") ? null : Base58.decode(json.getString("tradePrivateKey")), - json.isNull("acctName") ? null : json.getString("acctName"), - json.isNull("tradeState") ? null : json.getString("tradeState"), - json.isNull("tradeStateValue") ? null : json.getInt("tradeStateValue"), - json.isNull("creatorAddress") ? null : json.getString("creatorAddress"), - json.isNull("atAddress") ? null : json.getString("atAddress"), - json.isNull("timestamp") ? null : json.getLong("timestamp"), - json.isNull("qortAmount") ? null : json.getLong("qortAmount"), - json.isNull("tradeNativePublicKey") ? null : Base58.decode(json.getString("tradeNativePublicKey")), - json.isNull("tradeNativePublicKeyHash") ? null : Base58.decode(json.getString("tradeNativePublicKeyHash")), - json.isNull("tradeNativeAddress") ? null : json.getString("tradeNativeAddress"), - json.isNull("secret") ? null : Base58.decode(json.getString("secret")), - json.isNull("hashOfSecret") ? null : Base58.decode(json.getString("hashOfSecret")), - json.isNull("foreignBlockchain") ? null : json.getString("foreignBlockchain"), - json.isNull("tradeForeignPublicKey") ? null : Base58.decode(json.getString("tradeForeignPublicKey")), - json.isNull("tradeForeignPublicKeyHash") ? null : Base58.decode(json.getString("tradeForeignPublicKeyHash")), - json.isNull("foreignAmount") ? null : json.getLong("foreignAmount"), - json.isNull("foreignKey") ? null : json.getString("foreignKey"), - json.isNull("lastTransactionSignature") ? null : Base58.decode(json.getString("lastTransactionSignature")), - json.isNull("lockTimeA") ? null : json.getInt("lockTimeA"), - json.isNull("receivingAccountInfo") ? null : Base58.decode(json.getString("receivingAccountInfo")) - ); - } - - // Mostly for debugging - public String toString() { - return String.format("%s: %s (%d)", this.atAddress, this.tradeState, this.tradeStateValue); - } - -} diff --git a/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java b/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java deleted file mode 100644 index 001bd5b4..00000000 --- a/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.qortal.data.transaction; - -import javax.xml.bind.Unmarshaller; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - -import org.qortal.transaction.PresenceTransaction.PresenceType; -import org.qortal.transaction.Transaction.TransactionType; - -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.media.Schema.AccessMode; - -// All properties to be converted to JSON via JAXB -@XmlAccessorType(XmlAccessType.FIELD) -@Schema(allOf = { TransactionData.class }) -public class PresenceTransactionData extends TransactionData { - - // Properties - @Schema(description = "sender's public key", example = "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP") - private byte[] senderPublicKey; - - @Schema(accessMode = AccessMode.READ_ONLY) - private int nonce; - - private PresenceType presenceType; - - @Schema(description = "timestamp signature", example = "2yGEbwRFyhPZZckKA") - private byte[] timestampSignature; - - // Constructors - - // For JAXB - protected PresenceTransactionData() { - super(TransactionType.PRESENCE); - } - - public void afterUnmarshal(Unmarshaller u, Object parent) { - this.creatorPublicKey = this.senderPublicKey; - } - - public PresenceTransactionData(BaseTransactionData baseTransactionData, - int nonce, PresenceType presenceType, byte[] timestampSignature) { - super(TransactionType.PRESENCE, baseTransactionData); - - this.senderPublicKey = baseTransactionData.creatorPublicKey; - this.nonce = nonce; - this.presenceType = presenceType; - this.timestampSignature = timestampSignature; - } - - // Getters/Setters - - public byte[] getSenderPublicKey() { - return this.senderPublicKey; - } - - public int getNonce() { - return this.nonce; - } - - public void setNonce(int nonce) { - this.nonce = nonce; - } - - public PresenceType getPresenceType() { - return this.presenceType; - } - - public byte[] getTimestampSignature() { - return this.timestampSignature; - } - -} diff --git a/src/main/java/org/qortal/data/transaction/TransactionData.java b/src/main/java/org/qortal/data/transaction/TransactionData.java index 060901f2..397693b8 100644 --- a/src/main/java/org/qortal/data/transaction/TransactionData.java +++ b/src/main/java/org/qortal/data/transaction/TransactionData.java @@ -40,7 +40,7 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode; GroupApprovalTransactionData.class, SetGroupTransactionData.class, UpdateAssetTransactionData.class, AccountFlagsTransactionData.class, RewardShareTransactionData.class, - AccountLevelTransactionData.class, ChatTransactionData.class, PresenceTransactionData.class + AccountLevelTransactionData.class, ChatTransactionData.class }) //All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) diff --git a/src/main/java/org/qortal/repository/CrossChainRepository.java b/src/main/java/org/qortal/repository/CrossChainRepository.java deleted file mode 100644 index 70ebdbf9..00000000 --- a/src/main/java/org/qortal/repository/CrossChainRepository.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.qortal.repository; - -import java.util.List; - -import org.qortal.data.crosschain.TradeBotData; - -public interface CrossChainRepository { - - public TradeBotData getTradeBotData(byte[] tradePrivateKey) throws DataException; - - /** Returns true if there is an existing trade-bot entry relating to given AT address, excluding trade-bot entries with given states. */ - public boolean existsTradeWithAtExcludingStates(String atAddress, List excludeStates) throws DataException; - - public List getAllTradeBotData() throws DataException; - - public void save(TradeBotData tradeBotData) throws DataException; - - /** Delete trade-bot states using passed private key. */ - public int delete(byte[] tradePrivateKey) throws DataException; - -} diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index 656e6e1e..9cdfe26c 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -14,8 +14,6 @@ public interface Repository extends AutoCloseable { public ChatRepository getChatRepository(); - public CrossChainRepository getCrossChainRepository(); - public GroupRepository getGroupRepository(); public MessageRepository getMessageRepository(); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java deleted file mode 100644 index 29f2994c..00000000 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java +++ /dev/null @@ -1,202 +0,0 @@ -package org.qortal.repository.hsqldb; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.qortal.data.crosschain.TradeBotData; -import org.qortal.repository.CrossChainRepository; -import org.qortal.repository.DataException; - -public class HSQLDBCrossChainRepository implements CrossChainRepository { - - protected HSQLDBRepository repository; - - public HSQLDBCrossChainRepository(HSQLDBRepository repository) { - this.repository = repository; - } - - @Override - public TradeBotData getTradeBotData(byte[] tradePrivateKey) throws DataException { - String sql = "SELECT acct_name, trade_state, trade_state_value, " - + "creator_address, at_address, " - + "updated_when, qort_amount, " - + "trade_native_public_key, trade_native_public_key_hash, " - + "trade_native_address, secret, hash_of_secret, " - + "foreign_blockchain, trade_foreign_public_key, trade_foreign_public_key_hash, " - + "foreign_amount, foreign_key, last_transaction_signature, locktime_a, receiving_account_info " - + "FROM TradeBotStates " - + "WHERE trade_private_key = ?"; - - try (ResultSet resultSet = this.repository.checkedExecute(sql, tradePrivateKey)) { - if (resultSet == null) - return null; - - String acctName = resultSet.getString(1); - String tradeState = resultSet.getString(2); - int tradeStateValue = resultSet.getInt(3); - String creatorAddress = resultSet.getString(4); - String atAddress = resultSet.getString(5); - long timestamp = resultSet.getLong(6); - long qortAmount = resultSet.getLong(7); - byte[] tradeNativePublicKey = resultSet.getBytes(8); - byte[] tradeNativePublicKeyHash = resultSet.getBytes(9); - String tradeNativeAddress = resultSet.getString(10); - byte[] secret = resultSet.getBytes(11); - byte[] hashOfSecret = resultSet.getBytes(12); - String foreignBlockchain = resultSet.getString(13); - byte[] tradeForeignPublicKey = resultSet.getBytes(14); - byte[] tradeForeignPublicKeyHash = resultSet.getBytes(15); - long foreignAmount = resultSet.getLong(16); - String foreignKey = resultSet.getString(17); - byte[] lastTransactionSignature = resultSet.getBytes(18); - Integer lockTimeA = resultSet.getInt(19); - if (lockTimeA == 0 && resultSet.wasNull()) - lockTimeA = null; - byte[] receivingAccountInfo = resultSet.getBytes(20); - - return new TradeBotData(tradePrivateKey, acctName, - tradeState, tradeStateValue, - creatorAddress, atAddress, timestamp, qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - secret, hashOfSecret, - foreignBlockchain, tradeForeignPublicKey, tradeForeignPublicKeyHash, - foreignAmount, foreignKey, lastTransactionSignature, lockTimeA, receivingAccountInfo); - } catch (SQLException e) { - throw new DataException("Unable to fetch trade-bot trading state from repository", e); - } - } - - @Override - public boolean existsTradeWithAtExcludingStates(String atAddress, List excludeStates) throws DataException { - if (excludeStates == null) - excludeStates = Collections.emptyList(); - - StringBuilder whereClause = new StringBuilder(256); - whereClause.append("at_address = ?"); - - Object[] bindParams = new Object[1 + excludeStates.size()]; - bindParams[0] = atAddress; - - if (!excludeStates.isEmpty()) { - whereClause.append(" AND trade_state NOT IN (?"); - bindParams[1] = excludeStates.get(0); - - for (int i = 1; i < excludeStates.size(); ++i) { - whereClause.append(", ?"); - bindParams[1 + i] = excludeStates.get(i); - } - - whereClause.append(")"); - } - - try { - return this.repository.exists("TradeBotStates", whereClause.toString(), bindParams); - } catch (SQLException e) { - throw new DataException("Unable to check for trade-bot state in repository", e); - } - } - - @Override - public List getAllTradeBotData() throws DataException { - String sql = "SELECT trade_private_key, acct_name, trade_state, trade_state_value, " - + "creator_address, at_address, " - + "updated_when, qort_amount, " - + "trade_native_public_key, trade_native_public_key_hash, " - + "trade_native_address, secret, hash_of_secret, " - + "foreign_blockchain, trade_foreign_public_key, trade_foreign_public_key_hash, " - + "foreign_amount, foreign_key, last_transaction_signature, locktime_a, receiving_account_info " - + "FROM TradeBotStates"; - - List allTradeBotData = new ArrayList<>(); - - try (ResultSet resultSet = this.repository.checkedExecute(sql)) { - if (resultSet == null) - return allTradeBotData; - - do { - byte[] tradePrivateKey = resultSet.getBytes(1); - String acctName = resultSet.getString(2); - String tradeState = resultSet.getString(3); - int tradeStateValue = resultSet.getInt(4); - String creatorAddress = resultSet.getString(5); - String atAddress = resultSet.getString(6); - long timestamp = resultSet.getLong(7); - long qortAmount = resultSet.getLong(8); - byte[] tradeNativePublicKey = resultSet.getBytes(9); - byte[] tradeNativePublicKeyHash = resultSet.getBytes(10); - String tradeNativeAddress = resultSet.getString(11); - byte[] secret = resultSet.getBytes(12); - byte[] hashOfSecret = resultSet.getBytes(13); - String foreignBlockchain = resultSet.getString(14); - byte[] tradeForeignPublicKey = resultSet.getBytes(15); - byte[] tradeForeignPublicKeyHash = resultSet.getBytes(16); - long foreignAmount = resultSet.getLong(17); - String foreignKey = resultSet.getString(18); - byte[] lastTransactionSignature = resultSet.getBytes(19); - Integer lockTimeA = resultSet.getInt(20); - if (lockTimeA == 0 && resultSet.wasNull()) - lockTimeA = null; - byte[] receivingAccountInfo = resultSet.getBytes(21); - - TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, acctName, - tradeState, tradeStateValue, - creatorAddress, atAddress, timestamp, qortAmount, - tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, - secret, hashOfSecret, - foreignBlockchain, tradeForeignPublicKey, tradeForeignPublicKeyHash, - foreignAmount, foreignKey, lastTransactionSignature, lockTimeA, receivingAccountInfo); - allTradeBotData.add(tradeBotData); - } while (resultSet.next()); - - return allTradeBotData; - } catch (SQLException e) { - throw new DataException("Unable to fetch trade-bot trading states from repository", e); - } - } - - @Override - public void save(TradeBotData tradeBotData) throws DataException { - HSQLDBSaver saveHelper = new HSQLDBSaver("TradeBotStates"); - - saveHelper.bind("trade_private_key", tradeBotData.getTradePrivateKey()) - .bind("acct_name", tradeBotData.getAcctName()) - .bind("trade_state", tradeBotData.getState()) - .bind("trade_state_value", tradeBotData.getStateValue()) - .bind("creator_address", tradeBotData.getCreatorAddress()) - .bind("at_address", tradeBotData.getAtAddress()) - .bind("updated_when", tradeBotData.getTimestamp()) - .bind("qort_amount", tradeBotData.getQortAmount()) - .bind("trade_native_public_key", tradeBotData.getTradeNativePublicKey()) - .bind("trade_native_public_key_hash", tradeBotData.getTradeNativePublicKeyHash()) - .bind("trade_native_address", tradeBotData.getTradeNativeAddress()) - .bind("secret", tradeBotData.getSecret()) - .bind("hash_of_secret", tradeBotData.getHashOfSecret()) - .bind("foreign_blockchain", tradeBotData.getForeignBlockchain()) - .bind("trade_foreign_public_key", tradeBotData.getTradeForeignPublicKey()) - .bind("trade_foreign_public_key_hash", tradeBotData.getTradeForeignPublicKeyHash()) - .bind("foreign_amount", tradeBotData.getForeignAmount()) - .bind("foreign_key", tradeBotData.getForeignKey()) - .bind("last_transaction_signature", tradeBotData.getLastTransactionSignature()) - .bind("locktime_a", tradeBotData.getLockTimeA()) - .bind("receiving_account_info", tradeBotData.getReceivingAccountInfo()); - - try { - saveHelper.execute(this.repository); - } catch (SQLException e) { - throw new DataException("Unable to save trade bot data into repository", e); - } - } - - @Override - public int delete(byte[] tradePrivateKey) throws DataException { - try { - return this.repository.delete("TradeBotStates", "trade_private_key = ?", tradePrivateKey); - } catch (SQLException e) { - throw new DataException("Unable to delete trade-bot states from repository", e); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index b82f55c3..1dbac289 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -9,7 +9,6 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.controller.tradebot.BitcoinACCTv1TradeBot; public class HSQLDBDatabaseUpdates { @@ -620,17 +619,6 @@ public class HSQLDBDatabaseUpdates { break; case 20: - // Trade bot - // See case 25 below for changes - stmt.execute("CREATE TABLE TradeBotStates (trade_private_key QortalKeySeed NOT NULL, trade_state TINYINT NOT NULL, " - + "creator_address QortalAddress NOT NULL, at_address QortalAddress, updated_when BIGINT NOT NULL, qort_amount QortalAmount NOT NULL, " - + "trade_native_public_key QortalPublicKey NOT NULL, trade_native_public_key_hash VARBINARY(32) NOT NULL, " - + "trade_native_address QortalAddress NOT NULL, secret VARBINARY(32) NOT NULL, hash_of_secret VARBINARY(32) NOT NULL, " - + "trade_foreign_public_key VARBINARY(33) NOT NULL, trade_foreign_public_key_hash VARBINARY(32) NOT NULL, " - + "bitcoin_amount BIGINT NOT NULL, xprv58 VARCHAR(200), last_transaction_signature Signature, locktime_a BIGINT, " - + "receiving_account_info VARBINARY(32) NOT NULL, PRIMARY KEY (trade_private_key))"); - break; - case 21: // AT functionality index stmt.execute("CREATE INDEX IF NOT EXISTS ATCodeHashIndex ON ATs (code_hash, is_finished)"); @@ -712,14 +700,6 @@ public class HSQLDBDatabaseUpdates { } } - try (ResultSet resultSet = stmt.executeQuery("SELECT COUNT(*) FROM TradeBotStates")) { - int rowCount = resultSet.next() ? resultSet.getInt(1) : 0; - if (rowCount > 0) { - stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'"); - LOGGER.info("Exported sensitive/node-local trade-bot states into TradeBotStates.script"); - } - } - LOGGER.info("If following reshape takes too long, use bootstrap and import node-local data using API's POST /admin/repository/data"); } @@ -784,37 +764,6 @@ public class HSQLDBDatabaseUpdates { break; case 32: - // Multiple blockchains, ACCTs and trade-bots - stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN acct_name VARCHAR(40) BEFORE trade_state"); - stmt.execute("UPDATE TradeBotStates SET acct_name = 'BitcoinACCTv1' WHERE acct_name IS NULL"); - stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN acct_name SET NOT NULL"); - - stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN trade_state RENAME TO trade_state_value"); - - stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN trade_state VARCHAR(40) BEFORE trade_state_value"); - // Any existing values will be BitcoinACCTv1 - StringBuilder updateTradeBotStatesSql = new StringBuilder(1024); - updateTradeBotStatesSql.append("UPDATE TradeBotStates SET (trade_state) = (") - .append("SELECT state_name FROM (VALUES ") - .append( - Arrays.stream(BitcoinACCTv1TradeBot.State.values()) - .map(state -> String.format("(%d, '%s')", state.value, state.name())) - .collect(Collectors.joining(", "))) - .append(") AS BitcoinACCTv1States (state_value, state_name) ") - .append("WHERE state_value = trade_state_value)"); - stmt.execute(updateTradeBotStatesSql.toString()); - stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN trade_state SET NOT NULL"); - - stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN foreign_blockchain VARCHAR(40) BEFORE trade_foreign_public_key"); - - stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN bitcoin_amount RENAME TO foreign_amount"); - - stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN xprv58 RENAME TO foreign_key"); - - stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN secret SET NULL"); - stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN hash_of_secret SET NULL"); - break; - case 33: // PRESENCE transactions stmt.execute("CREATE TABLE IF NOT EXISTS PresenceTransactions (" diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 09c6a6d4..02de9f5f 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -28,7 +28,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.PrivateKeyAccount; import org.qortal.crypto.Crypto; -import org.qortal.data.crosschain.TradeBotData; import org.qortal.globalization.Translator; import org.qortal.gui.SysTray; import org.qortal.repository.ATRepository; @@ -37,7 +36,6 @@ import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.AssetRepository; import org.qortal.repository.BlockRepository; import org.qortal.repository.ChatRepository; -import org.qortal.repository.CrossChainRepository; import org.qortal.repository.DataException; import org.qortal.repository.GroupRepository; import org.qortal.repository.MessageRepository; @@ -76,7 +74,6 @@ public class HSQLDBRepository implements Repository { private final AssetRepository assetRepository = new HSQLDBAssetRepository(this); private final BlockRepository blockRepository = new HSQLDBBlockRepository(this); private final ChatRepository chatRepository = new HSQLDBChatRepository(this); - private final CrossChainRepository crossChainRepository = new HSQLDBCrossChainRepository(this); private final GroupRepository groupRepository = new HSQLDBGroupRepository(this); private final MessageRepository messageRepository = new HSQLDBMessageRepository(this); private final NameRepository nameRepository = new HSQLDBNameRepository(this); @@ -147,11 +144,6 @@ public class HSQLDBRepository implements Repository { return this.chatRepository; } - @Override - public CrossChainRepository getCrossChainRepository() { - return this.crossChainRepository; - } - @Override public GroupRepository getGroupRepository() { return this.groupRepository; @@ -458,68 +450,12 @@ public class HSQLDBRepository implements Repository { @Override public void exportNodeLocalData() throws DataException { - // Create the qortal-backup folder if it doesn't exist - Path backupPath = Paths.get("qortal-backup"); - try { - Files.createDirectories(backupPath); - } catch (IOException e) { - LOGGER.info("Unable to create backup folder"); - throw new DataException("Unable to create backup folder"); - } - - try { - // Load trade bot data - List allTradeBotData = this.getCrossChainRepository().getAllTradeBotData(); - JSONArray allTradeBotDataJson = new JSONArray(); - for (TradeBotData tradeBotData : allTradeBotData) { - JSONObject tradeBotDataJson = tradeBotData.toJson(); - allTradeBotDataJson.put(tradeBotDataJson); - } - - // We need to combine existing TradeBotStates data before overwriting - String fileName = "qortal-backup/TradeBotStates.json"; - File tradeBotStatesBackupFile = new File(fileName); - if (tradeBotStatesBackupFile.exists()) { - String jsonString = new String(Files.readAllBytes(Paths.get(fileName))); - JSONArray allExistingTradeBotData = new JSONArray(jsonString); - Iterator iterator = allExistingTradeBotData.iterator(); - while(iterator.hasNext()) { - JSONObject existingTradeBotData = (JSONObject)iterator.next(); - String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey"); - // Check if we already have an entry for this trade - boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey)); - if (found == false) - // We need to add this to our list - allTradeBotDataJson.put(existingTradeBotData); - } - } - - FileWriter writer = new FileWriter(fileName); - writer.write(allTradeBotDataJson.toString()); - writer.close(); - LOGGER.info("Exported sensitive/node-local data: trade bot states"); - - } catch (DataException | IOException e) { - throw new DataException("Unable to export trade bot states from repository"); - } + // TODO } @Override public void importDataFromFile(String filename) throws DataException { - LOGGER.info(() -> String.format("Importing data into repository from %s", filename)); - try { - String jsonString = new String(Files.readAllBytes(Paths.get(filename))); - JSONArray tradeBotDataToImport = new JSONArray(jsonString); - Iterator iterator = tradeBotDataToImport.iterator(); - while(iterator.hasNext()) { - JSONObject tradeBotDataJson = (JSONObject)iterator.next(); - TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson); - this.getCrossChainRepository().save(tradeBotData); - } - } catch (IOException e) { - throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage()); - } - LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename)); + // TODO } @Override @@ -1056,4 +992,4 @@ public class HSQLDBRepository implements Repository { return DEADLOCK_ERROR_CODE.equals(e.getErrorCode()); } -} \ No newline at end of file +} diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java deleted file mode 100644 index 309ffcad..00000000 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.qortal.repository.hsqldb.transaction; - -import java.sql.ResultSet; -import java.sql.SQLException; - -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.PresenceTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.repository.DataException; -import org.qortal.repository.hsqldb.HSQLDBRepository; -import org.qortal.repository.hsqldb.HSQLDBSaver; -import org.qortal.transaction.PresenceTransaction.PresenceType; - -public class HSQLDBPresenceTransactionRepository extends HSQLDBTransactionRepository { - - public HSQLDBPresenceTransactionRepository(HSQLDBRepository repository) { - this.repository = repository; - } - - TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException { - String sql = "SELECT nonce, presence_type, timestamp_signature FROM PresenceTransactions WHERE signature = ?"; - - try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) { - if (resultSet == null) - return null; - - int nonce = resultSet.getInt(1); - int presenceTypeValue = resultSet.getInt(2); - PresenceType presenceType = PresenceType.valueOf(presenceTypeValue); - - byte[] timestampSignature = resultSet.getBytes(3); - - return new PresenceTransactionData(baseTransactionData, nonce, presenceType, timestampSignature); - } catch (SQLException e) { - throw new DataException("Unable to fetch presence transaction from repository", e); - } - } - - @Override - public void save(TransactionData transactionData) throws DataException { - PresenceTransactionData presenceTransactionData = (PresenceTransactionData) transactionData; - - HSQLDBSaver saveHelper = new HSQLDBSaver("PresenceTransactions"); - - saveHelper.bind("signature", presenceTransactionData.getSignature()) - .bind("nonce", presenceTransactionData.getNonce()) - .bind("presence_type", presenceTransactionData.getPresenceType().value) - .bind("timestamp_signature", presenceTransactionData.getTimestampSignature()); - - try { - saveHelper.execute(this.repository); - } catch (SQLException e) { - throw new DataException("Unable to save chat transaction into repository", e); - } - } - -} diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 4743c9f4..f17324df 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -22,8 +22,6 @@ import org.eclipse.persistence.exceptions.XMLMarshalException; import org.eclipse.persistence.jaxb.JAXBContextFactory; import org.eclipse.persistence.jaxb.UnmarshallerProperties; import org.qortal.block.BlockChain; -import org.qortal.crosschain.Bitcoin.BitcoinNet; -import org.qortal.crosschain.Litecoin.LitecoinNet; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @@ -146,8 +144,6 @@ public class Settings { // Which blockchains this node is running private String blockchainConfig = null; // use default from resources - private BitcoinNet bitcoinNet = BitcoinNet.MAIN; - private LitecoinNet litecoinNet = LitecoinNet.MAIN; // Also crosschain-related: /** Whether to show SysTray pop-up notifications when trade-bot entries change state */ private boolean tradebotSystrayEnabled = false; @@ -440,14 +436,6 @@ public class Settings { return this.blockchainConfig; } - public BitcoinNet getBitcoinNet() { - return this.bitcoinNet; - } - - public LitecoinNet getLitecoinNet() { - return this.litecoinNet; - } - public boolean isTradebotSystrayEnabled() { return this.tradebotSystrayEnabled; } diff --git a/src/main/java/org/qortal/transaction/PresenceTransaction.java b/src/main/java/org/qortal/transaction/PresenceTransaction.java deleted file mode 100644 index 729270e0..00000000 --- a/src/main/java/org/qortal/transaction/PresenceTransaction.java +++ /dev/null @@ -1,256 +0,0 @@ -package org.qortal.transaction; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.qortal.account.Account; -import org.qortal.controller.Controller; -import org.qortal.crosschain.ACCT; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.crypto.Crypto; -import org.qortal.crypto.MemoryPoW; -import org.qortal.data.at.ATData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.PresenceTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.PresenceTransactionTransformer; -import org.qortal.transform.transaction.TransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.ByteArray; - -import com.google.common.primitives.Longs; - -public class PresenceTransaction extends Transaction { - - private static final Logger LOGGER = LogManager.getLogger(PresenceTransaction.class); - - // Properties - private PresenceTransactionData presenceTransactionData; - - // Other useful constants - public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes - public static final int POW_DIFFICULTY = 8; // leading zero bits - - public enum PresenceType { - REWARD_SHARE(0) { - @Override - public long getLifetime() { - return Controller.ONLINE_TIMESTAMP_MODULUS; - } - }, - TRADE_BOT(1) { - @Override - public long getLifetime() { - return 30 * 60 * 1000L; // 30 minutes in milliseconds - } - }; - - public final int value; - private static final Map map = stream(PresenceType.values()).collect(toMap(type -> type.value, type -> type)); - - PresenceType(int value) { - this.value = value; - } - - public abstract long getLifetime(); - - public static PresenceType valueOf(int value) { - return map.get(value); - } - - /** Returns PresenceType with matching name or null (instead of throwing IllegalArgumentException). */ - public static PresenceType fromString(String name) { - try { - return PresenceType.valueOf(name); - } catch (IllegalArgumentException e) { - return null; - } - } - } - - // Constructors - - public PresenceTransaction(Repository repository, TransactionData transactionData) { - super(repository, transactionData); - - this.presenceTransactionData = (PresenceTransactionData) this.transactionData; - } - - // More information - - @Override - public long getDeadline() { - return this.transactionData.getTimestamp() + this.presenceTransactionData.getPresenceType().getLifetime(); - } - - @Override - public List getRecipientAddresses() throws DataException { - return Collections.emptyList(); - } - - // Navigation - - public Account getSender() { - return this.getCreator(); - } - - // Processing - - public void computeNonce() throws DataException { - byte[] transactionBytes; - - try { - transactionBytes = TransactionTransformer.toBytesForSigning(this.transactionData); - } catch (TransformationException e) { - throw new RuntimeException("Unable to transform transaction to byte array for verification", e); - } - - // Clear nonce from transactionBytes - PresenceTransactionTransformer.clearNonce(transactionBytes); - - // Calculate nonce - this.presenceTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, POW_DIFFICULTY)); - } - - /** - * Returns whether PRESENCE transaction has valid txGroupId. - *

- * We insist on NO_GROUP. - */ - @Override - protected boolean isValidTxGroupId() throws DataException { - int txGroupId = this.transactionData.getTxGroupId(); - - return txGroupId == Group.NO_GROUP; - } - - @Override - public ValidationResult isFeeValid() throws DataException { - if (this.transactionData.getFee() < 0) - return ValidationResult.NEGATIVE_FEE; - - return ValidationResult.OK; - } - - @Override - public boolean hasValidReference() throws DataException { - return true; - } - - @Override - public ValidationResult isValid() throws DataException { - // Nonce checking is done via isSignatureValid() as that method is only called once per import - - // If we exist in the repository then we've been imported as unconfirmed, - // but we don't want to make it into a block, so return fake non-OK result. - if (this.repository.getTransactionRepository().exists(this.presenceTransactionData.getSignature())) - return ValidationResult.INVALID_BUT_OK; - - // We only support TRADE_BOT-type PRESENCE at this time - if (PresenceType.TRADE_BOT != this.presenceTransactionData.getPresenceType()) - return ValidationResult.NOT_YET_RELEASED; - - // Check timestamp signature - byte[] timestampSignature = this.presenceTransactionData.getTimestampSignature(); - byte[] timestampBytes = Longs.toByteArray(this.presenceTransactionData.getTimestamp()); - if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), timestampSignature, timestampBytes)) - return ValidationResult.INVALID_TIMESTAMP_SIGNATURE; - - Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); - Set codeHashes = acctSuppliersByCodeHash.keySet(); - boolean isExecutable = true; - - List atsData = repository.getATRepository().getAllATsByFunctionality(codeHashes, isExecutable); - - // Convert signer's public key to address form - String signerAddress = Crypto.toAddress(this.transactionData.getCreatorPublicKey()); - - for (ATData atData : atsData) { - ByteArray atCodeHash = new ByteArray(atData.getCodeHash()); - Supplier acctSupplier = acctSuppliersByCodeHash.get(atCodeHash); - if (acctSupplier == null) - continue; - - CrossChainTradeData crossChainTradeData = acctSupplier.get().populateTradeData(repository, atData); - - // OK if signer's public key (in address form) matches Bob's trade public key (in address form) - if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress)) - return ValidationResult.OK; - - // OK if signer's public key (in address form) matches Alice's trade public key (in address form) - if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress)) - return ValidationResult.OK; - } - - return ValidationResult.AT_UNKNOWN; - } - - @Override - public boolean isSignatureValid() { - byte[] signature = this.transactionData.getSignature(); - if (signature == null) - return false; - - byte[] transactionBytes; - - try { - transactionBytes = PresenceTransactionTransformer.toBytesForSigning(this.transactionData); - } catch (TransformationException e) { - throw new RuntimeException("Unable to transform transaction to byte array for verification", e); - } - - if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), signature, transactionBytes)) - return false; - - int nonce = this.presenceTransactionData.getNonce(); - - // Clear nonce from transactionBytes - PresenceTransactionTransformer.clearNonce(transactionBytes); - - // Check nonce - return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, POW_DIFFICULTY, nonce); - } - - /** - * Remove any PRESENCE transactions by the same signer that have older timestamps. - */ - @Override - protected void onImportAsUnconfirmed() throws DataException { - byte[] creatorPublicKey = this.transactionData.getCreatorPublicKey(); - List creatorsPresenceTransactions = this.repository.getTransactionRepository().getUnconfirmedTransactions(TransactionType.PRESENCE, creatorPublicKey); - - if (creatorsPresenceTransactions.isEmpty()) - return; - - for (TransactionData transactionData : creatorsPresenceTransactions) { - if (transactionData.getTimestamp() >= this.transactionData.getTimestamp()) - continue; - - LOGGER.debug(() -> String.format("Deleting older PRESENCE transaction %s", Base58.encode(transactionData.getSignature()))); - this.repository.getTransactionRepository().delete(transactionData); - } - } - - @Override - public void process() throws DataException { - throw new DataException("PRESENCE transactions should never be processed"); - } - - @Override - public void orphan() throws DataException { - throw new DataException("PRESENCE transactions should never be orphaned"); - } - -} diff --git a/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java deleted file mode 100644 index bf69d102..00000000 --- a/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java +++ /dev/null @@ -1,108 +0,0 @@ -package org.qortal.transform.transaction; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.PresenceTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.transaction.PresenceTransaction.PresenceType; -import org.qortal.transaction.Transaction.TransactionType; -import org.qortal.transform.TransformationException; -import org.qortal.utils.Serialization; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; - -public class PresenceTransactionTransformer extends TransactionTransformer { - - // Property lengths - private static final int NONCE_LENGTH = INT_LENGTH; - private static final int PRESENCE_TYPE_LENGTH = BYTE_LENGTH; - private static final int TIMESTAMP_SIGNATURE_LENGTH = SIGNATURE_LENGTH; - - private static final int EXTRAS_LENGTH = NONCE_LENGTH + PRESENCE_TYPE_LENGTH + TIMESTAMP_SIGNATURE_LENGTH; - - protected static final TransactionLayout layout; - - static { - layout = new TransactionLayout(); - layout.add("txType: " + TransactionType.PRESENCE.valueString, TransformationType.INT); - layout.add("timestamp", TransformationType.TIMESTAMP); - layout.add("transaction's groupID", TransformationType.INT); - layout.add("reference", TransformationType.SIGNATURE); - layout.add("sender's public key", TransformationType.PUBLIC_KEY); - layout.add("proof-of-work nonce", TransformationType.INT); - layout.add("presence type (reward-share=0, trade-bot=1)", TransformationType.BYTE); - layout.add("timestamp-signature", TransformationType.SIGNATURE); - layout.add("fee", TransformationType.AMOUNT); - layout.add("signature", TransformationType.SIGNATURE); - } - - public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { - long timestamp = byteBuffer.getLong(); - - int txGroupId = byteBuffer.getInt(); - - byte[] reference = new byte[REFERENCE_LENGTH]; - byteBuffer.get(reference); - - byte[] senderPublicKey = Serialization.deserializePublicKey(byteBuffer); - - int nonce = byteBuffer.getInt(); - - PresenceType presenceType = PresenceType.valueOf(byteBuffer.get()); - - byte[] timestampSignature = new byte[SIGNATURE_LENGTH]; - byteBuffer.get(timestampSignature); - - long fee = byteBuffer.getLong(); - - byte[] signature = new byte[SIGNATURE_LENGTH]; - byteBuffer.get(signature); - - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature); - - return new PresenceTransactionData(baseTransactionData, nonce, presenceType, timestampSignature); - } - - public static int getDataLength(TransactionData transactionData) { - return getBaseLength(transactionData) + EXTRAS_LENGTH; - } - - public static byte[] toBytes(TransactionData transactionData) throws TransformationException { - try { - PresenceTransactionData presenceTransactionData = (PresenceTransactionData) transactionData; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - transformCommonBytes(transactionData, bytes); - - bytes.write(Ints.toByteArray(presenceTransactionData.getNonce())); - - bytes.write(presenceTransactionData.getPresenceType().value); - - bytes.write(presenceTransactionData.getTimestampSignature()); - - bytes.write(Longs.toByteArray(presenceTransactionData.getFee())); - - if (presenceTransactionData.getSignature() != null) - bytes.write(presenceTransactionData.getSignature()); - - return bytes.toByteArray(); - } catch (IOException | ClassCastException e) { - throw new TransformationException(e); - } - } - - public static void clearNonce(byte[] transactionBytes) { - int nonceIndex = TYPE_LENGTH + TIMESTAMP_LENGTH + GROUPID_LENGTH + REFERENCE_LENGTH + PUBLIC_KEY_LENGTH; - - transactionBytes[nonceIndex++] = (byte) 0; - transactionBytes[nonceIndex++] = (byte) 0; - transactionBytes[nonceIndex++] = (byte) 0; - transactionBytes[nonceIndex++] = (byte) 0; - } - -} diff --git a/src/test/java/org/qortal/test/PresenceTests.java b/src/test/java/org/qortal/test/PresenceTests.java deleted file mode 100644 index b53b72cb..00000000 --- a/src/test/java/org/qortal/test/PresenceTests.java +++ /dev/null @@ -1,133 +0,0 @@ -package org.qortal.test; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.asset.Asset; -import org.qortal.crosschain.BitcoinACCTv1; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.PresenceTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.test.common.BlockUtils; -import org.qortal.test.common.Common; -import org.qortal.test.common.TransactionUtils; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.PresenceTransaction; -import org.qortal.transaction.PresenceTransaction.PresenceType; -import org.qortal.transaction.Transaction; -import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.utils.NTP; - -import com.google.common.primitives.Longs; - -import static org.junit.Assert.*; - -public class PresenceTests extends Common { - - private static final byte[] BITCOIN_PKH = new byte[20]; - private static final byte[] HASH_OF_SECRET_B = new byte[32]; - - private PrivateKeyAccount signer; - private Repository repository; - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); - - this.repository = RepositoryManager.getRepository(); - this.signer = Common.getTestAccount(this.repository, "bob"); - - // We need to create corresponding test trade offer - byte[] creationBytes = BitcoinACCTv1.buildQortalAT(this.signer.getAddress(), BITCOIN_PKH, HASH_OF_SECRET_B, - 0L, 0L, - 7 * 24 * 60 * 60); - - long txTimestamp = NTP.getTime(); - byte[] lastReference = this.signer.getLastReference(); - - long fee = 0; - String name = "QORT-BTC cross-chain trade"; - String description = "Qortal-Bitcoin cross-chain trade"; - String atType = "ACCT"; - String tags = "QORT-BTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, this.signer.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, 1L, Asset.QORT); - - Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - TransactionUtils.signAndImportValid(this.repository, deployAtTransactionData, this.signer); - BlockUtils.mintBlock(this.repository); - } - - @After - public void afterTest() throws DataException { - if (this.repository != null) - this.repository.close(); - - this.repository = null; - } - - @Test - public void validityTests() throws DataException { - long timestamp = System.currentTimeMillis(); - byte[] timestampBytes = Longs.toByteArray(timestamp); - - byte[] timestampSignature = this.signer.sign(timestampBytes); - - assertTrue(isValid(Group.NO_GROUP, this.signer, timestamp, timestampSignature)); - - PrivateKeyAccount nonTrader = Common.getTestAccount(repository, "alice"); - assertFalse(isValid(Group.NO_GROUP, nonTrader, timestamp, timestampSignature)); - } - - @Test - public void newestOnlyTests() throws DataException { - long OLDER_TIMESTAMP = System.currentTimeMillis() - 2000L; - long NEWER_TIMESTAMP = OLDER_TIMESTAMP + 1000L; - - PresenceTransaction older = buildPresenceTransaction(Group.NO_GROUP, this.signer, OLDER_TIMESTAMP, null); - older.computeNonce(); - TransactionUtils.signAndImportValid(repository, older.getTransactionData(), this.signer); - - assertTrue(this.repository.getTransactionRepository().exists(older.getTransactionData().getSignature())); - - PresenceTransaction newer = buildPresenceTransaction(Group.NO_GROUP, this.signer, NEWER_TIMESTAMP, null); - newer.computeNonce(); - TransactionUtils.signAndImportValid(repository, newer.getTransactionData(), this.signer); - - assertTrue(this.repository.getTransactionRepository().exists(newer.getTransactionData().getSignature())); - assertFalse(this.repository.getTransactionRepository().exists(older.getTransactionData().getSignature())); - } - - private boolean isValid(int txGroupId, PrivateKeyAccount signer, long timestamp, byte[] timestampSignature) throws DataException { - Transaction transaction = buildPresenceTransaction(txGroupId, signer, timestamp, timestampSignature); - return transaction.isValidUnconfirmed() == ValidationResult.OK; - } - - private PresenceTransaction buildPresenceTransaction(int txGroupId, PrivateKeyAccount signer, long timestamp, byte[] timestampSignature) throws DataException { - int nonce = 0; - - byte[] reference = signer.getLastReference(); - byte[] creatorPublicKey = signer.getPublicKey(); - long fee = 0L; - - if (timestampSignature == null) - timestampSignature = this.signer.sign(Longs.toByteArray(timestamp)); - - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, null); - PresenceTransactionData transactionData = new PresenceTransactionData(baseTransactionData, nonce, PresenceType.TRADE_BOT, timestampSignature); - - return new PresenceTransaction(this.repository, transactionData); - } - -} diff --git a/src/test/java/org/qortal/test/RepositoryTests.java b/src/test/java/org/qortal/test/RepositoryTests.java index 434e03f0..269e2aa3 100644 --- a/src/test/java/org/qortal/test/RepositoryTests.java +++ b/src/test/java/org/qortal/test/RepositoryTests.java @@ -4,7 +4,6 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.Account; import org.qortal.asset.Asset; -import org.qortal.crosschain.BitcoinACCTv1; import org.qortal.crypto.Crypto; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -394,25 +393,6 @@ public class RepositoryTests extends Common { } } - /** Specifically test LATERAL() usage in AT repository */ - @Test - public void testAtLateral() { - try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { - byte[] codeHash = BitcoinACCTv1.CODE_BYTES_HASH; - Boolean isFinished = null; - Integer dataByteOffset = null; - Long expectedValue = null; - Integer minimumFinalHeight = 2; - Integer limit = null; - Integer offset = null; - Boolean reverse = null; - - hsqldb.getATRepository().getMatchingFinalATStates(codeHash, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, limit, offset, reverse); - } catch (DataException e) { - fail("HSQLDB bug #1580"); - } - } - /** Specifically test LATERAL() usage in Chat repository */ @Test public void testChatLateral() { diff --git a/src/test/java/org/qortal/test/api/CrossChainApiTests.java b/src/test/java/org/qortal/test/api/CrossChainApiTests.java deleted file mode 100644 index d4f25bce..00000000 --- a/src/test/java/org/qortal/test/api/CrossChainApiTests.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.qortal.test.api; - -import org.junit.Before; -import org.junit.Test; -import org.qortal.api.ApiError; -import org.qortal.api.resource.CrossChainResource; -import org.qortal.crosschain.SupportedBlockchain; -import org.qortal.test.common.ApiCommon; - -public class CrossChainApiTests extends ApiCommon { - - private static final SupportedBlockchain SPECIFIC_BLOCKCHAIN = null; - - private CrossChainResource crossChainResource; - - @Before - public void buildResource() { - this.crossChainResource = (CrossChainResource) ApiCommon.buildResource(CrossChainResource.class); - } - - @Test - public void testGetTradeOffers() { - assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getTradeOffers(SPECIFIC_BLOCKCHAIN, limit, offset, reverse)); - } - - @Test - public void testGetCompletedTrades() { - long minimumTimestamp = System.currentTimeMillis(); - assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, minimumTimestamp, limit, offset, reverse)); - } - - @Test - public void testInvalidGetCompletedTrades() { - Integer limit = null; - Integer offset = null; - Boolean reverse = null; - - assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, -1L /*minimumTimestamp*/, limit, offset, reverse)); - assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, 0L /*minimumTimestamp*/, limit, offset, reverse)); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/BitcoinTests.java b/src/test/java/org/qortal/test/crosschain/BitcoinTests.java deleted file mode 100644 index af879e08..00000000 --- a/src/test/java/org/qortal/test/crosschain/BitcoinTests.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.qortal.test.crosschain; - -import static org.junit.Assert.*; - -import java.util.Arrays; - -import org.bitcoinj.core.Transaction; -import org.bitcoinj.store.BlockStoreException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.repository.DataException; -import org.qortal.test.common.Common; - -public class BitcoinTests extends Common { - - private Bitcoin bitcoin; - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); // TestNet3 - bitcoin = Bitcoin.getInstance(); - } - - @After - public void afterTest() { - Bitcoin.resetForTesting(); - bitcoin = null; - } - - @Test - public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { - System.out.println(String.format("Starting BTC instance...")); - System.out.println(String.format("BTC instance started")); - - long before = System.currentTimeMillis(); - System.out.println(String.format("Bitcoin median blocktime: %d", bitcoin.getMedianBlockTime())); - long afterFirst = System.currentTimeMillis(); - - System.out.println(String.format("Bitcoin median blocktime: %d", bitcoin.getMedianBlockTime())); - long afterSecond = System.currentTimeMillis(); - - long firstPeriod = afterFirst - before; - long secondPeriod = afterSecond - afterFirst; - - System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); - - assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); - assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); - } - - @Test - public void testFindHtlcSecret() throws ForeignBlockchainException { - // This actually exists on TEST3 but can take a while to fetch - String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - - byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); - byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); - - assertNotNull(secret); - assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); - } - - @Test - public void testBuildSpend() { - String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; - - String recipient = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - long amount = 1000L; - - Transaction transaction = bitcoin.buildSpend(xprv58, recipient, amount); - assertNotNull(transaction); - - // Check spent key caching doesn't affect outcome - - transaction = bitcoin.buildSpend(xprv58, recipient, amount); - assertNotNull(transaction); - } - - @Test - public void testGetWalletBalance() { - String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; - - Long balance = bitcoin.getWalletBalance(xprv58); - - assertNotNull(balance); - - System.out.println(bitcoin.format(balance)); - - // Check spent key caching doesn't affect outcome - - Long repeatBalance = bitcoin.getWalletBalance(xprv58); - - assertNotNull(repeatBalance); - - System.out.println(bitcoin.format(repeatBalance)); - - assertEquals(balance, repeatBalance); - } - - @Test - public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { - String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; - - String address = bitcoin.getUnusedReceiveAddress(xprv58); - - assertNotNull(address); - - System.out.println(address); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java b/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java deleted file mode 100644 index b7e57cf3..00000000 --- a/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.qortal.test.crosschain; - -import static org.junit.Assert.*; - -import java.security.Security; -import java.util.EnumMap; -import java.util.List; -import java.util.Map; - -import org.bitcoinj.core.Address; -import org.bitcoinj.params.TestNet3Params; -import org.bitcoinj.script.ScriptBuilder; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; -import org.junit.Test; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.BitcoinyTransaction; -import org.qortal.crosschain.ElectrumX; -import org.qortal.crosschain.TransactionHash; -import org.qortal.crosschain.UnspentOutput; -import org.qortal.crosschain.Bitcoin.BitcoinNet; -import org.qortal.crosschain.ElectrumX.Server.ConnectionType; -import org.qortal.utils.BitTwiddling; - -import com.google.common.hash.HashCode; - -public class ElectrumXTests { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); - } - - private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); - static { - DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); - DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); - } - - private ElectrumX getInstance() { - return new ElectrumX("Bitcoin-" + BitcoinNet.TEST3.name(), BitcoinNet.TEST3.getGenesisHash(), BitcoinNet.TEST3.getServers(), DEFAULT_ELECTRUMX_PORTS); - } - - @Test - public void testInstance() { - ElectrumX electrumX = getInstance(); - assertNotNull(electrumX); - } - - @Test - public void testGetCurrentHeight() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - int height = electrumX.getCurrentHeight(); - - assertTrue(height > 10000); - System.out.println("Current TEST3 height: " + height); - } - - @Test - public void testInvalidRequest() { - ElectrumX electrumX = getInstance(); - try { - electrumX.getRawBlockHeaders(-1, -1); - } catch (ForeignBlockchainException e) { - // Should throw due to negative start block height - return; - } - - fail("Negative start block height should cause error"); - } - - @Test - public void testGetRecentBlocks() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - int height = electrumX.getCurrentHeight(); - assertTrue(height > 10000); - - List recentBlockHeaders = electrumX.getRawBlockHeaders(height - 11, 11); - - System.out.println(String.format("Returned %d recent blocks", recentBlockHeaders.size())); - for (int i = 0; i < recentBlockHeaders.size(); ++i) { - byte[] blockHeader = recentBlockHeaders.get(i); - - // Timestamp(int) is at 4 + 32 + 32 = 68 bytes offset - int offset = 4 + 32 + 32; - int timestamp = BitTwiddling.intFromLEBytes(blockHeader, offset); - System.out.println(String.format("Block %d timestamp: %d", height + i, timestamp)); - } - } - - @Test - public void testGetP2PKHBalance() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - Address address = Address.fromString(TestNet3Params.get(), "n3GNqMveyvaPvUbH469vDRadqpJMPc84JA"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - long balance = electrumX.getConfirmedBalance(script); - - assertTrue(balance > 0L); - - System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); - } - - @Test - public void testGetP2SHBalance() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - long balance = electrumX.getConfirmedBalance(script); - - assertTrue(balance > 0L); - - System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); - } - - @Test - public void testGetUnspentOutputs() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - List unspentOutputs = electrumX.getUnspentOutputs(script, false); - - assertFalse(unspentOutputs.isEmpty()); - - for (UnspentOutput unspentOutput : unspentOutputs) - System.out.println(String.format("TestNet address %s has unspent output at tx %s, output index %d", address, HashCode.fromBytes(unspentOutput.hash), unspentOutput.index)); - } - - @Test - public void testGetRawTransaction() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - byte[] txHash = HashCode.fromString("7653fea9ffcd829d45ed2672938419a94951b08175982021e77d619b553f29af").asBytes(); - - byte[] rawTransactionBytes = electrumX.getRawTransaction(txHash); - - assertFalse(rawTransactionBytes.length == 0); - } - - @Test - public void testGetUnknownRawTransaction() { - ElectrumX electrumX = getInstance(); - - byte[] txHash = HashCode.fromString("f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0").asBytes(); - - try { - electrumX.getRawTransaction(txHash); - fail("Bitcoin transaction should be unknown and hence throw exception"); - } catch (ForeignBlockchainException e) { - if (!(e instanceof ForeignBlockchainException.NotFoundException)) - fail("Bitcoin transaction should be unknown and hence throw NotFoundException"); - } - } - - @Test - public void testGetTransaction() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - String txHash = "7653fea9ffcd829d45ed2672938419a94951b08175982021e77d619b553f29af"; - - BitcoinyTransaction transaction = electrumX.getTransaction(txHash); - - assertNotNull(transaction); - assertTrue(transaction.txHash.equals(txHash)); - } - - @Test - public void testGetUnknownTransaction() { - ElectrumX electrumX = getInstance(); - - String txHash = "f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0"; - - try { - electrumX.getTransaction(txHash); - fail("Bitcoin transaction should be unknown and hence throw exception"); - } catch (ForeignBlockchainException e) { - if (!(e instanceof ForeignBlockchainException.NotFoundException)) - fail("Bitcoin transaction should be unknown and hence throw NotFoundException"); - } - } - - @Test - public void testGetAddressTransactions() throws ForeignBlockchainException { - ElectrumX electrumX = getInstance(); - - Address address = Address.fromString(TestNet3Params.get(), "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - - List transactionHashes = electrumX.getAddressTransactions(script, false); - - assertFalse(transactionHashes.isEmpty()); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/HtlcTests.java b/src/test/java/org/qortal/test/crosschain/HtlcTests.java deleted file mode 100644 index 75b290bf..00000000 --- a/src/test/java/org/qortal/test/crosschain/HtlcTests.java +++ /dev/null @@ -1,128 +0,0 @@ -package org.qortal.test.crosschain; - -import static org.junit.Assert.*; - -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crypto.Crypto; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.repository.DataException; -import org.qortal.test.common.Common; - -import com.google.common.primitives.Longs; - -public class HtlcTests extends Common { - - private Bitcoin bitcoin; - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); // TestNet3 - bitcoin = Bitcoin.getInstance(); - } - - @After - public void afterTest() { - Bitcoin.resetForTesting(); - bitcoin = null; - } - - @Test - public void testFindHtlcSecret() throws ForeignBlockchainException { - // This actually exists on TEST3 but can take a while to fetch - String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - - byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); - byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); - - assertNotNull(secret); - assertArrayEquals("secret incorrect", expectedSecret, secret); - } - - @Test - @Ignore(value = "Doesn't work, to be fixed later") - public void testHtlcSecretCaching() throws ForeignBlockchainException { - String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); - - do { - // We need to perform fresh setup for 1st test - Bitcoin.resetForTesting(); - bitcoin = Bitcoin.getInstance(); - - long now = System.currentTimeMillis(); - long timestampBoundary = now / 30_000L; - - byte[] secret1 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); - long executionPeriod1 = System.currentTimeMillis() - now; - - assertNotNull(secret1); - assertArrayEquals("secret1 incorrect", expectedSecret, secret1); - - assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); - - byte[] secret2 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); - long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; - - assertNotNull(secret2); - assertArrayEquals("secret2 incorrect", expectedSecret, secret2); - - // Test is only valid if we've called within same timestampBoundary - if (System.currentTimeMillis() / 30_000L != timestampBoundary) - continue; - - assertArrayEquals(secret1, secret2); - - assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10); - } while (false); - } - - @Test - public void testDetermineHtlcStatus() throws ForeignBlockchainException { - // This actually exists on TEST3 but can take a while to fetch - String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - - BitcoinyHTLC.Status htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); - assertNotNull(htlcStatus); - - System.out.println(String.format("HTLC %s status: %s", p2shAddress, htlcStatus.name())); - } - - @Test - public void testHtlcStatusCaching() throws ForeignBlockchainException { - do { - // We need to perform fresh setup for 1st test - Bitcoin.resetForTesting(); - bitcoin = Bitcoin.getInstance(); - - long now = System.currentTimeMillis(); - long timestampBoundary = now / 30_000L; - - // Won't ever exist - String p2shAddress = bitcoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now))); - - BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); - long executionPeriod1 = System.currentTimeMillis() - now; - - assertNotNull(htlcStatus1); - assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); - - BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); - long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; - - assertNotNull(htlcStatus2); - assertEquals(htlcStatus1, htlcStatus2); - - // Test is only valid if we've called within same timestampBoundary - if (System.currentTimeMillis() / 30_000L != timestampBoundary) - continue; - - assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10); - } while (false); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/LitecoinTests.java b/src/test/java/org/qortal/test/crosschain/LitecoinTests.java deleted file mode 100644 index 64837347..00000000 --- a/src/test/java/org/qortal/test/crosschain/LitecoinTests.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.qortal.test.crosschain; - -import static org.junit.Assert.*; - -import java.util.Arrays; - -import org.bitcoinj.core.Transaction; -import org.bitcoinj.store.BlockStoreException; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.Litecoin; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.repository.DataException; -import org.qortal.test.common.Common; - -public class LitecoinTests extends Common { - - private Litecoin litecoin; - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); // TestNet3 - litecoin = Litecoin.getInstance(); - } - - @After - public void afterTest() { - Litecoin.resetForTesting(); - litecoin = null; - } - - @Test - public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { - long before = System.currentTimeMillis(); - System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); - long afterFirst = System.currentTimeMillis(); - - System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); - long afterSecond = System.currentTimeMillis(); - - long firstPeriod = afterFirst - before; - long secondPeriod = afterSecond - afterFirst; - - System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); - - assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); - assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); - } - - @Test - @Ignore(value = "Doesn't work, to be fixed later") - public void testFindHtlcSecret() throws ForeignBlockchainException { - // This actually exists on TEST3 but can take a while to fetch - String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - - byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); - byte[] secret = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress); - - assertNotNull("secret not found", secret); - assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); - } - - @Test - public void testBuildSpend() { - String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; - - String recipient = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; - long amount = 1000L; - - Transaction transaction = litecoin.buildSpend(xprv58, recipient, amount); - assertNotNull("insufficient funds", transaction); - - // Check spent key caching doesn't affect outcome - - transaction = litecoin.buildSpend(xprv58, recipient, amount); - assertNotNull("insufficient funds", transaction); - } - - @Test - public void testGetWalletBalance() { - String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; - - Long balance = litecoin.getWalletBalance(xprv58); - - assertNotNull(balance); - - System.out.println(litecoin.format(balance)); - - // Check spent key caching doesn't affect outcome - - Long repeatBalance = litecoin.getWalletBalance(xprv58); - - assertNotNull(repeatBalance); - - System.out.println(litecoin.format(repeatBalance)); - - assertEquals(balance, repeatBalance); - } - - @Test - public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { - String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; - - String address = litecoin.getUnusedReceiveAddress(xprv58); - - assertNotNull(address); - - System.out.println(address); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java deleted file mode 100644 index fa92fde7..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.crosschain.Litecoin; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.BitcoinyHTLC; - -import com.google.common.hash.HashCode; - -public class BuildHTLC { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: BuildHTLC (-b | -l) ")); - System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); - System.err.println(String.format("example: BuildHTLC -l " - + "msAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" - + "\t0.00008642 \\\n" - + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t1600000000")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 6 || args.length > 6) - usage(null); - - Common.init(); - - Bitcoiny bitcoiny = null; - NetworkParameters params = null; - - Address refundAddress = null; - Coin amount = null; - Address redeemAddress = null; - byte[] hashOfSecret = null; - int lockTime = 0; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - params = bitcoiny.getNetworkParameters(); - - refundAddress = Address.fromString(params, args[argIndex++]); - if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Refund address must be in P2PKH form"); - - amount = Coin.parseCoin(args[argIndex++]); - - redeemAddress = Address.fromString(params, args[argIndex++]); - if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Redeem address must be in P2PKH form"); - - hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); - if (hashOfSecret.length != 20) - usage("Hash of secret must be 20 bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - int refundTimeoutDelay = lockTime - (int) (System.currentTimeMillis() / 1000L); - if (refundTimeoutDelay < 600 || refundTimeoutDelay > 30 * 24 * 60 * 60) - usage("Locktime (seconds) should be at between 10 minutes and 1 month from now"); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); - if (p2shFee.isZero()) - return; - - System.out.println(String.format("Refund address: %s", refundAddress)); - System.out.println(String.format("Amount: %s", amount.toPlainString())); - System.out.println(String.format("Redeem address: %s", redeemAddress)); - System.out.println(String.format("Refund/redeem miner's fee: %s", bitcoiny.format(p2shFee))); - System.out.println(String.format("Script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); - System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(hashOfSecret))); - - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); - System.out.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); - - String p2shAddress = bitcoiny.deriveP2shAddress(redeemScriptBytes); - System.out.println(String.format("P2SH address: %s", p2shAddress)); - - amount = amount.add(p2shFee); - - // Fund P2SH - System.out.println(String.format("\nYou need to fund %s with %s (includes redeem/refund fee of %s)", - p2shAddress, bitcoiny.format(amount), bitcoiny.format(p2shFee))); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java deleted file mode 100644 index 8b1cc423..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java +++ /dev/null @@ -1,135 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.Litecoin; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.crypto.Crypto; - -import com.google.common.hash.HashCode; - -public class CheckHTLC { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: CheckHTLC (-b | -l) ")); - System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); - System.err.println(String.format("example: CheckP2SH -l " - + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" - + "msAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" - + "\t0.00008642 \\\n" - + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t1600184800")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 7 || args.length > 7) - usage(null); - - Common.init(); - - Bitcoiny bitcoiny = null; - NetworkParameters params = null; - - Address p2shAddress = null; - Address refundAddress = null; - Coin amount = null; - Address redeemAddress = null; - byte[] hashOfSecret = null; - int lockTime = 0; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - params = bitcoiny.getNetworkParameters(); - - p2shAddress = Address.fromString(params, args[argIndex++]); - if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) - usage("P2SH address invalid"); - - refundAddress = Address.fromString(params, args[argIndex++]); - if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Refund address must be in P2PKH form"); - - amount = Coin.parseCoin(args[argIndex++]); - - redeemAddress = Address.fromString(params, args[argIndex++]); - if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Redeem address must be in P2PKH form"); - - hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); - if (hashOfSecret.length != 20) - usage("Hash of secret must be 20 bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); - if (p2shFee.isZero()) - return; - - System.out.println(String.format("P2SH address: %s", p2shAddress)); - System.out.println(String.format("Refund PKH: %s", refundAddress)); - System.out.println(String.format("Redeem/refund amount: %s", amount.toPlainString())); - System.out.println(String.format("Redeem PKH: %s", redeemAddress)); - System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(hashOfSecret))); - System.out.println(String.format("Script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); - - System.out.println(String.format("Redeem/refund miner's fee: %s", bitcoiny.format(p2shFee))); - - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); - System.out.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!derivedP2shAddress.equals(p2shAddress)) { - System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); - System.exit(2); - } - - amount = amount.add(p2shFee); - - // Check network's median block time - int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, null); - if (medianBlockTime == 0) - return; - - // Check P2SH is funded - Common.getBalance(bitcoiny, p2shAddress.toString()); - - // Grab all unspent outputs - Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); - - Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), amount.value); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/Common.java b/src/test/java/org/qortal/test/crosschain/apps/Common.java deleted file mode 100644 index 78066fe7..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/Common.java +++ /dev/null @@ -1,158 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.security.Security; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.Collections; -import java.util.List; - -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.settings.Settings; -import org.qortal.utils.NTP; - -import com.google.common.hash.HashCode; - -public abstract class Common { - - public static void init() { - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); - - Settings.fileInstance("settings-test.json"); - - NTP.setFixedOffset(0L); - } - - public static long getP2shFee(Bitcoiny bitcoiny) { - long p2shFee; - - try { - p2shFee = bitcoiny.getP2shFee(null); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Unable to determine P2SH fee: %s", e.getMessage())); - return 0; - } - - return p2shFee; - } - - public static int checkMedianBlockTime(Bitcoiny bitcoiny, Integer lockTime) { - int medianBlockTime; - - try { - medianBlockTime = bitcoiny.getMedianBlockTime(); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Unable to determine median block time: %s", e.getMessage())); - return 0; - } - - System.out.println(String.format("Median block time: %s", LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - - long now = System.currentTimeMillis(); - - if (now < medianBlockTime * 1000L) { - System.out.println(String.format("Too soon (%s) based on median block time %s", - LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), - LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); - return 0; - } - - if (lockTime != null && now < lockTime * 1000L) { - System.err.println(String.format("Too soon (%s) based on lockTime %s", - LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), - LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC))); - return 0; - } - - return medianBlockTime; - } - - public static long getBalance(Bitcoiny bitcoiny, String address58) { - long balance; - - try { - balance = bitcoiny.getConfirmedBalance(address58); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Unable to check address %s balance: %s", address58, e.getMessage())); - return 0; - } - - System.out.println(String.format("Address %s balance: %s", address58, bitcoiny.format(balance))); - - return balance; - } - - public static List getUnspentOutputs(Bitcoiny bitcoiny, String address58) { - List unspentOutputs = Collections.emptyList(); - - try { - unspentOutputs = bitcoiny.getUnspentOutputs(address58); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Can't find unspent outputs for %s: %s", address58, e.getMessage())); - return unspentOutputs; - } - - System.out.println(String.format("Found %d output%s for %s", - unspentOutputs.size(), - (unspentOutputs.size() != 1 ? "s" : ""), - address58)); - - for (TransactionOutput fundingOutput : unspentOutputs) - System.out.println(String.format("Output %s:%d amount %s", - HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex(), - bitcoiny.format(fundingOutput.getValue()))); - - if (unspentOutputs.isEmpty()) - System.err.println(String.format("Can't use spent/unfunded %s", address58)); - - if (unspentOutputs.size() != 1) - System.err.println(String.format("Expecting only one unspent output?")); - - return unspentOutputs; - } - - public static BitcoinyHTLC.Status determineHtlcStatus(Bitcoiny bitcoiny, String address58, long minimumAmount) { - BitcoinyHTLC.Status htlcStatus = null; - - try { - htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), address58, minimumAmount); - - System.out.println(String.format("HTLC status: %s", htlcStatus.name())); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Unable to determine HTLC status: %s", e.getMessage())); - } - - return htlcStatus; - } - - public static void broadcastTransaction(Bitcoiny bitcoiny, Transaction transaction) { - byte[] rawTransactionBytes = transaction.bitcoinSerialize(); - - System.out.println(String.format("%nRaw transaction bytes:%n%s%n", HashCode.fromBytes(rawTransactionBytes).toString())); - - for (int countDown = 5; countDown >= 1; --countDown) { - System.out.print(String.format("\rBroadcasting transaction in %d second%s... use CTRL-C to abort ", countDown, (countDown != 1 ? "s" : ""))); - try { - Thread.sleep(1000L); - } catch (InterruptedException e) { - System.exit(0); - } - } - System.out.println("Broadcasting transaction... "); - - try { - bitcoiny.broadcastTransaction(transaction); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Failed to broadcast transaction: %s", e.getMessage())); - System.exit(1); - } - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java b/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java deleted file mode 100644 index ef22355b..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.security.Security; - -import org.bitcoinj.core.AddressFormatException; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.Litecoin; -import org.qortal.settings.Settings; - -public class GetNextReceiveAddress { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: GetNextReceiveAddress (-b | -l) ")); - System.err.println(String.format("example (testnet): GetNextReceiveAddress -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 2) - usage(null); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); - - Settings.fileInstance("settings-test.json"); - - Bitcoiny bitcoiny = null; - String key58 = null; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - - key58 = args[argIndex++]; - - if (!bitcoiny.isValidDeterministicKey(key58)) - usage("Not valid xprv/xpub/tprv/tpub"); - } catch (NumberFormatException | AddressFormatException e) { - usage(String.format("Argument format exception: %s", e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - String receiveAddress = null; - try { - receiveAddress = bitcoiny.getUnusedReceiveAddress(key58); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Failed to determine next receive address: %s", e.getMessage())); - System.exit(1); - } - - System.out.println(String.format("Next receive address: %s", receiveAddress)); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java b/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java deleted file mode 100644 index 9d903a56..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.security.Security; -import java.util.List; - -import org.bitcoinj.core.AddressFormatException; -import org.bitcoinj.core.TransactionOutput; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.ForeignBlockchainException; -import org.qortal.crosschain.Litecoin; -import org.qortal.settings.Settings; - -import com.google.common.hash.HashCode; - -public class GetTransaction { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: GetTransaction (-b | -l) ")); - System.err.println(String.format("example (mainnet): GetTransaction -b 816407e79568f165f13e09e9912c5f2243e0a23a007cec425acedc2e89284660")); - System.err.println(String.format("example (testnet): GetTransaction -b 3bfd17a492a4e3d6cb7204e17e20aca6c1ab82e1828bd1106eefbaf086fb8a4e")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 2) - usage(null); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); - - Settings.fileInstance("settings-test.json"); - - Bitcoiny bitcoiny = null; - byte[] transactionId = null; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - - transactionId = HashCode.fromString(args[argIndex++]).asBytes(); - } catch (NumberFormatException | AddressFormatException e) { - usage(String.format("Argument format exception: %s", e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - // Grab all outputs from transaction - List fundingOutputs; - try { - fundingOutputs = bitcoiny.getOutputs(transactionId); - } catch (ForeignBlockchainException e) { - System.out.println(String.format("Transaction not found (or error occurred)")); - return; - } - - System.out.println(String.format("Found %d output%s", fundingOutputs.size(), (fundingOutputs.size() != 1 ? "s" : ""))); - - for (TransactionOutput fundingOutput : fundingOutputs) - System.out.println(String.format("Output %d: %s", fundingOutput.getIndex(), fundingOutput.getValue().toPlainString())); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java b/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java deleted file mode 100644 index 7a880b1a..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java +++ /dev/null @@ -1,82 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.security.Security; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - -import org.bitcoinj.core.AddressFormatException; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; -import org.qortal.crosschain.*; -import org.qortal.settings.Settings; - -public class GetWalletTransactions { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: GetWalletTransactions (-b | -l) ")); - System.err.println(String.format("example (testnet): GetWalletTransactions -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 2) - usage(null); - - Security.insertProviderAt(new BouncyCastleProvider(), 0); - Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); - - Settings.fileInstance("settings-test.json"); - - Bitcoiny bitcoiny = null; - String key58 = null; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - - key58 = args[argIndex++]; - - if (!bitcoiny.isValidDeterministicKey(key58)) - usage("Not valid xprv/xpub/tprv/tpub"); - } catch (NumberFormatException | AddressFormatException e) { - usage(String.format("Argument format exception: %s", e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - // Grab all outputs from transaction - List transactions = null; - try { - transactions = bitcoiny.getWalletTransactions(key58); - } catch (ForeignBlockchainException e) { - System.err.println(String.format("Failed to obtain wallet transactions: %s", e.getMessage())); - System.exit(1); - } - - System.out.println(String.format("Found %d transaction%s", transactions.size(), (transactions.size() != 1 ? "s" : ""))); - - for (SimpleTransaction transaction : transactions.stream().sorted(Comparator.comparingInt(SimpleTransaction::getTimestamp)).collect(Collectors.toList())) - System.out.println(String.format("%s", transaction)); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/Pay.java b/src/test/java/org/qortal/test/crosschain/apps/Pay.java deleted file mode 100644 index 93c7aede..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/Pay.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.Litecoin; - -public class Pay { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: Pay (-b | -l) ")); - System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); - System.err.println(String.format("example: Pay -l " - + "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ \\\n" - + "\tmsAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" - + "\t0.00008642")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 4 || args.length > 4) - usage(null); - - Common.init(); - - Bitcoiny bitcoiny = null; - NetworkParameters params = null; - - String xprv58 = null; - Address address = null; - Coin amount = null; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - params = bitcoiny.getNetworkParameters(); - - xprv58 = args[argIndex++]; - if (!bitcoiny.isValidDeterministicKey(xprv58)) - usage("xprv invalid"); - - address = Address.fromString(params, args[argIndex++]); - - amount = Coin.parseCoin(args[argIndex++]); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - System.out.println(String.format("Address: %s", address)); - System.out.println(String.format("Amount: %s", amount.toPlainString())); - - Transaction transaction = bitcoiny.buildSpend(xprv58, address.toString(), amount.value); - if (transaction == null) { - System.err.println("Insufficent funds"); - System.exit(1); - } - - Common.broadcastTransaction(bitcoiny, transaction); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java deleted file mode 100644 index d4f1bcf1..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java +++ /dev/null @@ -1,166 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.util.Arrays; -import java.util.List; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.Litecoin; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.crypto.Crypto; - -import com.google.common.hash.HashCode; - -public class RedeemHTLC { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: Redeem (-b | -l) ")); - System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); - System.err.println(String.format("example: Redeem -l " - + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" - + "\tmsAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" - + "\tefdaed23c4bc85c8ccae40d774af3c2a10391c648b6420cdd83cd44c27fcb5955201c64e372d \\\n" - + "\t5468697320737472696e672069732065786163746c7920333220627974657321 \\\n" - + "\t1600184800 \\\n" - + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 7 || args.length > 7) - usage(null); - - Common.init(); - - Bitcoiny bitcoiny = null; - NetworkParameters params = null; - - Address p2shAddress = null; - Address refundAddress = null; - byte[] redeemPrivateKey = null; - byte[] secret = null; - int lockTime = 0; - Address outputAddress = null; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - params = bitcoiny.getNetworkParameters(); - - p2shAddress = Address.fromString(params, args[argIndex++]); - if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) - usage("P2SH address invalid"); - - refundAddress = Address.fromString(params, args[argIndex++]); - if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Refund address must be in P2PKH form"); - - redeemPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); - // Auto-trim - if (redeemPrivateKey.length >= 37 && redeemPrivateKey.length <= 38) - redeemPrivateKey = Arrays.copyOfRange(redeemPrivateKey, 1, 33); - if (redeemPrivateKey.length != 32) - usage("Redeem private key must be 32 bytes"); - - secret = HashCode.fromString(args[argIndex++]).asBytes(); - if (secret.length == 0) - usage("Invalid secret bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - - outputAddress = Address.fromString(params, args[argIndex++]); - if (outputAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Output address invalid"); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); - if (p2shFee.isZero()) - return; - - System.out.println(String.format("Attempting to redeem HTLC %s to %s", p2shAddress, outputAddress)); - - byte[] hashOfSecret = Crypto.hash160(secret); - - ECKey redeemKey = ECKey.fromPrivate(redeemPrivateKey); - Address redeemAddress = Address.fromKey(params, redeemKey, ScriptType.P2PKH); - - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!derivedP2shAddress.equals(p2shAddress)) { - System.err.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); - System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); - System.exit(2); - return; - } - - // Actual live processing... - - int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, null); - if (medianBlockTime == 0) - return; - - // Check P2SH is funded - long p2shBalance = Common.getBalance(bitcoiny, p2shAddress.toString()); - if (p2shBalance == 0) - return; - - // Grab all unspent outputs - List unspentOutputs = Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); - if (unspentOutputs.isEmpty()) - return; - - Coin redeemAmount = Coin.valueOf(p2shBalance).subtract(p2shFee); - - BitcoinyHTLC.Status htlcStatus = Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), redeemAmount.value); - if (htlcStatus == null) - return; - - if (htlcStatus != BitcoinyHTLC.Status.FUNDED) { - System.err.println(String.format("Expecting %s HTLC status, but actual status is %s", "FUNDED", htlcStatus.name())); - System.exit(2); - return; - } - - System.out.println(String.format("Spending %s of outputs, with %s as mining fee", bitcoiny.format(redeemAmount), bitcoiny.format(p2shFee))); - - Transaction redeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoiny.getNetworkParameters(), redeemAmount, redeemKey, - unspentOutputs, redeemScriptBytes, secret, outputAddress.getHash()); - - Common.broadcastTransaction(bitcoiny, redeemTransaction); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java deleted file mode 100644 index 723185f0..00000000 --- a/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java +++ /dev/null @@ -1,163 +0,0 @@ -package org.qortal.test.crosschain.apps; - -import java.util.Arrays; -import java.util.List; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.Coin; -import org.bitcoinj.core.ECKey; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.bitcoinj.core.Transaction; -import org.bitcoinj.core.TransactionOutput; -import org.bitcoinj.script.Script.ScriptType; -import org.qortal.crosschain.Litecoin; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.crosschain.BitcoinyHTLC; -import org.qortal.crypto.Crypto; - -import com.google.common.hash.HashCode; - -public class RefundHTLC { - - static { - // This must go before any calls to LogManager/Logger - System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); - } - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: RefundHTLC (-b | -l) ")); - System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); - System.err.println(String.format("example: RefundHTLC -l " - + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" - + "\tef8f31b49c31b4a140aebcd9605fded88cc2dad0844c4b984f9191a5a416f72d3801e16447b0 \\\n" - + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t1600184800 \\\n" - + "\tmoJtbbhs7T4Z5hmBH2iyKhGrCWBzQWS2CL")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length < 7 || args.length > 7) - usage(null); - - Common.init(); - - Bitcoiny bitcoiny = null; - NetworkParameters params = null; - - Address p2shAddress = null; - byte[] refundPrivateKey = null; - Address redeemAddress = null; - byte[] hashOfSecret = null; - int lockTime = 0; - Address outputAddress = null; - - int argIndex = 0; - try { - switch (args[argIndex++]) { - case "-b": - bitcoiny = Bitcoin.getInstance(); - break; - - case "-l": - bitcoiny = Litecoin.getInstance(); - break; - - default: - usage("Only Bitcoin (-b) or Litecoin (-l) supported"); - } - params = bitcoiny.getNetworkParameters(); - - p2shAddress = Address.fromString(params, args[argIndex++]); - if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) - usage("P2SH address invalid"); - - refundPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); - // Auto-trim - if (refundPrivateKey.length >= 37 && refundPrivateKey.length <= 38) - refundPrivateKey = Arrays.copyOfRange(refundPrivateKey, 1, 33); - if (refundPrivateKey.length != 32) - usage("Refund private key must be 32 bytes"); - - redeemAddress = Address.fromString(params, args[argIndex++]); - if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Redeem address must be in P2PKH form"); - - hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); - if (hashOfSecret.length != 20) - usage("HASH160 of secret must be 20 bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - - outputAddress = Address.fromString(params, args[argIndex++]); - if (outputAddress.getOutputScriptType() != ScriptType.P2PKH) - usage("Output address invalid"); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); - - Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); - if (p2shFee.isZero()) - return; - - System.out.println(String.format("Attempting to refund HTLC %s to %s", p2shAddress, outputAddress)); - - ECKey refundKey = ECKey.fromPrivate(refundPrivateKey); - Address refundAddress = Address.fromKey(params, refundKey, ScriptType.P2PKH); - - byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); - - byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); - Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); - - if (!derivedP2shAddress.equals(p2shAddress)) { - System.err.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); - System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); - System.exit(2); - } - - // Actual live processing... - - int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, lockTime); - if (medianBlockTime == 0) - return; - - // Check P2SH is funded - long p2shBalance = Common.getBalance(bitcoiny, p2shAddress.toString()); - if (p2shBalance == 0) - return; - - // Grab all unspent outputs - List unspentOutputs = Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); - if (unspentOutputs.isEmpty()) - return; - - Coin refundAmount = Coin.valueOf(p2shBalance).subtract(p2shFee); - - BitcoinyHTLC.Status htlcStatus = Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), refundAmount.value); - if (htlcStatus == null) - return; - - if (htlcStatus != BitcoinyHTLC.Status.FUNDED) { - System.err.println(String.format("Expecting %s HTLC status, but actual status is %s", "FUNDED", htlcStatus.name())); - System.exit(2); - return; - } - - System.out.println(String.format("Spending %s of outputs, with %s as mining fee", bitcoiny.format(refundAmount), bitcoiny.format(p2shFee))); - - Transaction refundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoiny.getNetworkParameters(), refundAmount, refundKey, - unspentOutputs, redeemScriptBytes, lockTime, outputAddress.getHash()); - - Common.broadcastTransaction(bitcoiny, refundTransaction); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java b/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java deleted file mode 100644 index 4487e874..00000000 --- a/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java +++ /dev/null @@ -1,795 +0,0 @@ -package org.qortal.test.crosschain.bitcoinv1; - -import static org.junit.Assert.*; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.Arrays; -import java.util.List; -import java.util.Random; -import java.util.function.Function; - -import org.junit.Before; -import org.junit.Test; -import org.qortal.account.Account; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.asset.Asset; -import org.qortal.block.Block; -import org.qortal.crosschain.BitcoinACCTv1; -import org.qortal.crosschain.AcctMode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.test.common.BlockUtils; -import org.qortal.test.common.Common; -import org.qortal.test.common.TransactionUtils; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.utils.Amounts; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -public class BitcoinACCTv1Tests extends Common { - - public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); - public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a - public static final byte[] secretB = "This string is roughly 32 bytes?".getBytes(); - public static final byte[] hashOfSecretB = Crypto.hash160(secretB); // 31f0dd71decf59bbc8ef0661f4030479255cfa58 - public static final byte[] bitcoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); - public static final int tradeTimeout = 20; // blocks - public static final long redeemAmount = 80_40200000L; - public static final long fundingAmount = 123_45600000L; - public static final long bitcoinAmount = 864200L; // 0.00864200 BTC - - private static final Random RANDOM = new Random(); - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); - } - - @Test - public void testCompile() { - PrivateKeyAccount tradeAccount = createTradeAccount(null); - - byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeAccount.getAddress(), bitcoinPublicKeyHash, hashOfSecretB, redeemAmount, bitcoinAmount, tradeTimeout); - assertNotNull(creationBytes); - - System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); - } - - @Test - public void testDeploy() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - - long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = fundingAmount; - actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); - - assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = partnersInitialBalance; - actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - - expectedBalance = deployersInitialBalance; - actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = 0; - actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); - - assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = partnersInitialBalance; - actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testOfferCancel() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - // Send creator's address to AT, instead of typical partner's address - byte[] messageData = BitcoinACCTv1.getInstance().buildCancelMessage(deployer.getAddress()); - MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); - long messageFee = messageTransaction.getTransactionData().getFee(); - - // AT should process 'cancel' message in next block - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in CANCELLED mode - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.CANCELLED, tradeData.mode); - - // Check balances - long expectedMinimumBalance = deployersPostDeploymentBalance; - long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; - - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); - assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - - // Check balances - long expectedBalance = deployersPostDeploymentBalance - messageFee; - actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testOfferCancelInvalidLength() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - // Instead of sending creator's address to AT, send too-short/invalid message - byte[] messageData = new byte[7]; - RANDOM.nextBytes(messageData); - MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); - long messageFee = messageTransaction.getTransactionData().getFee(); - - // AT should process 'cancel' message in next block - // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in CANCELLED mode - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.CANCELLED, tradeData.mode); - } - } - - @SuppressWarnings("unused") - @Test - public void testTradingInfoProcessing() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - Block postDeploymentBlock = BlockUtils.mintBlock(repository); - int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - describeAt(repository, atAddress); - - ATData atData = repository.getATRepository().fromATAddress(atAddress); - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - - // AT should be in TRADE mode - assertEquals(AcctMode.TRADING, tradeData.mode); - - // Check hashOfSecretA was extracted correctly - assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); - - // Check trade partner Qortal address was extracted correctly - assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); - - // Check trade partner's Bitcoin PKH was extracted correctly - assertTrue(Arrays.equals(bitcoinPublicKeyHash, tradeData.partnerForeignPKH)); - - // Test orphaning - BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); - - // Check balances - long expectedBalance = deployersPostDeploymentBalance; - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) - @SuppressWarnings("unused") - @Test - public void testIncorrectTradeSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT BUT NOT FROM AT CREATOR - byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); - - BlockUtils.mintBlock(repository); - - long expectedBalance = partnersInitialBalance; - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); - - describeAt(repository, atAddress); - - ATData atData = repository.getATRepository().fromATAddress(atAddress); - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - - // AT should still be in OFFER mode - assertEquals(AcctMode.OFFERING, tradeData.mode); - } - } - - @SuppressWarnings("unused") - @Test - public void testAutomaticTradeRefund() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - Block postDeploymentBlock = BlockUtils.mintBlock(repository); - int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); - - // Check refund - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in REFUNDED mode - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.REFUNDED, tradeData.mode); - - // Test orphaning - BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); - - // Check balances - long expectedBalance = deployersPostDeploymentBalance; - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretsCorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send correct secrets to AT, from correct account - messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, partner.getAddress()); - messageTransaction = sendMessage(repository, partner, messageData, atAddress); - - // AT should send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in REDEEMED mode - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.REDEEMED, tradeData.mode); - - // Check balances - long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); - - // Orphan redeem - BlockUtils.orphanLastBlock(repository); - - // Check balances - expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); - actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); - - // Check AT state - ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); - - assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretsIncorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send correct secrets to AT, but from wrong account - messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, partner.getAddress()); - messageTransaction = sendMessage(repository, bystander, messageData, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is NOT finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertFalse(atData.getIsFinished()); - - // AT should still be in TRADE mode - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.TRADING, tradeData.mode); - - // Check balances - long expectedBalance = partnersInitialBalance; - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); - - // Check eventual refund - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - } - } - - @SuppressWarnings("unused") - @Test - public void testIncorrectSecretsCorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send incorrect secrets to AT, from correct account - byte[] wrongSecret = new byte[32]; - RANDOM.nextBytes(wrongSecret); - messageData = BitcoinACCTv1.buildRedeemMessage(wrongSecret, secretB, partner.getAddress()); - messageTransaction = sendMessage(repository, partner, messageData, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is NOT finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertFalse(atData.getIsFinished()); - - // AT should still be in TRADE mode - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.TRADING, tradeData.mode); - - long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); - - // Send incorrect secrets to AT, from correct account - messageData = BitcoinACCTv1.buildRedeemMessage(secretA, wrongSecret, partner.getAddress()); - messageTransaction = sendMessage(repository, partner, messageData, atAddress); - - // AT should NOT send funds in the next block - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is NOT finished - atData = repository.getATRepository().fromATAddress(atAddress); - assertFalse(atData.getIsFinished()); - - // AT should still be in TRADE mode - tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.TRADING, tradeData.mode); - - // Check balances - expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() * 2; - actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); - - // Check eventual refund - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretsCorrectSenderInvalidMessageLength() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send correct secrets to AT, from correct account, but missing receive address, hence incorrect length - messageData = Bytes.concat(secretA, secretB); - messageTransaction = sendMessage(repository, partner, messageData, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is NOT finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertFalse(atData.getIsFinished()); - - // AT should be in TRADING mode - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.TRADING, tradeData.mode); - } - } - - @SuppressWarnings("unused") - @Test - public void testDescribeDeployed() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - - List executableAts = repository.getATRepository().getAllExecutableATs(); - - for (ATData atData : executableAts) { - String atAddress = atData.getATAddress(); - byte[] codeBytes = atData.getCodeBytes(); - byte[] codeHash = Crypto.digest(codeBytes); - - System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", - atAddress, - codeBytes.length, - (codeBytes.length != 1 ? "s": ""), - HashCode.fromBytes(codeHash))); - - // Not one of ours? - if (!Arrays.equals(codeHash, BitcoinACCTv1.CODE_BYTES_HASH)) - continue; - - describeAt(repository, atAddress); - } - } - } - - private int calcTestLockTimeA(long messageTimestamp) { - return (int) (messageTimestamp / 1000L + tradeTimeout * 60); - } - - private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { - byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeAddress, bitcoinPublicKeyHash, hashOfSecretB, redeemAmount, bitcoinAmount, tradeTimeout); - - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = deployer.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); - System.exit(2); - } - - Long fee = null; - String name = "QORT-BTC cross-chain trade"; - String description = String.format("Qortal-Bitcoin cross-chain trade"); - String atType = "ACCT"; - String tags = "QORT-BTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); - - return deployAtTransaction; - } - - private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = sender.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); - System.exit(2); - } - - Long fee = null; - int version = 4; - int nonce = 0; - long amount = 0; - Long assetId = null; // because amount is zero - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); - TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); - - MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); - - fee = messageTransaction.calcRecommendedFee(); - messageTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, messageTransactionData, sender); - - return messageTransaction; - } - - private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - int refundTimeout = tradeTimeout * 3 / 4 + 1; // close enough - - // AT should automatically refund deployer after 'refundTimeout' blocks - for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) - BlockUtils.mintBlock(repository); - - // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range - long expectedMinimumBalance = deployersPostDeploymentBalance; - long expectedMaximumBalance = deployersInitialBalance - deployAtFee; - - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); - assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); - } - - private void describeAt(Repository repository, String atAddress) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); - - Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); - int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); - - System.out.print(String.format("%s:\n" - + "\tmode: %s\n" - + "\tcreator: %s,\n" - + "\tcreation timestamp: %s,\n" - + "\tcurrent balance: %s QORT,\n" - + "\tis finished: %b,\n" - + "\tHASH160 of secret-B: %s,\n" - + "\tredeem payout: %s QORT,\n" - + "\texpected bitcoin: %s BTC,\n" - + "\tcurrent block height: %d,\n", - tradeData.qortalAtAddress, - tradeData.mode, - tradeData.qortalCreator, - epochMilliFormatter.apply(tradeData.creationTimestamp), - Amounts.prettyAmount(tradeData.qortBalance), - atData.getIsFinished(), - HashCode.fromBytes(tradeData.hashOfSecretB).toString().substring(0, 40), - Amounts.prettyAmount(tradeData.qortAmount), - Amounts.prettyAmount(tradeData.expectedForeignAmount), - currentBlockHeight)); - - if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { - System.out.println(String.format("\trefund height: block %d,\n" - + "\tHASH160 of secret-A: %s,\n" - + "\tBitcoin P2SH-A nLockTime: %d (%s),\n" - + "\tBitcoin P2SH-B nLockTime: %d (%s),\n" - + "\ttrade partner: %s", - tradeData.tradeRefundHeight, - HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), - tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), - tradeData.lockTimeB, epochMilliFormatter.apply(tradeData.lockTimeB * 1000L), - tradeData.qortalPartnerAddress)); - } - } - - private PrivateKeyAccount createTradeAccount(Repository repository) { - // We actually use a known test account with funds to avoid PoW compute - return Common.getTestAccount(repository, "alice"); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java b/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java deleted file mode 100644 index f27f7a7b..00000000 --- a/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java +++ /dev/null @@ -1,169 +0,0 @@ -package org.qortal.test.crosschain.bitcoinv1; - -import org.bitcoinj.core.Address; -import org.bitcoinj.core.AddressFormatException; -import org.bitcoinj.core.LegacyAddress; -import org.bitcoinj.core.NetworkParameters; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.asset.Asset; -import org.qortal.controller.Controller; -import org.qortal.crosschain.Bitcoin; -import org.qortal.crosschain.BitcoinACCTv1; -import org.qortal.crosschain.Bitcoiny; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.test.crosschain.apps.Common; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.Transaction; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.TransactionTransformer; -import org.qortal.utils.Amounts; -import org.qortal.utils.Base58; - -import com.google.common.hash.HashCode; - -public class DeployAT { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: DeployAT ")); - System.err.println(String.format("example: DeployAT " - + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" - + "\t10 \\\n" - + "\t10.1 \\\n" - + "\t0.00864200 \\\n" - + "\t750b06757a2448b8a4abebaa6e4662833fd5ddbb (or mrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h) \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t10080")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 7) - usage(null); - - Common.init(); - - Bitcoiny bitcoiny = Bitcoin.getInstance(); - NetworkParameters params = bitcoiny.getNetworkParameters(); - - byte[] refundPrivateKey = null; - long redeemAmount = 0; - long fundingAmount = 0; - long expectedBitcoin = 0; - byte[] bitcoinPublicKeyHash = null; - byte[] hashOfSecret = null; - int tradeTimeout = 0; - - int argIndex = 0; - try { - refundPrivateKey = Base58.decode(args[argIndex++]); - if (refundPrivateKey.length != 32) - usage("Refund private key must be 32 bytes"); - - redeemAmount = Long.parseLong(args[argIndex++]); - if (redeemAmount <= 0) - usage("QORT amount must be positive"); - - fundingAmount = Long.parseLong(args[argIndex++]); - if (fundingAmount <= redeemAmount) - usage("AT funding amount must be greater than QORT redeem amount"); - - expectedBitcoin = Long.parseLong(args[argIndex++]); - if (expectedBitcoin <= 0) - usage("Expected BTC amount must be positive"); - - String bitcoinPKHish = args[argIndex++]; - // Try P2PKH first - try { - Address bitcoinAddress = LegacyAddress.fromBase58(params, bitcoinPKHish); - bitcoinPublicKeyHash = bitcoinAddress.getHash(); - } catch (AddressFormatException e) { - // Try parsing as PKH hex string instead - bitcoinPublicKeyHash = HashCode.fromString(bitcoinPKHish).asBytes(); - } - if (bitcoinPublicKeyHash.length != 20) - usage("Bitcoin PKH must be 20 bytes"); - - hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); - if (hashOfSecret.length != 20) - usage("Hash of secret must be 20 bytes"); - - tradeTimeout = Integer.parseInt(args[argIndex++]); - if (tradeTimeout < 60 || tradeTimeout > 50000) - usage("Trade timeout (minutes) must be between 60 and 50000"); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); - System.exit(2); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount refundAccount = new PrivateKeyAccount(repository, refundPrivateKey); - System.out.println(String.format("Refund Qortal address: %s", refundAccount.getAddress())); - - System.out.println(String.format("QORT redeem amount: %s", Amounts.prettyAmount(redeemAmount))); - - System.out.println(String.format("AT funding amount: %s", Amounts.prettyAmount(fundingAmount))); - - System.out.println(String.format("HASH160 of secret: %s", HashCode.fromBytes(hashOfSecret))); - - // Deploy AT - byte[] creationBytes = BitcoinACCTv1.buildQortalAT(refundAccount.getAddress(), bitcoinPublicKeyHash, hashOfSecret, redeemAmount, expectedBitcoin, tradeTimeout); - System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); - - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = refundAccount.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", refundAccount.getAddress())); - System.exit(2); - } - - Long fee = null; - String name = "QORT-BTC cross-chain trade"; - String description = String.format("Qortal-Bitcoin cross-chain trade"); - String atType = "ACCT"; - String tags = "QORT-BTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, refundAccount.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); - - Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - deployAtTransaction.sign(refundAccount); - - byte[] signedBytes = null; - try { - signedBytes = TransactionTransformer.toBytes(deployAtTransactionData); - } catch (TransformationException e) { - System.err.println(String.format("Unable to convert transaction to base58: %s", e.getMessage())); - System.exit(2); - } - - System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); - } catch (DataException e) { - System.err.println(String.format("Repository issue: %s", e.getMessage())); - System.exit(2); - } - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java deleted file mode 100644 index 3a1f9208..00000000 --- a/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java +++ /dev/null @@ -1,150 +0,0 @@ -package org.qortal.test.crosschain.litecoinv1; - -import java.math.BigDecimal; - -import org.bitcoinj.core.ECKey; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.asset.Asset; -import org.qortal.controller.Controller; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.test.crosschain.apps.Common; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.TransactionTransformer; -import org.qortal.utils.Amounts; -import org.qortal.utils.Base58; - -import com.google.common.hash.HashCode; - -public class DeployAT { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: DeployAT ")); - System.err.println("A trading key-pair will be generated for you!"); - System.err.println(String.format("example: DeployAT " - + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" - + "\t10 \\\n" - + "\t10.1 \\\n" - + "\t0.00864200 \\\n" - + "\t120")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 5) - usage(null); - - Common.init(); - - byte[] creatorPrivateKey = null; - long redeemAmount = 0; - long fundingAmount = 0; - long expectedLitecoin = 0; - int tradeTimeout = 0; - - int argIndex = 0; - try { - creatorPrivateKey = Base58.decode(args[argIndex++]); - if (creatorPrivateKey.length != 32) - usage("Refund private key must be 32 bytes"); - - redeemAmount = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); - if (redeemAmount <= 0) - usage("QORT amount must be positive"); - - fundingAmount = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); - if (fundingAmount <= redeemAmount) - usage("AT funding amount must be greater than QORT redeem amount"); - - expectedLitecoin = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); - if (expectedLitecoin <= 0) - usage("Expected LTC amount must be positive"); - - tradeTimeout = Integer.parseInt(args[argIndex++]); - if (tradeTimeout < 60 || tradeTimeout > 50000) - usage("Trade timeout (minutes) must be between 60 and 50000"); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); - System.exit(2); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount creatorAccount = new PrivateKeyAccount(repository, creatorPrivateKey); - System.out.println(String.format("Creator Qortal address: %s", creatorAccount.getAddress())); - System.out.println(String.format("QORT redeem amount: %s", Amounts.prettyAmount(redeemAmount))); - System.out.println(String.format("AT funding amount: %s", Amounts.prettyAmount(fundingAmount))); - - // Generate trading key-pair - byte[] tradePrivateKey = new ECKey().getPrivKeyBytes(); - PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); - byte[] litecoinPublicKeyHash = ECKey.fromPrivate(tradePrivateKey).getPubKeyHash(); - - System.out.println(String.format("Trade private key: %s", HashCode.fromBytes(tradePrivateKey))); - - // Deploy AT - byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAccount.getAddress(), litecoinPublicKeyHash, redeemAmount, expectedLitecoin, tradeTimeout); - System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); - - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = creatorAccount.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", creatorAccount.getAddress())); - System.exit(2); - } - - Long fee = null; - String name = "QORT-LTC cross-chain trade"; - String description = String.format("Qortal-Litecoin cross-chain trade"); - String atType = "ACCT"; - String tags = "QORT-LTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, creatorAccount.getPublicKey(), fee, null); - DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - deployAtTransaction.sign(creatorAccount); - - byte[] signedBytes = null; - try { - signedBytes = TransactionTransformer.toBytes(deployAtTransactionData); - } catch (TransformationException e) { - System.err.println(String.format("Unable to convert transaction to base58: %s", e.getMessage())); - System.exit(2); - } - - DeployAtTransaction.ensureATAddress(deployAtTransactionData); - String atAddress = deployAtTransactionData.getAtAddress(); - - System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); - - System.out.println(String.format("AT address: %s", atAddress)); - } catch (DataException e) { - System.err.println(String.format("Repository issue: %s", e.getMessage())); - System.exit(2); - } - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java deleted file mode 100644 index 609ff5f3..00000000 --- a/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java +++ /dev/null @@ -1,770 +0,0 @@ -package org.qortal.test.crosschain.litecoinv1; - -import static org.junit.Assert.*; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.Arrays; -import java.util.List; -import java.util.Random; -import java.util.function.Function; - -import org.junit.Before; -import org.junit.Test; -import org.qortal.account.Account; -import org.qortal.account.PrivateKeyAccount; -import org.qortal.asset.Asset; -import org.qortal.block.Block; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.crosschain.AcctMode; -import org.qortal.crypto.Crypto; -import org.qortal.data.at.ATData; -import org.qortal.data.at.ATStateData; -import org.qortal.data.crosschain.CrossChainTradeData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.DeployAtTransactionData; -import org.qortal.data.transaction.MessageTransactionData; -import org.qortal.data.transaction.TransactionData; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.test.common.BlockUtils; -import org.qortal.test.common.Common; -import org.qortal.test.common.TransactionUtils; -import org.qortal.transaction.DeployAtTransaction; -import org.qortal.transaction.MessageTransaction; -import org.qortal.utils.Amounts; - -import com.google.common.hash.HashCode; -import com.google.common.primitives.Bytes; - -public class LitecoinACCTv1Tests extends Common { - - public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); - public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a - public static final byte[] litecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); - public static final int tradeTimeout = 20; // blocks - public static final long redeemAmount = 80_40200000L; - public static final long fundingAmount = 123_45600000L; - public static final long litecoinAmount = 864200L; // 0.00864200 LTC - - private static final Random RANDOM = new Random(); - - @Before - public void beforeTest() throws DataException { - Common.useDefaultSettings(); - } - - @Test - public void testCompile() { - PrivateKeyAccount tradeAccount = createTradeAccount(null); - - byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAccount.getAddress(), litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); - assertNotNull(creationBytes); - - System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); - } - - @Test - public void testDeploy() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - - long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = fundingAmount; - actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); - - assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = partnersInitialBalance; - actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - - expectedBalance = deployersInitialBalance; - actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = 0; - actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); - - assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - - expectedBalance = partnersInitialBalance; - actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testOfferCancel() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - // Send creator's address to AT, instead of typical partner's address - byte[] messageData = LitecoinACCTv1.getInstance().buildCancelMessage(deployer.getAddress()); - MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); - long messageFee = messageTransaction.getTransactionData().getFee(); - - // AT should process 'cancel' message in next block - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in CANCELLED mode - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.CANCELLED, tradeData.mode); - - // Check balances - long expectedMinimumBalance = deployersPostDeploymentBalance; - long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; - - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); - assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); - - // Test orphaning - BlockUtils.orphanLastBlock(repository); - - // Check balances - long expectedBalance = deployersPostDeploymentBalance - messageFee; - actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testOfferCancelInvalidLength() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - // Instead of sending creator's address to AT, send too-short/invalid message - byte[] messageData = new byte[7]; - RANDOM.nextBytes(messageData); - MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); - long messageFee = messageTransaction.getTransactionData().getFee(); - - // AT should process 'cancel' message in next block - // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in CANCELLED mode - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.CANCELLED, tradeData.mode); - } - } - - @SuppressWarnings("unused") - @Test - public void testTradingInfoProcessing() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - Block postDeploymentBlock = BlockUtils.mintBlock(repository); - int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); - - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - describeAt(repository, atAddress); - - ATData atData = repository.getATRepository().fromATAddress(atAddress); - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - - // AT should be in TRADE mode - assertEquals(AcctMode.TRADING, tradeData.mode); - - // Check hashOfSecretA was extracted correctly - assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); - - // Check trade partner Qortal address was extracted correctly - assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); - - // Check trade partner's Litecoin PKH was extracted correctly - assertTrue(Arrays.equals(litecoinPublicKeyHash, tradeData.partnerForeignPKH)); - - // Test orphaning - BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); - - // Check balances - long expectedBalance = deployersPostDeploymentBalance; - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) - @SuppressWarnings("unused") - @Test - public void testIncorrectTradeSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT BUT NOT FROM AT CREATOR - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); - - BlockUtils.mintBlock(repository); - - long expectedBalance = partnersInitialBalance; - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); - - describeAt(repository, atAddress); - - ATData atData = repository.getATRepository().fromATAddress(atAddress); - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - - // AT should still be in OFFER mode - assertEquals(AcctMode.OFFERING, tradeData.mode); - } - } - - @SuppressWarnings("unused") - @Test - public void testAutomaticTradeRefund() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - Block postDeploymentBlock = BlockUtils.mintBlock(repository); - int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); - - // Check refund - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in REFUNDED mode - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.REFUNDED, tradeData.mode); - - // Test orphaning - BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); - - // Check balances - long expectedBalance = deployersPostDeploymentBalance; - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretCorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send correct secret to AT, from correct account - messageData = LitecoinACCTv1.buildRedeemMessage(secretA, partner.getAddress()); - messageTransaction = sendMessage(repository, partner, messageData, atAddress); - - // AT should send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertTrue(atData.getIsFinished()); - - // AT should be in REDEEMED mode - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.REDEEMED, tradeData.mode); - - // Check balances - long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); - - // Orphan redeem - BlockUtils.orphanLastBlock(repository); - - // Check balances - expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); - actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); - - // Check AT state - ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); - - assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretIncorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send correct secret to AT, but from wrong account - messageData = LitecoinACCTv1.buildRedeemMessage(secretA, partner.getAddress()); - messageTransaction = sendMessage(repository, bystander, messageData, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is NOT finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertFalse(atData.getIsFinished()); - - // AT should still be in TRADE mode - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.TRADING, tradeData.mode); - - // Check balances - long expectedBalance = partnersInitialBalance; - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); - - // Check eventual refund - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - } - } - - @SuppressWarnings("unused") - @Test - public void testIncorrectSecretCorrectSender() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - long deployAtFee = deployAtTransaction.getTransactionData().getFee(); - - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send incorrect secret to AT, from correct account - byte[] wrongSecret = new byte[32]; - RANDOM.nextBytes(wrongSecret); - messageData = LitecoinACCTv1.buildRedeemMessage(wrongSecret, partner.getAddress()); - messageTransaction = sendMessage(repository, partner, messageData, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is NOT finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertFalse(atData.getIsFinished()); - - // AT should still be in TRADE mode - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.TRADING, tradeData.mode); - - long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); - long actualBalance = partner.getConfirmedBalance(Asset.QORT); - - assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); - - // Check eventual refund - checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); - } - } - - @SuppressWarnings("unused") - @Test - public void testCorrectSecretCorrectSenderInvalidMessageLength() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - Account at = deployAtTransaction.getATAccount(); - String atAddress = at.getAddress(); - - long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); - int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); - - // Send trade info to AT - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); - MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); - - // Give AT time to process message - BlockUtils.mintBlock(repository); - - // Send correct secret to AT, from correct account, but missing receive address, hence incorrect length - messageData = Bytes.concat(secretA); - messageTransaction = sendMessage(repository, partner, messageData, atAddress); - - // AT should NOT send funds in the next block - ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); - BlockUtils.mintBlock(repository); - - describeAt(repository, atAddress); - - // Check AT is NOT finished - ATData atData = repository.getATRepository().fromATAddress(atAddress); - assertFalse(atData.getIsFinished()); - - // AT should be in TRADING mode - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - assertEquals(AcctMode.TRADING, tradeData.mode); - } - } - - @SuppressWarnings("unused") - @Test - public void testDescribeDeployed() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); - PrivateKeyAccount tradeAccount = createTradeAccount(repository); - - PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); - - long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); - long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); - - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); - - List executableAts = repository.getATRepository().getAllExecutableATs(); - - for (ATData atData : executableAts) { - String atAddress = atData.getATAddress(); - byte[] codeBytes = atData.getCodeBytes(); - byte[] codeHash = Crypto.digest(codeBytes); - - System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", - atAddress, - codeBytes.length, - (codeBytes.length != 1 ? "s": ""), - HashCode.fromBytes(codeHash))); - - // Not one of ours? - if (!Arrays.equals(codeHash, LitecoinACCTv1.CODE_BYTES_HASH)) - continue; - - describeAt(repository, atAddress); - } - } - } - - private int calcTestLockTimeA(long messageTimestamp) { - return (int) (messageTimestamp / 1000L + tradeTimeout * 60); - } - - private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { - byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAddress, litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); - - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = deployer.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); - System.exit(2); - } - - Long fee = null; - String name = "QORT-LTC cross-chain trade"; - String description = String.format("Qortal-Litecoin cross-chain trade"); - String atType = "ACCT"; - String tags = "QORT-LTC ACCT"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); - - return deployAtTransaction; - } - - private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = sender.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); - System.exit(2); - } - - Long fee = null; - int version = 4; - int nonce = 0; - long amount = 0; - Long assetId = null; // because amount is zero - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); - TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); - - MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); - - fee = messageTransaction.calcRecommendedFee(); - messageTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, messageTransactionData, sender); - - return messageTransaction; - } - - private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { - long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; - int refundTimeout = tradeTimeout / 2 + 1; // close enough - - // AT should automatically refund deployer after 'refundTimeout' blocks - for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) - BlockUtils.mintBlock(repository); - - // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range - long expectedMinimumBalance = deployersPostDeploymentBalance; - long expectedMaximumBalance = deployersInitialBalance - deployAtFee; - - long actualBalance = deployer.getConfirmedBalance(Asset.QORT); - - assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); - assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); - } - - private void describeAt(Repository repository, String atAddress) throws DataException { - ATData atData = repository.getATRepository().fromATAddress(atAddress); - CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); - - Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); - int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); - - System.out.print(String.format("%s:\n" - + "\tmode: %s\n" - + "\tcreator: %s,\n" - + "\tcreation timestamp: %s,\n" - + "\tcurrent balance: %s QORT,\n" - + "\tis finished: %b,\n" - + "\tredeem payout: %s QORT,\n" - + "\texpected Litecoin: %s LTC,\n" - + "\tcurrent block height: %d,\n", - tradeData.qortalAtAddress, - tradeData.mode, - tradeData.qortalCreator, - epochMilliFormatter.apply(tradeData.creationTimestamp), - Amounts.prettyAmount(tradeData.qortBalance), - atData.getIsFinished(), - Amounts.prettyAmount(tradeData.qortAmount), - Amounts.prettyAmount(tradeData.expectedForeignAmount), - currentBlockHeight)); - - if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { - System.out.println(String.format("\trefund timeout: %d minutes,\n" - + "\trefund height: block %d,\n" - + "\tHASH160 of secret-A: %s,\n" - + "\tLitecoin P2SH-A nLockTime: %d (%s),\n" - + "\ttrade partner: %s\n" - + "\tpartner's receiving address: %s", - tradeData.refundTimeout, - tradeData.tradeRefundHeight, - HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), - tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), - tradeData.qortalPartnerAddress, - tradeData.qortalPartnerReceivingAddress)); - } - } - - private PrivateKeyAccount createTradeAccount(Repository repository) { - // We actually use a known test account with funds to avoid PoW compute - return Common.getTestAccount(repository, "alice"); - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java deleted file mode 100644 index 2d04098c..00000000 --- a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java +++ /dev/null @@ -1,90 +0,0 @@ -package org.qortal.test.crosschain.litecoinv1; - -import org.qortal.account.PrivateKeyAccount; -import org.qortal.controller.Controller; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.crypto.Crypto; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.test.crosschain.apps.Common; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.TransactionTransformer; -import org.qortal.utils.Base58; - -public class SendCancelMessage { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: SendCancelMessage ")); - System.err.println(String.format("example: SendCancelMessage " - + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" - + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 2) - usage(null); - - Common.init(); - - byte[] qortalPrivateKey = null; - String atAddress = null; - - int argIndex = 0; - try { - qortalPrivateKey = Base58.decode(args[argIndex++]); - if (qortalPrivateKey.length != 32) - usage("Refund private key must be 32 bytes"); - - atAddress = args[argIndex++]; - if (!Crypto.isValidAtAddress(atAddress)) - usage("Invalid AT address"); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); - System.exit(2); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount qortalAccount = new PrivateKeyAccount(repository, qortalPrivateKey); - - String creatorQortalAddress = qortalAccount.getAddress(); - System.out.println(String.format("Qortal address: %s", creatorQortalAddress)); - - byte[] messageData = LitecoinACCTv1.getInstance().buildCancelMessage(creatorQortalAddress); - MessageTransaction messageTransaction = MessageTransaction.build(repository, qortalAccount, Group.NO_GROUP, atAddress, messageData, false, false); - - System.out.println("Computing nonce..."); - messageTransaction.computeNonce(); - messageTransaction.sign(qortalAccount); - - byte[] signedBytes = null; - try { - signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); - } catch (TransformationException e) { - System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); - System.exit(2); - } - - System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); - } catch (DataException e) { - System.err.println(String.format("Repository issue: %s", e.getMessage())); - System.exit(2); - } - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java deleted file mode 100644 index 20386d2a..00000000 --- a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.qortal.test.crosschain.litecoinv1; - -import org.qortal.account.PrivateKeyAccount; -import org.qortal.controller.Controller; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.crypto.Crypto; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.test.crosschain.apps.Common; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.TransactionTransformer; -import org.qortal.utils.Base58; - -import com.google.common.hash.HashCode; - -public class SendRedeemMessage { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: SendRedeemMessage ")); - System.err.println(String.format("example: SendRedeemMessage " - + "dbfe739f5a3ecf7b0a22cea71f73d86ec71355b740e5972bcdf9e8bb4721ab9d \\\n" - + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \\\n" - + "\t5468697320737472696e672069732065786163746c7920333220627974657321 \\\n" - + "\tQqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 4) - usage(null); - - Common.init(); - - byte[] tradePrivateKey = null; - String atAddress = null; - byte[] secret = null; - String receiveAddress = null; - - int argIndex = 0; - try { - tradePrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); - if (tradePrivateKey.length != 32) - usage("Refund private key must be 32 bytes"); - - atAddress = args[argIndex++]; - if (!Crypto.isValidAtAddress(atAddress)) - usage("Invalid AT address"); - - secret = HashCode.fromString(args[argIndex++]).asBytes(); - if (secret.length != 32) - usage("Secret must be 32 bytes"); - - receiveAddress = args[argIndex++]; - if (!Crypto.isValidAddress(receiveAddress)) - usage("Invalid Qortal receive address"); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); - System.exit(2); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); - - byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secret, receiveAddress); - MessageTransaction messageTransaction = MessageTransaction.build(repository, tradeAccount, Group.NO_GROUP, atAddress, messageData, false, false); - - System.out.println("Computing nonce..."); - messageTransaction.computeNonce(); - messageTransaction.sign(tradeAccount); - - byte[] signedBytes = null; - try { - signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); - } catch (TransformationException e) { - System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); - System.exit(2); - } - - System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); - } catch (DataException e) { - System.err.println(String.format("Repository issue: %s", e.getMessage())); - System.exit(2); - } - } - -} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java deleted file mode 100644 index 83e9a20e..00000000 --- a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.qortal.test.crosschain.litecoinv1; - -import org.qortal.account.PrivateKeyAccount; -import org.qortal.controller.Controller; -import org.qortal.crosschain.LitecoinACCTv1; -import org.qortal.crypto.Crypto; -import org.qortal.group.Group; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; -import org.qortal.test.crosschain.apps.Common; -import org.qortal.transaction.MessageTransaction; -import org.qortal.transform.TransformationException; -import org.qortal.transform.transaction.TransactionTransformer; -import org.qortal.utils.Base58; -import org.qortal.utils.NTP; - -import com.google.common.hash.HashCode; - -public class SendTradeMessage { - - private static void usage(String error) { - if (error != null) - System.err.println(error); - - System.err.println(String.format("usage: SendTradeMessage ")); - System.err.println(String.format("example: SendTradeMessage " - + "ed77aa2c62d785a9428725fc7f95b907be8a1cc43213239876a62cf70fdb6ecb \\\n" - + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \\\n" - + "\tQqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq \\\n" - + "\tffffffffffffffffffffffffffffffffffffffff \\\n" - + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" - + "\t1600184800")); - System.exit(1); - } - - public static void main(String[] args) { - if (args.length != 6) - usage(null); - - Common.init(); - - byte[] tradePrivateKey = null; - String atAddress = null; - String partnerTradeAddress = null; - byte[] partnerTradePublicKeyHash = null; - byte[] hashOfSecret = null; - int lockTime = 0; - - int argIndex = 0; - try { - tradePrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); - if (tradePrivateKey.length != 32) - usage("Refund private key must be 32 bytes"); - - atAddress = args[argIndex++]; - if (!Crypto.isValidAtAddress(atAddress)) - usage("Invalid AT address"); - - partnerTradeAddress = args[argIndex++]; - if (!Crypto.isValidAddress(partnerTradeAddress)) - usage("Invalid partner trade Qortal address"); - - partnerTradePublicKeyHash = HashCode.fromString(args[argIndex++]).asBytes(); - if (partnerTradePublicKeyHash.length != 20) - usage("Partner trade PKH must be 20 bytes"); - - hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); - if (hashOfSecret.length != 20) - usage("HASH160 of secret must be 20 bytes"); - - lockTime = Integer.parseInt(args[argIndex++]); - } catch (IllegalArgumentException e) { - usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); - } - - try { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(repositoryFactory); - } catch (DataException e) { - System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); - System.exit(2); - } - - try (final Repository repository = RepositoryManager.getRepository()) { - PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); - - int refundTimeout = LitecoinACCTv1.calcRefundTimeout(NTP.getTime(), lockTime); - if (refundTimeout < 1) { - System.err.println("Refund timeout too small. Is locktime in the past?"); - System.exit(2); - } - - byte[] messageData = LitecoinACCTv1.buildTradeMessage(partnerTradeAddress, partnerTradePublicKeyHash, hashOfSecret, lockTime, refundTimeout); - MessageTransaction messageTransaction = MessageTransaction.build(repository, tradeAccount, Group.NO_GROUP, atAddress, messageData, false, false); - - System.out.println("Computing nonce..."); - messageTransaction.computeNonce(); - messageTransaction.sign(tradeAccount); - - byte[] signedBytes = null; - try { - signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); - } catch (TransformationException e) { - System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); - System.exit(2); - } - - System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); - } catch (DataException e) { - System.err.println(String.format("Repository issue: %s", e.getMessage())); - System.exit(2); - } - } - -} From b917da765c9406adacbb5abd261c3b6ed236ec74 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 15 Jun 2021 09:29:07 +0100 Subject: [PATCH 028/505] Removed block 212937 --- src/main/java/org/qortal/block/Block.java | 11 -- .../java/org/qortal/block/Block212937.java | 153 ------------------ 2 files changed, 164 deletions(-) delete mode 100644 src/main/java/org/qortal/block/Block212937.java diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 798a4f91..5f6e1641 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1104,10 +1104,6 @@ public class Block { // Create repository savepoint here so we can rollback to it after testing transactions repository.setSavepoint(); - if (this.blockData.getHeight() == 212937) - // Apply fix for block 212937 but fix will be rolled back before we exit method - Block212937.processFix(this); - for (Transaction transaction : this.getTransactions()) { TransactionData transactionData = transaction.getTransactionData(); @@ -1312,9 +1308,6 @@ public class Block { // Distribute block rewards, including transaction fees, before transactions processed processBlockRewards(); - if (this.blockData.getHeight() == 212937) - // Apply fix for block 212937 - Block212937.processFix(this); } // We're about to (test-)process a batch of transactions, @@ -1549,10 +1542,6 @@ public class Block { // Invalidate expandedAccounts as they may have changed due to orphaning TRANSFER_PRIVS transactions, etc. this.cachedExpandedAccounts = null; - if (this.blockData.getHeight() == 212937) - // Revert fix for block 212937 - Block212937.orphanFix(this); - // Block rewards, including transaction fees, removed after transactions undone orphanBlockRewards(); diff --git a/src/main/java/org/qortal/block/Block212937.java b/src/main/java/org/qortal/block/Block212937.java deleted file mode 100644 index a53c9d31..00000000 --- a/src/main/java/org/qortal/block/Block212937.java +++ /dev/null @@ -1,153 +0,0 @@ -package org.qortal.block; - -import java.io.InputStream; -import java.util.List; -import java.util.stream.Collectors; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.UnmarshalException; -import javax.xml.bind.Unmarshaller; -import javax.xml.transform.stream.StreamSource; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.eclipse.persistence.jaxb.JAXBContextFactory; -import org.eclipse.persistence.jaxb.UnmarshallerProperties; -import org.qortal.data.account.AccountBalanceData; -import org.qortal.repository.DataException; - -/** - * Block 212937 - *

- * Somehow a node minted a version of block 212937 that contained one transaction: - * a PAYMENT transaction that attempted to spend more QORT than that account had as QORT balance. - *

- * This invalid transaction made block 212937 (rightly) invalid to several nodes, - * which refused to use that block. - * However, it seems there were no other nodes minting an alternative, valid block at that time - * and so the chain stalled for several nodes in the network. - *

- * Additionally, the invalid block 212937 affected all new installations, regardless of whether - * they synchronized from scratch (block 1) or used an 'official release' bootstrap. - *

- * After lengthy diagnosis, it was discovered that - * the invalid transaction seemed to rely on incorrect balances in a corrupted database. - * Copies of DB files containing the broken chain were also shared around, exacerbating the problem. - *

- * There were three options: - *

    - *
  1. roll back the chain to last known valid block 212936 and re-mint empty blocks to current height
    2. - *
  2. keep existing chain, but apply database edits at block 212937 to allow current chain to be valid
    3. - *
  3. attempt to mint an alternative chain, retaining as many valid transactions as possible
    4. - *
- *

- * Option 1 was highly undesirable due to knock-on effects from wiping 700+ transactions, some of which - * might have affect cross-chain trades, although there were no cross-chain trade completed during - * the decision period. - *

- * Option 3 was essentially a slightly better version of option 1 and rejected for similar reasons. - * Attempts at option 3 also rapidly hit cumulative problems with every replacement block due to - * differing block timestamps making some transactions, and then even some blocks themselves, invalid. - *

- * This class is the implementation of option 2. - *

- * The change in account balances are relatively small, see block-212937-deltas.json resource - * for actual values. These values were obtained by exporting the AccountBalances table from - * both versions of the database with chain at block 212936, and then comparing. The values were also - * tested by syncing both databases up to block 225500, re-exporting and re-comparing. - *

- * The invalid block 212937 signature is: 2J3GVJjv...qavh6KkQ. - *

- * The invalid transaction in block 212937 is: - *

- * 

-   {
-      "amount" : "0.10788294",
-      "approvalStatus" : "NOT_REQUIRED",
-      "blockHeight" : 212937,
-      "creatorAddress" : "QLdw5uabviLJgRGkRiydAFmAtZzxHfNXSs",
-      "fee" : "0.00100000",
-      "recipient" : "QZi1mNHDbiLvsytxTgxDr9nhJe4pNZaWpw",
-      "reference" : "J6JukdTVuXZ3JYbHatfZzwxG2vSiZwVCPDzW5K7PsVQKRj8XZeDtqnkGCGGjaSQZ9bQMtV44ky88NnGM4YBQKU6",
-      "senderPublicKey" : "DBFfbD2M3uh4jPE5PaUcZVvNPfrrJzVB7seeEtBn5SPs",
-      "signature" : "qkitxdCEEnKt8w6wRfFixtErbXsxWE6zG2ESNhpqBdScikV1WxeA6WZTTMJVV4tCeZdBFXw3V1X5NVztv6LirWK",
-      "timestamp" : 1607863074904,
-      "txGroupId" : 0,
-      "type" : "PAYMENT"
-   }
-
- *

- * Account QLdw5uabviLJgRGkRiydAFmAtZzxHfNXSs attempted to spend 0.10888294 (including fees) - * when their QORT balance was really only 0.10886665. - *

- * However, on the broken DB nodes, their balance - * seemed to be 0.10890293 which was sufficient to make the transaction valid. - */ -public final class Block212937 { - - private static final Logger LOGGER = LogManager.getLogger(Block212937.class); - private static final String ACCOUNT_DELTAS_SOURCE = "block-212937-deltas.json"; - - private static final List accountDeltas = readAccountDeltas(); - - private Block212937() { - /* Do not instantiate */ - } - - @SuppressWarnings("unchecked") - private static List readAccountDeltas() { - Unmarshaller unmarshaller; - - try { - // Create JAXB context aware of classes we need to unmarshal - JAXBContext jc = JAXBContextFactory.createContext(new Class[] { - AccountBalanceData.class - }, null); - - // Create unmarshaller - unmarshaller = jc.createUnmarshaller(); - - // Set the unmarshaller media type to JSON - unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json"); - - // Tell unmarshaller that there's no JSON root element in the JSON input - unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false); - } catch (JAXBException e) { - String message = "Failed to setup unmarshaller to read block 212937 deltas"; - LOGGER.error(message, e); - throw new RuntimeException(message, e); - } - - ClassLoader classLoader = BlockChain.class.getClassLoader(); - InputStream in = classLoader.getResourceAsStream(ACCOUNT_DELTAS_SOURCE); - StreamSource jsonSource = new StreamSource(in); - - try { - // Attempt to unmarshal JSON stream to BlockChain config - return (List) unmarshaller.unmarshal(jsonSource, AccountBalanceData.class).getValue(); - } catch (UnmarshalException e) { - String message = "Failed to parse block 212937 deltas"; - LOGGER.error(message, e); - throw new RuntimeException(message, e); - } catch (JAXBException e) { - String message = "Unexpected JAXB issue while processing block 212937 deltas"; - LOGGER.error(message, e); - throw new RuntimeException(message, e); - } - } - - public static void processFix(Block block) throws DataException { - block.repository.getAccountRepository().modifyAssetBalances(accountDeltas); - } - - public static void orphanFix(Block block) throws DataException { - // Create inverse deltas - List inverseDeltas = accountDeltas.stream() - .map(delta -> new AccountBalanceData(delta.getAddress(), delta.getAssetId(), 0 - delta.getBalance())) - .collect(Collectors.toList()); - - block.repository.getAccountRepository().modifyAssetBalances(inverseDeltas); - } - -} From 9fb58c7ae353248b8ee74fb3dbb1bb79543d705c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 16 Jun 2021 19:10:35 +0100 Subject: [PATCH 029/505] Added the beginnings of an upload API, with some basic data file management. --- .gitignore | 1 + .../java/org/qortal/storage/DataFile.java | 260 ++++++++++++++++++ .../org/qortal/storage/DataFileChunk.java | 70 +++++ 3 files changed, 331 insertions(+) create mode 100644 src/main/java/org/qortal/storage/DataFile.java create mode 100644 src/main/java/org/qortal/storage/DataFileChunk.java diff --git a/.gitignore b/.gitignore index 890f8cb2..8db57002 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ /run.pid /run.log /WindowsInstaller/Install Files/qortal.jar +/data/* diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java new file mode 100644 index 00000000..36e2b09d --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -0,0 +1,260 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; +import org.qortal.utils.Base58; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + + +public class DataFile { + + // Validation results + public enum ValidationResult { + OK(1), + FILE_TOO_LARGE(10), + FILE_NOT_FOUND(11); + + public final int value; + + private static final Map map = stream(DataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result)); + + ValidationResult(int value) { + this.value = value; + } + + public static DataFile.ValidationResult valueOf(int value) { + return map.get(value); + } + } + + private static final Logger LOGGER = LogManager.getLogger(DataFile.class); + + public static final long MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MiB + public static final int CHUNK_SIZE = 2 * 1024 * 1024; // 2MiB + public static int SHORT_DIGEST_LENGTH = 8; + + protected String filePath; + private ArrayList chunks; + protected String base58Digest; + + public DataFile() { + } + + public DataFile(String filePath) { + this.createDataDirectory(); + this.filePath = filePath; + + if (!this.isInBaseDirectory(filePath)) { + // Copy file to base directory + LOGGER.debug("Copying file to data directory..."); + this.filePath = this.copyToDataDirectory(); + if (this.filePath == null) { + throw new IllegalStateException("Invalid file path after copy"); + } + } + } + + public DataFile(File file) { + this(file.getPath()); + } + + private boolean createDataDirectory() { + // Create the data directory if it doesn't exist + Path dataDirectory = Paths.get("data"); // TODO: allow custom directory in settings + try { + Files.createDirectories(dataDirectory); + } catch (IOException e) { + LOGGER.error("Unable to create data directory"); + return false; + } + return true; + } + + private String copyToDataDirectory() { + String outputFilePath = this.getOutputFilePath(this.base58Digest()); + Path source = Paths.get(this.filePath).toAbsolutePath(); + Path dest = Paths.get(outputFilePath).toAbsolutePath(); + try { + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + return dest.toString(); + } catch (IOException e) { + throw new IllegalStateException("Unable to copy file to data directory"); + } + } + + protected String getOutputFilePath(String base58Digest) { + String base58Digest2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); + String outputDirectory = String.format("data/%s", base58Digest2Chars); // TODO: allow custom directory in settings + Path outputDirectoryPath = Paths.get(outputDirectory); + + try { + Files.createDirectories(outputDirectoryPath); + } catch (IOException e) { + throw new IllegalStateException("Unable to create data subdirectory"); + } + return String.format("%s/%s.dat", outputDirectory, base58Digest); + } + + public ValidationResult isValid() { + try { + // Ensure the file exists on disk + Path path = Paths.get(this.filePath); + if (!Files.exists(path)) { + LOGGER.error("File doesn't exist at path {}", this.filePath); + return ValidationResult.FILE_NOT_FOUND; + } + + // Validate the file size + long fileSize = Files.size(path); + if (fileSize > MAX_FILE_SIZE) { + LOGGER.error(String.format("DataFile is too large: %d bytes (max chunk size: %d bytes)", fileSize, MAX_FILE_SIZE)); + return DataFile.ValidationResult.FILE_TOO_LARGE; + } + + } catch (IOException e) { + return ValidationResult.FILE_NOT_FOUND; + } + + return ValidationResult.OK; + } + + public int split() { + try { + + File file = this.getFile(); + byte[] buffer = new byte[CHUNK_SIZE]; + this.chunks = new ArrayList<>(); + + if (file != null) { + try (FileInputStream fileInputStream = new FileInputStream(file); + BufferedInputStream bis = new BufferedInputStream(fileInputStream)) { + + int numberOfBytes; + while ((numberOfBytes = bis.read(buffer)) > 0) { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + out.write(buffer, 0, numberOfBytes); + out.flush(); + + DataFileChunk chunk = new DataFileChunk(out.toByteArray()); + ValidationResult validationResult = chunk.isValid(); + if (validationResult == ValidationResult.OK) { + this.chunks.add(chunk); + } else { + throw new IllegalStateException(String.format("Chunk %s is invalid", chunk)); + } + } + } + } + } + } catch (Exception e) { + throw new IllegalStateException("Unable to split file into chunks"); + } + + return this.chunks.size(); + } + + public void delete() { + // Delete the complete file + Path path = Paths.get(this.filePath); + if (Files.exists(path)) { + try { + Files.delete(path); + this.cleanupFilesystem(); + LOGGER.debug("Deleted file {}", path.toString()); + } catch (IOException e) { + LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath); + } + } + } + + public void deleteAll() { + // Delete the complete file + this.delete(); + + // Delete the individual chunks + if (this.chunks != null && this.chunks.size() > 0) { + Iterator iterator = this.chunks.iterator(); + while (iterator.hasNext()) { + DataFileChunk chunk = (DataFileChunk) iterator.next(); + chunk.delete(); + iterator.remove(); + } + } + } + + protected void cleanupFilesystem() { + Path directory = Paths.get(this.filePath).getParent().toAbsolutePath(); + try (Stream files = Files.list(directory)) { + final long count = files.count(); + if (count == 0) { + Files.delete(directory); + } + } catch (IOException e) { + LOGGER.warn("Unable to count files in directory", e); + } + } + + + /* Helper methods */ + + private boolean isInBaseDirectory(String filePath) { + Path path = Paths.get(filePath).toAbsolutePath(); + String basePath = Paths.get("data").toAbsolutePath().toString(); // TODO: allow custom directory in settings + if (path.startsWith(basePath)) { + return true; + } + return false; + } + + private File getFile() { + File file = new File(this.filePath); + if (file.exists()) { + return file; + } + return null; + } + + public byte[] digest() { + File file = this.getFile(); + if (file != null && file.exists()) { + try { + byte[] fileContent = Files.readAllBytes(file.toPath()); + return Crypto.digest(fileContent); + + } catch (IOException e) { + LOGGER.error("Couldn't compute digest for DataFile"); + } + } + return null; + } + + public String base58Digest() { + if (this.base58Digest == null) { + this.base58Digest = Base58.encode(this.digest()); + } + return this.base58Digest; + } + + public String shortDigest() { + if (this.base58Digest() == null) { + return null; + } + return this.base58Digest().substring(0, Math.min(this.base58Digest().length(), SHORT_DIGEST_LENGTH)); + } + public String toString() { + return this.shortDigest(); + } +} diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/DataFileChunk.java new file mode 100644 index 00000000..82fd2891 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileChunk.java @@ -0,0 +1,70 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; +import org.qortal.utils.Base58; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + + +public class DataFileChunk extends DataFile { + + private static final Logger LOGGER = LogManager.getLogger(DataFileChunk.class); + + public DataFileChunk() { + } + + public DataFileChunk(byte[] fileContent) { + if (fileContent == null) { + LOGGER.error("Chunk fileContent is null"); + return; + } + + String base58Digest = Base58.encode(Crypto.digest(fileContent)); + LOGGER.debug(String.format("Chunk digest: %s, size: %d bytes", base58Digest, fileContent.length)); + + String outputFilePath = this.getOutputFilePath(base58Digest); + File outputFile = new File(outputFilePath); + try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { + outputStream.write(fileContent); + this.filePath = outputFilePath; + // Verify hash + if (!base58Digest.equals(this.base58Digest())) { + LOGGER.error("Digest {} does not match file digest {}", base58Digest, this.base58Digest()); + throw new IllegalStateException("DataFileChunk digest validation failed"); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to write chunk data to file"); + } + } + + @Override + public ValidationResult isValid() { + // DataChunk validation applies here too + ValidationResult superclassValidationResult = super.isValid(); + if (superclassValidationResult != ValidationResult.OK) { + return superclassValidationResult; + } + + Path path = Paths.get(this.filePath); + try { + // Validate the file size (chunks have stricter limits) + long fileSize = Files.size(path); + if (fileSize > CHUNK_SIZE) { + LOGGER.error(String.format("DataFileChunk is too large: %d bytes (max chunk size: %d bytes)", fileSize, CHUNK_SIZE)); + return ValidationResult.FILE_TOO_LARGE; + } + + } catch (IOException e) { + return ValidationResult.FILE_NOT_FOUND; + } + + return ValidationResult.OK; + } +} From 120552b36eb7924fcf98925b576b86bbd4d6343f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 16 Jun 2021 19:13:24 +0100 Subject: [PATCH 030/505] Added resource file missing from last commit. --- .../org/qortal/api/resource/DataResource.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 src/main/java/org/qortal/api/resource/DataResource.java diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java new file mode 100644 index 00000000..ccc1547f --- /dev/null +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -0,0 +1,98 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.storage.DataFile; +import org.qortal.storage.DataFile.ValidationResult; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Path; +import javax.ws.rs.POST; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + + +@Path("/data") +@Tag(name = "Data") +public class DataResource { + + private static final Logger LOGGER = LogManager.getLogger(DataResource.class); + + @Context + HttpServletRequest request; + + @POST + @Path("/upload/path") + @Operation( + summary = "Build raw, unsigned, UPLOAD_DATA transaction, based on a user-supplied file path", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "qortal.jar" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, UPLOAD_DATA transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public String uploadFile(String filePath) { + Security.checkApiCallAllowed(request); + + // It's too dangerous to allow user-supplied filenames in weaker security contexts + if (Settings.getInstance().isApiRestricted()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); + + try (final Repository repository = RepositoryManager.getRepository()) { + + DataFile dataFile = new DataFile(filePath); + ValidationResult validationResult = dataFile.isValid(); + if (validationResult != DataFile.ValidationResult.OK) { + LOGGER.error("Invalid file: {}", validationResult); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); + + int chunkCount = dataFile.split(); + if (chunkCount > 0) { + LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + return "true"; + } + + return "false"; + + } catch (DataException e) { + LOGGER.error("Repository issue when uploading data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (IllegalStateException e) { + LOGGER.error("Invalid upload data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); + } + } + +} From 9407e7e41870312f3cfb4a069c112022c9125a92 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 16 Jun 2021 19:22:50 +0100 Subject: [PATCH 031/505] Data storage location moved to settings ("dataPath") --- src/main/java/org/qortal/settings/Settings.java | 12 ++++++++++++ src/main/java/org/qortal/storage/DataFile.java | 9 ++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index f17324df..ae8c6275 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -179,6 +179,14 @@ public class Settings { /** Additional offset added to values returned by NTP.getTime() */ private Long testNtpOffset = null; + + // Data storage + + /** Data storage path. */ + private String dataPath = "data"; + + + // Constructors private Settings() { @@ -525,4 +533,8 @@ public class Settings { public List getFixedNetwork() { return fixedNetwork; } + + public String getDataPath() { + return this.dataPath; + } } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 36e2b09d..b4afce7b 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -3,6 +3,7 @@ package org.qortal.storage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.crypto.Crypto; +import org.qortal.settings.Settings; import org.qortal.utils.Base58; import java.io.*; @@ -73,7 +74,8 @@ public class DataFile { private boolean createDataDirectory() { // Create the data directory if it doesn't exist - Path dataDirectory = Paths.get("data"); // TODO: allow custom directory in settings + String dataPath = Settings.getInstance().getDataPath(); + Path dataDirectory = Paths.get(dataPath); try { Files.createDirectories(dataDirectory); } catch (IOException e) { @@ -97,7 +99,7 @@ public class DataFile { protected String getOutputFilePath(String base58Digest) { String base58Digest2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); - String outputDirectory = String.format("data/%s", base58Digest2Chars); // TODO: allow custom directory in settings + String outputDirectory = String.format("%s/%s", Settings.getInstance().getDataPath(), base58Digest2Chars); Path outputDirectoryPath = Paths.get(outputDirectory); try { @@ -212,7 +214,8 @@ public class DataFile { private boolean isInBaseDirectory(String filePath) { Path path = Paths.get(filePath).toAbsolutePath(); - String basePath = Paths.get("data").toAbsolutePath().toString(); // TODO: allow custom directory in settings + String dataPath = Settings.getInstance().getDataPath(); + String basePath = Paths.get(dataPath).toAbsolutePath().toString(); if (path.startsWith(basePath)) { return true; } From 76742c386998cacec5e1a2f4bcb9682e3f72ec72 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 16 Jun 2021 19:49:12 +0100 Subject: [PATCH 032/505] Removed .dat extension, and use an extra level in the directory structure (data/aB/cD/aBcDeF... etc) --- .../java/org/qortal/storage/DataFile.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index b4afce7b..f3d09c50 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -98,8 +98,9 @@ public class DataFile { } protected String getOutputFilePath(String base58Digest) { - String base58Digest2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); - String outputDirectory = String.format("%s/%s", Settings.getInstance().getDataPath(), base58Digest2Chars); + String base58DigestFirst2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); + String base58DigestNext2Chars = base58Digest.substring(2, Math.min(base58Digest.length(), 4)); + String outputDirectory = String.format("%s/%s/%s", Settings.getInstance().getDataPath(), base58DigestFirst2Chars, base58DigestNext2Chars); Path outputDirectoryPath = Paths.get(outputDirectory); try { @@ -107,7 +108,7 @@ public class DataFile { } catch (IOException e) { throw new IllegalStateException("Unable to create data subdirectory"); } - return String.format("%s/%s.dat", outputDirectory, base58Digest); + return String.format("%s/%s", outputDirectory, base58Digest); } public ValidationResult isValid() { @@ -198,14 +199,20 @@ public class DataFile { } protected void cleanupFilesystem() { - Path directory = Paths.get(this.filePath).getParent().toAbsolutePath(); - try (Stream files = Files.list(directory)) { - final long count = files.count(); - if (count == 0) { - Files.delete(directory); + String path = this.filePath; + + // Iterate through two levels of parent directories, and delete if empty + for (int i=0; i<2; i++) { + Path directory = Paths.get(path).getParent().toAbsolutePath(); + try (Stream files = Files.list(directory)) { + final long count = files.count(); + if (count == 0) { + Files.delete(directory); + } + } catch (IOException e) { + LOGGER.warn("Unable to count files in directory", e); } - } catch (IOException e) { - LOGGER.warn("Unable to count files in directory", e); + path = directory.toString(); } } From f82f2bd2874b36c5224d1e683cc5be0cd9c581d0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 16 Jun 2021 20:03:15 +0100 Subject: [PATCH 033/505] Added DELETE /data/file API endpoint This deletes a file referenced by a user supplied SHA256 digest string (which we will use as the file's "ID" in the Qortal data system). In the future this could be extended to delete all associated chunks, but first we need to build out the data chain so we have a way to look up chunks associated with a file hash. --- .../org/qortal/api/resource/DataResource.java | 37 +++++++++++++++++++ .../java/org/qortal/storage/DataFile.java | 17 +++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index ccc1547f..b17ec0ad 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -20,6 +20,7 @@ import org.qortal.storage.DataFile; import org.qortal.storage.DataFile.ValidationResult; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; import javax.ws.rs.Path; import javax.ws.rs.POST; import javax.ws.rs.core.Context; @@ -95,4 +96,40 @@ public class DataResource { } } + @DELETE + @Path("/file") + @Operation( + summary = "Delete file using supplied base58 encoded SHA256 digest string", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" + ) + ) + ), + responses = { + @ApiResponse( + description = "true if deleted, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public String deleteFile(String base58Digest) { + Security.checkApiCallAllowed(request); + + DataFile dataFile = DataFile.fromBase58Digest(base58Digest); + if (dataFile.delete()) { + return "true"; + } + return "false"; + } + } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index f3d09c50..ad35aeb7 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -72,6 +72,11 @@ public class DataFile { this(file.getPath()); } + public static DataFile fromBase58Digest(String base58Digest) { + String filePath = DataFile.getOutputFilePath(base58Digest); + return new DataFile(filePath); + } + private boolean createDataDirectory() { // Create the data directory if it doesn't exist String dataPath = Settings.getInstance().getDataPath(); @@ -97,7 +102,7 @@ public class DataFile { } } - protected String getOutputFilePath(String base58Digest) { + public static String getOutputFilePath(String base58Digest) { String base58DigestFirst2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); String base58DigestNext2Chars = base58Digest.substring(2, Math.min(base58Digest.length(), 4)); String outputDirectory = String.format("%s/%s/%s", Settings.getInstance().getDataPath(), base58DigestFirst2Chars, base58DigestNext2Chars); @@ -169,7 +174,7 @@ public class DataFile { return this.chunks.size(); } - public void delete() { + public boolean delete() { // Delete the complete file Path path = Paths.get(this.filePath); if (Files.exists(path)) { @@ -177,15 +182,17 @@ public class DataFile { Files.delete(path); this.cleanupFilesystem(); LOGGER.debug("Deleted file {}", path.toString()); + return true; } catch (IOException e) { LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath); } } + return false; } - public void deleteAll() { + public boolean deleteAll() { // Delete the complete file - this.delete(); + boolean success = this.delete(); // Delete the individual chunks if (this.chunks != null && this.chunks.size() > 0) { @@ -194,8 +201,10 @@ public class DataFile { DataFileChunk chunk = (DataFileChunk) iterator.next(); chunk.delete(); iterator.remove(); + success = true; } } + return success; } protected void cleanupFilesystem() { From 1e8dbfe4b7f834db9165787ae5cb83d990df3fef Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 17 Jun 2021 18:18:19 +0100 Subject: [PATCH 034/505] Delete chunk if it fails the hash validation in the constructor. --- src/main/java/org/qortal/storage/DataFileChunk.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/DataFileChunk.java index 82fd2891..d020c086 100644 --- a/src/main/java/org/qortal/storage/DataFileChunk.java +++ b/src/main/java/org/qortal/storage/DataFileChunk.java @@ -37,6 +37,7 @@ public class DataFileChunk extends DataFile { // Verify hash if (!base58Digest.equals(this.base58Digest())) { LOGGER.error("Digest {} does not match file digest {}", base58Digest, this.base58Digest()); + this.delete(); throw new IllegalStateException("DataFileChunk digest validation failed"); } } catch (IOException e) { From fa11f4f45b7a4a3eaef43efb7dee36519983d3e6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 17 Jun 2021 18:20:11 +0100 Subject: [PATCH 035/505] Moved DataFileChunk(byte[] fileContent) constructor to the superclass so it can be used by regular data files too. --- .../java/org/qortal/storage/DataFile.java | 25 +++++++++++++++++++ .../org/qortal/storage/DataFileChunk.java | 23 +---------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index ad35aeb7..e4d727d1 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -72,6 +72,31 @@ public class DataFile { this(file.getPath()); } + public DataFile(byte[] fileContent) { + if (fileContent == null) { + LOGGER.error("fileContent is null"); + return; + } + + String base58Digest = Base58.encode(Crypto.digest(fileContent)); + LOGGER.debug(String.format("File digest: %s, size: %d bytes", base58Digest, fileContent.length)); + + String outputFilePath = this.getOutputFilePath(base58Digest); + File outputFile = new File(outputFilePath); + try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { + outputStream.write(fileContent); + this.filePath = outputFilePath; + // Verify hash + if (!base58Digest.equals(this.base58Digest())) { + LOGGER.error("Digest {} does not match file digest {}", base58Digest, this.base58Digest()); + this.delete(); + throw new IllegalStateException("Data file digest validation failed"); + } + } catch (IOException e) { + throw new IllegalStateException("Unable to write data to file"); + } + } + public static DataFile fromBase58Digest(String base58Digest) { String filePath = DataFile.getOutputFilePath(base58Digest); return new DataFile(filePath); diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/DataFileChunk.java index d020c086..e7805d48 100644 --- a/src/main/java/org/qortal/storage/DataFileChunk.java +++ b/src/main/java/org/qortal/storage/DataFileChunk.java @@ -21,28 +21,7 @@ public class DataFileChunk extends DataFile { } public DataFileChunk(byte[] fileContent) { - if (fileContent == null) { - LOGGER.error("Chunk fileContent is null"); - return; - } - - String base58Digest = Base58.encode(Crypto.digest(fileContent)); - LOGGER.debug(String.format("Chunk digest: %s, size: %d bytes", base58Digest, fileContent.length)); - - String outputFilePath = this.getOutputFilePath(base58Digest); - File outputFile = new File(outputFilePath); - try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { - outputStream.write(fileContent); - this.filePath = outputFilePath; - // Verify hash - if (!base58Digest.equals(this.base58Digest())) { - LOGGER.error("Digest {} does not match file digest {}", base58Digest, this.base58Digest()); - this.delete(); - throw new IllegalStateException("DataFileChunk digest validation failed"); - } - } catch (IOException e) { - throw new IllegalStateException("Unable to write chunk data to file"); - } + super(fileContent); } @Override From abfe0a925a1733748abb5e7dabafd12fbeef8c24 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 17 Jun 2021 18:20:42 +0100 Subject: [PATCH 036/505] More DataFile methods and improvements --- .../java/org/qortal/storage/DataFile.java | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index e4d727d1..0a54555b 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -102,6 +102,10 @@ public class DataFile { return new DataFile(filePath); } + public static DataFile fromDigest(byte[] digest) { + return DataFile.fromBase58Digest(Base58.encode(digest)); + } + private boolean createDataDirectory() { // Create the data directory if it doesn't exist String dataPath = Settings.getInstance().getDataPath(); @@ -153,7 +157,7 @@ public class DataFile { // Validate the file size long fileSize = Files.size(path); if (fileSize > MAX_FILE_SIZE) { - LOGGER.error(String.format("DataFile is too large: %d bytes (max chunk size: %d bytes)", fileSize, MAX_FILE_SIZE)); + LOGGER.error(String.format("DataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE)); return DataFile.ValidationResult.FILE_TOO_LARGE; } @@ -250,6 +254,18 @@ public class DataFile { } } + public byte[] getBytes() { + Path path = Paths.get(this.filePath); + try { + byte[] bytes = Files.readAllBytes(path); + LOGGER.info("getBytes: %d", bytes); + return bytes; + } catch (IOException e) { + LOGGER.error("Unable to read bytes for file"); + return null; + } + } + /* Helper methods */ @@ -263,6 +279,20 @@ public class DataFile { return false; } + public boolean exists() { + File file = new File(this.filePath); + return file.exists(); + } + + public long size() { + Path path = Paths.get(this.filePath); + try { + return Files.size(path); + } catch (IOException e) { + return 0; + } + } + private File getFile() { File file = new File(this.filePath); if (file.exists()) { @@ -298,6 +328,8 @@ public class DataFile { } return this.base58Digest().substring(0, Math.min(this.base58Digest().length(), SHORT_DIGEST_LENGTH)); } + + @Override public String toString() { return this.shortDigest(); } From 5ac676d20100b7da30ffe6c4cf403f76ba0b65c4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 17 Jun 2021 19:05:00 +0100 Subject: [PATCH 037/505] Added DataFileMessage and GetDataFileMessage, used for requesting and sending files between peers. --- .../network/message/DataFileMessage.java | 44 ++++++++++++++++ .../network/message/GetDataFileMessage.java | 52 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 src/main/java/org/qortal/network/message/DataFileMessage.java create mode 100644 src/main/java/org/qortal/network/message/GetDataFileMessage.java diff --git a/src/main/java/org/qortal/network/message/DataFileMessage.java b/src/main/java/org/qortal/network/message/DataFileMessage.java new file mode 100644 index 00000000..6ea7e05c --- /dev/null +++ b/src/main/java/org/qortal/network/message/DataFileMessage.java @@ -0,0 +1,44 @@ +package org.qortal.network.message; + +import org.qortal.storage.DataFile; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +public class DataFileMessage extends Message { + + private final DataFile dataFile; + + public DataFileMessage(DataFile dataFile) { + super(MessageType.DATA_FILE); + + this.dataFile = dataFile; + } + + public DataFile getDataFile() { + return this.dataFile; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + DataFile dataFile = new DataFile(bytes); + + return new DataFileMessage(dataFile); + } + + @Override + protected byte[] toData() { + if (this.dataFile == null) + return null; + + return this.dataFile.getBytes(); + } + + public DataFileMessage cloneWithNewId(int newId) { + DataFileMessage clone = new DataFileMessage(this.dataFile); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/GetDataFileMessage.java b/src/main/java/org/qortal/network/message/GetDataFileMessage.java new file mode 100644 index 00000000..d4171b42 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetDataFileMessage.java @@ -0,0 +1,52 @@ +package org.qortal.network.message; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +public class GetDataFileMessage extends Message { + + private static final int DIGEST_LENGTH = 32; + + private final byte[] digest; + + public GetDataFileMessage(byte[] digest) { + this(-1, digest); + } + + private GetDataFileMessage(int id, byte[] digest) { + super(id, MessageType.GET_DATA_FILE); + + this.digest = digest; + } + + public byte[] getDigest() { + return this.digest; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + if (bytes.remaining() != DIGEST_LENGTH) + return null; + + byte[] digest = new byte[DIGEST_LENGTH]; + + bytes.get(digest); + + return new GetDataFileMessage(id, digest); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.digest); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + +} From 592490d70944b4a3c684c6a19db700abe789cd19 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Jun 2021 07:59:46 +0100 Subject: [PATCH 038/505] Added GET /data/file/frompeer API endpoint This requests a file from the supplied peer address, and stores a copy locally if successful. Still a work in progress. --- .../org/qortal/api/resource/DataResource.java | 80 +++++++++++++++++-- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index b17ec0ad..639d5ed8 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -8,23 +8,26 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiExceptionFactory; -import org.qortal.api.Security; +import org.qortal.api.*; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.PeerAddress; +import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.storage.DataFile.ValidationResult; +import org.qortal.utils.Base58; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.DELETE; -import javax.ws.rs.Path; -import javax.ws.rs.POST; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.List; @Path("/data") @@ -121,7 +124,6 @@ public class DataResource { ) } ) - @ApiErrors({ApiError.REPOSITORY_ISSUE}) public String deleteFile(String base58Digest) { Security.checkApiCallAllowed(request); @@ -132,4 +134,66 @@ public class DataResource { return "false"; } + @GET + @Path("/file/frompeer") + @Operation( + summary = "Request file from a given peer, using supplied base58 encoded SHA256 digest string", + responses = { + @ApiResponse( + description = "true if retrieved, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public String getFileFromPeer(@QueryParam("base58Digest") String base58Digest, + @QueryParam("peer") String targetPeerAddress) { + + try (final Repository repository = RepositoryManager.getRepository()) { + + if (base58Digest == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + if (targetPeerAddress == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + // Try to resolve passed address to make things easier + PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); + InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); + + List peers = Network.getInstance().getHandshakedPeers(); + Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().equals(resolvedAddress)).findFirst().orElse(null); + + if (targetPeer == null) { + LOGGER.error("Peer not connected"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + DataFile dataFile = DataFile.fromBase58Digest(base58Digest); + if (dataFile.exists()) { + LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); + } + Message getDataFileMessage = new GetDataFileMessage(Base58.decode(base58Digest)); + + Message message = targetPeer.getResponse(getDataFileMessage); + if (message == null || message.getType() != Message.MessageType.DATA_FILE) + return "invalid file received"; + + DataFileMessage dataFileMessage = (DataFileMessage) message; + + return String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size()); + } catch (ApiException e) { + throw e; + } catch (DataException | InterruptedException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (UnknownHostException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } } From f4ba7b2a0c95b1c878db2d7a4a32b9db279b3d2c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Jun 2021 08:02:13 +0100 Subject: [PATCH 039/505] Added onNetworkGetDataFileMessage() handler --- .../org/qortal/controller/Controller.java | 135 ++++++++++-------- 1 file changed, 74 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 1e74f365..67835dd2 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1,39 +1,6 @@ package org.qortal.controller; -import java.awt.TrayIcon.MessageType; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.security.SecureRandom; -import java.security.Security; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Deque; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; - +import com.google.common.primitives.Longs; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; @@ -55,52 +22,50 @@ import org.qortal.data.network.OnlineAccountData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.DataType; +import org.qortal.data.transaction.ChatTransactionData; +import org.qortal.data.transaction.TransactionData; import org.qortal.event.Event; import org.qortal.event.EventBus; -import org.qortal.data.transaction.ChatTransactionData; import org.qortal.globalization.Translator; import org.qortal.gui.Gui; import org.qortal.gui.SysTray; import org.qortal.network.Network; import org.qortal.network.Peer; -import org.qortal.network.message.ArbitraryDataMessage; -import org.qortal.network.message.BlockSummariesMessage; -import org.qortal.network.message.BlocksMessage; -import org.qortal.network.message.CachedBlockMessage; -import org.qortal.network.message.GetArbitraryDataMessage; -import org.qortal.network.message.GetBlockMessage; -import org.qortal.network.message.GetBlocksMessage; -import org.qortal.network.message.GetBlockSummariesMessage; -import org.qortal.network.message.GetOnlineAccountsMessage; -import org.qortal.network.message.GetPeersMessage; -import org.qortal.network.message.GetSignaturesV2Message; -import org.qortal.network.message.GetTransactionMessage; -import org.qortal.network.message.GetUnconfirmedTransactionsMessage; -import org.qortal.network.message.HeightV2Message; -import org.qortal.network.message.Message; -import org.qortal.network.message.OnlineAccountsMessage; -import org.qortal.network.message.SignaturesMessage; -import org.qortal.network.message.TransactionMessage; -import org.qortal.network.message.TransactionSignaturesMessage; +import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryFactory; import org.qortal.repository.RepositoryManager; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; +import org.qortal.storage.DataFile; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.utils.Base58; -import org.qortal.utils.ByteArray; -import org.qortal.utils.DaemonThreadFactory; -import org.qortal.utils.NTP; -import org.qortal.utils.Triple; +import org.qortal.utils.*; -import com.google.common.primitives.Longs; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.awt.TrayIcon.MessageType; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.security.SecureRandom; +import java.security.Security; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT; @@ -257,6 +222,15 @@ public class Controller extends Thread { } public GetBlockSignaturesV2Stats getBlockSignaturesV2Stats = new GetBlockSignaturesV2Stats(); + public static class GetDataFileMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownFiles = new AtomicLong(); + + public GetDataFileMessageStats() { + } + } + public GetDataFileMessageStats getDataFileMessageStats = new GetDataFileMessageStats(); + public AtomicLong latestBlocksCacheRefills = new AtomicLong(); public StatsSnapshot() { @@ -1259,6 +1233,10 @@ public class Controller extends Thread { onNetworkOnlineAccountsMessage(peer, message); break; + case GET_DATA_FILE: + onNetworkGetDataFileMessage(peer, message); + break; + default: LOGGER.debug(() -> String.format("Unhandled %s message [ID %d] from peer %s", message.getType().name(), message.getId(), peer)); break; @@ -1762,6 +1740,41 @@ public class Controller extends Thread { } } + private void onNetworkGetDataFileMessage(Peer peer, Message message) { + GetDataFileMessage getDataFileMessage = (GetDataFileMessage) message; + byte[] digest = getDataFileMessage.getDigest(); + this.stats.getDataFileMessageStats.requests.incrementAndGet(); + + DataFile dataFile = DataFile.fromDigest(digest); + if (dataFile.exists()) { + DataFileMessage dataFileMessage = new DataFileMessage(dataFile); + dataFileMessage.setId(message.getId()); + if (!peer.sendMessage(dataFileMessage)) { + LOGGER.info("Couldn't sent file"); + peer.disconnect("failed to send file"); + } + LOGGER.info("Sent file {}", dataFile); + } + else { + + // We don't have this file + this.stats.getDataFileMessageStats.unknownFiles.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, dataFile)); + + // We'll send empty block summaries message as it's very short + // TODO: use a different message type here + Message fileUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + fileUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(fileUnknownMessage)) { + LOGGER.info("Couldn't sent file-unknown response"); + peer.disconnect("failed to send file-unknown response"); + } + LOGGER.info("Sent file-unknown response for file {}", dataFile); + } + } + // Utilities private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException { From 652f30bdbd6118eb5675cfdabf3f853bc9a368ce Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Jun 2021 08:02:57 +0100 Subject: [PATCH 040/505] Added DATA_FILE and GET_DATA_FILE message types. --- src/main/java/org/qortal/network/message/Message.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index 07c44c7b..4483caee 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -83,7 +83,10 @@ public abstract class Message { GET_ARBITRARY_DATA(91), BLOCKS(100), - GET_BLOCKS(101); + GET_BLOCKS(101), + + DATA_FILE(110), + GET_DATA_FILE(111); public final int value; public final Method fromByteBufferMethod; From aafb9d7e4fb32a934720a26246ac858bd007ff00 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Jun 2021 08:05:35 +0100 Subject: [PATCH 041/505] Include running total in "Sent X bytes" log entry. --- src/main/java/org/qortal/network/Peer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index c2535118..bb6dd148 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -553,12 +553,14 @@ public class Peer { synchronized (this.socketChannel) { final long sendStart = System.currentTimeMillis(); + long totalBytes = 0; while (outputBuffer.hasRemaining()) { int bytesWritten = this.socketChannel.write(outputBuffer); + totalBytes += bytesWritten; - LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {}", this.peerConnectionId, - bytesWritten, message.getType().name(), message.getId(), this); + LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {} ({} total)", this.peerConnectionId, + bytesWritten, message.getType().name(), message.getId(), this, totalBytes); if (bytesWritten == 0) { // Underlying socket's internal buffer probably full, From 8e35f131d576715f9d1c305aafc1fa54825c5baa Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 18 Jun 2021 08:33:51 +0100 Subject: [PATCH 042/505] Removed debugging log that wasn't intended to be committed. --- src/main/java/org/qortal/storage/DataFile.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 0a54555b..2d20cb90 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -258,7 +258,6 @@ public class DataFile { Path path = Paths.get(this.filePath); try { byte[] bytes = Files.readAllBytes(path); - LOGGER.info("getBytes: %d", bytes); return bytes; } catch (IOException e) { LOGGER.error("Unable to read bytes for file"); From 64d19e480bba13bb08ac37e77da86ed8c743f7bd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 17:32:48 +0100 Subject: [PATCH 043/505] Chunk size set to 1MB for now, as it seems that our networking code has problems when transferring 2MB chunks. We can increase this later once that problem has been fixed. --- src/main/java/org/qortal/storage/DataFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 2d20cb90..37f5a2ea 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -44,7 +44,7 @@ public class DataFile { private static final Logger LOGGER = LogManager.getLogger(DataFile.class); public static final long MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MiB - public static final int CHUNK_SIZE = 2 * 1024 * 1024; // 2MiB + public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static int SHORT_DIGEST_LENGTH = 8; protected String filePath; From f296ec46c8033733887a20f80943ca7a00f9c3f5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 17:32:55 +0100 Subject: [PATCH 044/505] Removed unused headers. --- src/main/java/org/qortal/storage/DataFileChunk.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/DataFileChunk.java index e7805d48..d79eea5d 100644 --- a/src/main/java/org/qortal/storage/DataFileChunk.java +++ b/src/main/java/org/qortal/storage/DataFileChunk.java @@ -2,11 +2,7 @@ package org.qortal.storage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.crypto.Crypto; -import org.qortal.utils.Base58; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; From d0f9d478c26613211b67e6b368171261711f7f79 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 19:05:11 +0100 Subject: [PATCH 045/505] Fixed bug which prevented the DATA_FILE message ID from making it through to the reply queue. --- .../java/org/qortal/network/message/DataFileMessage.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/DataFileMessage.java b/src/main/java/org/qortal/network/message/DataFileMessage.java index 6ea7e05c..06bace5f 100644 --- a/src/main/java/org/qortal/network/message/DataFileMessage.java +++ b/src/main/java/org/qortal/network/message/DataFileMessage.java @@ -15,6 +15,12 @@ public class DataFileMessage extends Message { this.dataFile = dataFile; } + public DataFileMessage(int id, DataFile dataFile) { + super(id, MessageType.DATA_FILE); + + this.dataFile = dataFile; + } + public DataFile getDataFile() { return this.dataFile; } @@ -24,7 +30,7 @@ public class DataFileMessage extends Message { byteBuffer.get(bytes); DataFile dataFile = new DataFile(bytes); - return new DataFileMessage(dataFile); + return new DataFileMessage(id, dataFile); } @Override From 33d9c51b6fa09a9090c5919c9d570029167a65ca Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 19:26:13 +0100 Subject: [PATCH 046/505] Validate supplied base58 string in /data/file/frompeer API endpoint --- .../java/org/qortal/api/resource/DataResource.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index 639d5ed8..1ce23f56 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -179,7 +179,15 @@ public class DataResource { if (dataFile.exists()) { LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); } - Message getDataFileMessage = new GetDataFileMessage(Base58.decode(base58Digest)); + + byte[] digest = null; + try { + digest = Base58.decode(base58Digest); + } catch (NumberFormatException e) { + LOGGER.info("Invalid base58 encoded string"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + Message getDataFileMessage = new GetDataFileMessage(digest); Message message = targetPeer.getResponse(getDataFileMessage); if (message == null || message.getType() != Message.MessageType.DATA_FILE) From 5070c4eea9495b2e1593bb31a109bc268e14689d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 20:23:33 +0100 Subject: [PATCH 047/505] Better handling of data file responses in the /data/file/frompeer API endpoint. --- src/main/java/org/qortal/api/ApiError.java | 8 ++++++-- .../org/qortal/api/resource/DataResource.java | 17 +++++++++++------ src/main/resources/i18n/ApiError_en.properties | 4 ++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/ApiError.java b/src/main/java/org/qortal/api/ApiError.java index dd7fc4b0..d43aa197 100644 --- a/src/main/java/org/qortal/api/ApiError.java +++ b/src/main/java/org/qortal/api/ApiError.java @@ -129,7 +129,11 @@ public enum ApiError { // Foreign blockchain FOREIGN_BLOCKCHAIN_NETWORK_ISSUE(1201, 500), FOREIGN_BLOCKCHAIN_BALANCE_ISSUE(1202, 402), - FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408); + FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408), + + // Data + FILE_NOT_FOUND(1301, 404), + NO_REPLY(1302, 404); private static final Map map = stream(ApiError.values()).collect(toMap(apiError -> apiError.code, apiError -> apiError)); @@ -157,4 +161,4 @@ public enum ApiError { return this.status; } -} \ No newline at end of file +} diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index 1ce23f56..e9eae796 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -25,6 +25,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.List; @@ -150,9 +151,9 @@ public class DataResource { ) } ) - @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public String getFileFromPeer(@QueryParam("base58Digest") String base58Digest, - @QueryParam("peer") String targetPeerAddress) { + @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) + public Response getFileFromPeer(@QueryParam("base58Digest") String base58Digest, + @QueryParam("peer") String targetPeerAddress) { try (final Repository repository = RepositoryManager.getRepository()) { @@ -190,12 +191,16 @@ public class DataResource { Message getDataFileMessage = new GetDataFileMessage(digest); Message message = targetPeer.getResponse(getDataFileMessage); - if (message == null || message.getType() != Message.MessageType.DATA_FILE) - return "invalid file received"; + if (message == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NO_REPLY); + } + else if (message.getType() == Message.MessageType.BLOCK_SUMMARIES) { // TODO: use dedicated message type here + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FILE_NOT_FOUND); + } DataFileMessage dataFileMessage = (DataFileMessage) message; - return String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size()); + return Response.ok(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())).build(); } catch (ApiException e) { throw e; } catch (DataException | InterruptedException e) { diff --git a/src/main/resources/i18n/ApiError_en.properties b/src/main/resources/i18n/ApiError_en.properties index 5acf2373..2a6ec002 100644 --- a/src/main/resources/i18n/ApiError_en.properties +++ b/src/main/resources/i18n/ApiError_en.properties @@ -64,3 +64,7 @@ TRANSACTION_UNKNOWN = transaction unknown TRANSFORMATION_ERROR = could not transform JSON into transaction UNAUTHORIZED = API call unauthorized + +FILE_NOT_FOUND = file not found + +NO_REPLY = peer didn't reply within the allowed time From 7e9b1d5e1627bbfc301aa2ba8f75e5995cd24bac Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 20:25:25 +0100 Subject: [PATCH 048/505] Rework of DataFile.base58Digest() This fixes an NPE when trying to send a file that doesn't exist. It also removes the caching, which we can add again later if it turns out to be needed. --- src/main/java/org/qortal/storage/DataFile.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 37f5a2ea..9e446966 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -49,7 +49,6 @@ public class DataFile { protected String filePath; private ArrayList chunks; - protected String base58Digest; public DataFile() { } @@ -315,10 +314,10 @@ public class DataFile { } public String base58Digest() { - if (this.base58Digest == null) { - this.base58Digest = Base58.encode(this.digest()); + if (this.digest() != null) { + return Base58.encode(this.digest()); } - return this.base58Digest; + return null; } public String shortDigest() { From f5c9807a48f3f0b7923af11a517ddf3c0c5210f2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 20:29:05 +0100 Subject: [PATCH 049/505] Use contains() rather than equals() when matching a peer in /data/file/frompeer, so that the port can be optionally left out. --- src/main/java/org/qortal/api/resource/DataResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index e9eae796..f459c5fd 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -169,7 +169,7 @@ public class DataResource { InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); List peers = Network.getInstance().getHandshakedPeers(); - Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().equals(resolvedAddress)).findFirst().orElse(null); + Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); if (targetPeer == null) { LOGGER.error("Peer not connected"); From c2d0c63db0020e3384b676c2b1482c375a301908 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 19 Jun 2021 20:31:04 +0100 Subject: [PATCH 050/505] Improved error logging. --- src/main/java/org/qortal/api/resource/DataResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index f459c5fd..49eaa4e8 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -172,7 +172,7 @@ public class DataResource { Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); if (targetPeer == null) { - LOGGER.error("Peer not connected"); + LOGGER.info("Peer {} isn't connected", targetPeerAddress); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } From 16dc5b53276956110834156a19e1306774d91d6c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 20 Jun 2021 19:49:10 +0100 Subject: [PATCH 051/505] Include the data length in DataFileMessage, which is more similar to the approach used by ArbitraryDataMessage. These two message types are very similar, except arbitrary code currently has a requirement of one piece of data per signature, whereas DataFile code is independent and can support multiple files per transaction. Maybe the two can be combined at some point, but for now I'll keep them separate. --- .../network/message/DataFileMessage.java | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/network/message/DataFileMessage.java b/src/main/java/org/qortal/network/message/DataFileMessage.java index 06bace5f..e31dde67 100644 --- a/src/main/java/org/qortal/network/message/DataFileMessage.java +++ b/src/main/java/org/qortal/network/message/DataFileMessage.java @@ -1,7 +1,10 @@ package org.qortal.network.message; +import com.google.common.primitives.Ints; import org.qortal.storage.DataFile; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; @@ -26,19 +29,40 @@ public class DataFileMessage extends Message { } public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - byte[] bytes = new byte[byteBuffer.remaining()]; - byteBuffer.get(bytes); - DataFile dataFile = new DataFile(bytes); + int dataLength = byteBuffer.getInt(); + + if (byteBuffer.remaining() != dataLength) + return null; + + byte[] data = new byte[dataLength]; + byteBuffer.get(data); + DataFile dataFile = new DataFile(data); return new DataFileMessage(id, dataFile); } @Override protected byte[] toData() { - if (this.dataFile == null) + if (this.dataFile == null) { return null; + } - return this.dataFile.getBytes(); + byte[] data = this.dataFile.getBytes(); + if (data == null) { + return null; + } + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } } public DataFileMessage cloneWithNewId(int newId) { From b915d0aed50c37c22e0b95032f02ce8500db3f79 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 21 Jun 2021 08:40:52 +0100 Subject: [PATCH 052/505] Only create the output file directories when we are actually writing a file there. This should prevent empty directories being created when initializing a nonexistent DataFile using a hash. --- src/main/java/org/qortal/storage/DataFile.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 9e446966..160af290 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -80,7 +80,7 @@ public class DataFile { String base58Digest = Base58.encode(Crypto.digest(fileContent)); LOGGER.debug(String.format("File digest: %s, size: %d bytes", base58Digest, fileContent.length)); - String outputFilePath = this.getOutputFilePath(base58Digest); + String outputFilePath = this.getOutputFilePath(base58Digest, true); File outputFile = new File(outputFilePath); try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { outputStream.write(fileContent); @@ -97,7 +97,7 @@ public class DataFile { } public static DataFile fromBase58Digest(String base58Digest) { - String filePath = DataFile.getOutputFilePath(base58Digest); + String filePath = DataFile.getOutputFilePath(base58Digest, false); return new DataFile(filePath); } @@ -119,7 +119,7 @@ public class DataFile { } private String copyToDataDirectory() { - String outputFilePath = this.getOutputFilePath(this.base58Digest()); + String outputFilePath = this.getOutputFilePath(this.base58Digest(), true); Path source = Paths.get(this.filePath).toAbsolutePath(); Path dest = Paths.get(outputFilePath).toAbsolutePath(); try { @@ -130,16 +130,18 @@ public class DataFile { } } - public static String getOutputFilePath(String base58Digest) { + public static String getOutputFilePath(String base58Digest, boolean createDirectories) { String base58DigestFirst2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); String base58DigestNext2Chars = base58Digest.substring(2, Math.min(base58Digest.length(), 4)); String outputDirectory = String.format("%s/%s/%s", Settings.getInstance().getDataPath(), base58DigestFirst2Chars, base58DigestNext2Chars); Path outputDirectoryPath = Paths.get(outputDirectory); - try { - Files.createDirectories(outputDirectoryPath); - } catch (IOException e) { - throw new IllegalStateException("Unable to create data subdirectory"); + if (createDirectories) { + try { + Files.createDirectories(outputDirectoryPath); + } catch (IOException e) { + throw new IllegalStateException("Unable to create data subdirectory"); + } } return String.format("%s/%s", outputDirectory, base58Digest); } From 787ef957d2e7220ff0f1824e468dcea7898c4b13 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 21 Jun 2021 19:02:49 +0100 Subject: [PATCH 053/505] Added support for uploading an entire directory via POST /data/upload/path If a directory is specified instead of a file, the directory is automatically zipped before being split into chunks. --- .../org/qortal/api/resource/DataResource.java | 81 ++++++++++++-- src/main/java/org/qortal/utils/ZipUtils.java | 105 ++++++++++++++++++ 2 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/qortal/utils/ZipUtils.java diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index 49eaa4e8..49cd395a 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -20,14 +20,19 @@ import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.storage.DataFile.ValidationResult; import org.qortal.utils.Base58; +import org.qortal.utils.ZipUtils; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.io.File; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.List; @@ -66,7 +71,7 @@ public class DataResource { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public String uploadFile(String filePath) { + public String uploadDataAtPath(String path) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts @@ -75,7 +80,65 @@ public class DataResource { try (final Repository repository = RepositoryManager.getRepository()) { - DataFile dataFile = new DataFile(filePath); + // Check if a file or directory has been supplied + File file = new File(path); + if (file.isFile()) { + return this.uploadFile(path); + } + else if (file.isDirectory()) { + return this.uploadDirectory(path); + } + + LOGGER.info("No file or folder found at path: {}", path); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + } catch (DataException e) { + LOGGER.error("Repository issue when uploading data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (IllegalStateException e) { + LOGGER.error("Invalid upload data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); + } + } + + private String uploadFile(String filePath) { + DataFile dataFile = new DataFile(filePath); + ValidationResult validationResult = dataFile.isValid(); + if (validationResult != DataFile.ValidationResult.OK) { + LOGGER.error("Invalid file: {}", validationResult); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); + + int chunkCount = dataFile.split(); + if (chunkCount > 0) { + LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + return "true"; + } + + return "false"; + } + + private String uploadDirectory(String directoryPath) { + // Ensure temp folder exists + try { + Files.createDirectories(Paths.get("temp")); + } catch (IOException e) { + LOGGER.error("Unable to create temp directory"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } + + // Firstly zip up the directory + String outputFilePath = "temp/zipped.zip"; + try { + ZipUtils.zip(directoryPath, outputFilePath); + } catch (IOException e) { + LOGGER.info("Unable to zip directory", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + try { + DataFile dataFile = new DataFile(outputFilePath); ValidationResult validationResult = dataFile.isValid(); if (validationResult != DataFile.ValidationResult.OK) { LOGGER.error("Invalid file: {}", validationResult); @@ -90,13 +153,13 @@ public class DataResource { } return "false"; - - } catch (DataException e) { - LOGGER.error("Repository issue when uploading data", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } catch (IllegalStateException e) { - LOGGER.error("Invalid upload data", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); + } + finally { + // Clean up by deleting the zipped file + File zippedFile = new File(outputFilePath); + if (zippedFile.exists()) { + zippedFile.delete(); + } } } diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java new file mode 100644 index 00000000..a459304b --- /dev/null +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -0,0 +1,105 @@ +/* + * MIT License + * Copyright (c) 2017 Eugen Paraschiv + * + * Based on code taken from: https://github.com/eugenp/tutorials/tree/master/core-java-modules/core-java-io/src/main/java/com/baeldung + * + */ + +package org.qortal.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +public class ZipUtils { + + public static void zip(String sourcePath, String destFilePath) throws IOException { + File sourceFile = new File(sourcePath); + FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); + ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); + ZipUtils.zip(sourceFile, sourceFile.getName(), zipOutputStream); + zipOutputStream.close(); + fileOutputStream.close(); + } + + public static void zip(final File fileToZip, final String fileName, final ZipOutputStream zipOut) throws IOException { + if (fileToZip.isHidden()) { + return; + } + if (fileToZip.isDirectory()) { + if (fileName.endsWith("/")) { + zipOut.putNextEntry(new ZipEntry(fileName)); + zipOut.closeEntry(); + } else { + zipOut.putNextEntry(new ZipEntry(fileName + "/")); + zipOut.closeEntry(); + } + final File[] children = fileToZip.listFiles(); + for (final File childFile : children) { + ZipUtils.zip(childFile, fileName + "/" + childFile.getName(), zipOut); + } + return; + } + final FileInputStream fis = new FileInputStream(fileToZip); + final ZipEntry zipEntry = new ZipEntry(fileName); + zipOut.putNextEntry(zipEntry); + final byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + fis.close(); + } + + public static void unzip(String sourcePath, String destPath) throws IOException { + final File destDir = new File(destPath); + final byte[] buffer = new byte[1024]; + final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath)); + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + final File newFile = ZipUtils.newFile(destDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + File parent = newFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + final FileOutputStream fos = new FileOutputStream(newFile); + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } + fos.close(); + } + zipEntry = zis.getNextEntry(); + } + zis.closeEntry(); + zis.close(); + } + + /** + * See: https://snyk.io/research/zip-slip-vulnerability + */ + public static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException { + File destFile = new File(destinationDir, zipEntry.getName()); + + String destDirPath = destinationDir.getCanonicalPath(); + String destFilePath = destFile.getCanonicalPath(); + + if (!destFilePath.startsWith(destDirPath + File.separator)) { + throw new IOException("Entry is outside of the target dir: " + zipEntry.getName()); + } + + return destFile; + } + +} From 1613375cc0d971dd0b127b1dfbc8727028962c22 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 21 Jun 2021 19:03:34 +0100 Subject: [PATCH 054/505] Added more validation of files received in GET /data/file/frompeer --- src/main/java/org/qortal/api/resource/DataResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index 49cd395a..ff4e3fda 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -262,6 +262,10 @@ public class DataResource { } DataFileMessage dataFileMessage = (DataFileMessage) message; + dataFile = dataFileMessage.getDataFile(); + if (dataFile == null || !dataFile.exists()) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FILE_NOT_FOUND); + } return Response.ok(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())).build(); } catch (ApiException e) { From 808b36e088a75dbc1449fd2ff6ef29fafe460fc2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 22 Jun 2021 07:44:34 +0100 Subject: [PATCH 055/505] Specify chunk size when splitting. --- src/main/java/org/qortal/api/resource/DataResource.java | 4 ++-- src/main/java/org/qortal/storage/DataFile.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index ff4e3fda..ebfe20a9 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -110,7 +110,7 @@ public class DataResource { } LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); - int chunkCount = dataFile.split(); + int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); if (chunkCount > 0) { LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); return "true"; @@ -146,7 +146,7 @@ public class DataResource { } LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); - int chunkCount = dataFile.split(); + int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); if (chunkCount > 0) { LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); return "true"; diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 160af290..ce1e6203 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -170,10 +170,11 @@ public class DataFile { } public int split() { + public int split(int chunkSize) { try { File file = this.getFile(); - byte[] buffer = new byte[CHUNK_SIZE]; + byte[] buffer = new byte[chunkSize]; this.chunks = new ArrayList<>(); if (file != null) { From aca620241a05a62e088936f3c13b86d4b8cb69f2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 22 Jun 2021 08:58:16 +0100 Subject: [PATCH 056/505] More work on data file split/join, and added a test. --- .../java/org/qortal/storage/DataFile.java | 59 ++++++++++++++++++- src/test/java/org/qortal/test/DataTests.java | 45 ++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 src/test/java/org/qortal/test/DataTests.java diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index ce1e6203..9473487f 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -56,6 +56,7 @@ public class DataFile { public DataFile(String filePath) { this.createDataDirectory(); this.filePath = filePath; + this.chunks = new ArrayList<>(); if (!this.isInBaseDirectory(filePath)) { // Copy file to base directory @@ -123,8 +124,7 @@ public class DataFile { Path source = Paths.get(this.filePath).toAbsolutePath(); Path dest = Paths.get(outputFilePath).toAbsolutePath(); try { - Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); - return dest.toString(); + return Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING).toString(); } catch (IOException e) { throw new IllegalStateException("Unable to copy file to data directory"); } @@ -169,7 +169,10 @@ public class DataFile { return ValidationResult.OK; } - public int split() { + public void addChunk(DataFileChunk chunk) { + this.chunks.add(chunk); + } + public int split(int chunkSize) { try { @@ -205,6 +208,48 @@ public class DataFile { return this.chunks.size(); } + public boolean join() { + // Ensure we have chunks + if (this.chunks != null && this.chunks.size() > 0) { + + // Create temporary path for joined file + Path tempPath; + try { + tempPath = Files.createTempFile(this.chunks.get(0).base58Digest(), ".tmp"); + } catch (IOException e) { + return false; + } + this.filePath = tempPath.toString(); + + // Join the chunks + File outputFile = new File(this.filePath); + try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) { + for (DataFileChunk chunk : this.chunks) { + File sourceFile = new File(chunk.filePath); + BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); + byte[] buffer = new byte[2048]; + int inSize = -1; + while ((inSize = in.read(buffer)) != -1) { + out.write(buffer, 0, inSize); + } + in.close(); + } + out.close(); + + // Copy temporary file to data directory + this.filePath = this.copyToDataDirectory(); + Files.delete(tempPath); + + return true; + } catch (FileNotFoundException e) { + return false; + } catch (IOException e) { + return false; + } + } + return false; + } + public boolean delete() { // Delete the complete file Path path = Paths.get(this.filePath); @@ -294,6 +339,10 @@ public class DataFile { } } + public int chunkCount() { + return this.chunks.size(); + } + private File getFile() { File file = new File(this.filePath); if (file.exists()) { @@ -302,6 +351,10 @@ public class DataFile { return null; } + public String getFilePath() { + return this.filePath; + } + public byte[] digest() { File file = this.getFile(); if (file != null && file.exists()) { diff --git a/src/test/java/org/qortal/test/DataTests.java b/src/test/java/org/qortal/test/DataTests.java new file mode 100644 index 00000000..2d68d41b --- /dev/null +++ b/src/test/java/org/qortal/test/DataTests.java @@ -0,0 +1,45 @@ +package org.qortal.test; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.repository.DataException; +import org.qortal.storage.DataFile; +import org.qortal.test.common.Common; + +import static org.junit.Assert.*; + +public class DataTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testSplitAndJoin() { + String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; + DataFile dataFile = new DataFile(dummyDataString.getBytes()); + assertTrue(dataFile.exists()); + assertEquals(62, dataFile.size()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.base58Digest()); + + // Split into 7 chunks, each 10 bytes long + dataFile.split(10); + assertEquals(7, dataFile.chunkCount()); + + // Delete the original file + dataFile.delete(); + assertFalse(dataFile.exists()); + assertEquals(0, dataFile.size()); + + // Now rebuild the original file from the chunks + assertEquals(7, dataFile.chunkCount()); + dataFile.join(); + + // Validate that the original file is intact + assertTrue(dataFile.exists()); + assertEquals(62, dataFile.size()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.base58Digest()); + } + +} From 1c6428dd3b24e5c95969d2a56b9c8dc3e219991f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Jun 2021 08:07:35 +0100 Subject: [PATCH 057/505] Added equivalent split and join test but this time using a 5.5MiB source file. --- src/test/java/org/qortal/test/DataTests.java | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/java/org/qortal/test/DataTests.java b/src/test/java/org/qortal/test/DataTests.java index 2d68d41b..a628443f 100644 --- a/src/test/java/org/qortal/test/DataTests.java +++ b/src/test/java/org/qortal/test/DataTests.java @@ -6,6 +6,8 @@ import org.qortal.repository.DataException; import org.qortal.storage.DataFile; import org.qortal.test.common.Common; +import java.util.Random; + import static org.junit.Assert.*; public class DataTests extends Common { @@ -42,4 +44,34 @@ public class DataTests extends Common { assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.base58Digest()); } + @Test + public void testSplitAndJoinWithLargeFiles() { + int fileSize = (int) (5.5f * 1024 * 1024); // 5.5MiB + byte[] randomData = new byte[fileSize]; + new Random().nextBytes(randomData); // No need for SecureRandom here + + DataFile dataFile = new DataFile(randomData); + assertTrue(dataFile.exists()); + assertEquals(fileSize, dataFile.size()); + String originalFileDigest = dataFile.base58Digest(); + + // Split into chunks using 1MiB chunk size + dataFile.split(1 * 1024 * 1024); + assertEquals(6, dataFile.chunkCount()); + + // Delete the original file + dataFile.delete(); + assertFalse(dataFile.exists()); + assertEquals(0, dataFile.size()); + + // Now rebuild the original file from the chunks + assertEquals(6, dataFile.chunkCount()); + dataFile.join(); + + // Validate that the original file is intact + assertTrue(dataFile.exists()); + assertEquals(fileSize, dataFile.size()); + assertEquals(originalFileDigest, dataFile.base58Digest()); + } + } From 3f20fadb812c0feeb8c8ea5ed315e8bf9bd87911 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Jun 2021 08:09:59 +0100 Subject: [PATCH 058/505] When zipping files, rename the outer folder to "data" instead of using the original folder name. This means that the data can be accessed deterministically without the need to first lookup the folder name. --- src/main/java/org/qortal/api/resource/DataResource.java | 2 +- src/main/java/org/qortal/utils/ZipUtils.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index ebfe20a9..dafdfff7 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -131,7 +131,7 @@ public class DataResource { // Firstly zip up the directory String outputFilePath = "temp/zipped.zip"; try { - ZipUtils.zip(directoryPath, outputFilePath); + ZipUtils.zip(directoryPath, outputFilePath, "data"); } catch (IOException e) { LOGGER.info("Unable to zip directory", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index a459304b..0b6a892b 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -18,11 +18,14 @@ import java.util.zip.ZipOutputStream; public class ZipUtils { - public static void zip(String sourcePath, String destFilePath) throws IOException { + public static void zip(String sourcePath, String destFilePath, String fileName) throws IOException { File sourceFile = new File(sourcePath); + if (fileName == null) { + fileName = sourceFile.getName(); + } FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); - ZipUtils.zip(sourceFile, sourceFile.getName(), zipOutputStream); + ZipUtils.zip(sourceFile, fileName, zipOutputStream); zipOutputStream.close(); fileOutputStream.close(); } From cd3a1e0159da07497e74b3bfa1140105b7d5b352 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Jun 2021 08:33:53 +0100 Subject: [PATCH 059/505] Increased max file size to 1GiB. Will review this again later. --- src/main/java/org/qortal/storage/DataFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 9473487f..8984bc8a 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -43,7 +43,7 @@ public class DataFile { private static final Logger LOGGER = LogManager.getLogger(DataFile.class); - public static final long MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MiB + public static final long MAX_FILE_SIZE = 1 * 1024 * 1024 * 1024; // 1GiB public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static int SHORT_DIGEST_LENGTH = 8; From f77ec1faf6b2df3be40a96c3fde4f8349890d4e2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Jun 2021 08:43:21 +0100 Subject: [PATCH 060/505] A very crude proof of concept which serves a website from a zipped (and in future, chunked) data blob. This forms the beginnings of the "website hosting" layer on top of the data storage. It needs a significant rework - most importantly so that we aren't serving every asset from memory, and also so that the correct content-type headers are returned, etc. --- .../qortal/api/resource/WebsiteResource.java | 148 ++++++++++++++++++ .../org/qortal/storage/DataFileChunk.java | 14 ++ 2 files changed, 162 insertions(+) create mode 100644 src/main/java/org/qortal/api/resource/WebsiteResource.java diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java new file mode 100644 index 00000000..63bca899 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -0,0 +1,148 @@ +package org.qortal.api.resource; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Response; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.qortal.storage.DataFile; +import org.qortal.utils.ZipUtils; + + +@Path("/site") +public class WebsiteResource { + + private static final Logger LOGGER = LogManager.getLogger(WebsiteResource.class); + + @GET + @Path("{resource}") + public Response getResourceIndex(@PathParam("resource") String resourceId) { + return this.get(resourceId, "/"); + } + + @GET + @Path("{resource}/{path:.*}") + public Response getResourcePath(@PathParam("resource") String resourceId, @PathParam("path") String inPath) { + return this.get(resourceId, inPath); + } + + private Response get(String resourceId, String inPath) { + if (!inPath.startsWith(File.separator)) { + inPath = File.separator + inPath; + } + + String tempDirectory = System.getProperty("java.io.tmpdir"); + String destPath = tempDirectory + "qortal-sites" + File.separator + resourceId; + String unzippedPath = destPath + File.separator + "data"; + + if (!Files.exists(Paths.get(unzippedPath))) { + + // Load file + DataFile dataFile = DataFile.fromBase58Digest(resourceId); + if (dataFile == null || !dataFile.exists()) { + LOGGER.info("Unable to validate complete file hash"); + return Response.serverError().build(); + } + + String newHash = dataFile.base58Digest(); + LOGGER.info("newHash: {}", newHash); + if (!dataFile.base58Digest().equals(resourceId)) { + LOGGER.info("Unable to validate complete file hash"); + return Response.serverError().build(); + } + + try { + ZipUtils.unzip(dataFile.getFilePath(), destPath); + } catch (IOException e) { + LOGGER.info("Unable to unzip file"); + } + } + + try { + String filename = this.getFilename(unzippedPath, inPath); + byte[] data = Files.readAllBytes(Paths.get(unzippedPath + File.separator + filename)); // TODO: limit file size that can be read into memory + data = this.replaceRelativeLinks(filename, data, resourceId); + return Response.ok(data).build(); + } catch (IOException e) { + LOGGER.info("Unable to serve file at path: {}", inPath); + } + + return Response.serverError().build(); + } + + private String getFilename(String directory, String userPath) { + if (userPath == null || userPath.endsWith("/") || userPath.equals("")) { + // Locate index file + List indexFiles = this.indexFiles(); + for (String indexFile : indexFiles) { + String filePath = directory + File.separator + indexFile; + if (Files.exists(Paths.get(filePath))) { + return userPath + indexFile; + } + } + } + return userPath; + } + + /** + * Find relative links and prefix them with the resource ID, using Jsoup + * @param path + * @param data + * @param resourceId + * @return The data with links replaced + */ + private byte[] replaceRelativeLinks(String path, byte[] data, String resourceId) { + if (this.isHtmlFile(path)) { + String fileContents = new String(data); + Document document = Jsoup.parse(fileContents); + + Elements href = document.select("[href]"); + for (Element element : href) { + String elementHtml = element.attr("href"); + if (elementHtml.startsWith("/") && !elementHtml.startsWith("//")) { + element.attr("href", "/site/" +resourceId + element.attr("href")); + } + } + Elements src = document.select("[src]"); + for (Element element : src) { + String elementHtml = element.attr("src"); + if (elementHtml.startsWith("/") && !elementHtml.startsWith("//")) { + element.attr("src", "/site/" +resourceId + element.attr("src")); + } + } + return document.html().getBytes(); + } + return data; + } + + private List indexFiles() { + List indexFiles = new ArrayList<>(); + indexFiles.add("index.html"); + indexFiles.add("index.htm"); + indexFiles.add("default.html"); + indexFiles.add("default.htm"); + indexFiles.add("home.html"); + indexFiles.add("home.htm"); + return indexFiles; + } + + private boolean isHtmlFile(String path) { + if (path.endsWith(".html") || path.endsWith(".htm")) { + return true; + } + return false; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/DataFileChunk.java index d79eea5d..544a150c 100644 --- a/src/main/java/org/qortal/storage/DataFileChunk.java +++ b/src/main/java/org/qortal/storage/DataFileChunk.java @@ -3,6 +3,7 @@ package org.qortal.storage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -16,10 +17,23 @@ public class DataFileChunk extends DataFile { public DataFileChunk() { } + public DataFileChunk(String filePath) { + super(filePath); + } + + public DataFileChunk(File file) { + super(file); + } + public DataFileChunk(byte[] fileContent) { super(fileContent); } + public static DataFileChunk fromBase58Digest(String base58Digest) { + String filePath = DataFile.getOutputFilePath(base58Digest, false); + return new DataFileChunk(filePath); + } + @Override public ValidationResult isValid() { // DataChunk validation applies here too From 39f5dce51c1f69afbe481d452c7a59123d9e6d1f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Jun 2021 09:10:06 +0100 Subject: [PATCH 061/505] Moved "directory" data uploads to new POST /site/upload API. Directory uploads don't make much sense outside of website hosting, so it's best to make this API specific to that purpose. --- .../org/qortal/api/resource/DataResource.java | 77 +++---------- .../qortal/api/resource/WebsiteResource.java | 101 ++++++++++++++++++ 2 files changed, 114 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index dafdfff7..fce2f0dc 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -20,7 +20,6 @@ import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.storage.DataFile.ValidationResult; import org.qortal.utils.Base58; -import org.qortal.utils.ZipUtils; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -71,7 +70,7 @@ public class DataResource { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public String uploadDataAtPath(String path) { + public String uploadFileAtPath(String path) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts @@ -82,63 +81,12 @@ public class DataResource { // Check if a file or directory has been supplied File file = new File(path); - if (file.isFile()) { - return this.uploadFile(path); - } - else if (file.isDirectory()) { - return this.uploadDirectory(path); + if (!file.isFile()) { + LOGGER.info("Not a file: {}", path); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } - LOGGER.info("No file or folder found at path: {}", path); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - - } catch (DataException e) { - LOGGER.error("Repository issue when uploading data", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } catch (IllegalStateException e) { - LOGGER.error("Invalid upload data", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); - } - } - - private String uploadFile(String filePath) { - DataFile dataFile = new DataFile(filePath); - ValidationResult validationResult = dataFile.isValid(); - if (validationResult != DataFile.ValidationResult.OK) { - LOGGER.error("Invalid file: {}", validationResult); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); - - int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); - if (chunkCount > 0) { - LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - return "true"; - } - - return "false"; - } - - private String uploadDirectory(String directoryPath) { - // Ensure temp folder exists - try { - Files.createDirectories(Paths.get("temp")); - } catch (IOException e) { - LOGGER.error("Unable to create temp directory"); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } - - // Firstly zip up the directory - String outputFilePath = "temp/zipped.zip"; - try { - ZipUtils.zip(directoryPath, outputFilePath, "data"); - } catch (IOException e) { - LOGGER.info("Unable to zip directory", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - - try { - DataFile dataFile = new DataFile(outputFilePath); + DataFile dataFile = new DataFile(path); ValidationResult validationResult = dataFile.isValid(); if (validationResult != DataFile.ValidationResult.OK) { LOGGER.error("Invalid file: {}", validationResult); @@ -153,16 +101,17 @@ public class DataResource { } return "false"; - } - finally { - // Clean up by deleting the zipped file - File zippedFile = new File(outputFilePath); - if (zippedFile.exists()) { - zippedFile.delete(); - } + + } catch (DataException e) { + LOGGER.error("Repository issue when uploading data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (IllegalStateException e) { + LOGGER.error("Invalid upload data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); } } + @DELETE @Path("/file") @Operation( diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 63bca899..8696c293 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -1,8 +1,12 @@ package org.qortal.api.resource; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.File; import java.io.IOException; @@ -11,21 +15,118 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.qortal.api.ApiError; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.utils.ZipUtils; @Path("/site") +@Tag(name = "Website") public class WebsiteResource { private static final Logger LOGGER = LogManager.getLogger(WebsiteResource.class); + @Context + HttpServletRequest request; + + @POST + @Path("/upload") + @Operation( + summary = "Build raw, unsigned, UPLOAD_DATA transaction, based on a user-supplied path to a static website", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "/Users/user/Documents/MyStaticWebsite" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, UPLOAD_DATA transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + public String hostWebsite(String directoryPath) { + Security.checkApiCallAllowed(request); + + // It's too dangerous to allow user-supplied filenames in weaker security contexts + if (Settings.getInstance().isApiRestricted()) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); + } + + // Check if a file or directory has been supplied + File file = new File(directoryPath); + if (!file.isDirectory()) { + LOGGER.info("Not a directory: {}", directoryPath); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + // Ensure temp folder exists + try { + Files.createDirectories(Paths.get("temp")); + } catch (IOException e) { + LOGGER.error("Unable to create temp directory"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } + + // Firstly zip up the directory + String outputFilePath = "temp/zipped.zip"; + try { + ZipUtils.zip(directoryPath, outputFilePath, "data"); + } catch (IOException e) { + LOGGER.info("Unable to zip directory", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + try { + DataFile dataFile = new DataFile(outputFilePath); + DataFile.ValidationResult validationResult = dataFile.isValid(); + if (validationResult != DataFile.ValidationResult.OK) { + LOGGER.error("Invalid file: {}", validationResult); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); + + int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); + if (chunkCount > 0) { + LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + return "true"; + } + + return "false"; + } + finally { + // Clean up by deleting the zipped file + File zippedFile = new File(outputFilePath); + if (zippedFile.exists()) { + zippedFile.delete(); + } + } + } + @GET @Path("{resource}") public Response getResourceIndex(@PathParam("resource") String resourceId) { From b65c7a75feda58d29f9af4022fdb58313b4c5d37 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Jun 2021 09:26:41 +0100 Subject: [PATCH 062/505] Handle relative links when parsing HTML. --- .../java/org/qortal/api/resource/WebsiteResource.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 8696c293..9b291089 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -212,15 +212,17 @@ public class WebsiteResource { Elements href = document.select("[href]"); for (Element element : href) { String elementHtml = element.attr("href"); - if (elementHtml.startsWith("/") && !elementHtml.startsWith("//")) { - element.attr("href", "/site/" +resourceId + element.attr("href")); + if (!elementHtml.startsWith("http") && !elementHtml.startsWith("//")) { + String slash = (elementHtml.startsWith("/") ? "" : File.separator); + element.attr("href", "/site/" +resourceId + slash + element.attr("href")); } } Elements src = document.select("[src]"); for (Element element : src) { String elementHtml = element.attr("src"); - if (elementHtml.startsWith("/") && !elementHtml.startsWith("//")) { - element.attr("src", "/site/" +resourceId + element.attr("src")); + if (!elementHtml.startsWith("http") && !elementHtml.startsWith("//")) { + String slash = (elementHtml.startsWith("/") ? "" : File.separator); + element.attr("src", "/site/" +resourceId + slash + element.attr("src")); } } return document.html().getBytes(); From ea5e2f5580fd922e3232bee07e25d415feda94e1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 23 Jun 2021 09:28:38 +0100 Subject: [PATCH 063/505] Added POST /site/preview API This can be used to preview a site before signing a transaction and announcing it to the network. The response will need reworking to return JSON (along with most of the other new APIs) --- .../qortal/api/resource/WebsiteResource.java | 56 ++++++++++++++++++- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 9b291089..d8b559eb 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -69,7 +69,7 @@ public class WebsiteResource { ) } ) - public String hostWebsite(String directoryPath) { + public String uploadWebsite(String directoryPath) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts @@ -77,6 +77,56 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } + String base58Digest = this.hostWebsite(directoryPath); + if (base58Digest != null) { + // TODO: build transaction + return "true"; + } + return "false"; + } + + @POST + @Path("/preview") + @Operation( + summary = "Generate preview URL based on a user-supplied path to a static website", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "/Users/user/Documents/MyStaticWebsite" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, UPLOAD_DATA transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + public String previewWebsite(String directoryPath) { + Security.checkApiCallAllowed(request); + + // It's too dangerous to allow user-supplied filenames in weaker security contexts + if (Settings.getInstance().isApiRestricted()) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); + } + + String base58Digest = this.hostWebsite(directoryPath); + if (base58Digest != null) { + return "http://localhost:12393/site/" + base58Digest; + } + return "Unable to generate preview URL"; + } + + private String hostWebsite(String directoryPath) { + // Check if a file or directory has been supplied File file = new File(directoryPath); if (!file.isDirectory()) { @@ -113,10 +163,10 @@ public class WebsiteResource { int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); if (chunkCount > 0) { LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - return "true"; + return dataFile.base58Digest(); } - return "false"; + return null; } finally { // Clean up by deleting the zipped file From b286c15c515dc52dd695de193d8803786256968b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Jun 2021 08:32:22 +0100 Subject: [PATCH 064/505] Optimized website serving, and added code to return the correct content types. This is probably the most efficient way to process the data on the fly, but it's still not very scalable. A better approach would be to pre-process the HTML when building the file structure, and then serve them completely statically (i.e. using a standard webserver rather than via application memory). But it makes sense to keep it this way for development and maybe early beta testing. --- .../qortal/api/resource/WebsiteResource.java | 64 +++++++++++++++---- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index d8b559eb..9fdf048f 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -1,15 +1,15 @@ package org.qortal.api.resource; +import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.io.File; -import java.io.IOException; +import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -41,8 +41,9 @@ public class WebsiteResource { private static final Logger LOGGER = LogManager.getLogger(WebsiteResource.class); - @Context - HttpServletRequest request; + @Context HttpServletRequest request; + @Context HttpServletResponse response; + @Context ServletContext context; @POST @Path("/upload") @@ -179,17 +180,17 @@ public class WebsiteResource { @GET @Path("{resource}") - public Response getResourceIndex(@PathParam("resource") String resourceId) { + public HttpServletResponse getResourceIndex(@PathParam("resource") String resourceId) { return this.get(resourceId, "/"); } @GET @Path("{resource}/{path:.*}") - public Response getResourcePath(@PathParam("resource") String resourceId, @PathParam("path") String inPath) { + public HttpServletResponse getResourcePath(@PathParam("resource") String resourceId, @PathParam("path") String inPath) { return this.get(resourceId, inPath); } - private Response get(String resourceId, String inPath) { + private HttpServletResponse get(String resourceId, String inPath) { if (!inPath.startsWith(File.separator)) { inPath = File.separator + inPath; } @@ -204,14 +205,14 @@ public class WebsiteResource { DataFile dataFile = DataFile.fromBase58Digest(resourceId); if (dataFile == null || !dataFile.exists()) { LOGGER.info("Unable to validate complete file hash"); - return Response.serverError().build(); + return this.get404Response(); } String newHash = dataFile.base58Digest(); LOGGER.info("newHash: {}", newHash); if (!dataFile.base58Digest().equals(resourceId)) { LOGGER.info("Unable to validate complete file hash"); - return Response.serverError().build(); + return this.get404Response(); } try { @@ -223,14 +224,36 @@ public class WebsiteResource { try { String filename = this.getFilename(unzippedPath, inPath); - byte[] data = Files.readAllBytes(Paths.get(unzippedPath + File.separator + filename)); // TODO: limit file size that can be read into memory - data = this.replaceRelativeLinks(filename, data, resourceId); - return Response.ok(data).build(); + String filePath = unzippedPath + File.separator + filename; + + if (this.isHtmlFile(filename)) { + // HTML file - needs to be parsed + byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory + data = this.replaceRelativeLinks(filename, data, resourceId); + response.setContentType(context.getMimeType(filename)); + response.setContentLength(data.length); + response.getOutputStream().write(data); + } + else { + // Regular file - can be streamed directly + File file = new File(filePath); + FileInputStream inputStream = new FileInputStream(file); + response.setContentType(context.getMimeType(filename)); + int bytesRead, length = 0; + byte[] buffer = new byte[1024]; + while ((bytesRead = inputStream.read(buffer)) != -1) { + response.getOutputStream().write(buffer, 0, bytesRead); + length += bytesRead; + } + response.setContentLength(length); + inputStream.close(); + } + return response; } catch (IOException e) { LOGGER.info("Unable to serve file at path: {}", inPath); } - return Response.serverError().build(); + return this.get404Response(); } private String getFilename(String directory, String userPath) { @@ -247,6 +270,19 @@ public class WebsiteResource { return userPath; } + private HttpServletResponse get404Response() { + try { + String responseString = "404: File Not Found"; + byte[] responseData = responseString.getBytes(); + response.setStatus(404); + response.setContentLength(responseData.length); + response.getOutputStream().write(responseData); + } catch (IOException e) { + LOGGER.info("Error writing 404 response"); + } + return response; + } + /** * Find relative links and prefix them with the resource ID, using Jsoup * @param path From b34066f57955a5b215be69682f6ec608adbf22a9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Jun 2021 08:34:44 +0100 Subject: [PATCH 065/505] More work on HTML parsing. The style tag parsing ideally needs rewriting using an actual CSS parser, but we can get away with this hacky approach in the short term. --- .../qortal/api/resource/WebsiteResource.java | 60 ++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 9fdf048f..80a4733c 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -298,7 +298,7 @@ public class WebsiteResource { Elements href = document.select("[href]"); for (Element element : href) { String elementHtml = element.attr("href"); - if (!elementHtml.startsWith("http") && !elementHtml.startsWith("//")) { + if (this.isRelativeLink(elementHtml)) { String slash = (elementHtml.startsWith("/") ? "" : File.separator); element.attr("href", "/site/" +resourceId + slash + element.attr("href")); } @@ -306,16 +306,72 @@ public class WebsiteResource { Elements src = document.select("[src]"); for (Element element : src) { String elementHtml = element.attr("src"); - if (!elementHtml.startsWith("http") && !elementHtml.startsWith("//")) { + if (this.isRelativeLink(elementHtml)) { String slash = (elementHtml.startsWith("/") ? "" : File.separator); element.attr("src", "/site/" +resourceId + slash + element.attr("src")); } } + Elements srcset = document.select("[srcset]"); + for (Element element : srcset) { + String elementHtml = element.attr("srcset").trim(); + if (this.isRelativeLink(elementHtml)) { + String[] parts = element.attr("srcset").split(","); + ArrayList newParts = new ArrayList<>(); + for (String part : parts) { + part = part.trim(); + String slash = (elementHtml.startsWith("/") ? "" : File.separator); + String newPart = "/site/" +resourceId + slash + part; + newParts.add(newPart); + } + String newString = String.join(",", newParts); + element.attr("srcset", newString); + } + } + Elements style = document.select("[style]"); + for (Element element : style) { + String elementHtml = element.attr("style"); + if (elementHtml.contains("url(")) { + String[] parts = elementHtml.split("url\\("); + String[] parts2 = parts[1].split("\\)"); + String link = parts2[0]; + if (link != null) { + link = this.removeQuotes(link); + if (this.isRelativeLink(link)) { + String slash = (link.startsWith("/") ? "" : File.separator); + String modifiedLink = "url('" + "/site/" + resourceId + slash + link + "')"; + element.attr("style", parts[0] + modifiedLink + parts2[1]); + } + } + } + } return document.html().getBytes(); } return data; } + private boolean isRelativeLink(String elementHtml) { + List prefixes = new ArrayList<>(); + prefixes.add("http"); + prefixes.add("//"); + prefixes.add("javascript:"); + for (String prefix : prefixes) { + if (elementHtml.startsWith(prefix)) { + return false; + } + } + return true; + } + + private String removeQuotes(String elementHtml) { + if (elementHtml.startsWith("\"") || elementHtml.startsWith("\'")) { + elementHtml = elementHtml.substring(1); + } + if (elementHtml.endsWith("\"") || elementHtml.endsWith("\'")) { + elementHtml = elementHtml.substring(0, elementHtml.length() - 1); + } + return elementHtml; + } + private List indexFiles() { List indexFiles = new ArrayList<>(); indexFiles.add("index.html"); From 71c247fe5631998af625a7a4755df95a6c1b36ed Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Jun 2021 19:28:01 +0100 Subject: [PATCH 066/505] Added POST /data/file/{hash}/build API, used to join multiple chunks together. --- .../org/qortal/api/resource/DataResource.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index fce2f0dc..c7894989 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -225,4 +225,59 @@ public class DataResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } } + + @POST + @Path("/file/{hash}/build") + @Operation( + summary = "Join multiple chunks into a single file, using supplied comma separated base58 encoded SHA256 digest strings", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4,FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" + ) + ) + ), + responses = { + @ApiResponse( + description = "true if joined, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) + public Response joinFiles(String files, @PathParam("combinedHash") String combinedHash) { + + if (combinedHash == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + DataFile dataFile = DataFile.fromBase58Digest(combinedHash); + if (dataFile.exists()) { + LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash); + } + + String base58DigestList[] = files.split(","); + for (String base58Digest : base58DigestList) { + if (base58Digest != null) { + DataFileChunk chunk = DataFileChunk.fromBase58Digest(base58Digest); + dataFile.addChunk(chunk); + } + } + boolean success = dataFile.join(); + if (success) { + if (combinedHash.equals(dataFile.base58Digest())) { + LOGGER.info("Valid hash {} after joining {} files", dataFile.base58Digest(), dataFile.chunkCount()); + return Response.ok("true").build(); + } + } + + return Response.ok("false").build(); + } } From 52829a244bc018045e60315be25db79d6863f116 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Jun 2021 19:28:39 +0100 Subject: [PATCH 067/505] More work on APIs to request files from peers. I won't spend too long making these perfect as they are mostly a temporary measure. --- .../org/qortal/api/resource/DataResource.java | 107 +++++++++++++++--- 1 file changed, 89 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index c7894989..326bc122 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -19,6 +19,7 @@ import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.storage.DataFile.ValidationResult; +import org.qortal.storage.DataFileChunk; import org.qortal.utils.Base58; import javax.servlet.http.HttpServletRequest; @@ -27,11 +28,8 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.File; -import java.io.IOException; import java.net.InetSocketAddress; import java.net.UnknownHostException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.List; @@ -148,7 +146,7 @@ public class DataResource { } @GET - @Path("/file/frompeer") + @Path("/file/{hash}/frompeer/{peer}") @Operation( summary = "Request file from a given peer, using supplied base58 encoded SHA256 digest string", responses = { @@ -164,11 +162,9 @@ public class DataResource { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) - public Response getFileFromPeer(@QueryParam("base58Digest") String base58Digest, - @QueryParam("peer") String targetPeerAddress) { - - try (final Repository repository = RepositoryManager.getRepository()) { - + public Response getFileFromPeer(@PathParam("hash") String base58Digest, + @PathParam("peer") String targetPeerAddress) { + try { if (base58Digest == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } @@ -179,15 +175,92 @@ public class DataResource { // Try to resolve passed address to make things easier PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); - List peers = Network.getInstance().getHandshakedPeers(); Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); if (targetPeer == null) { LOGGER.info("Peer {} isn't connected", targetPeerAddress); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } + boolean success = this.requestFile(base58Digest, targetPeer); + if (success) { + return Response.ok("true").build(); + } + return Response.ok("false").build(); + + } catch (UnknownHostException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + + @POST + @Path("/files/frompeer/{peer}") + @Operation( + summary = "Request multiple files from a given peer, using supplied comma separated base58 encoded SHA256 digest strings", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4,FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" + ) + ) + ), + responses = { + @ApiResponse( + description = "true if retrieved, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) + public Response getFilesFromPeer(String files, @PathParam("peer") String targetPeerAddress) { + try { + if (targetPeerAddress == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + // Try to resolve passed address to make things easier + PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); + InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); + List peers = Network.getInstance().getHandshakedPeers(); + Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); + + for (Peer peer : peers) { + LOGGER.info("peer: {}", peer); + } + + if (targetPeer == null) { + LOGGER.info("Peer {} isn't connected", targetPeerAddress); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + String base58DigestList[] = files.split(","); + for (String base58Digest : base58DigestList) { + if (base58Digest != null) { + boolean success = this.requestFile(base58Digest, targetPeer); + if (!success) { + LOGGER.info("Failed to request file {} from peer {}", base58Digest, targetPeerAddress); + } + } + } + return Response.ok("true").build(); + + } catch (UnknownHostException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + + + private boolean requestFile(String base58Digest, Peer targetPeer) { + try (final Repository repository = RepositoryManager.getRepository()) { + DataFile dataFile = DataFile.fromBase58Digest(base58Digest); if (dataFile.exists()) { LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); @@ -204,25 +277,23 @@ public class DataResource { Message message = targetPeer.getResponse(getDataFileMessage); if (message == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NO_REPLY); + return false; } else if (message.getType() == Message.MessageType.BLOCK_SUMMARIES) { // TODO: use dedicated message type here - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FILE_NOT_FOUND); + return false; } DataFileMessage dataFileMessage = (DataFileMessage) message; dataFile = dataFileMessage.getDataFile(); if (dataFile == null || !dataFile.exists()) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FILE_NOT_FOUND); + return false; } - - return Response.ok(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())).build(); + LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())); + return true; } catch (ApiException e) { throw e; } catch (DataException | InterruptedException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } catch (UnknownHostException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } } From ace5d999e28c47fb5189c69d01ecfcd4dfdc2727 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Jun 2021 19:30:45 +0100 Subject: [PATCH 068/505] Log a comma separated list of hashes after splitting a file into chunks, so they can easily be requested from another node using the //data/files/frompeer/{peer} API endpoint. Again temporary until the sync happens automatically. --- .../org/qortal/api/resource/WebsiteResource.java | 3 ++- src/main/java/org/qortal/storage/DataFile.java | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 80a4733c..6af3698c 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -163,7 +163,8 @@ public class WebsiteResource { int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); if (chunkCount > 0) { - LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s"))); + LOGGER.info("{}", dataFile.printChunks()); return dataFile.base58Digest(); } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 8984bc8a..9fa222d3 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -383,6 +383,19 @@ public class DataFile { return this.base58Digest().substring(0, Math.min(this.base58Digest().length(), SHORT_DIGEST_LENGTH)); } + public String printChunks() { + String outputString = ""; + if (this.chunkCount() > 0) { + for (DataFileChunk chunk : this.chunks) { + if (outputString.length() > 0) { + outputString = outputString.concat(","); + } + outputString = outputString.concat(chunk.base58Digest()); + } + } + return outputString; + } + @Override public String toString() { return this.shortDigest(); From 8973626a4bd8cd9fb8eda2de33ec6b67dbc1e660 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Jun 2021 19:31:13 +0100 Subject: [PATCH 069/505] Fixed issue with temp directory on Linux. --- src/main/java/org/qortal/api/resource/WebsiteResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 6af3698c..96521f6a 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -197,7 +197,7 @@ public class WebsiteResource { } String tempDirectory = System.getProperty("java.io.tmpdir"); - String destPath = tempDirectory + "qortal-sites" + File.separator + resourceId; + String destPath = tempDirectory + File.separator + "qortal-sites" + File.separator + resourceId; String unzippedPath = destPath + File.separator + "data"; if (!Files.exists(Paths.get(unzippedPath))) { From fe7c40cb7c3041cef967c00cf3325a45a3d6e01d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 25 Jun 2021 19:31:28 +0100 Subject: [PATCH 070/505] Reduced log spam. --- src/main/java/org/qortal/api/resource/WebsiteResource.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 96521f6a..f0060a4a 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -209,8 +209,6 @@ public class WebsiteResource { return this.get404Response(); } - String newHash = dataFile.base58Digest(); - LOGGER.info("newHash: {}", newHash); if (!dataFile.base58Digest().equals(resourceId)) { LOGGER.info("Unable to validate complete file hash"); return this.get404Response(); From 47c70eea9e815c97c6fd6d798c10bfa75747625e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Jun 2021 10:11:34 +0100 Subject: [PATCH 071/505] Use system temp directory instead of making a "temp" subfolder when zipping files. --- src/main/java/org/qortal/api/resource/WebsiteResource.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index f0060a4a..60d56bd6 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -136,15 +136,16 @@ public class WebsiteResource { } // Ensure temp folder exists + java.nio.file.Path tempDir = null; try { - Files.createDirectories(Paths.get("temp")); + tempDir = Files.createTempDirectory("qortal-zip"); } catch (IOException e) { LOGGER.error("Unable to create temp directory"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } // Firstly zip up the directory - String outputFilePath = "temp/zipped.zip"; + String outputFilePath = tempDir.toString() + File.separator + "zipped.zip"; try { ZipUtils.zip(directoryPath, outputFilePath, "data"); } catch (IOException e) { From ebfa941a4f8faf2e494dc165473c8b55ee847233 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Jun 2021 10:27:32 +0100 Subject: [PATCH 072/505] Fixed some file separators. --- src/main/java/org/qortal/api/resource/WebsiteResource.java | 2 +- src/main/java/org/qortal/storage/DataFile.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 60d56bd6..c6a10ec7 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -337,7 +337,7 @@ public class WebsiteResource { if (link != null) { link = this.removeQuotes(link); if (this.isRelativeLink(link)) { - String slash = (link.startsWith("/") ? "" : File.separator); + String slash = (link.startsWith("/") ? "" : "/"); String modifiedLink = "url('" + "/site/" + resourceId + slash + link + "')"; element.attr("style", parts[0] + modifiedLink + parts2[1]); } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 9fa222d3..1e2eb4e5 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -133,7 +133,7 @@ public class DataFile { public static String getOutputFilePath(String base58Digest, boolean createDirectories) { String base58DigestFirst2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); String base58DigestNext2Chars = base58Digest.substring(2, Math.min(base58Digest.length(), 4)); - String outputDirectory = String.format("%s/%s/%s", Settings.getInstance().getDataPath(), base58DigestFirst2Chars, base58DigestNext2Chars); + String outputDirectory = Settings.getInstance().getDataPath() + File.separator + base58DigestFirst2Chars + File.separator + base58DigestNext2Chars; Path outputDirectoryPath = Paths.get(outputDirectory); if (createDirectories) { @@ -143,7 +143,7 @@ public class DataFile { throw new IllegalStateException("Unable to create data subdirectory"); } } - return String.format("%s/%s", outputDirectory, base58Digest); + return outputDirectory + base58Digest; } public ValidationResult isValid() { From aac4fe37e8b126ebf3aaf9208d6c8b51c6fd703c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Jun 2021 10:28:22 +0100 Subject: [PATCH 073/505] Fixed API response description. --- src/main/java/org/qortal/api/resource/WebsiteResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index c6a10ec7..72d3166d 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -101,7 +101,7 @@ public class WebsiteResource { ), responses = { @ApiResponse( - description = "raw, unsigned, UPLOAD_DATA transaction encoded in Base58", + description = "a temporary URL to preview the website", content = @Content( mediaType = MediaType.TEXT_PLAIN, schema = @Schema( From f3e593359991cdf22110e757bc6d7aebaffde40b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 1 Jul 2021 11:18:46 +0100 Subject: [PATCH 074/505] Fixed naming error in joinFiles API. --- src/main/java/org/qortal/api/resource/DataResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java index 326bc122..0b665a46 100644 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ b/src/main/java/org/qortal/api/resource/DataResource.java @@ -323,7 +323,7 @@ public class DataResource { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) - public Response joinFiles(String files, @PathParam("combinedHash") String combinedHash) { + public Response joinFiles(String files, @PathParam("hash") String combinedHash) { if (combinedHash == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); From 8c3a22aa5c2b9098b5d9bbb68441f43ce5fe6c97 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 1 Jul 2021 11:21:40 +0100 Subject: [PATCH 075/505] Improved link replacement criteria. --- .../qortal/api/resource/WebsiteResource.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 72d3166d..f504e09b 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -298,7 +298,7 @@ public class WebsiteResource { Elements href = document.select("[href]"); for (Element element : href) { String elementHtml = element.attr("href"); - if (this.isRelativeLink(elementHtml)) { + if (this.shouldReplaceLink(elementHtml)) { String slash = (elementHtml.startsWith("/") ? "" : File.separator); element.attr("href", "/site/" +resourceId + slash + element.attr("href")); } @@ -306,7 +306,7 @@ public class WebsiteResource { Elements src = document.select("[src]"); for (Element element : src) { String elementHtml = element.attr("src"); - if (this.isRelativeLink(elementHtml)) { + if (this.shouldReplaceLink(elementHtml)) { String slash = (elementHtml.startsWith("/") ? "" : File.separator); element.attr("src", "/site/" +resourceId + slash + element.attr("src")); } @@ -314,7 +314,7 @@ public class WebsiteResource { Elements srcset = document.select("[srcset]"); for (Element element : srcset) { String elementHtml = element.attr("srcset").trim(); - if (this.isRelativeLink(elementHtml)) { + if (this.shouldReplaceLink(elementHtml)) { String[] parts = element.attr("srcset").split(","); ArrayList newParts = new ArrayList<>(); for (String part : parts) { @@ -336,7 +336,7 @@ public class WebsiteResource { String link = parts2[0]; if (link != null) { link = this.removeQuotes(link); - if (this.isRelativeLink(link)) { + if (this.shouldReplaceLink(link)) { String slash = (link.startsWith("/") ? "" : "/"); String modifiedLink = "url('" + "/site/" + resourceId + slash + link + "')"; element.attr("style", parts[0] + modifiedLink + parts2[1]); @@ -349,11 +349,12 @@ public class WebsiteResource { return data; } - private boolean isRelativeLink(String elementHtml) { + private boolean shouldReplaceLink(String elementHtml) { List prefixes = new ArrayList<>(); - prefixes.add("http"); - prefixes.add("//"); - prefixes.add("javascript:"); + prefixes.add("http"); // Don't modify absolute links + prefixes.add("//"); // Don't modify absolute links + prefixes.add("javascript:"); // Don't modify javascript + prefixes.add("../"); // Don't modify valid relative links for (String prefix : prefixes) { if (elementHtml.startsWith(prefix)) { return false; From 28425efbe74335e4225f22d37ac879fe055d7593 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 2 Jul 2021 08:52:38 +0100 Subject: [PATCH 076/505] Added jsoup dependency - this was missing from an earlier commit. --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index aac866b7..b60b2b23 100644 --- a/pom.xml +++ b/pom.xml @@ -25,6 +25,7 @@ 2.0.9 3.23.8 1.1.0 + 1.13.1 src/main/java @@ -649,5 +650,10 @@ bctls-jdk15on ${bouncycastle.version} + + org.jsoup + jsoup + ${jsoup.version} + From cc449f93045fbde174bc3573b2e0c2395ac81108 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 2 Jul 2021 08:54:05 +0100 Subject: [PATCH 077/505] Added DataFile.chunkHashes() method which appends all hashes into a single byte array (32 bytes / 256 bits each). --- .../java/org/qortal/storage/DataFile.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 1e2eb4e5..6f82799f 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -369,6 +369,31 @@ public class DataFile { return null; } + public byte[] chunkHashes() { + if (this.chunks != null && this.chunks.size() > 0) { + // Return null if we only have one chunk, with the same hash as the parent + if (this.digest().equals(this.chunks.get(0).digest())) { + return null; + } + + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (DataFileChunk chunk : this.chunks) { + byte[] chunkHash = chunk.digest(); + if (chunkHash.length != 32) { + LOGGER.info("Invalid chunk hash length: {}", chunkHash.length); + throw new IllegalStateException("Invalid chunk hash length"); + } + outputStream.write(chunk.digest()); + } + return outputStream.toByteArray(); + } catch (IOException e) { + return null; + } + } + return null; + } + public String base58Digest() { if (this.digest() != null) { return Base58.encode(this.digest()); From 7cc2c4f6211afd774cf775b5a32822f30bd97b32 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 2 Jul 2021 08:57:55 +0100 Subject: [PATCH 078/505] Progress on website API endpoints. --- .../qortal/api/resource/AdminResource.java | 3 +- .../qortal/api/resource/WebsiteResource.java | 77 +++++++++++++++---- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 719a3b9d..35fccd96 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -76,7 +76,8 @@ public class AdminResource { @Path("/unused") @Parameter(in = ParameterIn.PATH, name = "assetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer")) @Parameter(in = ParameterIn.PATH, name = "otherassetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer")) - @Parameter(in = ParameterIn.PATH, name = "address", description = "an account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v") + @Parameter(in = ParameterIn.PATH, name = "address", description = "An account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v") + @Parameter(in = ParameterIn.PATH, name = "path", description = "Local path to folder containing the files", schema = @Schema(type = "String", defaultValue = "/Users/user/Documents/MyStaticWebsite")) @Parameter(in = ParameterIn.QUERY, name = "count", description = "Maximum number of entries to return, 0 means none", schema = @Schema(type = "integer", defaultValue = "20")) @Parameter(in = ParameterIn.QUERY, name = "limit", description = "Maximum number of entries to return, 0 means unlimited", schema = @Schema(type = "integer", defaultValue = "20")) @Parameter(in = ParameterIn.QUERY, name = "offset", description = "Starting entry in results, 0 is first entry", schema = @Schema(type = "integer")) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index f504e09b..bf58492f 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -3,10 +3,7 @@ package org.qortal.api.resource; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.io.*; @@ -30,6 +27,10 @@ import org.jsoup.select.Elements; import org.qortal.api.ApiError; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.Security; +import org.qortal.data.account.AccountData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.utils.ZipUtils; @@ -46,9 +47,9 @@ public class WebsiteResource { @Context ServletContext context; @POST - @Path("/upload") + @Path("/upload/creator/{address}") @Operation( - summary = "Build raw, unsigned, UPLOAD_DATA transaction, based on a user-supplied path to a static website", + summary = "Build raw, unsigned, HASHED_DATA transaction, based on a user-supplied path to a static website", requestBody = @RequestBody( required = true, content = @Content( @@ -60,7 +61,7 @@ public class WebsiteResource { ), responses = { @ApiResponse( - description = "raw, unsigned, UPLOAD_DATA transaction encoded in Base58", + description = "raw, unsigned, HASHED_DATA transaction encoded in Base58", content = @Content( mediaType = MediaType.TEXT_PLAIN, schema = @Schema( @@ -70,7 +71,7 @@ public class WebsiteResource { ) } ) - public String uploadWebsite(String directoryPath) { + public String uploadWebsite(@PathParam("address") String creatorAddress, String path) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts @@ -78,10 +79,49 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } - String base58Digest = this.hostWebsite(directoryPath); - if (base58Digest != null) { - // TODO: build transaction - return "true"; + if (creatorAddress == null || path == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + DataFile dataFile = this.hostWebsite(path); + if (dataFile != null) { + String base58Digest = dataFile.base58Digest(); + if (base58Digest != null) { + try (final Repository repository = RepositoryManager.getRepository()) { + + AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); + if (accountData == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + } + byte[] creatorPublicKey = accountData.getPublicKey(); + byte[] lastReference = accountData.getReference(); + +// BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, +// lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); +// int size = (int)dataFile.size(); +// byte[] digest = dataFile.digest(); +// byte[] chunkHashes = dataFile.chunkHashes(); +// +// HashedDataTransactionData transactionData = new HashedDataTransactionData(baseTransactionData, +// 1, 2, 0, size, digest, chunkHashes); +// +// HashedDataTransaction transaction = (HashedDataTransaction)Transaction.fromData(repository, transactionData); +// transaction.computeNonce(); +// +// Transaction.ValidationResult result = transaction.isValidUnconfirmed(); +// if (result != Transaction.ValidationResult.OK) +// throw TransactionsResource.createTransactionInvalidException(request, result); +// +// byte[] bytes = HashedDataTransactionTransformer.toBytes(transactionData); +// return Base58.encode(bytes); + return "true"; + +// } catch (TransformationException e) { +// throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } } return "false"; } @@ -119,14 +159,17 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } - String base58Digest = this.hostWebsite(directoryPath); - if (base58Digest != null) { - return "http://localhost:12393/site/" + base58Digest; + DataFile dataFile = this.hostWebsite(directoryPath); + if (dataFile != null) { + String base58Digest = dataFile.base58Digest(); + if (base58Digest != null) { + return "http://localhost:12393/site/" + base58Digest; + } } return "Unable to generate preview URL"; } - private String hostWebsite(String directoryPath) { + private DataFile hostWebsite(String directoryPath) { // Check if a file or directory has been supplied File file = new File(directoryPath); @@ -166,7 +209,7 @@ public class WebsiteResource { if (chunkCount > 0) { LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s"))); LOGGER.info("{}", dataFile.printChunks()); - return dataFile.base58Digest(); + return dataFile; } return null; From 5f4649ee2bc0891dcd6f75a9e65efd7e90b8cf49 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 3 Jul 2021 17:40:02 +0100 Subject: [PATCH 079/505] Major upgrade of arbitrary transactions - Adds support for files up 500MiB per transaction (at 2MiB chunk sizes). Previously, the max data size was 4000 bytes. - Adds a nonce, giving us the option to remove the transaction fees altogether on the data chain. These features become enabled in version 5 of arbitrary transactions. --- .../transaction/ArbitraryTransactionData.java | 30 ++- .../hsqldb/HSQLDBArbitraryRepository.java | 191 ++++++++++-------- .../hsqldb/HSQLDBDatabaseUpdates.java | 14 ++ .../HSQLDBArbitraryTransactionRepository.java | 18 +- .../transaction/ArbitraryTransaction.java | 117 ++++++++++- .../ArbitraryTransactionTransformer.java | 74 ++++++- .../transaction/ArbitraryTestTransaction.java | 5 +- 7 files changed, 347 insertions(+), 102 deletions(-) diff --git a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java index 56529852..19c0d0dc 100644 --- a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java +++ b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java @@ -32,10 +32,14 @@ public class ArbitraryTransactionData extends TransactionData { private byte[] senderPublicKey; private int service; + private int nonce; + private int size; @Schema(example = "raw_data_in_base58") private byte[] data; private DataType dataType; + @Schema(example = "chunk_hashes_in_base58") + private byte[] chunkHashes; private List payments; // Constructors @@ -50,14 +54,18 @@ public class ArbitraryTransactionData extends TransactionData { } public ArbitraryTransactionData(BaseTransactionData baseTransactionData, - int version, int service, byte[] data, DataType dataType, List payments) { + int version, int service, int nonce, int size, byte[] data, + DataType dataType, byte[] chunkHashes, List payments) { super(TransactionType.ARBITRARY, baseTransactionData); this.senderPublicKey = baseTransactionData.creatorPublicKey; this.version = version; this.service = service; + this.nonce = nonce; + this.size = size; this.data = data; this.dataType = dataType; + this.chunkHashes = chunkHashes; this.payments = payments; } @@ -75,6 +83,18 @@ public class ArbitraryTransactionData extends TransactionData { return this.service; } + public int getNonce() { + return this.nonce; + } + + public void setNonce(int nonce) { + this.nonce = nonce; + } + + public int getSize() { + return this.size; + } + public byte[] getData() { return this.data; } @@ -91,6 +111,14 @@ public class ArbitraryTransactionData extends TransactionData { this.dataType = dataType; } + public byte[] getChunkHashes() { + return this.chunkHashes; + } + + public void setChunkHashes(byte[] chunkHashes) { + this.chunkHashes = chunkHashes; + } + public List getPayments() { return this.payments; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 3d99bbb3..332e711a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -1,21 +1,14 @@ package org.qortal.repository.hsqldb; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.DataType; import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.DataException; -import org.qortal.settings.Settings; -import org.qortal.utils.Base58; +import org.qortal.storage.DataFile; public class HSQLDBArbitraryRepository implements ArbitraryRepository { @@ -23,36 +16,12 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { protected HSQLDBRepository repository; + private static final Logger LOGGER = LogManager.getLogger(ArbitraryRepository.class); + public HSQLDBArbitraryRepository(HSQLDBRepository repository) { this.repository = repository; } - /** - * Returns pathname for saving arbitrary transaction data payloads. - *

- * Format: arbitrary//.raw - * - * @param arbitraryTransactionData - * @return - */ - public static String buildPathname(ArbitraryTransactionData arbitraryTransactionData) { - String senderAddress = Crypto.toAddress(arbitraryTransactionData.getSenderPublicKey()); - - StringBuilder stringBuilder = new StringBuilder(1024); - - stringBuilder.append(Settings.getInstance().getUserPath()); - stringBuilder.append("arbitrary"); - stringBuilder.append(File.separator); - stringBuilder.append(senderAddress); - stringBuilder.append(File.separator); - stringBuilder.append(arbitraryTransactionData.getService()); - stringBuilder.append(File.separator); - stringBuilder.append(Base58.encode(arbitraryTransactionData.getSignature())); - stringBuilder.append(".raw"); - - return stringBuilder.toString(); - } - private ArbitraryTransactionData getTransactionData(byte[] signature) throws DataException { TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature); if (transactionData == null) @@ -64,48 +33,89 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public boolean isDataLocal(byte[] signature) throws DataException { ArbitraryTransactionData transactionData = getTransactionData(signature); - if (transactionData == null) + if (transactionData == null) { return false; + } // Raw data is always available - if (transactionData.getDataType() == DataType.RAW_DATA) + if (transactionData.getDataType() == DataType.RAW_DATA) { return true; + } - String dataPathname = buildPathname(transactionData); + // Load hashes + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); - Path dataPath = Paths.get(dataPathname); - return Files.exists(dataPath); + // Load data file(s) + DataFile dataFile = DataFile.fromDigest(digest); + if (chunkHashes.length > 0) { + dataFile.addChunkHashes(chunkHashes); + } + + // Check if we already have the complete data file + if (dataFile.exists()) { + return true; + } + + // Alternatively, if we have all the chunks, then it's safe to assume the data is local + if (dataFile.allChunksExist(chunkHashes)) { + return true; + } + + return false; } @Override public byte[] fetchData(byte[] signature) throws DataException { ArbitraryTransactionData transactionData = getTransactionData(signature); - if (transactionData == null) - return null; - - // Raw data is always available - if (transactionData.getDataType() == DataType.RAW_DATA) - return transactionData.getData(); - - String dataPathname = buildPathname(transactionData); - - Path dataPath = Paths.get(dataPathname); - try { - return Files.readAllBytes(dataPath); - } catch (IOException e) { + if (transactionData == null) { return null; } + + // Raw data is always available + if (transactionData.getDataType() == DataType.RAW_DATA) { + return transactionData.getData(); + } + + // Load hashes + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + // Load data file(s) + DataFile dataFile = DataFile.fromDigest(digest); + if (chunkHashes.length > 0) { + dataFile.addChunkHashes(chunkHashes); + } + + // If we have the complete data file, return it + if (dataFile.exists()) { + return dataFile.getBytes(); + } + + // Alternatively, if we have all the chunks, combine them into a single file + if (dataFile.allChunksExist(chunkHashes)) { + dataFile.join(); + + // Verify that the combined hash matches the expected hash + if (digest.equals(dataFile.digest())) { + return dataFile.getBytes(); + } + } + + return null; } @Override public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException { // Already hashed? Nothing to do - if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH) + if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH) { return; + } // Trivial-sized payloads can remain in raw form - if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE) + if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE) { return; + } // Store non-trivial payloads in filesystem and convert transaction's data to hash form byte[] rawData = arbitraryTransactionData.getData(); @@ -115,48 +125,55 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryTransactionData.setData(dataHash); arbitraryTransactionData.setDataType(DataType.DATA_HASH); - String dataPathname = buildPathname(arbitraryTransactionData); + // Create DataFile + DataFile dataFile = new DataFile(rawData); - Path dataPath = Paths.get(dataPathname); - - // Make sure directory structure exists - try { - Files.createDirectories(dataPath.getParent()); - } catch (IOException e) { - throw new DataException("Unable to create arbitrary transaction directory", e); + // Verify that the data file is valid, and that it matches the expected hash + DataFile.ValidationResult validationResult = dataFile.isValid(); + if (validationResult != DataFile.ValidationResult.OK) { + dataFile.deleteAll(); + throw new DataException("Invalid data file when attempting to store arbitrary transaction data"); + } + if (!dataHash.equals(dataFile.digest())) { + dataFile.deleteAll(); + throw new DataException("Could not verify hash when attempting to store arbitrary transaction data"); } - // Output actual transaction data - try (OutputStream dataOut = Files.newOutputStream(dataPath)) { - dataOut.write(rawData); - } catch (IOException e) { - throw new DataException("Unable to store arbitrary transaction data", e); + // Now create chunks if needed + int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); + if (chunkCount > 0) { + LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s"))); + LOGGER.info("{}", dataFile.printChunks()); + + // Verify that the chunk hashes match those in the transaction + byte[] chunkHashes = dataFile.chunkHashes(); + if (!chunkHashes.equals(arbitraryTransactionData.getChunkHashes())) { + dataFile.deleteAll(); + throw new DataException("Could not verify chunk hashes when attempting to store arbitrary transaction data"); + } + } } @Override public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException { // No need to do anything if we still only have raw data, and hence nothing saved in filesystem - if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) + if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) { return; - - String dataPathname = buildPathname(arbitraryTransactionData); - Path dataPath = Paths.get(dataPathname); - try { - Files.deleteIfExists(dataPath); - - // Also attempt to delete parent directory if empty - Path servicePath = dataPath.getParent(); - Files.deleteIfExists(servicePath); - - // Also attempt to delete parent directory if empty - Path senderpath = servicePath.getParent(); - Files.deleteIfExists(senderpath); - } catch (DirectoryNotEmptyException e) { - // One of the parent service/sender directories still has data from other transactions - this is OK - } catch (IOException e) { - throw new DataException("Unable to delete arbitrary transaction data", e); } + + // Load hashes + byte[] digest = arbitraryTransactionData.getData(); + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + // Load data file(s) + DataFile dataFile = DataFile.fromDigest(digest); + if (chunkHashes.length > 0) { + dataFile.addChunkHashes(chunkHashes); + } + + // Delete file and chunks + dataFile.deleteAll(); } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 1dbac289..5e906d75 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -770,6 +770,20 @@ public class HSQLDBDatabaseUpdates { + "signature Signature, nonce INT NOT NULL, presence_type INT NOT NULL, " + "timestamp_signature Signature NOT NULL, " + TRANSACTION_KEYS + ")"); break; + case 34: + // ARBITRARY transaction updates for off-chain data storage + stmt.execute("CREATE TYPE ArbitraryDataHashes AS VARBINARY(8000)"); + // We may want to use a nonce rather than a transaction fee on the data chain + stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0"); + // We need to know the total size of the data file(s) associated with each transaction + stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0"); + // Larger data files need to be split into chunks, for easier transmission and greater decentralization + stmt.execute("ALTER TABLE ArbitraryTransactions ADD chunk_hashes ArbitraryDataHashes"); + // For finding data files by hash + stmt.execute("CREATE INDEX ArbitraryDataIndex ON ArbitraryTransactions (is_data_raw, data)"); + + // TODO: resource ID, compression, layers + break; default: // nothing to do diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java index 804b2b10..97f52f61 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java @@ -20,21 +20,23 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos } TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException { - String sql = "SELECT version, service, is_data_raw, data from ArbitraryTransactions WHERE signature = ?"; + String sql = "SELECT version, nonce, service, size, is_data_raw, data, chunk_hashes from ArbitraryTransactions WHERE signature = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) { if (resultSet == null) return null; int version = resultSet.getInt(1); - int service = resultSet.getInt(2); - boolean isDataRaw = resultSet.getBoolean(3); // NOT NULL, so no null to false + int nonce = resultSet.getInt(2); + int service = resultSet.getInt(3); + int size = resultSet.getInt(4); + boolean isDataRaw = resultSet.getBoolean(5); // NOT NULL, so no null to false DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; - byte[] data = resultSet.getBytes(4); + byte[] data = resultSet.getBytes(6); + byte[] chunkHashes = resultSet.getBytes(7); List payments = this.getPaymentsFromSignature(baseTransactionData.getSignature()); - - return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, data, dataType, chunkHashes, payments); } catch (SQLException e) { throw new DataException("Unable to fetch arbitrary transaction from repository", e); } @@ -52,7 +54,9 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos saveHelper.bind("signature", arbitraryTransactionData.getSignature()).bind("sender", arbitraryTransactionData.getSenderPublicKey()) .bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService()) - .bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData()); + .bind("nonce", arbitraryTransactionData.getNonce()).bind("size", arbitraryTransactionData.getSize()) + .bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData()) + .bind("chunk_hashes", arbitraryTransactionData.getChunkHashes()); try { saveHelper.execute(this.repository); diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 04ecc09f..ae6e7e05 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -4,12 +4,19 @@ import java.util.List; import java.util.stream.Collectors; import org.qortal.account.Account; +import org.qortal.crypto.Crypto; +import org.qortal.crypto.MemoryPoW; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.payment.Payment; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.storage.DataFile; +import org.qortal.storage.DataFileChunk; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.ArbitraryTransactionTransformer; +import org.qortal.transform.transaction.TransactionTransformer; public class ArbitraryTransaction extends Transaction { @@ -18,6 +25,10 @@ public class ArbitraryTransaction extends Transaction { // Other useful constants public static final int MAX_DATA_SIZE = 4000; + public static final int MAX_CHUNK_HASHES_LENGTH = 8000; + public static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; + public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes + public static final int POW_DIFFICULTY = 10; // leading zero bits // Constructors @@ -42,20 +53,122 @@ public class ArbitraryTransaction extends Transaction { // Processing + public void computeNonce() throws DataException { + byte[] transactionBytes; + + try { + transactionBytes = TransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + // Clear nonce from transactionBytes + ArbitraryTransactionTransformer.clearNonce(transactionBytes); + + int difficulty = POW_DIFFICULTY; + + // Calculate nonce + this.arbitraryTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, difficulty)); + } + @Override public ValidationResult isValid() throws DataException { - // Check data length - if (arbitraryTransactionData.getData().length < 1 || arbitraryTransactionData.getData().length > MAX_DATA_SIZE) + // Check that some data - or a data hash - has been supplied + if (arbitraryTransactionData.getData() == null) { return ValidationResult.INVALID_DATA_LENGTH; + } + + // Check data length + if (arbitraryTransactionData.getData().length < 1 || arbitraryTransactionData.getData().length > MAX_DATA_SIZE) { + return ValidationResult.INVALID_DATA_LENGTH; + } + + // Check hashes + if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.DATA_HASH) { + // Check length of data hash + if (arbitraryTransactionData.getData().length != HASH_LENGTH) { + return ValidationResult.INVALID_DATA_LENGTH; + } + + // Version 5+ + if (arbitraryTransactionData.getVersion() >= 5) { + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + // Check maximum length of chunk hashes + if (chunkHashes != null && chunkHashes.length > MAX_CHUNK_HASHES_LENGTH) { + return ValidationResult.INVALID_DATA_LENGTH; + } + + // Check expected length of chunk hashes + int chunkCount = arbitraryTransactionData.getSize() / DataFileChunk.CHUNK_SIZE; + int expectedChunkHashesSize = (chunkCount > 1) ? chunkCount * HASH_LENGTH : 0; + if (chunkHashes == null && expectedChunkHashesSize > 0) { + return ValidationResult.INVALID_DATA_LENGTH; + } + if (chunkHashes.length != expectedChunkHashesSize) { + return ValidationResult.INVALID_DATA_LENGTH; + } + } + } + + // Check raw data + if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.RAW_DATA) { + // Version 5+ + if (arbitraryTransactionData.getVersion() >= 5) { + // Check reported length of the raw data + // We should not download the raw data, so validation of that will be performed later + if (arbitraryTransactionData.getSize() > DataFile.MAX_FILE_SIZE) { + return ValidationResult.INVALID_DATA_LENGTH; + } + } + } // Wrap and delegate final payment validity checks to Payment class + // TODO: we won't be able to do this if we are on the data chain where fees may start as zero return new Payment(this.repository).isValid(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments(), arbitraryTransactionData.getFee()); } + @Override + public boolean isSignatureValid() { + byte[] signature = this.transactionData.getSignature(); + if (signature == null) { + return false; + } + + byte[] transactionBytes; + + try { + transactionBytes = ArbitraryTransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), signature, transactionBytes)) { + return false; + } + + // Nonce wasn't added until version 5+ + if (arbitraryTransactionData.getVersion() >= 5) { + + int nonce = arbitraryTransactionData.getNonce(); + + // Clear nonce from transactionBytes + ArbitraryTransactionTransformer.clearNonce(transactionBytes); + + int difficulty = POW_DIFFICULTY; + + // Check nonce + return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, difficulty, nonce); + } + + return true; + } + @Override public ValidationResult isProcessable() throws DataException { // Wrap and delegate final payment processable checks to Payment class + // TODO: we won't be able to do this if we are on the data chain where fees may start as zero return new Payment(this.repository).isProcessable(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments(), arbitraryTransactionData.getFee()); } diff --git a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java index 3402ca66..f9df86af 100644 --- a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java @@ -26,11 +26,14 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { // Property lengths private static final int SERVICE_LENGTH = INT_LENGTH; + private static final int NONCE_LENGTH = INT_LENGTH; private static final int DATA_TYPE_LENGTH = BYTE_LENGTH; private static final int DATA_SIZE_LENGTH = INT_LENGTH; + private static final int RAW_DATA_SIZE_LENGTH = INT_LENGTH; + private static final int CHUNKS_SIZE_LENGTH = INT_LENGTH; private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH; - private static final int EXTRAS_LENGTH = SERVICE_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH; + private static final int EXTRAS_LENGTH = SERVICE_LENGTH + NONCE_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH + RAW_DATA_SIZE_LENGTH + CHUNKS_SIZE_LENGTH; protected static final TransactionLayout layout; @@ -41,8 +44,9 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("transaction's groupID", TransformationType.INT); layout.add("reference", TransformationType.SIGNATURE); layout.add("sender's public key", TransformationType.PUBLIC_KEY); - layout.add("number of payments", TransformationType.INT); + layout.add("nonce", TransformationType.INT); // Version 5+ + layout.add("number of payments", TransformationType.INT); layout.add("* recipient", TransformationType.ADDRESS); layout.add("* asset ID of payment", TransformationType.LONG); layout.add("* payment amount", TransformationType.AMOUNT); @@ -51,6 +55,11 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("is data raw?", TransformationType.BOOLEAN); layout.add("data length", TransformationType.INT); layout.add("data", TransformationType.DATA); + + layout.add("raw data size", TransformationType.INT); // Version 5+ + layout.add("chunk count", TransformationType.INT); // Version 5+ + layout.add("chunk hashes", TransformationType.DATA); // Version 5+ + layout.add("fee", TransformationType.AMOUNT); layout.add("signature", TransformationType.SIGNATURE); } @@ -67,6 +76,11 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { byte[] senderPublicKey = Serialization.deserializePublicKey(byteBuffer); + int nonce = 0; + if (version >= 5) { + nonce = byteBuffer.getInt(); + } + // Always return a list of payments, even if empty List payments = new ArrayList<>(); if (version != 1) { @@ -91,6 +105,18 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { byte[] data = new byte[dataSize]; byteBuffer.get(data); + int size = 0; + byte[] chunkHashes = null; + + if (version >= 5) { + size = byteBuffer.getInt(); + + int chunkHashesSize = byteBuffer.getInt(); + + chunkHashes = new byte[chunkHashesSize]; + byteBuffer.get(chunkHashes); + } + long fee = byteBuffer.getLong(); byte[] signature = new byte[SIGNATURE_LENGTH]; @@ -98,13 +124,16 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature); - return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, data, dataType, chunkHashes, payments); } public static int getDataLength(TransactionData transactionData) throws TransformationException { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - int length = getBaseLength(transactionData) + EXTRAS_LENGTH + arbitraryTransactionData.getData().length; + int dataLength = (arbitraryTransactionData.getData() != null) ? arbitraryTransactionData.getData().length : 0; + int chunkHashesLength = (arbitraryTransactionData.getChunkHashes() != null) ? arbitraryTransactionData.getChunkHashes().length : 0; + + int length = getBaseLength(transactionData) + EXTRAS_LENGTH + dataLength + chunkHashesLength; // Optional payments length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength(); @@ -120,6 +149,10 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { transformCommonBytes(transactionData, bytes); + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce())); + } + List payments = arbitraryTransactionData.getPayments(); bytes.write(Ints.toByteArray(payments.size())); @@ -133,6 +166,16 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length)); bytes.write(arbitraryTransactionData.getData()); + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getSize())); + + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + int chunkHashesLength = (chunkHashes != null) ? chunkHashes.length : 0; + bytes.write(Ints.toByteArray(chunkHashesLength)); + + bytes.write(arbitraryTransactionData.getChunkHashes()); + } + bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee())); if (arbitraryTransactionData.getSignature() != null) @@ -159,6 +202,10 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { transformCommonBytes(arbitraryTransactionData, bytes); + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce())); + } + if (arbitraryTransactionData.getVersion() != 1) { List payments = arbitraryTransactionData.getPayments(); bytes.write(Ints.toByteArray(payments.size())); @@ -182,6 +229,16 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { break; } + if (arbitraryTransactionData.getVersion() >= 5) { + bytes.write(Ints.toByteArray(arbitraryTransactionData.getSize())); + + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + int chunkHashesLength = (chunkHashes != null) ? chunkHashes.length : 0; + bytes.write(Ints.toByteArray(chunkHashesLength)); + + bytes.write(arbitraryTransactionData.getChunkHashes()); + } + bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee())); // Never append signature @@ -192,4 +249,13 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { } } + public static void clearNonce(byte[] transactionBytes) { + int nonceIndex = TYPE_LENGTH + TIMESTAMP_LENGTH + GROUPID_LENGTH + REFERENCE_LENGTH + PUBLIC_KEY_LENGTH; + + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + } + } diff --git a/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java b/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java index 0b48748d..9e9814f8 100644 --- a/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java +++ b/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java @@ -18,6 +18,9 @@ public class ArbitraryTestTransaction extends TestTransaction { public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { final int version = 4; final int service = 123; + final int nonce = 0; // Version 4 doesn't need a nonce + final int size = 0; // Version 4 doesn't need a size + final byte[] chunkHashes = null; // Version 4 doesn't use chunk hashes byte[] data = new byte[1024]; random.nextBytes(data); @@ -31,7 +34,7 @@ public class ArbitraryTestTransaction extends TestTransaction { List payments = new ArrayList<>(); payments.add(new PaymentData(recipient, assetId, amount)); - return new ArbitraryTransactionData(generateBase(account), version, service, data, dataType, payments); + return new ArbitraryTransactionData(generateBase(account), version, service, nonce, size, data, dataType, chunkHashes, payments); } } From 56da7deb4c1aa5d5d1e6bf869dd7839e54cc5538 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 3 Jul 2021 17:41:52 +0100 Subject: [PATCH 080/505] DataFile updates to simplify integration with arbitrary transactions. --- .../java/org/qortal/storage/DataFile.java | 52 +++++++++++++++++++ .../org/qortal/storage/DataFileChunk.java | 5 ++ 2 files changed, 57 insertions(+) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 6f82799f..e473d23c 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -4,9 +4,11 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; +import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Base58; import java.io.*; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -173,6 +175,21 @@ public class DataFile { this.chunks.add(chunk); } + public void addChunkHashes(byte[] chunks) { + ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); + while (byteBuffer.remaining() > 0) { + byte[] chunkData = new byte[TransactionTransformer.SHA256_LENGTH]; + byteBuffer.get(chunkData); + if (chunkData.length == TransactionTransformer.SHA256_LENGTH) { + DataFileChunk chunk = new DataFileChunk(chunkData); + this.addChunk(chunk); + } + else { + throw new IllegalStateException(String.format("Invalid chunk hash length: %d", chunkData.length)); + } + } + } + public int split(int chunkSize) { try { @@ -252,7 +269,14 @@ public class DataFile { public boolean delete() { // Delete the complete file + // ... but only if it's inside the Qortal data directory Path path = Paths.get(this.filePath); + String dataPath = Settings.getInstance().getDataPath(); + Path dataDirectory = Paths.get(dataPath); + if (!path.toAbsolutePath().startsWith(dataDirectory)) { + return false; + } + if (Files.exists(path)) { try { Files.delete(path); @@ -330,6 +354,34 @@ public class DataFile { return file.exists(); } + public boolean chunkExists(byte[] digest) { + for (DataFileChunk chunk : this.chunks) { + if (digest.equals(chunk.digest())) { // TODO: this is too heavy on the filesystem. We need a cache + return chunk.exists(); + } + } + File file = new File(this.filePath); + return file.exists(); + } + + public boolean allChunksExist(byte[] chunks) { + ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); + while (byteBuffer.remaining() > 0) { + byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; + byteBuffer.get(chunkDigest); + if (chunkDigest.length == TransactionTransformer.SHA256_LENGTH) { + DataFileChunk chunk = DataFileChunk.fromDigest(chunkDigest); + if (chunk.exists() == false) { + return false; + } + } + else { + throw new IllegalStateException(String.format("Invalid chunk hash length: %d", chunkDigest.length)); + } + } + return true; + } + public long size() { Path path = Paths.get(this.filePath); try { diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/DataFileChunk.java index 544a150c..50a232a8 100644 --- a/src/main/java/org/qortal/storage/DataFileChunk.java +++ b/src/main/java/org/qortal/storage/DataFileChunk.java @@ -2,6 +2,7 @@ package org.qortal.storage; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.utils.Base58; import java.io.File; import java.io.IOException; @@ -34,6 +35,10 @@ public class DataFileChunk extends DataFile { return new DataFileChunk(filePath); } + public static DataFileChunk fromDigest(byte[] digest) { + return DataFileChunk.fromBase58Digest(Base58.encode(digest)); + } + @Override public ValidationResult isValid() { // DataChunk validation applies here too From e46c735efa01a52bf57ec882e1c61b8c3e9ef546 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 3 Jul 2021 18:05:17 +0100 Subject: [PATCH 081/505] Fixed recently introduced bugs with file management. --- src/main/java/org/qortal/storage/DataFile.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index e473d23c..56fb5327 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -145,7 +145,7 @@ public class DataFile { throw new IllegalStateException("Unable to create data subdirectory"); } } - return outputDirectory + base58Digest; + return outputDirectory + File.separator + base58Digest; } public ValidationResult isValid() { @@ -273,7 +273,7 @@ public class DataFile { Path path = Paths.get(this.filePath); String dataPath = Settings.getInstance().getDataPath(); Path dataDirectory = Paths.get(dataPath); - if (!path.toAbsolutePath().startsWith(dataDirectory)) { + if (!path.toAbsolutePath().startsWith(dataDirectory.toAbsolutePath())) { return false; } From 4b1de108d121e9ee2fdd3f1b78091221f9546b3e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 3 Jul 2021 18:42:42 +0100 Subject: [PATCH 082/505] Fixed bug in expected chunk count. --- src/main/java/org/qortal/transaction/ArbitraryTransaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index ae6e7e05..869efe85 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -100,7 +100,7 @@ public class ArbitraryTransaction extends Transaction { } // Check expected length of chunk hashes - int chunkCount = arbitraryTransactionData.getSize() / DataFileChunk.CHUNK_SIZE; + int chunkCount = (int)Math.ceil((double)arbitraryTransactionData.getSize() / (double)DataFileChunk.CHUNK_SIZE); int expectedChunkHashesSize = (chunkCount > 1) ? chunkCount * HASH_LENGTH : 0; if (chunkHashes == null && expectedChunkHashesSize > 0) { return ValidationResult.INVALID_DATA_LENGTH; From 49eddc9da509eae2895f08a96602e696d05fb59f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 09:31:51 +0100 Subject: [PATCH 083/505] Allow zero fee transactions if the fee is zero in blockchain.json Until now it wasn't possible to set up a chain with zero transaction fees due to a hardcoded zero check in Payment.isValid(), and a divide by zero error in Transaction.hasMinimumFeePerByte() --- src/main/java/org/qortal/payment/Payment.java | 5 +++-- src/main/java/org/qortal/transaction/Transaction.java | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/payment/Payment.java b/src/main/java/org/qortal/payment/Payment.java index cd7f1118..8b6070ee 100644 --- a/src/main/java/org/qortal/payment/Payment.java +++ b/src/main/java/org/qortal/payment/Payment.java @@ -40,8 +40,9 @@ public class Payment { public ValidationResult isValid(byte[] senderPublicKey, List payments, long fee, boolean isZeroAmountValid) throws DataException { AssetRepository assetRepository = this.repository.getAssetRepository(); - // Check fee is positive - if (fee <= 0) + // Check fee is positive or zero + // We have already checked that the fee is correct in the Transaction superclass + if (fee < 0) return ValidationResult.NEGATIVE_FEE; // Total up payment amounts by assetId diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index 2a57649c..cc986f41 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -348,6 +348,10 @@ public abstract class Transaction { long unitFee = BlockChain.getInstance().getUnitFee(); int maxBytePerUnitFee = BlockChain.getInstance().getMaxBytesPerUnitFee(); + // If the unit fee is zero, any fee is enough to cover the byte-length of the transaction + if (unitFee == 0) { + return true; + } return this.feePerByte() >= maxBytePerUnitFee / unitFee; } From 7af973b60dbe048b6792390f94847d83241476ce Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 09:41:53 +0100 Subject: [PATCH 084/505] Removed TODO comments that are now done. --- src/main/java/org/qortal/transaction/ArbitraryTransaction.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 869efe85..c4b595bf 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -124,7 +124,6 @@ public class ArbitraryTransaction extends Transaction { } // Wrap and delegate final payment validity checks to Payment class - // TODO: we won't be able to do this if we are on the data chain where fees may start as zero return new Payment(this.repository).isValid(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments(), arbitraryTransactionData.getFee()); } @@ -168,7 +167,6 @@ public class ArbitraryTransaction extends Transaction { @Override public ValidationResult isProcessable() throws DataException { // Wrap and delegate final payment processable checks to Payment class - // TODO: we won't be able to do this if we are on the data chain where fees may start as zero return new Payment(this.repository).isProcessable(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments(), arbitraryTransactionData.getFee()); } From d73f5ed2b5ea93e00294fd26909581243d66a04a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 09:45:52 +0100 Subject: [PATCH 085/505] /site/upload/creator/{address} now returns an unsigned ARBITRARY transaction, currently with pre-computed nonce --- .../qortal/api/resource/WebsiteResource.java | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index bf58492f..4db2ca71 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -27,12 +27,23 @@ import org.jsoup.select.Elements; import org.qortal.api.ApiError; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.Security; +import org.qortal.block.BlockChain; +import org.qortal.data.PaymentData; import org.qortal.data.account.AccountData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.ArbitraryTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; import org.qortal.utils.ZipUtils; @@ -49,7 +60,7 @@ public class WebsiteResource { @POST @Path("/upload/creator/{address}") @Operation( - summary = "Build raw, unsigned, HASHED_DATA transaction, based on a user-supplied path to a static website", + summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied path to a static website", requestBody = @RequestBody( required = true, content = @Content( @@ -61,7 +72,7 @@ public class WebsiteResource { ), responses = { @ApiResponse( - description = "raw, unsigned, HASHED_DATA transaction encoded in Base58", + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", content = @Content( mediaType = MediaType.TEXT_PLAIN, schema = @Schema( @@ -90,38 +101,44 @@ public class WebsiteResource { try (final Repository repository = RepositoryManager.getRepository()) { AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); - if (accountData == null) { + if (accountData == null || accountData.getPublicKey() == null) { + dataFile.deleteAll(); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); } byte[] creatorPublicKey = accountData.getPublicKey(); byte[] lastReference = accountData.getReference(); -// BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, -// lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); -// int size = (int)dataFile.size(); -// byte[] digest = dataFile.digest(); -// byte[] chunkHashes = dataFile.chunkHashes(); -// -// HashedDataTransactionData transactionData = new HashedDataTransactionData(baseTransactionData, -// 1, 2, 0, size, digest, chunkHashes); -// -// HashedDataTransaction transaction = (HashedDataTransaction)Transaction.fromData(repository, transactionData); -// transaction.computeNonce(); -// -// Transaction.ValidationResult result = transaction.isValidUnconfirmed(); -// if (result != Transaction.ValidationResult.OK) -// throw TransactionsResource.createTransactionInvalidException(request, result); -// -// byte[] bytes = HashedDataTransactionTransformer.toBytes(transactionData); -// return Base58.encode(bytes); - return "true"; + BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, + lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); + int size = (int)dataFile.size(); + ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + byte[] digest = dataFile.digest(); + byte[] chunkHashes = dataFile.chunkHashes(); + List payments = new ArrayList<>(); -// } catch (TransformationException e) { -// throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + 5, 2, 0, size, digest, dataType, chunkHashes, payments); + + ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + transaction.computeNonce(); + + Transaction.ValidationResult result = transaction.isValidUnconfirmed(); + if (result != Transaction.ValidationResult.OK) { + dataFile.deleteAll(); + throw TransactionsResource.createTransactionInvalidException(request, result); + } + + byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData); + return Base58.encode(bytes); + + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } + // Something went wrong, so delete our copies of the data and chunks + dataFile.deleteAll(); } return "false"; } From ffb39ef074cea84194fe9b71252c091ed4cf7956 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 09:49:01 +0100 Subject: [PATCH 086/505] /data API endpoints moved to /arbitrary --- .../api/resource/ArbitraryResource.java | 339 ++++++++++++++++- .../org/qortal/api/resource/DataResource.java | 354 ------------------ 2 files changed, 329 insertions(+), 364 deletions(-) delete mode 100644 src/main/java/org/qortal/api/resource/DataResource.java diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 26604318..6f2290c0 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -9,30 +9,37 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.File; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; -import org.qortal.api.ApiError; -import org.qortal.api.ApiErrors; -import org.qortal.api.ApiException; -import org.qortal.api.ApiExceptionFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.*; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.DataType; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.PeerAddress; +import org.qortal.network.message.DataFileMessage; +import org.qortal.network.message.GetDataFileMessage; +import org.qortal.network.message.Message; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; +import org.qortal.storage.DataFile; +import org.qortal.storage.DataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; @@ -45,6 +52,8 @@ import org.qortal.utils.Base58; @Tag(name = "Arbitrary") public class ArbitraryResource { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryResource.class); + @Context HttpServletRequest request; @@ -209,4 +218,314 @@ public class ArbitraryResource { } } -} \ No newline at end of file + @POST + @Path("/upload/path") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied file path", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "qortal.jar" + ) + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public String uploadFileAtPath(String path) { + Security.checkApiCallAllowed(request); + + // It's too dangerous to allow user-supplied filenames in weaker security contexts + if (Settings.getInstance().isApiRestricted()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Check if a file or directory has been supplied + File file = new File(path); + if (!file.isFile()) { + LOGGER.info("Not a file: {}", path); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + DataFile dataFile = new DataFile(path); + DataFile.ValidationResult validationResult = dataFile.isValid(); + if (validationResult != DataFile.ValidationResult.OK) { + LOGGER.error("Invalid file: {}", validationResult); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); + + int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); + if (chunkCount > 0) { + LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + return "true"; + } + + return "false"; + + } catch (DataException e) { + LOGGER.error("Repository issue when uploading data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (IllegalStateException e) { + LOGGER.error("Invalid upload data", e); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); + } + } + + + @DELETE + @Path("/file") + @Operation( + summary = "Delete file using supplied base58 encoded SHA256 digest string", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" + ) + ) + ), + responses = { + @ApiResponse( + description = "true if deleted, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + public String deleteFile(String base58Digest) { + Security.checkApiCallAllowed(request); + + DataFile dataFile = DataFile.fromBase58Digest(base58Digest); + if (dataFile.delete()) { + return "true"; + } + return "false"; + } + + @GET + @Path("/file/{hash}/frompeer/{peer}") + @Operation( + summary = "Request file from a given peer, using supplied base58 encoded SHA256 digest string", + responses = { + @ApiResponse( + description = "true if retrieved, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) + public Response getFileFromPeer(@PathParam("hash") String base58Digest, + @PathParam("peer") String targetPeerAddress) { + try { + if (base58Digest == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + if (targetPeerAddress == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + // Try to resolve passed address to make things easier + PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); + InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); + List peers = Network.getInstance().getHandshakedPeers(); + Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); + + if (targetPeer == null) { + LOGGER.info("Peer {} isn't connected", targetPeerAddress); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + boolean success = this.requestFile(base58Digest, targetPeer); + if (success) { + return Response.ok("true").build(); + } + return Response.ok("false").build(); + + } catch (UnknownHostException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + + @POST + @Path("/files/frompeer/{peer}") + @Operation( + summary = "Request multiple files from a given peer, using supplied comma separated base58 encoded SHA256 digest strings", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4,FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" + ) + ) + ), + responses = { + @ApiResponse( + description = "true if retrieved, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) + public Response getFilesFromPeer(String files, @PathParam("peer") String targetPeerAddress) { + try { + if (targetPeerAddress == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + // Try to resolve passed address to make things easier + PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); + InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); + List peers = Network.getInstance().getHandshakedPeers(); + Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); + + for (Peer peer : peers) { + LOGGER.info("peer: {}", peer); + } + + if (targetPeer == null) { + LOGGER.info("Peer {} isn't connected", targetPeerAddress); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + String base58DigestList[] = files.split(","); + for (String base58Digest : base58DigestList) { + if (base58Digest != null) { + boolean success = this.requestFile(base58Digest, targetPeer); + if (!success) { + LOGGER.info("Failed to request file {} from peer {}", base58Digest, targetPeerAddress); + } + } + } + return Response.ok("true").build(); + + } catch (UnknownHostException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + } + + + private boolean requestFile(String base58Digest, Peer targetPeer) { + try (final Repository repository = RepositoryManager.getRepository()) { + + DataFile dataFile = DataFile.fromBase58Digest(base58Digest); + if (dataFile.exists()) { + LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); + } + + byte[] digest = null; + try { + digest = Base58.decode(base58Digest); + } catch (NumberFormatException e) { + LOGGER.info("Invalid base58 encoded string"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + Message getDataFileMessage = new GetDataFileMessage(digest); + + Message message = targetPeer.getResponse(getDataFileMessage); + if (message == null) { + return false; + } + else if (message.getType() == Message.MessageType.BLOCK_SUMMARIES) { // TODO: use dedicated message type here + return false; + } + + DataFileMessage dataFileMessage = (DataFileMessage) message; + dataFile = dataFileMessage.getDataFile(); + if (dataFile == null || !dataFile.exists()) { + return false; + } + LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())); + return true; + } catch (ApiException e) { + throw e; + } catch (DataException | InterruptedException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/file/{hash}/build") + @Operation( + summary = "Join multiple chunks into a single file, using supplied comma separated base58 encoded SHA256 digest strings", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4,FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" + ) + ) + ), + responses = { + @ApiResponse( + description = "true if joined, false if not", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) + public Response joinFiles(String files, @PathParam("hash") String combinedHash) { + + if (combinedHash == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + DataFile dataFile = DataFile.fromBase58Digest(combinedHash); + if (dataFile.exists()) { + LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash); + } + + String base58DigestList[] = files.split(","); + for (String base58Digest : base58DigestList) { + if (base58Digest != null) { + DataFileChunk chunk = DataFileChunk.fromBase58Digest(base58Digest); + dataFile.addChunk(chunk); + } + } + boolean success = dataFile.join(); + if (success) { + if (combinedHash.equals(dataFile.base58Digest())) { + LOGGER.info("Valid hash {} after joining {} files", dataFile.base58Digest(), dataFile.chunkCount()); + return Response.ok("true").build(); + } + } + + return Response.ok("false").build(); + } + +} diff --git a/src/main/java/org/qortal/api/resource/DataResource.java b/src/main/java/org/qortal/api/resource/DataResource.java deleted file mode 100644 index 0b665a46..00000000 --- a/src/main/java/org/qortal/api/resource/DataResource.java +++ /dev/null @@ -1,354 +0,0 @@ -package org.qortal.api.resource; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.media.Content; -import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.qortal.api.*; -import org.qortal.network.Network; -import org.qortal.network.Peer; -import org.qortal.network.PeerAddress; -import org.qortal.network.message.*; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.settings.Settings; -import org.qortal.storage.DataFile; -import org.qortal.storage.DataFile.ValidationResult; -import org.qortal.storage.DataFileChunk; -import org.qortal.utils.Base58; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.io.File; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.util.List; - - -@Path("/data") -@Tag(name = "Data") -public class DataResource { - - private static final Logger LOGGER = LogManager.getLogger(DataResource.class); - - @Context - HttpServletRequest request; - - @POST - @Path("/upload/path") - @Operation( - summary = "Build raw, unsigned, UPLOAD_DATA transaction, based on a user-supplied file path", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", example = "qortal.jar" - ) - ) - ), - responses = { - @ApiResponse( - description = "raw, unsigned, UPLOAD_DATA transaction encoded in Base58", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public String uploadFileAtPath(String path) { - Security.checkApiCallAllowed(request); - - // It's too dangerous to allow user-supplied filenames in weaker security contexts - if (Settings.getInstance().isApiRestricted()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); - - try (final Repository repository = RepositoryManager.getRepository()) { - - // Check if a file or directory has been supplied - File file = new File(path); - if (!file.isFile()) { - LOGGER.info("Not a file: {}", path); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - DataFile dataFile = new DataFile(path); - ValidationResult validationResult = dataFile.isValid(); - if (validationResult != DataFile.ValidationResult.OK) { - LOGGER.error("Invalid file: {}", validationResult); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); - - int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); - if (chunkCount > 0) { - LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - return "true"; - } - - return "false"; - - } catch (DataException e) { - LOGGER.error("Repository issue when uploading data", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } catch (IllegalStateException e) { - LOGGER.error("Invalid upload data", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); - } - } - - - @DELETE - @Path("/file") - @Operation( - summary = "Delete file using supplied base58 encoded SHA256 digest string", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" - ) - ) - ), - responses = { - @ApiResponse( - description = "true if deleted, false if not", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - public String deleteFile(String base58Digest) { - Security.checkApiCallAllowed(request); - - DataFile dataFile = DataFile.fromBase58Digest(base58Digest); - if (dataFile.delete()) { - return "true"; - } - return "false"; - } - - @GET - @Path("/file/{hash}/frompeer/{peer}") - @Operation( - summary = "Request file from a given peer, using supplied base58 encoded SHA256 digest string", - responses = { - @ApiResponse( - description = "true if retrieved, false if not", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) - public Response getFileFromPeer(@PathParam("hash") String base58Digest, - @PathParam("peer") String targetPeerAddress) { - try { - if (base58Digest == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - if (targetPeerAddress == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - // Try to resolve passed address to make things easier - PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); - InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); - List peers = Network.getInstance().getHandshakedPeers(); - Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); - - if (targetPeer == null) { - LOGGER.info("Peer {} isn't connected", targetPeerAddress); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - boolean success = this.requestFile(base58Digest, targetPeer); - if (success) { - return Response.ok("true").build(); - } - return Response.ok("false").build(); - - } catch (UnknownHostException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - } - - @POST - @Path("/files/frompeer/{peer}") - @Operation( - summary = "Request multiple files from a given peer, using supplied comma separated base58 encoded SHA256 digest strings", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4,FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" - ) - ) - ), - responses = { - @ApiResponse( - description = "true if retrieved, false if not", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) - public Response getFilesFromPeer(String files, @PathParam("peer") String targetPeerAddress) { - try { - if (targetPeerAddress == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - // Try to resolve passed address to make things easier - PeerAddress peerAddress = PeerAddress.fromString(targetPeerAddress); - InetSocketAddress resolvedAddress = peerAddress.toSocketAddress(); - List peers = Network.getInstance().getHandshakedPeers(); - Peer targetPeer = peers.stream().filter(peer -> peer.getResolvedAddress().toString().contains(resolvedAddress.toString())).findFirst().orElse(null); - - for (Peer peer : peers) { - LOGGER.info("peer: {}", peer); - } - - if (targetPeer == null) { - LOGGER.info("Peer {} isn't connected", targetPeerAddress); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - String base58DigestList[] = files.split(","); - for (String base58Digest : base58DigestList) { - if (base58Digest != null) { - boolean success = this.requestFile(base58Digest, targetPeer); - if (!success) { - LOGGER.info("Failed to request file {} from peer {}", base58Digest, targetPeerAddress); - } - } - } - return Response.ok("true").build(); - - } catch (UnknownHostException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - } - - - private boolean requestFile(String base58Digest, Peer targetPeer) { - try (final Repository repository = RepositoryManager.getRepository()) { - - DataFile dataFile = DataFile.fromBase58Digest(base58Digest); - if (dataFile.exists()) { - LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); - } - - byte[] digest = null; - try { - digest = Base58.decode(base58Digest); - } catch (NumberFormatException e) { - LOGGER.info("Invalid base58 encoded string"); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - Message getDataFileMessage = new GetDataFileMessage(digest); - - Message message = targetPeer.getResponse(getDataFileMessage); - if (message == null) { - return false; - } - else if (message.getType() == Message.MessageType.BLOCK_SUMMARIES) { // TODO: use dedicated message type here - return false; - } - - DataFileMessage dataFileMessage = (DataFileMessage) message; - dataFile = dataFileMessage.getDataFile(); - if (dataFile == null || !dataFile.exists()) { - return false; - } - LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())); - return true; - } catch (ApiException e) { - throw e; - } catch (DataException | InterruptedException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @POST - @Path("/file/{hash}/build") - @Operation( - summary = "Join multiple chunks into a single file, using supplied comma separated base58 encoded SHA256 digest strings", - requestBody = @RequestBody( - required = true, - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string", example = "FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4,FZdHKgF5CbN2tKihvop5Ts9vmWmA9ZyyPY6bC1zivjy4" - ) - ) - ), - responses = { - @ApiResponse( - description = "true if joined, false if not", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "string" - ) - ) - ) - } - ) - @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) - public Response joinFiles(String files, @PathParam("hash") String combinedHash) { - - if (combinedHash == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - DataFile dataFile = DataFile.fromBase58Digest(combinedHash); - if (dataFile.exists()) { - LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash); - } - - String base58DigestList[] = files.split(","); - for (String base58Digest : base58DigestList) { - if (base58Digest != null) { - DataFileChunk chunk = DataFileChunk.fromBase58Digest(base58Digest); - dataFile.addChunk(chunk); - } - } - boolean success = dataFile.join(); - if (success) { - if (combinedHash.equals(dataFile.base58Digest())) { - LOGGER.info("Valid hash {} after joining {} files", dataFile.base58Digest(), dataFile.chunkCount()); - return Response.ok("true").build(); - } - } - - return Response.ok("false").build(); - } -} From 60415b922275cb7afe33ad5eaf18e06b17c2418d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 10:25:15 +0100 Subject: [PATCH 087/505] /arbitrary/upload/creator/{address} now returns an unsigned ARBITRARY transaction, currently with pre-computed nonce (same as the /site equivalent) --- .../api/resource/ArbitraryResource.java | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 6f2290c0..ba1750aa 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -25,9 +25,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.*; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.block.BlockChain; +import org.qortal.data.PaymentData; +import org.qortal.data.account.AccountData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.DataType; +import org.qortal.group.Group; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.PeerAddress; @@ -47,6 +52,7 @@ import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.transform.TransformationException; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.utils.Base58; +import org.qortal.utils.NTP; @Path("/arbitrary") @Tag(name = "Arbitrary") @@ -219,9 +225,9 @@ public class ArbitraryResource { } @POST - @Path("/upload/path") + @Path("/upload/creator/{address}") @Operation( - summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied file path", + summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied path to a single file", requestBody = @RequestBody( required = true, content = @Content( @@ -244,7 +250,7 @@ public class ArbitraryResource { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public String uploadFileAtPath(String path) { + public String uploadFileAtPath(@PathParam("address") String creatorAddress, String path) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts @@ -271,7 +277,45 @@ public class ArbitraryResource { int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); if (chunkCount > 0) { LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - return "true"; + + String base58Digest = dataFile.base58Digest(); + if (base58Digest != null) { + + AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); + if (accountData == null || accountData.getPublicKey() == null) { + dataFile.deleteAll(); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + } + byte[] creatorPublicKey = accountData.getPublicKey(); + byte[] lastReference = accountData.getReference(); + + BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, + lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); + int size = (int)dataFile.size(); + ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + byte[] digest = dataFile.digest(); + byte[] chunkHashes = dataFile.chunkHashes(); + List payments = new ArrayList<>(); + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + 5, 2, 0, size, digest, dataType, chunkHashes, payments); + + ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + transaction.computeNonce(); + + Transaction.ValidationResult result = transaction.isValidUnconfirmed(); + if (result != Transaction.ValidationResult.OK) { + dataFile.deleteAll(); + throw TransactionsResource.createTransactionInvalidException(request, result); + } + + byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData); + return Base58.encode(bytes); + + } + // Something went wrong, so delete our copies of the data and chunks + dataFile.deleteAll(); + } return "false"; @@ -279,6 +323,8 @@ public class ArbitraryResource { } catch (DataException e) { LOGGER.error("Repository issue when uploading data", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); } catch (IllegalStateException e) { LOGGER.error("Invalid upload data", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); From a742fecf9caf6ba42bc1bc13cf34ec59eb8bdfb2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 13:38:20 +0100 Subject: [PATCH 088/505] API refactors to avoid generic unhandled states. --- .../api/resource/ArbitraryResource.java | 80 ++++++++-------- .../qortal/api/resource/WebsiteResource.java | 92 ++++++++++--------- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index ba1750aa..920cc6e1 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -275,50 +275,48 @@ public class ArbitraryResource { LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); - if (chunkCount > 0) { - LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - - String base58Digest = dataFile.base58Digest(); - if (base58Digest != null) { - - AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); - if (accountData == null || accountData.getPublicKey() == null) { - dataFile.deleteAll(); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - } - byte[] creatorPublicKey = accountData.getPublicKey(); - byte[] lastReference = accountData.getReference(); - - BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, - lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); - int size = (int)dataFile.size(); - ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; - byte[] digest = dataFile.digest(); - byte[] chunkHashes = dataFile.chunkHashes(); - List payments = new ArrayList<>(); - - ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, 2, 0, size, digest, dataType, chunkHashes, payments); - - ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); - transaction.computeNonce(); - - Transaction.ValidationResult result = transaction.isValidUnconfirmed(); - if (result != Transaction.ValidationResult.OK) { - dataFile.deleteAll(); - throw TransactionsResource.createTransactionInvalidException(request, result); - } - - byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData); - return Base58.encode(bytes); - - } - // Something went wrong, so delete our copies of the data and chunks - dataFile.deleteAll(); + if (chunkCount == 0) { + LOGGER.error("No chunks created"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + String base58Digest = dataFile.base58Digest(); + if (base58Digest == null) { + LOGGER.error("Unable to calculate digest"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - return "false"; + AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); + if (accountData == null || accountData.getPublicKey() == null) { + dataFile.deleteAll(); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + } + byte[] creatorPublicKey = accountData.getPublicKey(); + byte[] lastReference = accountData.getReference(); + + BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, + lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); + int size = (int)dataFile.size(); + ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + byte[] digest = dataFile.digest(); + byte[] chunkHashes = dataFile.chunkHashes(); + List payments = new ArrayList<>(); + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + 5, 2, 0, size, digest, dataType, chunkHashes, payments); + + ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + transaction.computeNonce(); + + Transaction.ValidationResult result = transaction.isValidUnconfirmed(); + if (result != Transaction.ValidationResult.OK) { + dataFile.deleteAll(); + throw TransactionsResource.createTransactionInvalidException(request, result); + } + + byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData); + return Base58.encode(bytes); } catch (DataException e) { LOGGER.error("Repository issue when uploading data", e); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 4db2ca71..21b3c4e8 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -95,52 +95,54 @@ public class WebsiteResource { } DataFile dataFile = this.hostWebsite(path); - if (dataFile != null) { - String base58Digest = dataFile.base58Digest(); - if (base58Digest != null) { - try (final Repository repository = RepositoryManager.getRepository()) { - - AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); - if (accountData == null || accountData.getPublicKey() == null) { - dataFile.deleteAll(); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - } - byte[] creatorPublicKey = accountData.getPublicKey(); - byte[] lastReference = accountData.getReference(); - - BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, - lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); - int size = (int)dataFile.size(); - ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; - byte[] digest = dataFile.digest(); - byte[] chunkHashes = dataFile.chunkHashes(); - List payments = new ArrayList<>(); - - ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, 2, 0, size, digest, dataType, chunkHashes, payments); - - ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); - transaction.computeNonce(); - - Transaction.ValidationResult result = transaction.isValidUnconfirmed(); - if (result != Transaction.ValidationResult.OK) { - dataFile.deleteAll(); - throw TransactionsResource.createTransactionInvalidException(request, result); - } - - byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData); - return Base58.encode(bytes); - - } catch (TransformationException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - // Something went wrong, so delete our copies of the data and chunks - dataFile.deleteAll(); + if (dataFile == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + String base58Digest = dataFile.base58Digest(); + if (base58Digest != null) { + LOGGER.error("Unable to calculate digest"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + + AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); + if (accountData == null || accountData.getPublicKey() == null) { + dataFile.deleteAll(); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + } + byte[] creatorPublicKey = accountData.getPublicKey(); + byte[] lastReference = accountData.getReference(); + + BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, + lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); + int size = (int)dataFile.size(); + ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + byte[] digest = dataFile.digest(); + byte[] chunkHashes = dataFile.chunkHashes(); + List payments = new ArrayList<>(); + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + 5, 2, 0, size, digest, dataType, chunkHashes, payments); + + ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + transaction.computeNonce(); + + Transaction.ValidationResult result = transaction.isValidUnconfirmed(); + if (result != Transaction.ValidationResult.OK) { + dataFile.deleteAll(); + throw TransactionsResource.createTransactionInvalidException(request, result); + } + + byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData); + return Base58.encode(bytes); + + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } - return "false"; } @POST From 6407b5452bf8753c252ca77d64b444bed8d66d05 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 13:39:00 +0100 Subject: [PATCH 089/505] Delete our copies of data if any exception is thrown. --- .../api/resource/ArbitraryResource.java | 23 ++++++++++++------- .../qortal/api/resource/WebsiteResource.java | 2 ++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 920cc6e1..63d590f3 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -257,16 +257,20 @@ public class ArbitraryResource { if (Settings.getInstance().isApiRestricted()) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); + // Check if a file or directory has been supplied + File file = new File(path); + if (!file.isFile()) { + LOGGER.info("Not a file: {}", path); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + DataFile dataFile = new DataFile(path); + if (dataFile == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + try (final Repository repository = RepositoryManager.getRepository()) { - // Check if a file or directory has been supplied - File file = new File(path); - if (!file.isFile()) { - LOGGER.info("Not a file: {}", path); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - DataFile dataFile = new DataFile(path); DataFile.ValidationResult validationResult = dataFile.isValid(); if (validationResult != DataFile.ValidationResult.OK) { LOGGER.error("Invalid file: {}", validationResult); @@ -319,11 +323,14 @@ public class ArbitraryResource { return Base58.encode(bytes); } catch (DataException e) { + dataFile.deleteAll(); LOGGER.error("Repository issue when uploading data", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } catch (TransformationException e) { + dataFile.deleteAll(); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); } catch (IllegalStateException e) { + dataFile.deleteAll(); LOGGER.error("Invalid upload data", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); } diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 21b3c4e8..59c56137 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -139,8 +139,10 @@ public class WebsiteResource { return Base58.encode(bytes); } catch (TransformationException e) { + dataFile.deleteAll(); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); } catch (DataException e) { + dataFile.deleteAll(); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } From bb5b62466e16ce53d938010dec5ae47019a08da4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 13:41:37 +0100 Subject: [PATCH 090/505] Fixed bug introduced in recent commit. --- src/main/java/org/qortal/api/resource/WebsiteResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 59c56137..6a963669 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -100,7 +100,7 @@ public class WebsiteResource { } String base58Digest = dataFile.base58Digest(); - if (base58Digest != null) { + if (base58Digest == null) { LOGGER.error("Unable to calculate digest"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } From cb4203b6dbe4991fd3a38e56700edde93335c251 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 14:53:54 +0100 Subject: [PATCH 091/505] Use public key as parameter instead of address, since we can obtain the address from the public key in all cases. --- .../api/resource/ArbitraryResource.java | 22 ++++++++++--------- .../qortal/api/resource/WebsiteResource.java | 17 ++++++-------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 63d590f3..3624c129 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -26,6 +26,7 @@ import org.apache.logging.log4j.Logger; import org.qortal.api.*; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.block.BlockChain; +import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; import org.qortal.data.account.AccountData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -225,7 +226,7 @@ public class ArbitraryResource { } @POST - @Path("/upload/creator/{address}") + @Path("/upload/creator/{publickey}") @Operation( summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied path to a single file", requestBody = @RequestBody( @@ -250,12 +251,18 @@ public class ArbitraryResource { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE}) - public String uploadFileAtPath(@PathParam("address") String creatorAddress, String path) { + public String uploadFileAtPath(@PathParam("publickey") String creatorPublicKeyBase58, String path) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts - if (Settings.getInstance().isApiRestricted()) + if (Settings.getInstance().isApiRestricted()) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); + } + + if (creatorPublicKeyBase58 == null || path == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); // Check if a file or directory has been supplied File file = new File(path); @@ -291,13 +298,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); - if (accountData == null || accountData.getPublicKey() == null) { - dataFile.deleteAll(); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - } - byte[] creatorPublicKey = accountData.getPublicKey(); - byte[] lastReference = accountData.getReference(); + String creatorAddress = Crypto.toAddress(creatorPublicKey); + byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 6a963669..aa25d463 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -28,6 +28,7 @@ import org.qortal.api.ApiError; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.Security; import org.qortal.block.BlockChain; +import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; import org.qortal.data.account.AccountData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -58,7 +59,7 @@ public class WebsiteResource { @Context ServletContext context; @POST - @Path("/upload/creator/{address}") + @Path("/upload/creator/{publickey}") @Operation( summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied path to a static website", requestBody = @RequestBody( @@ -82,7 +83,7 @@ public class WebsiteResource { ) } ) - public String uploadWebsite(@PathParam("address") String creatorAddress, String path) { + public String uploadWebsite(@PathParam("publickey") String creatorPublicKeyBase58, String path) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts @@ -90,9 +91,10 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } - if (creatorAddress == null || path == null) { + if (creatorPublicKeyBase58 == null || path == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } + byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); DataFile dataFile = this.hostWebsite(path); if (dataFile == null) { @@ -107,13 +109,8 @@ public class WebsiteResource { try (final Repository repository = RepositoryManager.getRepository()) { - AccountData accountData = repository.getAccountRepository().getAccount(creatorAddress); - if (accountData == null || accountData.getPublicKey() == null) { - dataFile.deleteAll(); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); - } - byte[] creatorPublicKey = accountData.getPublicKey(); - byte[] lastReference = accountData.getReference(); + String creatorAddress = Crypto.toAddress(creatorPublicKey); + byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); From 2f2c4964c5b7a48c779488125e5296514318cada Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 16:43:12 +0100 Subject: [PATCH 092/505] Transaction version temporarily bumped to 5. TODO: We will need to set a hard fork timestamp if this is ever merged back into the main Qortal core repo. --- src/main/java/org/qortal/transaction/Transaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index cc986f41..f026d0ae 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -380,7 +380,7 @@ public abstract class Transaction { * @return transaction version number */ public static int getVersionByTimestamp(long timestamp) { - return 4; + return 5; // TODO: hard fork timestamp!! } /** From 10dc19652e7781a76fc47a83d2a00281caee4a54 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 4 Jul 2021 16:52:20 +0100 Subject: [PATCH 093/505] Use "qortaldata-" version prefix --- src/main/java/org/qortal/controller/Controller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 67835dd2..8f56bd5a 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -78,7 +78,7 @@ public class Controller extends Thread { /** Controller start-up time (ms) taken using System.currentTimeMillis(). */ public static final long startTime = System.currentTimeMillis(); - public static final String VERSION_PREFIX = "qortal-"; + public static final String VERSION_PREFIX = "qortaldata-"; private static final Logger LOGGER = LogManager.getLogger(Controller.class); private static final long MISBEHAVIOUR_COOLOFF = 10 * 60 * 1000L; // ms From 0086c6373bdab82f2f7daac1bf32e7ee4afe33d1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 5 Jul 2021 07:26:20 +0100 Subject: [PATCH 094/505] Significant refactor of DataFile and DataFileChunk This introduces the hash58 property, which stores the base58 hash of the file passed in at initialization. It leaves digest() and digest58() for when we need to compute a new hash from the file itself. --- .../api/resource/ArbitraryResource.java | 53 +++-- .../qortal/api/resource/WebsiteResource.java | 19 +- .../java/org/qortal/storage/DataFile.java | 212 ++++++++++-------- .../org/qortal/storage/DataFileChunk.java | 21 +- src/test/java/org/qortal/test/DataTests.java | 8 +- 5 files changed, 159 insertions(+), 154 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 3624c129..8f4dad61 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -28,7 +28,6 @@ import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; -import org.qortal.data.account.AccountData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; @@ -271,7 +270,7 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } - DataFile dataFile = new DataFile(path); + DataFile dataFile = DataFile.fromPath(path); if (dataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } @@ -283,7 +282,7 @@ public class ArbitraryResource { LOGGER.error("Invalid file: {}", validationResult); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); + LOGGER.info("Whole file digest: {}", dataFile.digest58()); int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); if (chunkCount == 0) { @@ -292,8 +291,8 @@ public class ArbitraryResource { } LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - String base58Digest = dataFile.base58Digest(); - if (base58Digest == null) { + String digest58 = dataFile.digest58(); + if (digest58 == null) { LOGGER.error("Unable to calculate digest"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } @@ -364,10 +363,10 @@ public class ArbitraryResource { ) } ) - public String deleteFile(String base58Digest) { + public String deleteFile(String hash58) { Security.checkApiCallAllowed(request); - DataFile dataFile = DataFile.fromBase58Digest(base58Digest); + DataFile dataFile = DataFile.fromHash58(hash58); if (dataFile.delete()) { return "true"; } @@ -377,7 +376,7 @@ public class ArbitraryResource { @GET @Path("/file/{hash}/frompeer/{peer}") @Operation( - summary = "Request file from a given peer, using supplied base58 encoded SHA256 digest string", + summary = "Request file from a given peer, using supplied base58 encoded SHA256 hash", responses = { @ApiResponse( description = "true if retrieved, false if not", @@ -391,10 +390,10 @@ public class ArbitraryResource { } ) @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) - public Response getFileFromPeer(@PathParam("hash") String base58Digest, + public Response getFileFromPeer(@PathParam("hash") String hash58, @PathParam("peer") String targetPeerAddress) { try { - if (base58Digest == null) { + if (hash58 == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } if (targetPeerAddress == null) { @@ -412,7 +411,7 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } - boolean success = this.requestFile(base58Digest, targetPeer); + boolean success = this.requestFile(hash58, targetPeer); if (success) { return Response.ok("true").build(); } @@ -426,7 +425,7 @@ public class ArbitraryResource { @POST @Path("/files/frompeer/{peer}") @Operation( - summary = "Request multiple files from a given peer, using supplied comma separated base58 encoded SHA256 digest strings", + summary = "Request multiple files from a given peer, using supplied comma separated base58 encoded SHA256 hashes", requestBody = @RequestBody( required = true, content = @Content( @@ -470,12 +469,12 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } - String base58DigestList[] = files.split(","); - for (String base58Digest : base58DigestList) { - if (base58Digest != null) { - boolean success = this.requestFile(base58Digest, targetPeer); + String hash58List[] = files.split(","); + for (String hash58 : hash58List) { + if (hash58 != null) { + boolean success = this.requestFile(hash58, targetPeer); if (!success) { - LOGGER.info("Failed to request file {} from peer {}", base58Digest, targetPeerAddress); + LOGGER.info("Failed to request file {} from peer {}", hash58, targetPeerAddress); } } } @@ -487,17 +486,17 @@ public class ArbitraryResource { } - private boolean requestFile(String base58Digest, Peer targetPeer) { + private boolean requestFile(String hash58, Peer targetPeer) { try (final Repository repository = RepositoryManager.getRepository()) { - DataFile dataFile = DataFile.fromBase58Digest(base58Digest); + DataFile dataFile = DataFile.fromHash58(hash58); if (dataFile.exists()) { LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); } byte[] digest = null; try { - digest = Base58.decode(base58Digest); + digest = Base58.decode(hash58); } catch (NumberFormatException e) { LOGGER.info("Invalid base58 encoded string"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -558,22 +557,22 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } - DataFile dataFile = DataFile.fromBase58Digest(combinedHash); + DataFile dataFile = DataFile.fromHash58(combinedHash); if (dataFile.exists()) { LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash); } - String base58DigestList[] = files.split(","); - for (String base58Digest : base58DigestList) { - if (base58Digest != null) { - DataFileChunk chunk = DataFileChunk.fromBase58Digest(base58Digest); + String hash58List[] = files.split(","); + for (String hash58 : hash58List) { + if (hash58 != null) { + DataFileChunk chunk = DataFileChunk.fromHash58(hash58); dataFile.addChunk(chunk); } } boolean success = dataFile.join(); if (success) { - if (combinedHash.equals(dataFile.base58Digest())) { - LOGGER.info("Valid hash {} after joining {} files", dataFile.base58Digest(), dataFile.chunkCount()); + if (combinedHash.equals(dataFile.digest58())) { + LOGGER.info("Valid hash {} after joining {} files", dataFile.digest58(), dataFile.chunkCount()); return Response.ok("true").build(); } } diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index aa25d463..5965c5aa 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -30,7 +30,6 @@ import org.qortal.api.Security; import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; -import org.qortal.data.account.AccountData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.group.Group; @@ -101,8 +100,8 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - String base58Digest = dataFile.base58Digest(); - if (base58Digest == null) { + String digest58 = dataFile.digest58(); + if (digest58 == null) { LOGGER.error("Unable to calculate digest"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } @@ -179,9 +178,9 @@ public class WebsiteResource { DataFile dataFile = this.hostWebsite(directoryPath); if (dataFile != null) { - String base58Digest = dataFile.base58Digest(); - if (base58Digest != null) { - return "http://localhost:12393/site/" + base58Digest; + String digest58 = dataFile.digest58(); + if (digest58 != null) { + return "http://localhost:12393/site/" + digest58; } } return "Unable to generate preview URL"; @@ -215,13 +214,13 @@ public class WebsiteResource { } try { - DataFile dataFile = new DataFile(outputFilePath); + DataFile dataFile = DataFile.fromPath(outputFilePath); DataFile.ValidationResult validationResult = dataFile.isValid(); if (validationResult != DataFile.ValidationResult.OK) { LOGGER.error("Invalid file: {}", validationResult); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); + LOGGER.info("Whole file digest: {}", dataFile.digest58()); int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); if (chunkCount > 0) { @@ -265,13 +264,13 @@ public class WebsiteResource { if (!Files.exists(Paths.get(unzippedPath))) { // Load file - DataFile dataFile = DataFile.fromBase58Digest(resourceId); + DataFile dataFile = DataFile.fromHash58(resourceId); if (dataFile == null || !dataFile.exists()) { LOGGER.info("Unable to validate complete file hash"); return this.get404Response(); } - if (!dataFile.base58Digest().equals(resourceId)) { + if (!dataFile.digest58().equals(resourceId)) { LOGGER.info("Unable to validate complete file hash"); return this.get404Response(); } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 56fb5327..cfd9a843 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -13,9 +13,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.Map; +import java.util.*; import java.util.stream.Stream; import static java.util.Arrays.stream; @@ -50,28 +48,17 @@ public class DataFile { public static int SHORT_DIGEST_LENGTH = 8; protected String filePath; + protected String hash58; private ArrayList chunks; public DataFile() { } - public DataFile(String filePath) { + public DataFile(String hash58) { this.createDataDirectory(); - this.filePath = filePath; + this.filePath = DataFile.getOutputFilePath(hash58, false); this.chunks = new ArrayList<>(); - - if (!this.isInBaseDirectory(filePath)) { - // Copy file to base directory - LOGGER.debug("Copying file to data directory..."); - this.filePath = this.copyToDataDirectory(); - if (this.filePath == null) { - throw new IllegalStateException("Invalid file path after copy"); - } - } - } - - public DataFile(File file) { - this(file.getPath()); + this.hash58 = hash58; } public DataFile(byte[] fileContent) { @@ -80,17 +67,17 @@ public class DataFile { return; } - String base58Digest = Base58.encode(Crypto.digest(fileContent)); - LOGGER.debug(String.format("File digest: %s, size: %d bytes", base58Digest, fileContent.length)); + this.hash58 = Base58.encode(Crypto.digest(fileContent)); + LOGGER.debug(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); - String outputFilePath = this.getOutputFilePath(base58Digest, true); + String outputFilePath = getOutputFilePath(this.hash58, true); File outputFile = new File(outputFilePath); try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { outputStream.write(fileContent); this.filePath = outputFilePath; // Verify hash - if (!base58Digest.equals(this.base58Digest())) { - LOGGER.error("Digest {} does not match file digest {}", base58Digest, this.base58Digest()); + if (!this.hash58.equals(this.digest58())) { + LOGGER.error("Hash {} does not match file digest {}", this.hash58, this.digest58()); this.delete(); throw new IllegalStateException("Data file digest validation failed"); } @@ -99,13 +86,38 @@ public class DataFile { } } - public static DataFile fromBase58Digest(String base58Digest) { - String filePath = DataFile.getOutputFilePath(base58Digest, false); - return new DataFile(filePath); + public static DataFile fromHash58(String hash58) { + return new DataFile(hash58); } public static DataFile fromDigest(byte[] digest) { - return DataFile.fromBase58Digest(Base58.encode(digest)); + return DataFile.fromHash58(Base58.encode(digest)); + } + + public static DataFile fromPath(String path) { + File file = new File(path); + if (file.exists()) { + try { + byte[] fileContent = Files.readAllBytes(file.toPath()); + byte[] digest = Crypto.digest(fileContent); + DataFile dataFile = DataFile.fromDigest(digest); + + // Copy file to base directory if needed + Path filePath = Paths.get(path); + if (Files.exists(filePath) && !dataFile.isInBaseDirectory(path)) { + dataFile.copyToDataDirectory(filePath); + } + return dataFile; + + } catch (IOException e) { + LOGGER.error("Couldn't compute digest for DataFile"); + } + } + return null; + } + + public static DataFile fromFile(File file) { + return DataFile.fromPath(file.getPath()); } private boolean createDataDirectory() { @@ -121,21 +133,27 @@ public class DataFile { return true; } - private String copyToDataDirectory() { - String outputFilePath = this.getOutputFilePath(this.base58Digest(), true); - Path source = Paths.get(this.filePath).toAbsolutePath(); - Path dest = Paths.get(outputFilePath).toAbsolutePath(); + private String copyToDataDirectory(Path sourcePath) { + if (this.hash58 == null || this.filePath == null) { + return null; + } + String outputFilePath = getOutputFilePath(this.hash58, true); + sourcePath = sourcePath.toAbsolutePath(); + Path destPath = Paths.get(outputFilePath).toAbsolutePath(); try { - return Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING).toString(); + return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING).toString(); } catch (IOException e) { throw new IllegalStateException("Unable to copy file to data directory"); } } - public static String getOutputFilePath(String base58Digest, boolean createDirectories) { - String base58DigestFirst2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); - String base58DigestNext2Chars = base58Digest.substring(2, Math.min(base58Digest.length(), 4)); - String outputDirectory = Settings.getInstance().getDataPath() + File.separator + base58DigestFirst2Chars + File.separator + base58DigestNext2Chars; + public static String getOutputFilePath(String hash58, boolean createDirectories) { + if (hash58 == null) { + return null; + } + String hash58First2Chars = hash58.substring(0, 2); + String hash58Next2Chars = hash58.substring(2, 4); + String outputDirectory = Settings.getInstance().getDataPath() + File.separator + hash58First2Chars + File.separator + hash58Next2Chars; Path outputDirectoryPath = Paths.get(outputDirectory); if (createDirectories) { @@ -145,7 +163,7 @@ public class DataFile { throw new IllegalStateException("Unable to create data subdirectory"); } } - return outputDirectory + File.separator + base58Digest; + return outputDirectory + File.separator + hash58; } public ValidationResult isValid() { @@ -177,16 +195,11 @@ public class DataFile { public void addChunkHashes(byte[] chunks) { ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); - while (byteBuffer.remaining() > 0) { - byte[] chunkData = new byte[TransactionTransformer.SHA256_LENGTH]; - byteBuffer.get(chunkData); - if (chunkData.length == TransactionTransformer.SHA256_LENGTH) { - DataFileChunk chunk = new DataFileChunk(chunkData); - this.addChunk(chunk); - } - else { - throw new IllegalStateException(String.format("Invalid chunk hash length: %d", chunkData.length)); - } + while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { + byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; + byteBuffer.get(chunkDigest); + DataFileChunk chunk = DataFileChunk.fromHash(chunkDigest); + this.addChunk(chunk); } } @@ -232,7 +245,7 @@ public class DataFile { // Create temporary path for joined file Path tempPath; try { - tempPath = Files.createTempFile(this.chunks.get(0).base58Digest(), ".tmp"); + tempPath = Files.createTempFile(this.chunks.get(0).digest58(), ".tmp"); } catch (IOException e) { return false; } @@ -245,7 +258,7 @@ public class DataFile { File sourceFile = new File(chunk.filePath); BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); byte[] buffer = new byte[2048]; - int inSize = -1; + int inSize; while ((inSize = in.read(buffer)) != -1) { out.write(buffer, 0, inSize); } @@ -254,7 +267,7 @@ public class DataFile { out.close(); // Copy temporary file to data directory - this.filePath = this.copyToDataDirectory(); + this.filePath = this.copyToDataDirectory(tempPath); Files.delete(tempPath); return true; @@ -328,8 +341,7 @@ public class DataFile { public byte[] getBytes() { Path path = Paths.get(this.filePath); try { - byte[] bytes = Files.readAllBytes(path); - return bytes; + return Files.readAllBytes(path); } catch (IOException e) { LOGGER.error("Unable to read bytes for file"); return null; @@ -343,10 +355,7 @@ public class DataFile { Path path = Paths.get(filePath).toAbsolutePath(); String dataPath = Settings.getInstance().getDataPath(); String basePath = Paths.get(dataPath).toAbsolutePath().toString(); - if (path.startsWith(basePath)) { - return true; - } - return false; + return path.startsWith(basePath); } public boolean exists() { @@ -354,9 +363,9 @@ public class DataFile { return file.exists(); } - public boolean chunkExists(byte[] digest) { + public boolean chunkExists(byte[] hash) { for (DataFileChunk chunk : this.chunks) { - if (digest.equals(chunk.digest())) { // TODO: this is too heavy on the filesystem. We need a cache + if (Arrays.equals(hash, chunk.getHash())) { return chunk.exists(); } } @@ -366,17 +375,12 @@ public class DataFile { public boolean allChunksExist(byte[] chunks) { ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); - while (byteBuffer.remaining() > 0) { - byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; - byteBuffer.get(chunkDigest); - if (chunkDigest.length == TransactionTransformer.SHA256_LENGTH) { - DataFileChunk chunk = DataFileChunk.fromDigest(chunkDigest); - if (chunk.exists() == false) { - return false; - } - } - else { - throw new IllegalStateException(String.format("Invalid chunk hash length: %d", chunkDigest.length)); + while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { + byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; + byteBuffer.get(chunkHash); + DataFileChunk chunk = DataFileChunk.fromHash(chunkHash); + if (!chunk.exists()) { + return false; } } return true; @@ -395,6 +399,35 @@ public class DataFile { return this.chunks.size(); } + public List getChunks() { + return this.chunks; + } + + public byte[] chunkHashes() { + if (this.chunks != null && this.chunks.size() > 0) { + // Return null if we only have one chunk, with the same hash as the parent + if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) { + return null; + } + + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (DataFileChunk chunk : this.chunks) { + byte[] chunkHash = chunk.digest(); + if (chunkHash.length != 32) { + LOGGER.info("Invalid chunk hash length: {}", chunkHash.length); + throw new IllegalStateException("Invalid chunk hash length"); + } + outputStream.write(chunk.digest()); + } + return outputStream.toByteArray(); + } catch (IOException e) { + return null; + } + } + return null; + } + private File getFile() { File file = new File(this.filePath); if (file.exists()) { @@ -421,32 +454,7 @@ public class DataFile { return null; } - public byte[] chunkHashes() { - if (this.chunks != null && this.chunks.size() > 0) { - // Return null if we only have one chunk, with the same hash as the parent - if (this.digest().equals(this.chunks.get(0).digest())) { - return null; - } - - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - for (DataFileChunk chunk : this.chunks) { - byte[] chunkHash = chunk.digest(); - if (chunkHash.length != 32) { - LOGGER.info("Invalid chunk hash length: {}", chunkHash.length); - throw new IllegalStateException("Invalid chunk hash length"); - } - outputStream.write(chunk.digest()); - } - return outputStream.toByteArray(); - } catch (IOException e) { - return null; - } - } - return null; - } - - public String base58Digest() { + public String digest58() { if (this.digest() != null) { return Base58.encode(this.digest()); } @@ -454,10 +462,18 @@ public class DataFile { } public String shortDigest() { - if (this.base58Digest() == null) { + if (this.digest58() == null) { return null; } - return this.base58Digest().substring(0, Math.min(this.base58Digest().length(), SHORT_DIGEST_LENGTH)); + return this.digest58().substring(0, Math.min(this.digest58().length(), SHORT_DIGEST_LENGTH)); + } + + public String getHash58() { + return this.hash58; + } + + public byte[] getHash() { + return Base58.decode(this.hash58); } public String printChunks() { @@ -467,7 +483,7 @@ public class DataFile { if (outputString.length() > 0) { outputString = outputString.concat(","); } - outputString = outputString.concat(chunk.base58Digest()); + outputString = outputString.concat(chunk.digest58()); } } return outputString; diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/DataFileChunk.java index 50a232a8..ebc6641e 100644 --- a/src/main/java/org/qortal/storage/DataFileChunk.java +++ b/src/main/java/org/qortal/storage/DataFileChunk.java @@ -4,7 +4,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.utils.Base58; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -15,28 +14,20 @@ public class DataFileChunk extends DataFile { private static final Logger LOGGER = LogManager.getLogger(DataFileChunk.class); - public DataFileChunk() { - } - - public DataFileChunk(String filePath) { - super(filePath); - } - - public DataFileChunk(File file) { - super(file); + public DataFileChunk(String hash58) { + super(hash58); } public DataFileChunk(byte[] fileContent) { super(fileContent); } - public static DataFileChunk fromBase58Digest(String base58Digest) { - String filePath = DataFile.getOutputFilePath(base58Digest, false); - return new DataFileChunk(filePath); + public static DataFileChunk fromHash58(String hash58) { + return new DataFileChunk(hash58); } - public static DataFileChunk fromDigest(byte[] digest) { - return DataFileChunk.fromBase58Digest(Base58.encode(digest)); + public static DataFileChunk fromHash(byte[] hash) { + return DataFileChunk.fromHash58(Base58.encode(hash)); } @Override diff --git a/src/test/java/org/qortal/test/DataTests.java b/src/test/java/org/qortal/test/DataTests.java index a628443f..415025be 100644 --- a/src/test/java/org/qortal/test/DataTests.java +++ b/src/test/java/org/qortal/test/DataTests.java @@ -23,7 +23,7 @@ public class DataTests extends Common { DataFile dataFile = new DataFile(dummyDataString.getBytes()); assertTrue(dataFile.exists()); assertEquals(62, dataFile.size()); - assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.base58Digest()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58()); // Split into 7 chunks, each 10 bytes long dataFile.split(10); @@ -41,7 +41,7 @@ public class DataTests extends Common { // Validate that the original file is intact assertTrue(dataFile.exists()); assertEquals(62, dataFile.size()); - assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.base58Digest()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58()); } @Test @@ -53,7 +53,7 @@ public class DataTests extends Common { DataFile dataFile = new DataFile(randomData); assertTrue(dataFile.exists()); assertEquals(fileSize, dataFile.size()); - String originalFileDigest = dataFile.base58Digest(); + String originalFileDigest = dataFile.digest58(); // Split into chunks using 1MiB chunk size dataFile.split(1 * 1024 * 1024); @@ -71,7 +71,7 @@ public class DataTests extends Common { // Validate that the original file is intact assertTrue(dataFile.exists()); assertEquals(fileSize, dataFile.size()); - assertEquals(originalFileDigest, dataFile.base58Digest()); + assertEquals(originalFileDigest, dataFile.digest58()); } } From 7531fe14fe4ca1ad7abf7e7e59d20a7e337435bb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 5 Jul 2021 08:23:29 +0100 Subject: [PATCH 095/505] Fixed major performance issue in DataFile.toString() --- src/main/java/org/qortal/storage/DataFile.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index cfd9a843..585e3711 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -461,11 +461,11 @@ public class DataFile { return null; } - public String shortDigest() { - if (this.digest58() == null) { + public String shortHash58() { + if (this.hash58 == null) { return null; } - return this.digest58().substring(0, Math.min(this.digest58().length(), SHORT_DIGEST_LENGTH)); + return this.hash58.substring(0, Math.min(this.hash58.length(), SHORT_DIGEST_LENGTH)); } public String getHash58() { @@ -491,6 +491,6 @@ public class DataFile { @Override public String toString() { - return this.shortDigest(); + return this.shortHash58(); } } From 5319c5f8321467a59aa223992c3f2c6ab83892cb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 5 Jul 2021 09:05:45 +0100 Subject: [PATCH 096/505] Reworked existing unused ArbitraryDataManager. It's now capable of syncing chunks as well as complete files. This isn't production ready as it currently requests/receives the same file from multiple peers at once, which slows down the sync and wastes lots of bandwidth. Ideally we would find an appropriate peer first and then sync the file from them. --- .../controller/ArbitraryDataManager.java | 57 ++++++++++++++++++- .../org/qortal/controller/Controller.java | 30 ++++------ 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 61447dbc..1b3311d0 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -12,6 +12,8 @@ import org.qortal.data.transaction.TransactionData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.storage.DataFile; +import org.qortal.storage.DataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction.TransactionType; @@ -45,20 +47,69 @@ public class ArbitraryDataManager extends Thread { // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true); - if (signatures == null || signatures.isEmpty()) + if (signatures == null || signatures.isEmpty()) { continue; + } // Filter out those that already have local data signatures.removeIf(signature -> hasLocalData(repository, signature)); - if (signatures.isEmpty()) + if (signatures.isEmpty()) { continue; + } // Pick one at random final int index = new Random().nextInt(signatures.size()); byte[] signature = signatures.get(index); - Controller.getInstance().fetchArbitraryData(signature); + // Load the full transaction data so we can access the file hashes + ArbitraryTransactionData transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) { + signatures.remove(signature); + continue; + } + + // Load hashes + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + // Load data file(s) + DataFile dataFile = DataFile.fromDigest(digest); + if (chunkHashes.length > 0) { + dataFile.addChunkHashes(chunkHashes); + + // Now try and fetch each chunk in turn if we don't have them already + for (DataFileChunk dataFileChunk : dataFile.getChunks()) { + if (!dataFileChunk.exists()) { + LOGGER.info("Requesting chunk {}...", dataFileChunk); + boolean success = Controller.getInstance().fetchArbitraryDataFile(dataFileChunk.getHash()); + if (success) { + LOGGER.info("Chunk {} received", dataFileChunk); + } + else { + LOGGER.info("Couldn't retrieve chunk {}", dataFileChunk); + } + } + } + } + else if (transactionData.getSize() < DataFileChunk.CHUNK_SIZE) { + // Fetch the complete file, as it is less than the chunk size + LOGGER.info("Requesting file {}...", dataFile.getHash58()); + boolean success = Controller.getInstance().fetchArbitraryDataFile(dataFile.getHash()); + if (success) { + LOGGER.info("File {} received", dataFile); + } + else { + LOGGER.info("Couldn't retrieve file {}", dataFile); + } + } + else { + // Invalid transaction (should have already failed validation) + LOGGER.info(String.format("Invalid arbitrary transaction: %.8s", signature)); + } + + signatures.remove(signature); + } catch (DataException e) { LOGGER.error("Repository issue when fetching arbitrary transaction data", e); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 8f56bd5a..cd0d14ac 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -457,8 +457,8 @@ public class Controller extends Thread { blockMinter.start(); // Arbitrary transaction data manager - // LOGGER.info("Starting arbitrary-transaction data manager"); - // ArbitraryDataManager.getInstance().start(); + LOGGER.info("Starting arbitrary-transaction data manager"); + ArbitraryDataManager.getInstance().start(); // Auto-update service? if (Settings.getInstance().isAutoUpdateEnabled()) { @@ -920,8 +920,8 @@ public class Controller extends Thread { } // Arbitrary transaction data manager - // LOGGER.info("Shutting down arbitrary-transaction data manager"); - // ArbitraryDataManager.getInstance().shutdown(); + LOGGER.info("Shutting down arbitrary-transaction data manager"); + ArbitraryDataManager.getInstance().shutdown(); if (blockMinter != null) { LOGGER.info("Shutting down block minter"); @@ -2022,13 +2022,13 @@ public class Controller extends Thread { } } - public byte[] fetchArbitraryData(byte[] signature) throws InterruptedException { + public boolean fetchArbitraryDataFile(byte[] hash) throws InterruptedException { // Build request - Message getArbitraryDataMessage = new GetArbitraryDataMessage(signature); + Message getDataFileMessage = new GetDataFileMessage(hash); // Save our request into requests map - String signature58 = Base58.encode(signature); - Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); + String hash58 = Base58.encode(hash); + Triple requestEntry = new Triple<>(hash58, null, NTP.getTime()); // Assign random ID to this message int id; @@ -2038,10 +2038,10 @@ public class Controller extends Thread { // Put queue into map (keyed by message ID) so we can poll for a response // If putIfAbsent() doesn't return null, then this ID is already taken } while (arbitraryDataRequests.put(id, requestEntry) != null); - getArbitraryDataMessage.setId(id); + getDataFileMessage.setId(id); // Broadcast request - Network.getInstance().broadcast(peer -> getArbitraryDataMessage); + Network.getInstance().broadcast(peer -> getDataFileMessage); // Poll to see if data has arrived final long singleWait = 100; @@ -2051,20 +2051,14 @@ public class Controller extends Thread { requestEntry = arbitraryDataRequests.get(id); if (requestEntry == null) - return null; + return false; if (requestEntry.getA() == null) break; totalWait += singleWait; } - - try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getArbitraryRepository().fetchData(signature); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary transaction data"), e); - return null; - } + return true; } /** Returns a list of peers that are not misbehaving, and have a recent block. */ From f2feb127084ef02f9f9763f62dec6218efc4af3d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 5 Jul 2021 09:07:06 +0100 Subject: [PATCH 097/505] /site API endpoint now tales a signature rather than a file hash This allows it to verify that the data in a transaction, after which it will then build the complete file from its chunks if needed. --- .../qortal/api/resource/WebsiteResource.java | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 5965c5aa..1030e1e0 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -10,6 +10,7 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import io.swagger.v3.oas.annotations.Operation; @@ -38,6 +39,7 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; +import org.qortal.storage.DataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; @@ -263,22 +265,48 @@ public class WebsiteResource { if (!Files.exists(Paths.get(unzippedPath))) { - // Load file - DataFile dataFile = DataFile.fromHash58(resourceId); - if (dataFile == null || !dataFile.exists()) { - LOGGER.info("Unable to validate complete file hash"); - return this.get404Response(); - } + // Load the full transaction data so we can access the file hashes + try (final Repository repository = RepositoryManager.getRepository()) { - if (!dataFile.digest58().equals(resourceId)) { - LOGGER.info("Unable to validate complete file hash"); - return this.get404Response(); - } + ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); + if (!(transactionData instanceof ArbitraryTransactionData)) { + return this.get404Response(); + } - try { - ZipUtils.unzip(dataFile.getFilePath(), destPath); - } catch (IOException e) { - LOGGER.info("Unable to unzip file"); + // Load hashes + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + // Load data file(s) + DataFile dataFile = DataFile.fromDigest(digest); + if (!dataFile.exists()) { + if (!dataFile.allChunksExist(chunkHashes)) { + // TODO: fetch them? + return this.get404Response(); + } + // We have all the chunks but not the complete file, so join them + dataFile.addChunkHashes(chunkHashes); + dataFile.join(); + } + + // If the complete file still doesn't exist then something went wrong + if (!dataFile.exists()) { + return this.get404Response(); + } + + if (!Arrays.equals(dataFile.digest(), digest)) { + LOGGER.info("Unable to validate complete file hash"); + return this.get404Response(); + } + + try { + ZipUtils.unzip(dataFile.getFilePath(), destPath); + } catch (IOException e) { + LOGGER.info("Unable to unzip file"); + } + + } catch (DataException e) { + return this.get500Response(); } } @@ -343,6 +371,19 @@ public class WebsiteResource { return response; } + private HttpServletResponse get500Response() { + try { + String responseString = "500: Internal Server Error"; + byte[] responseData = responseString.getBytes(); + response.setStatus(500); + response.setContentLength(responseData.length); + response.getOutputStream().write(responseData); + } catch (IOException e) { + LOGGER.info("Error writing 500 response"); + } + return response; + } + /** * Find relative links and prefix them with the resource ID, using Jsoup * @param path From e64a3978e6f4829608ee288f9fc25dc20ddaeb05 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 6 Jul 2021 08:23:28 +0100 Subject: [PATCH 098/505] Moved HTML parsing to new class. --- src/main/java/org/qortal/api/HTMLParser.java | 115 ++++++++++++++++++ .../qortal/api/resource/WebsiteResource.java | 107 +--------------- 2 files changed, 121 insertions(+), 101 deletions(-) create mode 100644 src/main/java/org/qortal/api/HTMLParser.java diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java new file mode 100644 index 00000000..301a716d --- /dev/null +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -0,0 +1,115 @@ +package org.qortal.api; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class HTMLParser { + + private String linkPrefix; + + public HTMLParser(String resourceId, boolean usePrefix) { + this.linkPrefix = usePrefix ? "/site/" + resourceId : ""; + } + + /** + * Find relative links and prefix them with the resource ID, using Jsoup + * @param path + * @param data + * @return The data with links replaced + */ + public byte[] replaceRelativeLinks(String path, byte[] data) { + if (HTMLParser.isHtmlFile(path)) { + String fileContents = new String(data); + Document document = Jsoup.parse(fileContents); + + Elements href = document.select("[href]"); + for (Element element : href) { + String elementHtml = element.attr("href"); + if (this.shouldReplaceLink(elementHtml)) { + String slash = (elementHtml.startsWith("/") ? "" : File.separator); + element.attr("href", this.linkPrefix + slash + element.attr("href")); + } + } + Elements src = document.select("[src]"); + for (Element element : src) { + String elementHtml = element.attr("src"); + if (this.shouldReplaceLink(elementHtml)) { + String slash = (elementHtml.startsWith("/") ? "" : File.separator); + element.attr("src", this.linkPrefix + slash + element.attr("src")); + } + } + Elements srcset = document.select("[srcset]"); + for (Element element : srcset) { + String elementHtml = element.attr("srcset").trim(); + if (this.shouldReplaceLink(elementHtml)) { + String[] parts = element.attr("srcset").split(","); + ArrayList newParts = new ArrayList<>(); + for (String part : parts) { + part = part.trim(); + String slash = (elementHtml.startsWith("/") ? "" : File.separator); + String newPart = this.linkPrefix + slash + part; + newParts.add(newPart); + } + String newString = String.join(",", newParts); + element.attr("srcset", newString); + } + } + Elements style = document.select("[style]"); + for (Element element : style) { + String elementHtml = element.attr("style"); + if (elementHtml.contains("url(")) { + String[] parts = elementHtml.split("url\\("); + String[] parts2 = parts[1].split("\\)"); + String link = parts2[0]; + if (link != null) { + link = this.removeQuotes(link); + if (this.shouldReplaceLink(link)) { + String slash = (link.startsWith("/") ? "" : "/"); + String modifiedLink = "url('" + this.linkPrefix + slash + link + "')"; + element.attr("style", parts[0] + modifiedLink + parts2[1]); + } + } + } + } + return document.html().getBytes(); + } + return data; + } + + private boolean shouldReplaceLink(String elementHtml) { + List prefixes = new ArrayList<>(); + prefixes.add("http"); // Don't modify absolute links + prefixes.add("//"); // Don't modify absolute links + prefixes.add("javascript:"); // Don't modify javascript + prefixes.add("../"); // Don't modify valid relative links + for (String prefix : prefixes) { + if (elementHtml.startsWith(prefix)) { + return false; + } + } + return true; + } + + private String removeQuotes(String elementHtml) { + if (elementHtml.startsWith("\"") || elementHtml.startsWith("\'")) { + elementHtml = elementHtml.substring(1); + } + if (elementHtml.endsWith("\"") || elementHtml.endsWith("\'")) { + elementHtml = elementHtml.substring(0, elementHtml.length() - 1); + } + return elementHtml; + } + + public static boolean isHtmlFile(String path) { + if (path.endsWith(".html") || path.endsWith(".htm")) { + return true; + } + return false; + } +} diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 1030e1e0..b90b2717 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -27,6 +27,7 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.qortal.api.ApiError; import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.HTMLParser; import org.qortal.api.Security; import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; @@ -245,13 +246,13 @@ public class WebsiteResource { @GET @Path("{resource}") public HttpServletResponse getResourceIndex(@PathParam("resource") String resourceId) { - return this.get(resourceId, "/"); + return this.get(resourceId, "/", true); } @GET @Path("{resource}/{path:.*}") public HttpServletResponse getResourcePath(@PathParam("resource") String resourceId, @PathParam("path") String inPath) { - return this.get(resourceId, inPath); + return this.get(resourceId, inPath, true); } private HttpServletResponse get(String resourceId, String inPath) { @@ -314,10 +315,11 @@ public class WebsiteResource { String filename = this.getFilename(unzippedPath, inPath); String filePath = unzippedPath + File.separator + filename; - if (this.isHtmlFile(filename)) { + if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory - data = this.replaceRelativeLinks(filename, data, resourceId); + HTMLParser htmlParser = new HTMLParser(resourceId, usePrefix); + data = htmlParser.replaceRelativeLinks(filename, data); response.setContentType(context.getMimeType(filename)); response.setContentLength(data.length); response.getOutputStream().write(data); @@ -384,96 +386,6 @@ public class WebsiteResource { return response; } - /** - * Find relative links and prefix them with the resource ID, using Jsoup - * @param path - * @param data - * @param resourceId - * @return The data with links replaced - */ - private byte[] replaceRelativeLinks(String path, byte[] data, String resourceId) { - if (this.isHtmlFile(path)) { - String fileContents = new String(data); - Document document = Jsoup.parse(fileContents); - - Elements href = document.select("[href]"); - for (Element element : href) { - String elementHtml = element.attr("href"); - if (this.shouldReplaceLink(elementHtml)) { - String slash = (elementHtml.startsWith("/") ? "" : File.separator); - element.attr("href", "/site/" +resourceId + slash + element.attr("href")); - } - } - Elements src = document.select("[src]"); - for (Element element : src) { - String elementHtml = element.attr("src"); - if (this.shouldReplaceLink(elementHtml)) { - String slash = (elementHtml.startsWith("/") ? "" : File.separator); - element.attr("src", "/site/" +resourceId + slash + element.attr("src")); - } - } - Elements srcset = document.select("[srcset]"); - for (Element element : srcset) { - String elementHtml = element.attr("srcset").trim(); - if (this.shouldReplaceLink(elementHtml)) { - String[] parts = element.attr("srcset").split(","); - ArrayList newParts = new ArrayList<>(); - for (String part : parts) { - part = part.trim(); - String slash = (elementHtml.startsWith("/") ? "" : File.separator); - String newPart = "/site/" +resourceId + slash + part; - newParts.add(newPart); - } - String newString = String.join(",", newParts); - element.attr("srcset", newString); - } - } - Elements style = document.select("[style]"); - for (Element element : style) { - String elementHtml = element.attr("style"); - if (elementHtml.contains("url(")) { - String[] parts = elementHtml.split("url\\("); - String[] parts2 = parts[1].split("\\)"); - String link = parts2[0]; - if (link != null) { - link = this.removeQuotes(link); - if (this.shouldReplaceLink(link)) { - String slash = (link.startsWith("/") ? "" : "/"); - String modifiedLink = "url('" + "/site/" + resourceId + slash + link + "')"; - element.attr("style", parts[0] + modifiedLink + parts2[1]); - } - } - } - } - return document.html().getBytes(); - } - return data; - } - - private boolean shouldReplaceLink(String elementHtml) { - List prefixes = new ArrayList<>(); - prefixes.add("http"); // Don't modify absolute links - prefixes.add("//"); // Don't modify absolute links - prefixes.add("javascript:"); // Don't modify javascript - prefixes.add("../"); // Don't modify valid relative links - for (String prefix : prefixes) { - if (elementHtml.startsWith(prefix)) { - return false; - } - } - return true; - } - - private String removeQuotes(String elementHtml) { - if (elementHtml.startsWith("\"") || elementHtml.startsWith("\'")) { - elementHtml = elementHtml.substring(1); - } - if (elementHtml.endsWith("\"") || elementHtml.endsWith("\'")) { - elementHtml = elementHtml.substring(0, elementHtml.length() - 1); - } - return elementHtml; - } - private List indexFiles() { List indexFiles = new ArrayList<>(); indexFiles.add("index.html"); @@ -485,11 +397,4 @@ public class WebsiteResource { return indexFiles; } - private boolean isHtmlFile(String path) { - if (path.endsWith(".html") || path.endsWith(".htm")) { - return true; - } - return false; - } - } From cdc5348a06c3c1ca73c612ca1b2644376c1f5a33 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 6 Jul 2021 18:50:40 +0100 Subject: [PATCH 099/505] Added "domain map" server Domain names can be mapped to arbitrary transaction signatures via the node's settings, and then served over port 80 or 443. This allows Qortal hosted sites to be accessible via a traditional domain name. Example configuration to map two domains: "domainMapServiceEnabled": true, "domainMapServicePort": 80, "domainMap": [ { "domain": "example.com", "signature": "tEsw4kUn4ZJfPha7CotUL6BHkFPs79BwKXdY6yrf28YTpDn4KSY6ZKX3nwZCkqDF9RyXbgaVnB1rTEExY3h9CQA" }, { "domain": "demo.qortal.org", "signature": "ZdBWWPMhR7AZwSu5xZm89mQEacekqkNfAimSCqFP6rQGKaGnXR9G4SWYpY5awFGfhmNBWzvRnXkWZKCsj6EMgc8" } ] Each domain needs to be pointed to the Qortal data node via an A record or CNAME. You can add redundant nodes by adding multiple A records for the same domain (this is known as DNS Failover). Note that running a webserver on port 80 (or anything less than 1024) requires running the data node as root. There are workarounds to this, such as disabling privileged ports, or using a reverse proxy. I will investigate this more as time goes on, but this is okay for a proof of concept. --- .../java/org/qortal/api/DomainMapService.java | 174 ++++++++++++++++++ .../qortal/api/resource/WebsiteResource.java | 28 ++- .../org/qortal/controller/Controller.java | 14 ++ .../java/org/qortal/settings/Settings.java | 65 ++++++- 4 files changed, 274 insertions(+), 7 deletions(-) create mode 100644 src/main/java/org/qortal/api/DomainMapService.java diff --git a/src/main/java/org/qortal/api/DomainMapService.java b/src/main/java/org/qortal/api/DomainMapService.java new file mode 100644 index 00000000..5fc89b5f --- /dev/null +++ b/src/main/java/org/qortal/api/DomainMapService.java @@ -0,0 +1,174 @@ +package org.qortal.api; + +import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.rewrite.handler.RewriteHandler; +import org.eclipse.jetty.rewrite.handler.RewritePatternRule; +import org.eclipse.jetty.server.*; +import org.eclipse.jetty.server.handler.ErrorHandler; +import org.eclipse.jetty.server.handler.InetAccessHandler; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.servlets.CrossOriginFilter; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.servlet.ServletContainer; +import org.qortal.api.resource.AnnotationPostProcessor; +import org.qortal.api.resource.ApiDefinition; +import org.qortal.settings.Settings; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyStore; +import java.security.SecureRandom; + +public class DomainMapService { + + private static DomainMapService instance; + + private final ResourceConfig config; + private Server server; + + private DomainMapService() { + this.config = new ResourceConfig(); + this.config.packages("org.qortal.api.resource"); + this.config.register(OpenApiResource.class); + this.config.register(ApiDefinition.class); + this.config.register(AnnotationPostProcessor.class); + } + + public static DomainMapService getInstance() { + if (instance == null) + instance = new DomainMapService(); + + return instance; + } + + public Iterable> getResources() { + return this.config.getClasses(); + } + + public void start() { + try { + // Create API server + + // SSL support if requested + String keystorePathname = Settings.getInstance().getSslKeystorePathname(); + String keystorePassword = Settings.getInstance().getSslKeystorePassword(); + + if (keystorePathname != null && keystorePassword != null) { + // SSL version + if (!Files.isReadable(Path.of(keystorePathname))) + throw new RuntimeException("Failed to start SSL API due to broken keystore"); + + // BouncyCastle-specific SSLContext build + SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE"); + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE"); + + KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC"); + + try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) { + keyStore.load(keystoreStream, keystorePassword.toCharArray()); + } + + keyManagerFactory.init(keyStore, keystorePassword.toCharArray()); + sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom()); + + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); + sslContextFactory.setSslContext(sslContext); + + this.server = new Server(); + + HttpConfiguration httpConfig = new HttpConfiguration(); + httpConfig.setSecureScheme("https"); + httpConfig.setSecurePort(Settings.getInstance().getDomainMapServicePort()); + + SecureRequestCustomizer src = new SecureRequestCustomizer(); + httpConfig.addCustomizer(src); + + HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig); + SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()); + + ServerConnector portUnifiedConnector = new ServerConnector(this.server, + new DetectorConnectionFactory(sslConnectionFactory), + httpConnectionFactory); + portUnifiedConnector.setHost(Settings.getInstance().getBindAddress()); + portUnifiedConnector.setPort(Settings.getInstance().getDomainMapServicePort()); + + this.server.addConnector(portUnifiedConnector); + } else { + // Non-SSL + InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress()); + InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDomainMapServicePort()); + this.server = new Server(endpoint); + } + + // Error handler + ErrorHandler errorHandler = new ApiErrorHandler(); + this.server.setErrorHandler(errorHandler); + + // Request logging + if (Settings.getInstance().isDomainMapLoggingEnabled()) { + RequestLogWriter logWriter = new RequestLogWriter("domainmap-requests.log"); + logWriter.setAppend(true); + logWriter.setTimeZone("UTC"); + RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT); + this.server.setRequestLog(requestLog); + } + + // Access handler (currently no whitelist is used) + InetAccessHandler accessHandler = new InetAccessHandler(); + this.server.setHandler(accessHandler); + + // URL rewriting + RewriteHandler rewriteHandler = new RewriteHandler(); + accessHandler.setHandler(rewriteHandler); + + // Context + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.setContextPath("/"); + rewriteHandler.setHandler(context); + + // Cross-origin resource sharing + FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class); + corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*"); + corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE"); + corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false"); + context.addFilter(corsFilterHolder, "/*", null); + + // API servlet + ServletContainer container = new ServletContainer(this.config); + ServletHolder apiServlet = new ServletHolder(container); + apiServlet.setInitOrder(1); + context.addServlet(apiServlet, "/*"); + + // Rewrite URLs + rewriteHandler.addRule(new RewritePatternRule("/*", "/site/domainmap/")); // rewrite / as /site/domainmap/ + + // Start server + this.server.start(); + } catch (Exception e) { + // Failed to start + throw new RuntimeException("Failed to start API", e); + } + } + + public void stop() { + try { + // Stop server + this.server.stop(); + } catch (Exception e) { + // Failed to stop + } + + this.server = null; + } + +} diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index b90b2717..5d281407 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -12,6 +12,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; @@ -21,10 +22,6 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; import org.qortal.api.ApiError; import org.qortal.api.ApiExceptionFactory; import org.qortal.api.HTMLParser; @@ -40,7 +37,6 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; -import org.qortal.storage.DataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; @@ -255,7 +251,27 @@ public class WebsiteResource { return this.get(resourceId, inPath, true); } - private HttpServletResponse get(String resourceId, String inPath) { + @GET + @Path("/domainmap") + public HttpServletResponse getDomainMapIndex() { + Map domainMap = Settings.getInstance().getSimpleDomainMap(); + if (domainMap != null && domainMap.containsKey(request.getServerName())) { + return this.get(domainMap.get(request.getServerName()), "/", false); + } + return this.get404Response(); + } + + @GET + @Path("/domainmap/{path:.*}") + public HttpServletResponse getDomainMapPath(@PathParam("path") String inPath) { + Map domainMap = Settings.getInstance().getSimpleDomainMap(); + if (domainMap != null && domainMap.containsKey(request.getServerName())) { + return this.get(domainMap.get(request.getServerName()), inPath, false); + } + return this.get404Response(); + } + + private HttpServletResponse get(String resourceId, String inPath, boolean usePrefix) { if (!inPath.startsWith(File.separator)) { inPath = File.separator + inPath; } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index cd0d14ac..9e3f8903 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -9,6 +9,7 @@ import org.qortal.account.Account; import org.qortal.account.PrivateKeyAccount; import org.qortal.account.PublicKeyAccount; import org.qortal.api.ApiService; +import org.qortal.api.DomainMapService; import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; @@ -477,6 +478,19 @@ public class Controller extends Thread { return; // Not System.exit() so that GUI can display error } + if (Settings.getInstance().isDomainMapServiceEnabled()) { + LOGGER.info(String.format("Starting domain map service on port %d", Settings.getInstance().getDomainMapServicePort())); + try { + DomainMapService domainMapService = DomainMapService.getInstance(); + domainMapService.start(); + } catch (Exception e) { + LOGGER.error("Unable to start domain map service", e); + Controller.getInstance().shutdown(); + Gui.getInstance().fatalError("Domain map service failure", e); + return; // Not System.exit() so that GUI can display error + } + } + // If GUI is enabled, we're no longer starting up but actually running now Gui.getInstance().notifyRunning(); } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index ae8c6275..d0ff70e1 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -5,8 +5,10 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.io.Reader; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; @@ -33,6 +35,9 @@ public class Settings { private static final int MAINNET_API_PORT = 12393; private static final int TESTNET_API_PORT = 62393; + private static final int MAINNET_DOMAIN_MAP_SERVICE_PORT = 80; + private static final int TESTNET_DOMAIN_MAP_SERVICE_PORT = 8080; + private static final Logger LOGGER = LogManager.getLogger(Settings.class); private static final String SETTINGS_FILENAME = "settings.json"; @@ -72,6 +77,12 @@ public class Settings { private String sslKeystorePathname = null; private String sslKeystorePassword = null; + // Domain mapping + private Integer domainMapServicePort; + private boolean domainMapServiceEnabled = false; + private boolean domainMapLoggingEnabled = false; + private List domainMap = null; + // Specific to this node private boolean wipeUnconfirmedOnStart = false; /** Maximum number of unconfirmed transactions allowed per account */ @@ -184,7 +195,32 @@ public class Settings { /** Data storage path. */ private String dataPath = "data"; - + + + // Domain mapping + public static class DomainMap { + private String domain; + private String signature; + + private DomainMap() { // makes JAXB happy; will never be invoked + } + + public String getDomain() { + return domain; + } + + public void setDomain(String domain) { + this.domain = domain; + } + + public String getSignature() { + return signature; + } + + public void setSignature(String signature) { + this.signature = signature; + } + } // Constructors @@ -379,6 +415,33 @@ public class Settings { return this.sslKeystorePassword; } + public int getDomainMapServicePort() { + if (this.domainMapServicePort != null) + return this.domainMapServicePort; + + return this.isTestNet ? TESTNET_DOMAIN_MAP_SERVICE_PORT : MAINNET_DOMAIN_MAP_SERVICE_PORT; + } + + public boolean isDomainMapServiceEnabled() { + return this.domainMapServiceEnabled; + } + + public boolean isDomainMapLoggingEnabled() { + return this.domainMapLoggingEnabled; + } + + public List getDomainMap() { + return this.domainMap; + } + + public Map getSimpleDomainMap() { + HashMap map = new HashMap<>(); + for (DomainMap dMap : this.domainMap) { + map.put(dMap.getDomain(), dMap.getSignature()); + } + return map; + } + public boolean getWipeUnconfirmedOnStart() { return this.wipeUnconfirmedOnStart; } From bfc0122c1b355d616f352d68682822add912abfc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Jul 2021 13:53:07 +0100 Subject: [PATCH 100/505] Added warning to README --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9dd9ad60..e9001f9c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ -# Qortal Project - Official Repo +# Qortal Data Node + +## Important + +This code is unfinished, and we haven't had the official genesis block for the data chain yet. +Therefore it is only possible to use this code if you first create your own test chain. I would +highly recommend waiting until the code is in a more complete state before trying to run this. ## Build / run From f938d8c878eb03eebfc80f06c72c38235f13e898 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Jul 2021 14:47:51 +0100 Subject: [PATCH 101/505] More refactoring --- .../qortal/api/resource/WebsiteResource.java | 2 +- .../network/message/GetDataFileMessage.java | 28 ++++++++++--------- .../hsqldb/HSQLDBArbitraryRepository.java | 6 ++-- .../java/org/qortal/storage/DataFile.java | 9 ++++-- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 5d281407..69ccfba8 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -295,7 +295,7 @@ public class WebsiteResource { byte[] chunkHashes = transactionData.getChunkHashes(); // Load data file(s) - DataFile dataFile = DataFile.fromDigest(digest); + DataFile dataFile = DataFile.fromHash(digest); if (!dataFile.exists()) { if (!dataFile.allChunksExist(chunkHashes)) { // TODO: fetch them? diff --git a/src/main/java/org/qortal/network/message/GetDataFileMessage.java b/src/main/java/org/qortal/network/message/GetDataFileMessage.java index d4171b42..a7ab8d67 100644 --- a/src/main/java/org/qortal/network/message/GetDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/GetDataFileMessage.java @@ -1,5 +1,7 @@ package org.qortal.network.message; +import org.qortal.transform.transaction.TransactionTransformer; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -7,33 +9,33 @@ import java.nio.ByteBuffer; public class GetDataFileMessage extends Message { - private static final int DIGEST_LENGTH = 32; + private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; - private final byte[] digest; + private final byte[] hash; - public GetDataFileMessage(byte[] digest) { - this(-1, digest); + public GetDataFileMessage(byte[] hash) { + this(-1, hash); } - private GetDataFileMessage(int id, byte[] digest) { + private GetDataFileMessage(int id, byte[] hash) { super(id, MessageType.GET_DATA_FILE); - this.digest = digest; + this.hash = hash; } - public byte[] getDigest() { - return this.digest; + public byte[] getHash() { + return this.hash; } public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != DIGEST_LENGTH) + if (bytes.remaining() != HASH_LENGTH) return null; - byte[] digest = new byte[DIGEST_LENGTH]; + byte[] hash = new byte[HASH_LENGTH]; - bytes.get(digest); + bytes.get(hash); - return new GetDataFileMessage(id, digest); + return new GetDataFileMessage(id, hash); } @Override @@ -41,7 +43,7 @@ public class GetDataFileMessage extends Message { try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(this.digest); + bytes.write(this.hash); return bytes.toByteArray(); } catch (IOException e) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 332e711a..b3edf41a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -47,7 +47,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = transactionData.getChunkHashes(); // Load data file(s) - DataFile dataFile = DataFile.fromDigest(digest); + DataFile dataFile = DataFile.fromHash(digest); if (chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -82,7 +82,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = transactionData.getChunkHashes(); // Load data file(s) - DataFile dataFile = DataFile.fromDigest(digest); + DataFile dataFile = DataFile.fromHash(digest); if (chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -167,7 +167,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); // Load data file(s) - DataFile dataFile = DataFile.fromDigest(digest); + DataFile dataFile = DataFile.fromHash(digest); if (chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 585e3711..be78e6b0 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -90,8 +90,8 @@ public class DataFile { return new DataFile(hash58); } - public static DataFile fromDigest(byte[] digest) { - return DataFile.fromHash58(Base58.encode(digest)); + public static DataFile fromHash(byte[] hash) { + return DataFile.fromHash58(Base58.encode(hash)); } public static DataFile fromPath(String path) { @@ -100,7 +100,7 @@ public class DataFile { try { byte[] fileContent = Files.readAllBytes(file.toPath()); byte[] digest = Crypto.digest(fileContent); - DataFile dataFile = DataFile.fromDigest(digest); + DataFile dataFile = DataFile.fromHash(digest); // Copy file to base directory if needed Path filePath = Paths.get(path); @@ -194,6 +194,9 @@ public class DataFile { } public void addChunkHashes(byte[] chunks) { + if (chunks == null || chunks.length == 0) { + return; + } ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; From 5a95c827b4bab05bb281a6d71f97045d94107f90 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Jul 2021 15:51:46 +0100 Subject: [PATCH 102/505] When serving a website, delete the unzipped directory if the index file is not found. This is a quick solution to rebuild directory structures with missing files. This whole area of the code needs some reworking, as serving the site from a temporary folder is not a robust long term solution. --- pom.xml | 5 +++++ .../org/qortal/api/resource/WebsiteResource.java | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b60b2b23..2c0afe0e 100644 --- a/pom.xml +++ b/pom.xml @@ -450,6 +450,11 @@ commons-text ${commons-text.version} + + commons-io + commons-io + 2.6 + io.druid diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 69ccfba8..ef188dda 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -20,6 +20,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.ApiError; @@ -355,8 +356,19 @@ public class WebsiteResource { inputStream.close(); } return response; + } catch (FileNotFoundException e) { + LOGGER.info("File not found at path: {}", unzippedPath); + if (inPath.equals("/")) { + // Delete the unzipped folder if no index file was found + try { + FileUtils.deleteDirectory(new File(unzippedPath)); + } catch (IOException ioException) { + LOGGER.info("Unable to delete directory: {}", unzippedPath, e); + } + } + } catch (IOException e) { - LOGGER.info("Unable to serve file at path: {}", inPath); + LOGGER.info("Unable to serve file at path: {}", inPath, e); } return this.get404Response(); From 2679252b047d7cedf99e9cd23c1f90ec6a186cdd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 11 Jul 2021 10:00:09 +0100 Subject: [PATCH 103/505] Rework of arbitrary data requests Previously we would ask all connected peers for the file itself, but this caused the network to be swamped when multiple peers responded with the same file. This new approach instead asks all connected peers to send back a list of hashes for all files they have relating to a transaction signature. The requesting node then uses these lists to make separate requests for each missing file. --- .../controller/ArbitraryDataManager.java | 52 +---- .../org/qortal/controller/Controller.java | 179 ++++++++++++++---- .../message/ArbitraryDataFileListMessage.java | 90 +++++++++ .../GetArbitraryDataFileListMessage.java | 54 ++++++ .../org/qortal/network/message/Message.java | 5 +- .../java/org/qortal/storage/DataFile.java | 12 +- 6 files changed, 299 insertions(+), 93 deletions(-) create mode 100644 src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java create mode 100644 src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 1b3311d0..8292d2a5 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -12,8 +12,6 @@ import org.qortal.data.transaction.TransactionData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.storage.DataFile; -import org.qortal.storage.DataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction.TransactionType; @@ -62,53 +60,9 @@ public class ArbitraryDataManager extends Thread { final int index = new Random().nextInt(signatures.size()); byte[] signature = signatures.get(index); - // Load the full transaction data so we can access the file hashes - ArbitraryTransactionData transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) { - signatures.remove(signature); - continue; - } - - // Load hashes - byte[] digest = transactionData.getData(); - byte[] chunkHashes = transactionData.getChunkHashes(); - - // Load data file(s) - DataFile dataFile = DataFile.fromDigest(digest); - if (chunkHashes.length > 0) { - dataFile.addChunkHashes(chunkHashes); - - // Now try and fetch each chunk in turn if we don't have them already - for (DataFileChunk dataFileChunk : dataFile.getChunks()) { - if (!dataFileChunk.exists()) { - LOGGER.info("Requesting chunk {}...", dataFileChunk); - boolean success = Controller.getInstance().fetchArbitraryDataFile(dataFileChunk.getHash()); - if (success) { - LOGGER.info("Chunk {} received", dataFileChunk); - } - else { - LOGGER.info("Couldn't retrieve chunk {}", dataFileChunk); - } - } - } - } - else if (transactionData.getSize() < DataFileChunk.CHUNK_SIZE) { - // Fetch the complete file, as it is less than the chunk size - LOGGER.info("Requesting file {}...", dataFile.getHash58()); - boolean success = Controller.getInstance().fetchArbitraryDataFile(dataFile.getHash()); - if (success) { - LOGGER.info("File {} received", dataFile); - } - else { - LOGGER.info("Couldn't retrieve file {}", dataFile); - } - } - else { - // Invalid transaction (should have already failed validation) - LOGGER.info(String.format("Invalid arbitrary transaction: %.8s", signature)); - } - - signatures.remove(signature); + // Ask our connected peers if they have files for this signature + // This process automatically then fetches the files themselves if a peer is found + Controller.getInstance().fetchArbitraryDataFileList(signature); } catch (DataException e) { LOGGER.error("Repository issue when fetching arbitrary transaction data", e); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 9e3f8903..576356c4 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -14,7 +14,6 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; -import org.qortal.crypto.Crypto; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; @@ -23,7 +22,6 @@ import org.qortal.data.network.OnlineAccountData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.data.transaction.ArbitraryTransactionData.DataType; import org.qortal.data.transaction.ChatTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.event.Event; @@ -41,6 +39,7 @@ import org.qortal.repository.RepositoryManager; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; +import org.qortal.storage.DataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; @@ -167,7 +166,9 @@ public class Controller extends Thread { *

  • we have forwarded the data payload (and maybe also saved it locally)
    • * */ - private Map> arbitraryDataRequests = Collections.synchronizedMap(new HashMap<>()); + private Map> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>()); + + private List arbitraryDataFileRequests = Collections.synchronizedList(new ArrayList<>()); /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); @@ -232,6 +233,15 @@ public class Controller extends Thread { } public GetDataFileMessageStats getDataFileMessageStats = new GetDataFileMessageStats(); + public static class GetArbitraryDataFileListMessageStats { + public AtomicLong requests = new AtomicLong(); + public AtomicLong unknownFiles = new AtomicLong(); + + public GetArbitraryDataFileListMessageStats() { + } + } + public GetArbitraryDataFileListMessageStats getArbitraryDataFileListMessageStats = new GetArbitraryDataFileListMessageStats(); + public AtomicLong latestBlocksCacheRefills = new AtomicLong(); public StatsSnapshot() { @@ -552,7 +562,7 @@ public class Controller extends Thread { // Clean up arbitrary data request cache final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); // Time to 'checkpoint' uncommitted repository writes? if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) { @@ -1235,8 +1245,8 @@ public class Controller extends Thread { onNetworkGetArbitraryDataMessage(peer, message); break; - case ARBITRARY_DATA: - onNetworkArbitraryDataMessage(peer, message); + case ARBITRARY_DATA_FILE_LIST: + onNetworkArbitraryDataFileListMessage(peer, message); break; case GET_ONLINE_ACCOUNTS: @@ -1251,6 +1261,10 @@ public class Controller extends Thread { onNetworkGetDataFileMessage(peer, message); break; + case GET_ARBITRARY_DATA_FILE_LIST: + onNetworkGetArbitraryDataFileListMessage(peer, message); + break; + default: LOGGER.debug(() -> String.format("Unhandled %s message [ID %d] from peer %s", message.getType().name(), message.getId(), peer)); break; @@ -1619,7 +1633,7 @@ public class Controller extends Thread { Triple newEntry = new Triple<>(signature58, peer, timestamp); // If we've seen this request recently, then ignore - if (arbitraryDataRequests.putIfAbsent(message.getId(), newEntry) != null) + if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) return; // Do we even have this transaction? @@ -1638,7 +1652,7 @@ public class Controller extends Thread { // Update requests map to reflect that we've sent it newEntry = new Triple<>(signature58, null, timestamp); - arbitraryDataRequests.put(message.getId(), newEntry); + arbitraryDataFileListRequests.put(message.getId(), newEntry); Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data); arbitraryDataMessage.setId(message.getId()); @@ -1655,23 +1669,29 @@ public class Controller extends Thread { } } - private void onNetworkArbitraryDataMessage(Peer peer, Message message) { - ArbitraryDataMessage arbitraryDataMessage = (ArbitraryDataMessage) message; + private void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { + LOGGER.info("Received hash list from peer {}", peer); + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; // Do we have a pending request for this data? - Triple request = arbitraryDataRequests.get(message.getId()); - if (request == null || request.getA() == null) + Triple request = arbitraryDataFileListRequests.get(message.getId()); + if (request == null || request.getA() == null) { return; + } // Does this message's signature match what we're expecting? - byte[] signature = arbitraryDataMessage.getSignature(); + byte[] signature = arbitraryDataFileListMessage.getSignature(); String signature58 = Base58.encode(signature); - if (!request.getA().equals(signature58)) + if (!request.getA().equals(signature58)) { return; + } - byte[] data = arbitraryDataMessage.getData(); + List hashes = arbitraryDataFileListMessage.getHashes(); + if (hashes == null || hashes.isEmpty()) { + return; + } - // Check transaction exists and payload hash is correct + // Check transaction exists and hashes are correct try (final Repository repository = RepositoryManager.getRepository()) { TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); if (!(transactionData instanceof ArbitraryTransactionData)) @@ -1679,31 +1699,47 @@ public class Controller extends Thread { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - byte[] actualHash = Crypto.digest(data); + // Load data file(s) + DataFile dataFile = DataFile.fromHash(arbitraryTransactionData.getData()); + dataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); - // "data" from repository will always be hash of actual raw data - if (!Arrays.equals(arbitraryTransactionData.getData(), actualHash)) - return; + // Check all hashes exist + for (byte[] hash : hashes) { + if (!dataFile.containsChunk(hash)) { + LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58); + return; + } + } // Update requests map to reflect that we've received it Triple newEntry = new Triple<>(null, null, request.getC()); - arbitraryDataRequests.put(message.getId(), newEntry); + arbitraryDataFileListRequests.put(message.getId(), newEntry); - // Save payload locally - // TODO: storage policy - arbitraryTransactionData.setDataType(DataType.RAW_DATA); - arbitraryTransactionData.setData(data); - repository.getArbitraryRepository().save(arbitraryTransactionData); - repository.saveChanges(); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e); + // Now fetch actual data from this peer + for (byte[] hash : hashes) { + if (!dataFile.chunkExists(hash)) { + // Only request the file if we aren't already requesting it from someone else + if (!arbitraryDataFileRequests.contains(Base58.encode(hash))) { + DataFile receivedDataFile = fetchArbitraryDataFile(peer, hash); + LOGGER.info("Received data file {} from peer {}", receivedDataFile, peer); + } + else { + LOGGER.info("Already requesting data file {}", dataFile); + } + } + } + + } catch (DataException | InterruptedException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); } + // Forwarding (not yet used) Peer requestingPeer = request.getB(); if (requestingPeer != null) { // Forward to requesting peer; - if (!requestingPeer.sendMessage(arbitraryDataMessage)) - requestingPeer.disconnect("failed to forward arbitrary data"); + if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { + requestingPeer.disconnect("failed to forward arbitrary data file list"); + } } } @@ -1756,10 +1792,10 @@ public class Controller extends Thread { private void onNetworkGetDataFileMessage(Peer peer, Message message) { GetDataFileMessage getDataFileMessage = (GetDataFileMessage) message; - byte[] digest = getDataFileMessage.getDigest(); + byte[] hash = getDataFileMessage.getHash(); this.stats.getDataFileMessageStats.requests.incrementAndGet(); - DataFile dataFile = DataFile.fromDigest(digest); + DataFile dataFile = DataFile.fromHash(hash); if (dataFile.exists()) { DataFileMessage dataFileMessage = new DataFileMessage(dataFile); dataFileMessage.setId(message.getId()); @@ -1789,6 +1825,49 @@ public class Controller extends Thread { } } + private void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + byte[] signature = getArbitraryDataFileListMessage.getSignature(); + this.stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + + LOGGER.info("Received hash list request from peer {} for signature {}", peer, Base58.encode(signature)); + + List hashes = new ArrayList<>(); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get a list of its hashes + ArbitraryTransactionData transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); + if (transactionData instanceof ArbitraryTransactionData) { + + byte[] hash = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + // Load file(s) and add any that exist to the list of hashes + DataFile dataFile = DataFile.fromHash(hash); + if (chunkHashes.length > 0) { + dataFile.addChunkHashes(chunkHashes); + for (DataFileChunk dataFileChunk : dataFile.getChunks()) { + if (dataFileChunk.exists()) { + hashes.add(dataFileChunk.getHash()); + } + } + } + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); + } + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + arbitraryDataFileListMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileListMessage)) { + LOGGER.info("Couldn't send list of hashes"); + peer.disconnect("failed to send list of hashes"); + } + LOGGER.info("Sent list of hashes", hashes); + } + // Utilities private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException { @@ -2036,13 +2115,14 @@ public class Controller extends Thread { } } - public boolean fetchArbitraryDataFile(byte[] hash) throws InterruptedException { + public boolean fetchArbitraryDataFileList(byte[] signature) throws InterruptedException { + LOGGER.info(String.format("Sending data file list request for signature %s", Base58.encode(signature))); // Build request - Message getDataFileMessage = new GetDataFileMessage(hash); + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature); // Save our request into requests map - String hash58 = Base58.encode(hash); - Triple requestEntry = new Triple<>(hash58, null, NTP.getTime()); + String signature58 = Base58.encode(signature); + Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); // Assign random ID to this message int id; @@ -2051,11 +2131,11 @@ public class Controller extends Thread { // Put queue into map (keyed by message ID) so we can poll for a response // If putIfAbsent() doesn't return null, then this ID is already taken - } while (arbitraryDataRequests.put(id, requestEntry) != null); - getDataFileMessage.setId(id); + } while (arbitraryDataFileListRequests.put(id, requestEntry) != null); + getArbitraryDataFileListMessage.setId(id); // Broadcast request - Network.getInstance().broadcast(peer -> getDataFileMessage); + Network.getInstance().broadcast(peer -> getArbitraryDataFileListMessage); // Poll to see if data has arrived final long singleWait = 100; @@ -2063,7 +2143,7 @@ public class Controller extends Thread { while (totalWait < ARBITRARY_REQUEST_TIMEOUT) { Thread.sleep(singleWait); - requestEntry = arbitraryDataRequests.get(id); + requestEntry = arbitraryDataFileListRequests.get(id); if (requestEntry == null) return false; @@ -2075,6 +2155,23 @@ public class Controller extends Thread { return true; } + private DataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException { + String hash58 = Base58.encode(hash); + LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); + arbitraryDataFileRequests.add(hash58); + Message getDataFileMessage = new GetDataFileMessage(hash); + + Message message = peer.getResponse(getDataFileMessage); + arbitraryDataFileRequests.remove(hash58); + + if (message == null || message.getType() != Message.MessageType.DATA_FILE) { + return null; + } + + DataFileMessage dataFileMessage = (DataFileMessage) message; + return dataFileMessage.getDataFile(); + } + /** Returns a list of peers that are not misbehaving, and have a recent block. */ public List getRecentBehavingPeers() { final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java new file mode 100644 index 00000000..36658a9f --- /dev/null +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -0,0 +1,90 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.qortal.data.block.BlockSummaryData; +import org.qortal.transform.Transformer; +import org.qortal.transform.block.BlockTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class ArbitraryDataFileListMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + private static final int HASH_LENGTH = Transformer.SHA256_LENGTH; + + private final byte[] signature; + private final List hashes; + + public ArbitraryDataFileListMessage(byte[] signature, List hashes) { + super(MessageType.ARBITRARY_DATA_FILE_LIST); + + this.signature = signature; + this.hashes = hashes; + } + + public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes) { + super(id, MessageType.ARBITRARY_DATA_FILE_LIST); + + this.signature = signature; + this.hashes = hashes; + } + + public List getHashes() { + return this.hashes; + } + + public byte[] getSignature() { + return this.signature; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + byte[] signature = new byte[SIGNATURE_LENGTH]; + bytes.get(signature); + + int count = bytes.getInt(); + + if (bytes.remaining() != count * HASH_LENGTH) + return null; + + List hashes = new ArrayList<>(); + for (int i = 0; i < count; ++i) { + + byte[] hash = new byte[HASH_LENGTH]; + bytes.get(hash); + hashes.add(hash); + } + + return new ArbitraryDataFileListMessage(id, signature, hashes); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.signature); + + bytes.write(Ints.toByteArray(this.hashes.size())); + + for (byte[] hash : this.hashes) { + bytes.write(hash); + } + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + + public ArbitraryDataFileListMessage cloneWithNewId(int newId) { + ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes); + clone.setId(newId); + return clone; + } + +} diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java new file mode 100644 index 00000000..49fd60cb --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java @@ -0,0 +1,54 @@ +package org.qortal.network.message; + +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +public class GetArbitraryDataFileListMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + + private final byte[] signature; + + public GetArbitraryDataFileListMessage(byte[] signature) { + this(-1, signature); + } + + private GetArbitraryDataFileListMessage(int id, byte[] signature) { + super(id, MessageType.GET_ARBITRARY_DATA_FILE_LIST); + + this.signature = signature; + } + + public byte[] getSignature() { + return this.signature; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + if (bytes.remaining() != SIGNATURE_LENGTH) + return null; + + byte[] signature = new byte[SIGNATURE_LENGTH]; + + bytes.get(signature); + + return new GetArbitraryDataFileListMessage(id, signature); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.signature); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index 4483caee..3ecb151b 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -86,7 +86,10 @@ public abstract class Message { GET_BLOCKS(101), DATA_FILE(110), - GET_DATA_FILE(111); + GET_DATA_FILE(111), + + ARBITRARY_DATA_FILE_LIST(120), + GET_ARBITRARY_DATA_FILE_LIST(121); public final int value; public final Method fromByteBufferMethod; diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index be78e6b0..2e872a51 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -372,8 +372,7 @@ public class DataFile { return chunk.exists(); } } - File file = new File(this.filePath); - return file.exists(); + return false; } public boolean allChunksExist(byte[] chunks) { @@ -389,6 +388,15 @@ public class DataFile { return true; } + public boolean containsChunk(byte[] hash) { + for (DataFileChunk chunk : this.chunks) { + if (Arrays.equals(hash, chunk.getHash())) { + return true; + } + } + return false; + } + public long size() { Path path = Paths.get(this.filePath); try { From 483557163e56702c226346d127852bb01c6776c8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 11 Jul 2021 10:17:38 +0100 Subject: [PATCH 104/505] Refactor: moved arbitrary data code from Controller to ArbitraryDataManager --- .../controller/ArbitraryDataManager.java | 311 +++++++++++++++++- .../org/qortal/controller/Controller.java | 305 +---------------- 2 files changed, 319 insertions(+), 297 deletions(-) diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 8292d2a5..63455124 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -1,29 +1,62 @@ package org.qortal.controller; -import java.util.Arrays; -import java.util.List; -import java.util.Random; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.storage.DataFile; +import org.qortal.storage.DataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; +import org.qortal.utils.Triple; public class ArbitraryDataManager extends Thread { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class); private static final List ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY); + private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000L; // ms + private static ArbitraryDataManager instance; private volatile boolean isStopping = false; + + /** + * Map of recent requests for ARBITRARY transaction data file lists. + *

    + * Key is original request's message ID
    + * Value is Triple<transaction signature in base58, first requesting peer, first request's timestamp> + *

    + * If peer is null then either:
    + *

      + *
    • we are the original requesting peer
      • + *
    • we have already sent data payload to original requesting peer.
      • + *
    + * If signature is null then we have already received the file list and either:
    + *
      + *
    • we are the original requesting peer and have processed it
      • + *
    • we have forwarded the file list
      • + *
    + */ + public Map> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>()); + + /** + * Array to keep track of in progress arbitrary data file requests + */ + private List arbitraryDataFileRequests = Collections.synchronizedList(new ArrayList<>()); + private ArbitraryDataManager() { } @@ -62,7 +95,7 @@ public class ArbitraryDataManager extends Thread { // Ask our connected peers if they have files for this signature // This process automatically then fetches the files themselves if a peer is found - Controller.getInstance().fetchArbitraryDataFileList(signature); + fetchArbitraryDataFileList(signature); } catch (DataException e) { LOGGER.error("Repository issue when fetching arbitrary transaction data", e); @@ -93,4 +126,274 @@ public class ArbitraryDataManager extends Thread { } } + private boolean fetchArbitraryDataFileList(byte[] signature) throws InterruptedException { + LOGGER.info(String.format("Sending data file list request for signature %s", Base58.encode(signature))); + // Build request + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature); + + // Save our request into requests map + String signature58 = Base58.encode(signature); + Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); + + // Assign random ID to this message + int id; + do { + id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; + + // Put queue into map (keyed by message ID) so we can poll for a response + // If putIfAbsent() doesn't return null, then this ID is already taken + } while (arbitraryDataFileListRequests.put(id, requestEntry) != null); + getArbitraryDataFileListMessage.setId(id); + + // Broadcast request + Network.getInstance().broadcast(peer -> getArbitraryDataFileListMessage); + + // Poll to see if data has arrived + final long singleWait = 100; + long totalWait = 0; + while (totalWait < ARBITRARY_REQUEST_TIMEOUT) { + Thread.sleep(singleWait); + + requestEntry = arbitraryDataFileListRequests.get(id); + if (requestEntry == null) + return false; + + if (requestEntry.getA() == null) + break; + + totalWait += singleWait; + } + return true; + } + + private DataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException { + String hash58 = Base58.encode(hash); + LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); + arbitraryDataFileRequests.add(hash58); + Message getDataFileMessage = new GetDataFileMessage(hash); + + Message message = peer.getResponse(getDataFileMessage); + arbitraryDataFileRequests.remove(hash58); + LOGGER.info(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); + + if (message == null || message.getType() != Message.MessageType.DATA_FILE) { + return null; + } + + DataFileMessage dataFileMessage = (DataFileMessage) message; + return dataFileMessage.getDataFile(); + } + + public void cleanupRequestCache(long now) { + final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); + + // TODO: cleanup arbitraryDataFileRequests + } + + + // Network handlers + + public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { + GetArbitraryDataMessage getArbitraryDataMessage = (GetArbitraryDataMessage) message; + + byte[] signature = getArbitraryDataMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long timestamp = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peer, timestamp); + + // If we've seen this request recently, then ignore + if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) + return; + + // Do we even have this transaction? + try (final Repository repository = RepositoryManager.getRepository()) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY) + return; + + ArbitraryTransaction transaction = new ArbitraryTransaction(repository, transactionData); + + // If we have the data then send it + if (transaction.isDataLocal()) { + byte[] data = transaction.fetchData(); + if (data == null) + return; + + // Update requests map to reflect that we've sent it + newEntry = new Triple<>(signature58, null, timestamp); + arbitraryDataFileListRequests.put(message.getId(), newEntry); + + Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data); + arbitraryDataMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataMessage)) + peer.disconnect("failed to send arbitrary data"); + + return; + } + + // Ask our other peers if they have it + Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : message); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e); + } + } + + public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { + LOGGER.info("Received hash list from peer {}", peer); + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + + // Do we have a pending request for this data? + Triple request = arbitraryDataFileListRequests.get(message.getId()); + if (request == null || request.getA() == null) { + return; + } + + // Does this message's signature match what we're expecting? + byte[] signature = arbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + if (!request.getA().equals(signature58)) { + return; + } + + List hashes = arbitraryDataFileListMessage.getHashes(); + if (hashes == null || hashes.isEmpty()) { + return; + } + + // Check transaction exists and hashes are correct + try (final Repository repository = RepositoryManager.getRepository()) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) + return; + + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + + // Load data file(s) + DataFile dataFile = DataFile.fromHash(arbitraryTransactionData.getData()); + dataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); + + // Check all hashes exist + for (byte[] hash : hashes) { + //LOGGER.info("Received hash {}", Base58.encode(hash)); + if (!dataFile.containsChunk(hash)) { + LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58); + return; + } + } + + // Update requests map to reflect that we've received it + Triple newEntry = new Triple<>(null, null, request.getC()); + arbitraryDataFileListRequests.put(message.getId(), newEntry); + + // Now fetch actual data from this peer + for (byte[] hash : hashes) { + if (!dataFile.chunkExists(hash)) { + // Only request the file if we aren't already requesting it from someone else + if (!arbitraryDataFileRequests.contains(Base58.encode(hash))) { + DataFile receivedDataFile = fetchArbitraryDataFile(peer, hash); + LOGGER.info("Received data file {} from peer {}", receivedDataFile, peer); + } + else { + LOGGER.info("Already requesting data file {}", dataFile); + } + } + } + + } catch (DataException | InterruptedException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); + } + + // Forwarding (not yet used) + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { + // Forward to requesting peer; + if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { + requestingPeer.disconnect("failed to forward arbitrary data file list"); + } + } + } + + public void onNetworkGetDataFileMessage(Peer peer, Message message) { + GetDataFileMessage getDataFileMessage = (GetDataFileMessage) message; + byte[] hash = getDataFileMessage.getHash(); + Controller.getInstance().stats.getDataFileMessageStats.requests.incrementAndGet(); + + DataFile dataFile = DataFile.fromHash(hash); + if (dataFile.exists()) { + DataFileMessage dataFileMessage = new DataFileMessage(dataFile); + dataFileMessage.setId(message.getId()); + if (!peer.sendMessage(dataFileMessage)) { + LOGGER.info("Couldn't sent file"); + peer.disconnect("failed to send file"); + } + LOGGER.info("Sent file {}", dataFile); + } + else { + + // We don't have this file + Controller.getInstance().stats.getDataFileMessageStats.unknownFiles.getAndIncrement(); + + // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout + LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, dataFile)); + + // We'll send empty block summaries message as it's very short + // TODO: use a different message type here + Message fileUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); + fileUnknownMessage.setId(message.getId()); + if (!peer.sendMessage(fileUnknownMessage)) { + LOGGER.info("Couldn't sent file-unknown response"); + peer.disconnect("failed to send file-unknown response"); + } + LOGGER.info("Sent file-unknown response for file {}", dataFile); + } + } + + public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + byte[] signature = getArbitraryDataFileListMessage.getSignature(); + Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + + LOGGER.info("Received hash list request from peer {} for signature {}", peer, Base58.encode(signature)); + + List hashes = new ArrayList<>(); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get a list of its hashes + ArbitraryTransactionData transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); + if (transactionData instanceof ArbitraryTransactionData) { + + byte[] hash = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + // Load file(s) and add any that exist to the list of hashes + DataFile dataFile = DataFile.fromHash(hash); + if (chunkHashes.length > 0) { + dataFile.addChunkHashes(chunkHashes); + for (DataFileChunk dataFileChunk : dataFile.getChunks()) { + if (dataFileChunk.exists()) { + hashes.add(dataFileChunk.getHash()); + //LOGGER.info("Added hash {}", dataFileChunk.getHash58()); + } + else { + LOGGER.info("Couldn't add hash {} because it doesn't exist", dataFileChunk.getHash58()); + } + } + } + } + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); + } + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + arbitraryDataFileListMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileListMessage)) { + LOGGER.info("Couldn't send list of hashes"); + peer.disconnect("failed to send list of hashes"); + } + LOGGER.info("Sent list of hashes", hashes); + } + } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 576356c4..df6babd1 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -21,7 +21,6 @@ import org.qortal.data.block.BlockSummaryData; import org.qortal.data.network.OnlineAccountData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; -import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ChatTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.event.Event; @@ -38,9 +37,6 @@ import org.qortal.repository.RepositoryFactory; import org.qortal.repository.RepositoryManager; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; -import org.qortal.storage.DataFile; -import org.qortal.storage.DataFileChunk; -import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; @@ -85,7 +81,6 @@ public class Controller extends Thread { private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks private static final Object shutdownLock = new Object(); private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s" + File.separator + "blockchain;create=true;hsqldb.full_log_replay=true"; - private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000L; // ms private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms @@ -149,27 +144,6 @@ public class Controller extends Thread { private boolean peersAvailable = true; // peersAvailable must default to true private long timePeersLastAvailable = 0; - /** - * Map of recent requests for ARBITRARY transaction data payloads. - *

    - * Key is original request's message ID
    - * Value is Triple<transaction signature in base58, first requesting peer, first request's timestamp> - *

    - * If peer is null then either:
    - *

      - *
    • we are the original requesting peer
      • - *
    • we have already sent data payload to original requesting peer.
      • - *
    - * If signature is null then we have already received the data payload and either:
    - *
      - *
    • we are the original requesting peer and have saved it locally
      • - *
    • we have forwarded the data payload (and maybe also saved it locally)
      • - *
    - */ - private Map> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>()); - - private List arbitraryDataFileRequests = Collections.synchronizedList(new ArrayList<>()); - /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); @@ -247,7 +221,7 @@ public class Controller extends Thread { public StatsSnapshot() { } } - private final StatsSnapshot stats = new StatsSnapshot(); + public final StatsSnapshot stats = new StatsSnapshot(); // Constructors @@ -561,8 +535,7 @@ public class Controller extends Thread { } // Clean up arbitrary data request cache - final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); + ArbitraryDataManager.getInstance().cleanupRequestCache(now); // Time to 'checkpoint' uncommitted repository writes? if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) { @@ -1241,14 +1214,6 @@ public class Controller extends Thread { onNetworkTransactionSignaturesMessage(peer, message); break; - case GET_ARBITRARY_DATA: - onNetworkGetArbitraryDataMessage(peer, message); - break; - - case ARBITRARY_DATA_FILE_LIST: - onNetworkArbitraryDataFileListMessage(peer, message); - break; - case GET_ONLINE_ACCOUNTS: onNetworkGetOnlineAccountsMessage(peer, message); break; @@ -1257,12 +1222,20 @@ public class Controller extends Thread { onNetworkOnlineAccountsMessage(peer, message); break; + case GET_ARBITRARY_DATA: + ArbitraryDataManager.getInstance().onNetworkGetArbitraryDataMessage(peer, message); + break; + + case ARBITRARY_DATA_FILE_LIST: + ArbitraryDataManager.getInstance().onNetworkArbitraryDataFileListMessage(peer, message); + break; + case GET_DATA_FILE: - onNetworkGetDataFileMessage(peer, message); + ArbitraryDataManager.getInstance().onNetworkGetDataFileMessage(peer, message); break; case GET_ARBITRARY_DATA_FILE_LIST: - onNetworkGetArbitraryDataFileListMessage(peer, message); + ArbitraryDataManager.getInstance().onNetworkGetArbitraryDataFileListMessage(peer, message); break; default: @@ -1624,125 +1597,6 @@ public class Controller extends Thread { } } - private void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { - GetArbitraryDataMessage getArbitraryDataMessage = (GetArbitraryDataMessage) message; - - byte[] signature = getArbitraryDataMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long timestamp = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, timestamp); - - // If we've seen this request recently, then ignore - if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) - return; - - // Do we even have this transaction? - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY) - return; - - ArbitraryTransaction transaction = new ArbitraryTransaction(repository, transactionData); - - // If we have the data then send it - if (transaction.isDataLocal()) { - byte[] data = transaction.fetchData(); - if (data == null) - return; - - // Update requests map to reflect that we've sent it - newEntry = new Triple<>(signature58, null, timestamp); - arbitraryDataFileListRequests.put(message.getId(), newEntry); - - Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data); - arbitraryDataMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryDataMessage)) - peer.disconnect("failed to send arbitrary data"); - - return; - } - - // Ask our other peers if they have it - Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : message); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e); - } - } - - private void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { - LOGGER.info("Received hash list from peer {}", peer); - ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; - - // Do we have a pending request for this data? - Triple request = arbitraryDataFileListRequests.get(message.getId()); - if (request == null || request.getA() == null) { - return; - } - - // Does this message's signature match what we're expecting? - byte[] signature = arbitraryDataFileListMessage.getSignature(); - String signature58 = Base58.encode(signature); - if (!request.getA().equals(signature58)) { - return; - } - - List hashes = arbitraryDataFileListMessage.getHashes(); - if (hashes == null || hashes.isEmpty()) { - return; - } - - // Check transaction exists and hashes are correct - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) - return; - - ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - - // Load data file(s) - DataFile dataFile = DataFile.fromHash(arbitraryTransactionData.getData()); - dataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); - - // Check all hashes exist - for (byte[] hash : hashes) { - if (!dataFile.containsChunk(hash)) { - LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58); - return; - } - } - - // Update requests map to reflect that we've received it - Triple newEntry = new Triple<>(null, null, request.getC()); - arbitraryDataFileListRequests.put(message.getId(), newEntry); - - // Now fetch actual data from this peer - for (byte[] hash : hashes) { - if (!dataFile.chunkExists(hash)) { - // Only request the file if we aren't already requesting it from someone else - if (!arbitraryDataFileRequests.contains(Base58.encode(hash))) { - DataFile receivedDataFile = fetchArbitraryDataFile(peer, hash); - LOGGER.info("Received data file {} from peer {}", receivedDataFile, peer); - } - else { - LOGGER.info("Already requesting data file {}", dataFile); - } - } - } - - } catch (DataException | InterruptedException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); - } - - // Forwarding (not yet used) - Peer requestingPeer = request.getB(); - if (requestingPeer != null) { - // Forward to requesting peer; - if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { - requestingPeer.disconnect("failed to forward arbitrary data file list"); - } - } - } - private void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) { GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; @@ -1790,84 +1644,6 @@ public class Controller extends Thread { } } - private void onNetworkGetDataFileMessage(Peer peer, Message message) { - GetDataFileMessage getDataFileMessage = (GetDataFileMessage) message; - byte[] hash = getDataFileMessage.getHash(); - this.stats.getDataFileMessageStats.requests.incrementAndGet(); - - DataFile dataFile = DataFile.fromHash(hash); - if (dataFile.exists()) { - DataFileMessage dataFileMessage = new DataFileMessage(dataFile); - dataFileMessage.setId(message.getId()); - if (!peer.sendMessage(dataFileMessage)) { - LOGGER.info("Couldn't sent file"); - peer.disconnect("failed to send file"); - } - LOGGER.info("Sent file {}", dataFile); - } - else { - - // We don't have this file - this.stats.getDataFileMessageStats.unknownFiles.getAndIncrement(); - - // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout - LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, dataFile)); - - // We'll send empty block summaries message as it's very short - // TODO: use a different message type here - Message fileUnknownMessage = new BlockSummariesMessage(Collections.emptyList()); - fileUnknownMessage.setId(message.getId()); - if (!peer.sendMessage(fileUnknownMessage)) { - LOGGER.info("Couldn't sent file-unknown response"); - peer.disconnect("failed to send file-unknown response"); - } - LOGGER.info("Sent file-unknown response for file {}", dataFile); - } - } - - private void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { - GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; - byte[] signature = getArbitraryDataFileListMessage.getSignature(); - this.stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); - - LOGGER.info("Received hash list request from peer {} for signature {}", peer, Base58.encode(signature)); - - List hashes = new ArrayList<>(); - - try (final Repository repository = RepositoryManager.getRepository()) { - - // Firstly we need to lookup this file on chain to get a list of its hashes - ArbitraryTransactionData transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (transactionData instanceof ArbitraryTransactionData) { - - byte[] hash = transactionData.getData(); - byte[] chunkHashes = transactionData.getChunkHashes(); - - // Load file(s) and add any that exist to the list of hashes - DataFile dataFile = DataFile.fromHash(hash); - if (chunkHashes.length > 0) { - dataFile.addChunkHashes(chunkHashes); - for (DataFileChunk dataFileChunk : dataFile.getChunks()) { - if (dataFileChunk.exists()) { - hashes.add(dataFileChunk.getHash()); - } - } - } - } - - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); - } - - ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - arbitraryDataFileListMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryDataFileListMessage)) { - LOGGER.info("Couldn't send list of hashes"); - peer.disconnect("failed to send list of hashes"); - } - LOGGER.info("Sent list of hashes", hashes); - } - // Utilities private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException { @@ -2115,63 +1891,6 @@ public class Controller extends Thread { } } - public boolean fetchArbitraryDataFileList(byte[] signature) throws InterruptedException { - LOGGER.info(String.format("Sending data file list request for signature %s", Base58.encode(signature))); - // Build request - Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature); - - // Save our request into requests map - String signature58 = Base58.encode(signature); - Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); - - // Assign random ID to this message - int id; - do { - id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; - - // Put queue into map (keyed by message ID) so we can poll for a response - // If putIfAbsent() doesn't return null, then this ID is already taken - } while (arbitraryDataFileListRequests.put(id, requestEntry) != null); - getArbitraryDataFileListMessage.setId(id); - - // Broadcast request - Network.getInstance().broadcast(peer -> getArbitraryDataFileListMessage); - - // Poll to see if data has arrived - final long singleWait = 100; - long totalWait = 0; - while (totalWait < ARBITRARY_REQUEST_TIMEOUT) { - Thread.sleep(singleWait); - - requestEntry = arbitraryDataFileListRequests.get(id); - if (requestEntry == null) - return false; - - if (requestEntry.getA() == null) - break; - - totalWait += singleWait; - } - return true; - } - - private DataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException { - String hash58 = Base58.encode(hash); - LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); - arbitraryDataFileRequests.add(hash58); - Message getDataFileMessage = new GetDataFileMessage(hash); - - Message message = peer.getResponse(getDataFileMessage); - arbitraryDataFileRequests.remove(hash58); - - if (message == null || message.getType() != Message.MessageType.DATA_FILE) { - return null; - } - - DataFileMessage dataFileMessage = (DataFileMessage) message; - return dataFileMessage.getDataFile(); - } - /** Returns a list of peers that are not misbehaving, and have a recent block. */ public List getRecentBehavingPeers() { final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); From 00d4f35f2ced0668bea5ff435375b13092d80de8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 11 Jul 2021 10:28:14 +0100 Subject: [PATCH 105/505] Track the request time in arbitraryDataFileRequests and automatically remove those that have timed out. --- .../org/qortal/controller/ArbitraryDataManager.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 63455124..2968db3d 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -53,9 +53,9 @@ public class ArbitraryDataManager extends Thread { public Map> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>()); /** - * Array to keep track of in progress arbitrary data file requests + * Map to keep track of in progress arbitrary data file requests */ - private List arbitraryDataFileRequests = Collections.synchronizedList(new ArrayList<>()); + private Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); private ArbitraryDataManager() { } @@ -169,7 +169,7 @@ public class ArbitraryDataManager extends Thread { private DataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException { String hash58 = Base58.encode(hash); LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); - arbitraryDataFileRequests.add(hash58); + arbitraryDataFileRequests.put(hash58, NTP.getTime()); Message getDataFileMessage = new GetDataFileMessage(hash); Message message = peer.getResponse(getDataFileMessage); @@ -187,8 +187,7 @@ public class ArbitraryDataManager extends Thread { public void cleanupRequestCache(long now) { final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); - - // TODO: cleanup arbitraryDataFileRequests + arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); } @@ -290,7 +289,7 @@ public class ArbitraryDataManager extends Thread { for (byte[] hash : hashes) { if (!dataFile.chunkExists(hash)) { // Only request the file if we aren't already requesting it from someone else - if (!arbitraryDataFileRequests.contains(Base58.encode(hash))) { + if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { DataFile receivedDataFile = fetchArbitraryDataFile(peer, hash); LOGGER.info("Received data file {} from peer {}", receivedDataFile, peer); } From 9384a50879765c2b0813d28f3fefbe0fa77eff9c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 13 Jul 2021 22:18:21 +0100 Subject: [PATCH 106/505] Derive PoW difficulty from the file size. Exact values TBC. --- .../transaction/ArbitraryTransaction.java | 18 ++++- .../arbitrary/ArbitraryTransactionTests.java | 65 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index c4b595bf..6ee8d5ff 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -1,5 +1,6 @@ package org.qortal.transaction; +import java.math.BigInteger; import java.util.List; import java.util.stream.Collectors; @@ -28,7 +29,9 @@ public class ArbitraryTransaction extends Transaction { public static final int MAX_CHUNK_HASHES_LENGTH = 8000; public static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes - public static final int POW_DIFFICULTY = 10; // leading zero bits + public static final int POW_MIN_DIFFICULTY = 12; // leading zero bits + public static final int POW_MAX_DIFFICULTY = 19; // leading zero bits + public static final long MAX_FILE_SIZE = DataFile.MAX_FILE_SIZE; // Constructors @@ -65,7 +68,7 @@ public class ArbitraryTransaction extends Transaction { // Clear nonce from transactionBytes ArbitraryTransactionTransformer.clearNonce(transactionBytes); - int difficulty = POW_DIFFICULTY; + int difficulty = difficultyForFileSize(arbitraryTransactionData.getSize()); // Calculate nonce this.arbitraryTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, difficulty)); @@ -155,7 +158,7 @@ public class ArbitraryTransaction extends Transaction { // Clear nonce from transactionBytes ArbitraryTransactionTransformer.clearNonce(transactionBytes); - int difficulty = POW_DIFFICULTY; + int difficulty = difficultyForFileSize(arbitraryTransactionData.getSize()); // Check nonce return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, difficulty, nonce); @@ -213,4 +216,13 @@ public class ArbitraryTransaction extends Transaction { return null; } + // Helper methods + + public int difficultyForFileSize(long size) { + final BigInteger powRange = BigInteger.valueOf(POW_MAX_DIFFICULTY - POW_MIN_DIFFICULTY); + final BigInteger multiplier = BigInteger.valueOf(100); + final BigInteger percentage = BigInteger.valueOf(size).multiply(multiplier).divide(BigInteger.valueOf(MAX_FILE_SIZE)); + return POW_MIN_DIFFICULTY + powRange.multiply(percentage).divide(multiplier).intValue(); + } + } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java new file mode 100644 index 00000000..39204167 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java @@ -0,0 +1,65 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; + +import org.qortal.data.PaymentData; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.*; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.Transaction; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class ArbitraryTransactionTests extends Common { + + private static final int version = 4; + private static final String recipient = Common.getTestAccount(null, "bob").getAddress(); + + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testDifficultyCalculation() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + TestAccount alice = Common.getTestAccount(repository, "alice"); + ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + List payments = new ArrayList<>(); + + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(TestTransaction.generateBase(alice), + 5, ArbitraryTransaction.SERVICE_ARBITRARY_DATA, 0, 0, null, dataType, null, payments); + + ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + assertEquals(12, transaction.difficultyForFileSize(1)); + assertEquals(12, transaction.difficultyForFileSize(5123456)); + assertEquals(12, transaction.difficultyForFileSize(74 * 1024 * 1024)); + assertEquals(13, transaction.difficultyForFileSize(75 * 1024 * 1024)); + assertEquals(13, transaction.difficultyForFileSize(144 * 1024 * 1024)); + assertEquals(14, transaction.difficultyForFileSize(145 * 1024 * 1024)); + assertEquals(14, transaction.difficultyForFileSize(214 * 1024 * 1024)); + assertEquals(15, transaction.difficultyForFileSize(215 * 1024 * 1024)); + assertEquals(15, transaction.difficultyForFileSize(289 * 1024 * 1024)); + assertEquals(16, transaction.difficultyForFileSize(290 * 1024 * 1024)); + assertEquals(16, transaction.difficultyForFileSize(359 * 1024 * 1024)); + assertEquals(17, transaction.difficultyForFileSize(360 * 1024 * 1024)); + assertEquals(17, transaction.difficultyForFileSize(429 * 1024 * 1024)); + assertEquals(18, transaction.difficultyForFileSize(430 * 1024 * 1024)); + assertEquals(18, transaction.difficultyForFileSize(499 * 1024 * 1024)); + assertEquals(19, transaction.difficultyForFileSize(500 * 1024 * 1024)); + + } + } + +} From 2d272e0207acea84645f31e01bfccfeba8805d3c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 14 Jul 2021 17:50:55 +0100 Subject: [PATCH 107/505] Added some service constants. These combine some Qora services (SERVICE_NAME_STORAGE, SERVICE_BLOG_POST, and SERVICE_BLOG_COMMENT) with existing Qortal services (SERVICE_AUTO_UPDATE), and some new additions (SERVICE_ARBITRARY_DATA, SERVICE_WEBSITE, and SERVICE_GIT_REPOSITORY) --- .../java/org/qortal/api/resource/ArbitraryResource.java | 2 +- .../java/org/qortal/api/resource/WebsiteResource.java | 2 +- .../org/qortal/transaction/ArbitraryTransaction.java | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 8f4dad61..8e1b043d 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -309,7 +309,7 @@ public class ArbitraryResource { List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, 2, 0, size, digest, dataType, chunkHashes, payments); + 5, ArbitraryTransaction.SERVICE_ARBITRARY_DATA, 0, size, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); transaction.computeNonce(); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index ef188dda..93bcdf06 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -120,7 +120,7 @@ public class WebsiteResource { List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, 2, 0, size, digest, dataType, chunkHashes, payments); + 5, ArbitraryTransaction.SERVICE_WEBSITE, 0, size, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); transaction.computeNonce(); diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 6ee8d5ff..8f42aff9 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -24,6 +24,15 @@ public class ArbitraryTransaction extends Transaction { // Properties private ArbitraryTransactionData arbitraryTransactionData; + // Services + public static final int SERVICE_AUTO_UPDATE = 1; + public static final int SERVICE_NAME_STORAGE = 10; + public static final int SERVICE_ARBITRARY_DATA = 100; + public static final int SERVICE_WEBSITE = 200; + public static final int SERVICE_GIT_REPOSITORY = 300; + public static final int SERVICE_BLOG_POST = 777; + public static final int SERVICE_BLOG_COMMENT = 778; + // Other useful constants public static final int MAX_DATA_SIZE = 4000; public static final int MAX_CHUNK_HASHES_LENGTH = 8000; From 182dcc7e5f5f0baede132777c8057ee852a76b0e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 14 Jul 2021 18:03:35 +0100 Subject: [PATCH 108/505] MAX_FILE_SIZE reduced to 500MiB to match the difficulty calculation. --- src/main/java/org/qortal/storage/DataFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 2e872a51..6dc6d5c0 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -43,7 +43,7 @@ public class DataFile { private static final Logger LOGGER = LogManager.getLogger(DataFile.class); - public static final long MAX_FILE_SIZE = 1 * 1024 * 1024 * 1024; // 1GiB + public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static int SHORT_DIGEST_LENGTH = 8; From 53f44a40292b5e13463d2ae0e7f18805383e5db0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 14 Jul 2021 18:03:51 +0100 Subject: [PATCH 109/505] Added support for subdirectories in the HTML parser. --- src/main/java/org/qortal/api/HTMLParser.java | 5 +++-- src/main/java/org/qortal/api/resource/WebsiteResource.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index 301a716d..0c72f5da 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -13,8 +13,9 @@ public class HTMLParser { private String linkPrefix; - public HTMLParser(String resourceId, boolean usePrefix) { - this.linkPrefix = usePrefix ? "/site/" + resourceId : ""; + public HTMLParser(String resourceId, String inPath, boolean usePrefix) { + String inPathWithoutFilename = inPath.substring(0, inPath.lastIndexOf('/')); + this.linkPrefix = usePrefix ? String.format("/site/%s%s", resourceId, inPathWithoutFilename) : ""; } /** diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 93bcdf06..3d4984c3 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -335,7 +335,7 @@ public class WebsiteResource { if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory - HTMLParser htmlParser = new HTMLParser(resourceId, usePrefix); + HTMLParser htmlParser = new HTMLParser(resourceId, inPath, usePrefix); data = htmlParser.replaceRelativeLinks(filename, data); response.setContentType(context.getMimeType(filename)); response.setContentLength(data.length); From bb76fa80cdad885f8c7cdc8ca3482aaeadae36f2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 15 Jul 2021 09:27:49 +0100 Subject: [PATCH 110/505] Another significant upgrade of arbitrary transactions Adds "name", "method", "secret", and "compression" properties. These are the foundations needed in order to handle updates, encryption, and name registration. Compression has been added so that we have the option of switching to different algorithms whilst maintaining support for existing transactions. --- .../api/resource/ArbitraryResource.java | 9 +- .../qortal/api/resource/WebsiteResource.java | 14 ++- .../transaction/ArbitraryTransactionData.java | 101 +++++++++++++++++- .../hsqldb/HSQLDBDatabaseUpdates.java | 14 ++- .../HSQLDBArbitraryTransactionRepository.java | 16 ++- .../transaction/ArbitraryTransaction.java | 9 -- .../org/qortal/transform/Transformer.java | 1 + .../ArbitraryTransactionTransformer.java | 54 ++++++++-- .../arbitrary/ArbitraryTransactionTests.java | 6 +- .../transaction/ArbitraryTestTransaction.java | 9 +- 10 files changed, 199 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 8e1b043d..cefffd67 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -263,6 +263,12 @@ public class ArbitraryResource { } byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); + String name = null; + byte[] secret = null; + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; + ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.ARBITRARY_DATA; + ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.NONE; + // Check if a file or directory has been supplied File file = new File(path); if (!file.isFile()) { @@ -309,7 +315,8 @@ public class ArbitraryResource { List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, ArbitraryTransaction.SERVICE_ARBITRARY_DATA, 0, size, digest, dataType, chunkHashes, payments); + 5, service, 0, size, name, method, + secret, compression, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); transaction.computeNonce(); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 3d4984c3..ccfdc21f 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -95,6 +95,12 @@ public class WebsiteResource { } byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); + String name = null; + byte[] secret = null; + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; + ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; + ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; + DataFile dataFile = this.hostWebsite(path); if (dataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -120,7 +126,8 @@ public class WebsiteResource { List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, ArbitraryTransaction.SERVICE_WEBSITE, 0, size, digest, dataType, chunkHashes, payments); + 5, service, 0, size, name, method, + secret, compression, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); transaction.computeNonce(); @@ -318,7 +325,10 @@ public class WebsiteResource { } try { - ZipUtils.unzip(dataFile.getFilePath(), destPath); + // TODO: compression types + //if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) { + ZipUtils.unzip(dataFile.getFilePath(), destPath); + //} } catch (IOException e) { LOGGER.info("Unable to unzip file"); } diff --git a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java index 19c0d0dc..9fc91465 100644 --- a/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java +++ b/src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java @@ -1,6 +1,7 @@ package org.qortal.data.transaction; import java.util.List; +import java.util.Map; import javax.xml.bind.Unmarshaller; import javax.xml.bind.annotation.XmlAccessType; @@ -12,6 +13,9 @@ import org.qortal.transaction.Transaction.TransactionType; import io.swagger.v3.oas.annotations.media.Schema; +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @Schema(allOf = { TransactionData.class }) @@ -25,21 +29,87 @@ public class ArbitraryTransactionData extends TransactionData { DATA_HASH; } + // Service types + public enum Service { + AUTO_UPDATE(1), + ARBITRARY_DATA(100), + WEBSITE(200), + GIT_REPOSITORY(300), + BLOG_POST(777), + BLOG_COMMENT(778); + + public final int value; + + private static final Map map = stream(Service.values()) + .collect(toMap(service -> service.value, service -> service)); + + Service(int value) { + this.value = value; + } + + public static Service valueOf(int value) { + return map.get(value); + } + } + + // Methods + public enum Method { + PUT(0), // A complete replacement of a resource + PATCH(1); // An update / partial replacement of a resource + + public final int value; + + private static final Map map = stream(Method.values()) + .collect(toMap(method -> method.value, method -> method)); + + Method(int value) { + this.value = value; + } + + public static Method valueOf(int value) { + return map.get(value); + } + } + + // Compression types + public enum Compression { + NONE(0), + ZIP(1); + + public final int value; + + private static final Map map = stream(Compression.values()) + .collect(toMap(compression -> compression.value, compression -> compression)); + + Compression(int value) { + this.value = value; + } + + public static Compression valueOf(int value) { + return map.get(value); + } + } + // Properties private int version; - @Schema(example = "sender_public_key") private byte[] senderPublicKey; - private int service; + private Service service; private int nonce; private int size; + private String name; + private Method method; + private byte[] secret; + private Compression compression; + @Schema(example = "raw_data_in_base58") private byte[] data; private DataType dataType; @Schema(example = "chunk_hashes_in_base58") private byte[] chunkHashes; + private List payments; // Constructors @@ -54,8 +124,9 @@ public class ArbitraryTransactionData extends TransactionData { } public ArbitraryTransactionData(BaseTransactionData baseTransactionData, - int version, int service, int nonce, int size, byte[] data, - DataType dataType, byte[] chunkHashes, List payments) { + int version, Service service, int nonce, int size, + String name, Method method, byte[] secret, Compression compression, + byte[] data, DataType dataType, byte[] chunkHashes, List payments) { super(TransactionType.ARBITRARY, baseTransactionData); this.senderPublicKey = baseTransactionData.creatorPublicKey; @@ -63,6 +134,10 @@ public class ArbitraryTransactionData extends TransactionData { this.service = service; this.nonce = nonce; this.size = size; + this.name = name; + this.method = method; + this.secret = secret; + this.compression = compression; this.data = data; this.dataType = dataType; this.chunkHashes = chunkHashes; @@ -79,7 +154,7 @@ public class ArbitraryTransactionData extends TransactionData { return this.version; } - public int getService() { + public Service getService() { return this.service; } @@ -95,6 +170,22 @@ public class ArbitraryTransactionData extends TransactionData { return this.size; } + public String getName() { + return this.name; + } + + public Method getMethod() { + return this.method; + } + + public byte[] getSecret() { + return this.secret; + } + + public Compression getCompression() { + return this.compression; + } + public byte[] getData() { return this.data; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 5e906d75..2fcbd4e3 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -4,8 +4,6 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; -import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -781,10 +779,18 @@ public class HSQLDBDatabaseUpdates { stmt.execute("ALTER TABLE ArbitraryTransactions ADD chunk_hashes ArbitraryDataHashes"); // For finding data files by hash stmt.execute("CREATE INDEX ArbitraryDataIndex ON ArbitraryTransactions (is_data_raw, data)"); - - // TODO: resource ID, compression, layers break; + case 35: + // We need the ability for arbitrary transactions to be associated with a name + stmt.execute("ALTER TABLE ArbitraryTransactions ADD name RegisteredName"); + // A "method" specifies how the data should be applied (e.g. PUT or PATCH) + stmt.execute("ALTER TABLE ArbitraryTransactions ADD update_method INTEGER NOT NULL DEFAULT 0"); + // For public data, the AES shared secret needs to be available. This is more for data obfuscation as apposed to actual encryption. + stmt.execute("ALTER TABLE ArbitraryTransactions ADD secret VARBINARY(32)"); + // We want to support compressed and uncompressed data, as well as different compression algorithms + stmt.execute("ALTER TABLE ArbitraryTransactions ADD compression INTEGER NOT NULL DEFAULT 0"); + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java index 97f52f61..2a9aa764 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java @@ -20,7 +20,8 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos } TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException { - String sql = "SELECT version, nonce, service, size, is_data_raw, data, chunk_hashes from ArbitraryTransactions WHERE signature = ?"; + String sql = "SELECT version, nonce, service, size, is_data_raw, data, chunk_hashes, " + + "name, update_method, secret, compression from ArbitraryTransactions WHERE signature = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) { if (resultSet == null) @@ -28,15 +29,20 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos int version = resultSet.getInt(1); int nonce = resultSet.getInt(2); - int service = resultSet.getInt(3); + ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.valueOf(resultSet.getInt(3)); int size = resultSet.getInt(4); boolean isDataRaw = resultSet.getBoolean(5); // NOT NULL, so no null to false DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; byte[] data = resultSet.getBytes(6); byte[] chunkHashes = resultSet.getBytes(7); + String name = resultSet.getString(8); + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.valueOf(resultSet.getInt(9)); + byte[] secret = resultSet.getBytes(10); + ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.valueOf(resultSet.getInt(11)); List payments = this.getPaymentsFromSignature(baseTransactionData.getSignature()); - return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, data, dataType, chunkHashes, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, name, method, + secret, compression, data, dataType, chunkHashes, payments); } catch (SQLException e) { throw new DataException("Unable to fetch arbitrary transaction from repository", e); } @@ -56,7 +62,9 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos .bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService()) .bind("nonce", arbitraryTransactionData.getNonce()).bind("size", arbitraryTransactionData.getSize()) .bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData()) - .bind("chunk_hashes", arbitraryTransactionData.getChunkHashes()); + .bind("chunk_hashes", arbitraryTransactionData.getChunkHashes()).bind("name", arbitraryTransactionData.getName()) + .bind("method", arbitraryTransactionData.getMethod()).bind("secret", arbitraryTransactionData.getSecret()) + .bind("compression", arbitraryTransactionData.getCompression()); try { saveHelper.execute(this.repository); diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 8f42aff9..6ee8d5ff 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -24,15 +24,6 @@ public class ArbitraryTransaction extends Transaction { // Properties private ArbitraryTransactionData arbitraryTransactionData; - // Services - public static final int SERVICE_AUTO_UPDATE = 1; - public static final int SERVICE_NAME_STORAGE = 10; - public static final int SERVICE_ARBITRARY_DATA = 100; - public static final int SERVICE_WEBSITE = 200; - public static final int SERVICE_GIT_REPOSITORY = 300; - public static final int SERVICE_BLOG_POST = 777; - public static final int SERVICE_BLOG_COMMENT = 778; - // Other useful constants public static final int MAX_DATA_SIZE = 4000; public static final int MAX_CHUNK_HASHES_LENGTH = 8000; diff --git a/src/main/java/org/qortal/transform/Transformer.java b/src/main/java/org/qortal/transform/Transformer.java index 341d545b..e78d3284 100644 --- a/src/main/java/org/qortal/transform/Transformer.java +++ b/src/main/java/org/qortal/transform/Transformer.java @@ -20,5 +20,6 @@ public abstract class Transformer { public static final int MD5_LENGTH = 16; public static final int SHA256_LENGTH = 32; + public static final int AES256_LENGTH = 32; } diff --git a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java index f9df86af..4df2e148 100644 --- a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java @@ -6,12 +6,14 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import com.google.common.base.Utf8; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.DataType; +import org.qortal.naming.Name; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; @@ -32,8 +34,12 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { private static final int RAW_DATA_SIZE_LENGTH = INT_LENGTH; private static final int CHUNKS_SIZE_LENGTH = INT_LENGTH; private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH; + private static final int NAME_SIZE_LENGTH = INT_LENGTH; + private static final int SECRET_LENGTH = AES256_LENGTH; + private static final int COMPRESSION_LENGTH = INT_LENGTH; - private static final int EXTRAS_LENGTH = SERVICE_LENGTH + NONCE_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH + RAW_DATA_SIZE_LENGTH + CHUNKS_SIZE_LENGTH; + private static final int EXTRAS_LENGTH = SERVICE_LENGTH + NONCE_LENGTH + NAME_SIZE_LENGTH + SERVICE_LENGTH + + COMPRESSION_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH + RAW_DATA_SIZE_LENGTH + CHUNKS_SIZE_LENGTH; protected static final TransactionLayout layout; @@ -46,6 +52,11 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("sender's public key", TransformationType.PUBLIC_KEY); layout.add("nonce", TransformationType.INT); // Version 5+ + layout.add("name", TransformationType.DATA); // Version 5+ + layout.add("method", TransformationType.INT); // Version 5+ + layout.add("secret", TransformationType.DATA); // Version 5+ + layout.add("compression", TransformationType.INT); // Version 5+ + layout.add("number of payments", TransformationType.INT); layout.add("* recipient", TransformationType.ADDRESS); layout.add("* asset ID of payment", TransformationType.LONG); @@ -77,8 +88,22 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { byte[] senderPublicKey = Serialization.deserializePublicKey(byteBuffer); int nonce = 0; + String name = null; + ArbitraryTransactionData.Method method = null; + byte[] secret = null; + ArbitraryTransactionData.Compression compression = null; + if (version >= 5) { nonce = byteBuffer.getInt(); + + name = Serialization.deserializeSizedString(byteBuffer, Name.MAX_NAME_SIZE); + + method = ArbitraryTransactionData.Method.valueOf(byteBuffer.getInt()); + + secret = new byte[SECRET_LENGTH]; + byteBuffer.get(secret); + + compression = ArbitraryTransactionData.Compression.valueOf(byteBuffer.getInt()); } // Always return a list of payments, even if empty @@ -90,7 +115,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { payments.add(PaymentTransformer.fromByteBuffer(byteBuffer)); } - int service = byteBuffer.getInt(); + ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.valueOf(byteBuffer.getInt()); // We might be receiving hash of data instead of actual raw data boolean isRaw = byteBuffer.get() != 0; @@ -124,16 +149,17 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature); - return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, data, dataType, chunkHashes, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, name, method, secret, compression, data, dataType, chunkHashes, payments); } public static int getDataLength(TransactionData transactionData) throws TransformationException { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + int nameLength = Utf8.encodedLength(arbitraryTransactionData.getName()); int dataLength = (arbitraryTransactionData.getData() != null) ? arbitraryTransactionData.getData().length : 0; int chunkHashesLength = (arbitraryTransactionData.getChunkHashes() != null) ? arbitraryTransactionData.getChunkHashes().length : 0; - int length = getBaseLength(transactionData) + EXTRAS_LENGTH + dataLength + chunkHashesLength; + int length = getBaseLength(transactionData) + EXTRAS_LENGTH + nameLength + dataLength + chunkHashesLength; // Optional payments length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength(); @@ -151,6 +177,14 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { if (arbitraryTransactionData.getVersion() >= 5) { bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce())); + + Serialization.serializeSizedString(bytes, arbitraryTransactionData.getName()); + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getMethod().value)); + + bytes.write(arbitraryTransactionData.getSecret()); + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getCompression().value)); } List payments = arbitraryTransactionData.getPayments(); @@ -159,7 +193,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { for (PaymentData paymentData : payments) bytes.write(PaymentTransformer.toBytes(paymentData)); - bytes.write(Ints.toByteArray(arbitraryTransactionData.getService())); + bytes.write(Ints.toByteArray(arbitraryTransactionData.getService().value)); bytes.write((byte) (arbitraryTransactionData.getDataType() == DataType.RAW_DATA ? 1 : 0)); @@ -204,6 +238,14 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { if (arbitraryTransactionData.getVersion() >= 5) { bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce())); + + Serialization.serializeSizedString(bytes, arbitraryTransactionData.getName()); + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getMethod().value)); + + bytes.write(arbitraryTransactionData.getSecret()); + + bytes.write(Ints.toByteArray(arbitraryTransactionData.getCompression().value)); } if (arbitraryTransactionData.getVersion() != 1) { @@ -214,7 +256,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { bytes.write(PaymentTransformer.toBytes(paymentData)); } - bytes.write(Ints.toByteArray(arbitraryTransactionData.getService())); + bytes.write(Ints.toByteArray(arbitraryTransactionData.getService().value)); bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length)); diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java index 39204167..4d0280b6 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java @@ -36,10 +36,14 @@ public class ArbitraryTransactionTests extends Common { TestAccount alice = Common.getTestAccount(repository, "alice"); ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.ARBITRARY_DATA; + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; + ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.NONE; List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(TestTransaction.generateBase(alice), - 5, ArbitraryTransaction.SERVICE_ARBITRARY_DATA, 0, 0, null, dataType, null, payments); + 5, service, 0, 0, null, method, + null, compression, null, dataType, null, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); assertEquals(12, transaction.difficultyForFileSize(1)); diff --git a/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java b/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java index 9e9814f8..53cdf990 100644 --- a/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java +++ b/src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java @@ -17,9 +17,13 @@ public class ArbitraryTestTransaction extends TestTransaction { public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { final int version = 4; - final int service = 123; + final ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.ARBITRARY_DATA; final int nonce = 0; // Version 4 doesn't need a nonce final int size = 0; // Version 4 doesn't need a size + final String name = null; // Version 4 doesn't need a name + final ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; // Version 4 doesn't need a method + final byte[] secret = null; // Version 4 doesn't need a secret + final ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.NONE; // Version 4 doesn't use compression final byte[] chunkHashes = null; // Version 4 doesn't use chunk hashes byte[] data = new byte[1024]; @@ -34,7 +38,8 @@ public class ArbitraryTestTransaction extends TestTransaction { List payments = new ArrayList<>(); payments.add(new PaymentData(recipient, assetId, amount)); - return new ArbitraryTransactionData(generateBase(account), version, service, nonce, size, data, dataType, chunkHashes, payments); + return new ArbitraryTransactionData(generateBase(account), version, service, nonce, size, name, method, + secret, compression, data, dataType, chunkHashes, payments); } } From 8a654834acd317f2b45f907486058195717b5e91 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 15 Jul 2021 19:47:53 +0100 Subject: [PATCH 111/505] Small reorganization. --- .../api/resource/ArbitraryResource.java | 20 ++++++++++--------- .../qortal/api/resource/WebsiteResource.java | 20 ++++++++++--------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index cefffd67..3b6c2b6d 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -303,19 +303,21 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - String creatorAddress = Crypto.toAddress(creatorPublicKey); - byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); + final String creatorAddress = Crypto.toAddress(creatorPublicKey); + final byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); - BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, + final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); - int size = (int)dataFile.size(); - ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; - byte[] digest = dataFile.digest(); - byte[] chunkHashes = dataFile.chunkHashes(); - List payments = new ArrayList<>(); + final int size = (int)dataFile.size(); + final int version = 5; + final int nonce = 0; + final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + final byte[] digest = dataFile.digest(); + final byte[] chunkHashes = dataFile.chunkHashes(); + final List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, service, 0, size, name, method, + version, service, nonce, size, name, method, secret, compression, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index ccfdc21f..f8a32063 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -114,19 +114,21 @@ public class WebsiteResource { try (final Repository repository = RepositoryManager.getRepository()) { - String creatorAddress = Crypto.toAddress(creatorPublicKey); - byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); + final String creatorAddress = Crypto.toAddress(creatorPublicKey); + final byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); - BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, + final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); - int size = (int)dataFile.size(); - ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; - byte[] digest = dataFile.digest(); - byte[] chunkHashes = dataFile.chunkHashes(); - List payments = new ArrayList<>(); + final int size = (int)dataFile.size(); + final int version = 5; + final int nonce = 0; + final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; + final byte[] digest = dataFile.digest(); + final byte[] chunkHashes = dataFile.chunkHashes(); + final List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, service, 0, size, name, method, + 5, service, nonce, size, name, method, secret, compression, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); From 944e396823c8b0e10d9db78ca5fbbda2b8565c0c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 15 Jul 2021 19:50:42 +0100 Subject: [PATCH 112/505] Added AES utility class from baeldung and updated copyright notice for ZipUtils which was based on code from the same author. This code still needs reviewing and modifying but it's a good starting point for AES encryption and decryption. --- src/main/java/org/qortal/crypto/AES.java | 181 +++++++++++++++++++ src/main/java/org/qortal/utils/ZipUtils.java | 19 +- 2 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/qortal/crypto/AES.java diff --git a/src/main/java/org/qortal/crypto/AES.java b/src/main/java/org/qortal/crypto/AES.java new file mode 100644 index 00000000..e47e7676 --- /dev/null +++ b/src/main/java/org/qortal/crypto/AES.java @@ -0,0 +1,181 @@ +/* + * MIT License + * + * Copyright (c) 2017 Eugen Paraschiv + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ + +package org.qortal.crypto; + +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.BadPaddingException; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKeyFactory; +import javax.crypto.SealedObject; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.Serializable; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.KeySpec; +import java.util.Base64; + +public class AES { + + public static String encrypt(String algorithm, String input, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + byte[] cipherText = cipher.doFinal(input.getBytes()); + return Base64.getEncoder() + .encodeToString(cipherText); + } + + public static String decrypt(String algorithm, String cipherText, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + byte[] plainText = cipher.doFinal(Base64.getDecoder() + .decode(cipherText)); + return new String(plainText); + } + + public static SecretKey generateKey(int n) throws NoSuchAlgorithmException { + KeyGenerator keyGenerator = KeyGenerator.getInstance("AES"); + keyGenerator.init(n); + SecretKey key = keyGenerator.generateKey(); + return key; + } + + public static SecretKey getKeyFromPassword(String password, String salt) + throws NoSuchAlgorithmException, InvalidKeySpecException { + SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256"); + KeySpec spec = new PBEKeySpec(password.toCharArray(), salt.getBytes(), 65536, 256); + SecretKey secret = new SecretKeySpec(factory.generateSecret(spec) + .getEncoded(), "AES"); + return secret; + } + + public static IvParameterSpec generateIv() { + byte[] iv = new byte[16]; + new SecureRandom().nextBytes(iv); + return new IvParameterSpec(iv); + } + + public static void encryptFile(String algorithm, SecretKey key, IvParameterSpec iv, + File inputFile, File outputFile) throws IOException, NoSuchPaddingException, + NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, + BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + FileInputStream inputStream = new FileInputStream(inputFile); + FileOutputStream outputStream = new FileOutputStream(outputFile); + byte[] buffer = new byte[64]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + byte[] output = cipher.update(buffer, 0, bytesRead); + if (output != null) { + outputStream.write(output); + } + } + byte[] outputBytes = cipher.doFinal(); + if (outputBytes != null) { + outputStream.write(outputBytes); + } + inputStream.close(); + outputStream.close(); + } + + public static void decryptFile(String algorithm, SecretKey key, IvParameterSpec iv, + File encryptedFile, File decryptedFile) throws IOException, NoSuchPaddingException, + NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, + BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + FileInputStream inputStream = new FileInputStream(encryptedFile); + FileOutputStream outputStream = new FileOutputStream(decryptedFile); + byte[] buffer = new byte[64]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + byte[] output = cipher.update(buffer, 0, bytesRead); + if (output != null) { + outputStream.write(output); + } + } + byte[] output = cipher.doFinal(); + if (output != null) { + outputStream.write(output); + } + inputStream.close(); + outputStream.close(); + } + + public static SealedObject encryptObject(String algorithm, Serializable object, SecretKey key, + IvParameterSpec iv) throws NoSuchPaddingException, NoSuchAlgorithmException, + InvalidAlgorithmParameterException, InvalidKeyException, IOException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + SealedObject sealedObject = new SealedObject(object, cipher); + return sealedObject; + } + + public static Serializable decryptObject(String algorithm, SealedObject sealedObject, SecretKey key, + IvParameterSpec iv) throws NoSuchPaddingException, NoSuchAlgorithmException, + InvalidAlgorithmParameterException, InvalidKeyException, ClassNotFoundException, + BadPaddingException, IllegalBlockSizeException, IOException { + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + Serializable unsealObject = (Serializable) sealedObject.getObject(cipher); + return unsealObject; + } + + public static String encryptPasswordBased(String plainText, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + return Base64.getEncoder() + .encodeToString(cipher.doFinal(plainText.getBytes())); + } + + public static String decryptPasswordBased(String cipherText, SecretKey key, IvParameterSpec iv) + throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, + InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5PADDING"); + cipher.init(Cipher.DECRYPT_MODE, key, iv); + return new String(cipher.doFinal(Base64.getDecoder() + .decode(cipherText))); + } + +} diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index 0b6a892b..29da90da 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -1,8 +1,25 @@ /* * MIT License + * * Copyright (c) 2017 Eugen Paraschiv * - * Based on code taken from: https://github.com/eugenp/tutorials/tree/master/core-java-modules/core-java-io/src/main/java/com/baeldung + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. * */ From 93eede7c6b20cfc3af2f19d083558024e14526cc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 15 Jul 2021 19:56:09 +0100 Subject: [PATCH 113/505] Added missing break statement which caused a database version mismatch. --- .../java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 2fcbd4e3..5d169eb8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -790,6 +790,7 @@ public class HSQLDBDatabaseUpdates { stmt.execute("ALTER TABLE ArbitraryTransactions ADD secret VARBINARY(32)"); // We want to support compressed and uncompressed data, as well as different compression algorithms stmt.execute("ALTER TABLE ArbitraryTransactions ADD compression INTEGER NOT NULL DEFAULT 0"); + break; default: // nothing to do From 02016c77f1aefd2927560e5dbab0a9528e8fcdb5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Jul 2021 14:40:29 +0100 Subject: [PATCH 114/505] Fixed issue in HSQLDBSaver, introduced in recent commit. --- .../transaction/HSQLDBArbitraryTransactionRepository.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java index 2a9aa764..bcf4abd1 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java @@ -59,12 +59,12 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryTransactions"); saveHelper.bind("signature", arbitraryTransactionData.getSignature()).bind("sender", arbitraryTransactionData.getSenderPublicKey()) - .bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService()) + .bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService().value) .bind("nonce", arbitraryTransactionData.getNonce()).bind("size", arbitraryTransactionData.getSize()) .bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData()) .bind("chunk_hashes", arbitraryTransactionData.getChunkHashes()).bind("name", arbitraryTransactionData.getName()) - .bind("method", arbitraryTransactionData.getMethod()).bind("secret", arbitraryTransactionData.getSecret()) - .bind("compression", arbitraryTransactionData.getCompression()); + .bind("update_method", arbitraryTransactionData.getMethod().value).bind("secret", arbitraryTransactionData.getSecret()) + .bind("compression", arbitraryTransactionData.getCompression().value); try { saveHelper.execute(this.repository); From 6f05de2fcc33e14fb96c07335222c38010f91961 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Jul 2021 14:42:00 +0100 Subject: [PATCH 115/505] Fixed newly introduced issues with arbitrary transaction transformation --- .../ArbitraryTransactionTransformer.java | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java index 4df2e148..149843f8 100644 --- a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java @@ -35,7 +35,6 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { private static final int CHUNKS_SIZE_LENGTH = INT_LENGTH; private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH; private static final int NAME_SIZE_LENGTH = INT_LENGTH; - private static final int SECRET_LENGTH = AES256_LENGTH; private static final int COMPRESSION_LENGTH = INT_LENGTH; private static final int EXTRAS_LENGTH = SERVICE_LENGTH + NONCE_LENGTH + NAME_SIZE_LENGTH + SERVICE_LENGTH + @@ -54,6 +53,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("name", TransformationType.DATA); // Version 5+ layout.add("method", TransformationType.INT); // Version 5+ + layout.add("secret length", TransformationType.INT); // Version 5+ layout.add("secret", TransformationType.DATA); // Version 5+ layout.add("compression", TransformationType.INT); // Version 5+ @@ -100,8 +100,12 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { method = ArbitraryTransactionData.Method.valueOf(byteBuffer.getInt()); - secret = new byte[SECRET_LENGTH]; - byteBuffer.get(secret); + int secretLength = byteBuffer.getInt(); + + if (secretLength > 0) { + secret = new byte[secretLength]; + byteBuffer.get(secret); + } compression = ArbitraryTransactionData.Compression.valueOf(byteBuffer.getInt()); } @@ -136,10 +140,12 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { if (version >= 5) { size = byteBuffer.getInt(); - int chunkHashesSize = byteBuffer.getInt(); + int chunkHashesLength = byteBuffer.getInt(); - chunkHashes = new byte[chunkHashesSize]; - byteBuffer.get(chunkHashes); + if (chunkHashesLength > 0) { + chunkHashes = new byte[chunkHashesLength]; + byteBuffer.get(chunkHashes); + } } long fee = byteBuffer.getLong(); @@ -182,7 +188,13 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { bytes.write(Ints.toByteArray(arbitraryTransactionData.getMethod().value)); - bytes.write(arbitraryTransactionData.getSecret()); + byte[] secret = arbitraryTransactionData.getSecret(); + int secretLength = (secret != null) ? secret.length : 0; + bytes.write(Ints.toByteArray(secretLength)); + + if (secretLength > 0) { + bytes.write(secret); + } bytes.write(Ints.toByteArray(arbitraryTransactionData.getCompression().value)); } @@ -207,7 +219,9 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { int chunkHashesLength = (chunkHashes != null) ? chunkHashes.length : 0; bytes.write(Ints.toByteArray(chunkHashesLength)); - bytes.write(arbitraryTransactionData.getChunkHashes()); + if (chunkHashesLength > 0) { + bytes.write(arbitraryTransactionData.getChunkHashes()); + } } bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee())); @@ -243,7 +257,13 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { bytes.write(Ints.toByteArray(arbitraryTransactionData.getMethod().value)); - bytes.write(arbitraryTransactionData.getSecret()); + byte[] secret = arbitraryTransactionData.getSecret(); + int secretLength = (secret != null) ? secret.length : 0; + bytes.write(Ints.toByteArray(secretLength)); + + if (secretLength > 0) { + bytes.write(secret); + } bytes.write(Ints.toByteArray(arbitraryTransactionData.getCompression().value)); } @@ -278,7 +298,9 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { int chunkHashesLength = (chunkHashes != null) ? chunkHashes.length : 0; bytes.write(Ints.toByteArray(chunkHashesLength)); - bytes.write(arbitraryTransactionData.getChunkHashes()); + if (chunkHashesLength > 0) { + bytes.write(arbitraryTransactionData.getChunkHashes()); + } } bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee())); From f599aa485268b89ef68ae8fdbb08c3d3ea23ff08 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Jul 2021 14:43:02 +0100 Subject: [PATCH 116/505] Modified serializeSizedString() and deserializeSizedString() to cope with null strings. This affects various other parts of the system, not just arbitrary transactions. --- .../java/org/qortal/utils/Serialization.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/utils/Serialization.java b/src/main/java/org/qortal/utils/Serialization.java index e9bf6e0e..8c3c43ed 100644 --- a/src/main/java/org/qortal/utils/Serialization.java +++ b/src/main/java/org/qortal/utils/Serialization.java @@ -101,9 +101,17 @@ public class Serialization { } public static void serializeSizedString(ByteArrayOutputStream bytes, String string) throws UnsupportedEncodingException, IOException { - byte[] stringBytes = string.getBytes(StandardCharsets.UTF_8); - bytes.write(Ints.toByteArray(stringBytes.length)); - bytes.write(stringBytes); + byte[] stringBytes = null; + int stringBytesLength = 0; + + if (string != null) { + stringBytes = string.getBytes(StandardCharsets.UTF_8); + stringBytesLength = stringBytes.length; + } + bytes.write(Ints.toByteArray(stringBytesLength)); + if (stringBytesLength > 0) { + bytes.write(stringBytes); + } } public static String deserializeSizedString(ByteBuffer byteBuffer, int maxSize) throws TransformationException { @@ -114,6 +122,9 @@ public class Serialization { if (size > byteBuffer.remaining()) throw new TransformationException("Byte data too short for serialized string"); + if (size == 0) + return null; + byte[] bytes = new byte[size]; byteBuffer.get(bytes); From f5b29bad335ba2f2d922da2c832a9a907cf31c3a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Jul 2021 14:51:39 +0100 Subject: [PATCH 117/505] Encrypt websites with AES. This ensures that nodes are storing unreadable files, outside of the context of Qortal. For public data, the decryption keys themselves are on-chain, included in the "secret" field of arbitrary transactions. When we introduce the concept of private data, we can simply exclude the secret key from the transaction so that only the owner can decrypt it. When encrypting the file, I have added the 16 byte initialization vector as a prefix to the cyphertext, and it is then automatically extracted back out when decrypting. This gives us the option to encrypt more than one file with the same key, if we ever need it. Right now, we are using a unique key per file, so it's not actually needed, but it's good to have support. --- .../qortal/api/resource/WebsiteResource.java | 63 +++++++++++++-- src/main/java/org/qortal/crypto/AES.java | 40 ++++++++-- .../hsqldb/HSQLDBDatabaseUpdates.java | 1 - .../java/org/qortal/storage/DataFile.java | 9 +++ .../java/org/qortal/test/CryptoTests.java | 80 +++++++++++++++++++ 5 files changed, 176 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index f8a32063..4125c7c4 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -1,5 +1,10 @@ package org.qortal.api.resource; +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -9,6 +14,9 @@ import javax.ws.rs.core.MediaType; import java.io.*; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -28,6 +36,7 @@ import org.qortal.api.ApiExceptionFactory; import org.qortal.api.HTMLParser; import org.qortal.api.Security; import org.qortal.block.BlockChain; +import org.qortal.crypto.AES; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -41,6 +50,7 @@ import org.qortal.storage.DataFile; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -96,7 +106,6 @@ public class WebsiteResource { byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); String name = null; - byte[] secret = null; ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; @@ -122,13 +131,14 @@ public class WebsiteResource { final int size = (int)dataFile.size(); final int version = 5; final int nonce = 0; + byte[] secret = dataFile.getSecret(); final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; final byte[] digest = dataFile.digest(); final byte[] chunkHashes = dataFile.chunkHashes(); final List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, - 5, service, nonce, size, name, method, + version, service, nonce, size, name, method, secret, compression, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); @@ -214,16 +224,29 @@ public class WebsiteResource { } // Firstly zip up the directory - String outputFilePath = tempDir.toString() + File.separator + "zipped.zip"; + String zipOutputFilePath = tempDir.toString() + File.separator + "zipped.zip"; try { - ZipUtils.zip(directoryPath, outputFilePath, "data"); + ZipUtils.zip(directoryPath, zipOutputFilePath, "data"); } catch (IOException e) { LOGGER.info("Unable to zip directory", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } + // Next, encrypt the file with AES + String encryptedFilePath = tempDir.toString() + File.separator + "zipped_encrypted.zip"; + SecretKey aesKey; try { - DataFile dataFile = DataFile.fromPath(outputFilePath); + aesKey = AES.generateKey(256); + AES.encryptFile("AES", aesKey, zipOutputFilePath, encryptedFilePath); + Files.delete(Paths.get(zipOutputFilePath)); + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException + | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR); + } + + try { + DataFile dataFile = DataFile.fromPath(encryptedFilePath); + dataFile.setSecret(aesKey.getEncoded()); DataFile.ValidationResult validationResult = dataFile.isValid(); if (validationResult != DataFile.ValidationResult.OK) { LOGGER.error("Invalid file: {}", validationResult); @@ -241,11 +264,15 @@ public class WebsiteResource { return null; } finally { - // Clean up by deleting the zipped file - File zippedFile = new File(outputFilePath); + // Clean up + File zippedFile = new File(zipOutputFilePath); if (zippedFile.exists()) { zippedFile.delete(); } + File encryptedFile = new File(encryptedFilePath); + if (encryptedFile.exists()) { + encryptedFile.delete(); + } } } @@ -288,6 +315,7 @@ public class WebsiteResource { String tempDirectory = System.getProperty("java.io.tmpdir"); String destPath = tempDirectory + File.separator + "qortal-sites" + File.separator + resourceId; + String unencryptedPath = destPath + File.separator + "zipped.zip"; String unzippedPath = destPath + File.separator + "data"; if (!Files.exists(Paths.get(unzippedPath))) { @@ -304,6 +332,9 @@ public class WebsiteResource { byte[] digest = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); + // Load secret + byte[] secret = transactionData.getSecret(); + // Load data file(s) DataFile dataFile = DataFile.fromHash(digest); if (!dataFile.exists()) { @@ -326,10 +357,26 @@ public class WebsiteResource { return this.get404Response(); } + // Decrypt if we have the secret key. + if (secret != null && secret.length == Transformer.AES256_LENGTH) { + try { + SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES"); + AES.decryptFile("AES", aesKey, dataFile.getFilePath(), unencryptedPath); + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException + | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { + return this.get404Response(); + } + } + else { + // Assume it is unencrypted. We may block this. + unencryptedPath = dataFile.getFilePath(); + } + + // Unzip try { // TODO: compression types //if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) { - ZipUtils.unzip(dataFile.getFilePath(), destPath); + ZipUtils.unzip(unencryptedPath, destPath); //} } catch (IOException e) { LOGGER.info("Unable to unzip file"); diff --git a/src/main/java/org/qortal/crypto/AES.java b/src/main/java/org/qortal/crypto/AES.java index e47e7676..0e8018f5 100644 --- a/src/main/java/org/qortal/crypto/AES.java +++ b/src/main/java/org/qortal/crypto/AES.java @@ -2,6 +2,7 @@ * MIT License * * Copyright (c) 2017 Eugen Paraschiv + * Modified in 2021 by CalDescent * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -93,15 +94,24 @@ public class AES { return new IvParameterSpec(iv); } - public static void encryptFile(String algorithm, SecretKey key, IvParameterSpec iv, - File inputFile, File outputFile) throws IOException, NoSuchPaddingException, - NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, + public static void encryptFile(String algorithm, SecretKey key, + String inputFilePath, String outputFilePath) throws IOException, + NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, BadPaddingException, IllegalBlockSizeException { + + File inputFile = new File(inputFilePath); + File outputFile = new File(outputFilePath); + + IvParameterSpec iv = AES.generateIv(); Cipher cipher = Cipher.getInstance(algorithm); cipher.init(Cipher.ENCRYPT_MODE, key, iv); FileInputStream inputStream = new FileInputStream(inputFile); FileOutputStream outputStream = new FileOutputStream(outputFile); - byte[] buffer = new byte[64]; + + // Prepend the output stream with the 16 byte initialization vector + outputStream.write(iv.getIV()); + + byte[] buffer = new byte[1024]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); @@ -117,14 +127,28 @@ public class AES { outputStream.close(); } - public static void decryptFile(String algorithm, SecretKey key, IvParameterSpec iv, - File encryptedFile, File decryptedFile) throws IOException, NoSuchPaddingException, + public static void decryptFile(String algorithm, SecretKey key, String encryptedFilePath, + String decryptedFilePath) throws IOException, NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, BadPaddingException, IllegalBlockSizeException { - Cipher cipher = Cipher.getInstance(algorithm); - cipher.init(Cipher.DECRYPT_MODE, key, iv); + + File encryptedFile = new File(encryptedFilePath); + File decryptedFile = new File(decryptedFilePath); + + File parent = decryptedFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + FileInputStream inputStream = new FileInputStream(encryptedFile); FileOutputStream outputStream = new FileOutputStream(decryptedFile); + + // Read the initialization vector from the first 16 bytes of the file + byte[] iv = new byte[16]; + inputStream.read(iv); + Cipher cipher = Cipher.getInstance(algorithm); + cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv)); + byte[] buffer = new byte[64]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 5d169eb8..d6d48acc 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -276,7 +276,6 @@ public class HSQLDBDatabaseUpdates { + "service SMALLINT NOT NULL, is_data_raw BOOLEAN NOT NULL, data ArbitraryData NOT NULL, " + TRANSACTION_KEYS + ")"); // NB: Actual data payload stored elsewhere - // For the future: data payload should be encrypted, at the very least with transaction's reference as the seed for the encryption key break; case 8: diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 6dc6d5c0..a6a17385 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -50,6 +50,7 @@ public class DataFile { protected String filePath; protected String hash58; private ArrayList chunks; + private byte[] secret; public DataFile() { } @@ -500,6 +501,14 @@ public class DataFile { return outputString; } + public void setSecret(byte[] secret) { + this.secret = secret; + } + + public byte[] getSecret() { + return this.secret; + } + @Override public String toString() { return this.shortHash58(); diff --git a/src/test/java/org/qortal/test/CryptoTests.java b/src/test/java/org/qortal/test/CryptoTests.java index 46edc698..44ad03f9 100644 --- a/src/test/java/org/qortal/test/CryptoTests.java +++ b/src/test/java/org/qortal/test/CryptoTests.java @@ -3,6 +3,7 @@ package org.qortal.test; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.block.BlockChain; +import org.qortal.crypto.AES; import org.qortal.crypto.BouncyCastle25519; import org.qortal.crypto.Crypto; import org.qortal.test.common.Common; @@ -10,7 +11,17 @@ import org.qortal.utils.Base58; import static org.junit.Assert.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Random; import org.bouncycastle.crypto.agreement.X25519Agreement; import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; @@ -20,6 +31,11 @@ import org.bouncycastle.crypto.params.X25519PublicKeyParameters; import com.google.common.hash.HashCode; +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; + public class CryptoTests extends Common { @Test @@ -255,4 +271,68 @@ public class CryptoTests extends Common { assertEquals(expectedProxyPrivateKey, Base58.encode(proxyPrivateKey)); } + + @Test + public void testAESFileEncryption() throws NoSuchAlgorithmException, IOException, IllegalBlockSizeException, + InvalidKeyException, BadPaddingException, InvalidAlgorithmParameterException, NoSuchPaddingException { + + // Create temporary directory and file paths + java.nio.file.Path tempDir = Files.createTempDirectory("qortal-tests"); + String inputFilePath = tempDir.toString() + File.separator + "inputFile"; + String outputFilePath = tempDir.toString() + File.separator + "outputFile"; + String decryptedFilePath = tempDir.toString() + File.separator + "decryptedFile"; + String reencryptedFilePath = tempDir.toString() + File.separator + "reencryptedFile"; + + // Generate some dummy data + byte[] randomBytes = new byte[1024]; + new Random().nextBytes(randomBytes); + + // Write it to the input file + FileOutputStream outputStream = new FileOutputStream(inputFilePath); + outputStream.write(randomBytes); + + // Make sure only the input file exists + assertTrue(Files.exists(Paths.get(inputFilePath))); + assertFalse(Files.exists(Paths.get(outputFilePath))); + + // Encrypt + SecretKey aesKey = AES.generateKey(256); + AES.encryptFile("AES", aesKey, inputFilePath, outputFilePath); + assertTrue(Files.exists(Paths.get(outputFilePath))); + byte[] encryptedBytes = Files.readAllBytes(Paths.get(outputFilePath)); + + // Delete the input file + Files.delete(Paths.get(inputFilePath)); + assertFalse(Files.exists(Paths.get(inputFilePath))); + + // Decrypt + String encryptedFilePath = outputFilePath; + assertFalse(Files.exists(Paths.get(decryptedFilePath))); + AES.decryptFile("AES", aesKey, encryptedFilePath, decryptedFilePath); + assertTrue(Files.exists(Paths.get(decryptedFilePath))); + + // Delete the output file + Files.delete(Paths.get(outputFilePath)); + assertFalse(Files.exists(Paths.get(outputFilePath))); + + // Check that the decrypted file contents matches the original data + byte[] decryptedBytes = Files.readAllBytes(Paths.get(decryptedFilePath)); + assertTrue(Arrays.equals(decryptedBytes, randomBytes)); + assertEquals(1024, decryptedBytes.length); + + // Write the original data back to the input file + outputStream = new FileOutputStream(inputFilePath); + outputStream.write(randomBytes); + + // Now encrypt the data one more time using the same key + // This is to ensure the initialization vector produces a different result + AES.encryptFile("AES", aesKey, inputFilePath, reencryptedFilePath); + assertTrue(Files.exists(Paths.get(reencryptedFilePath))); + + // Make sure the ciphertexts do not match + byte[] reencryptedBytes = Files.readAllBytes(Paths.get(reencryptedFilePath)); + assertFalse(Arrays.equals(encryptedBytes, reencryptedBytes)); + + } + } From dc83e32173fdc368d103e1a0c4994a7270e5cba8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Jul 2021 15:23:24 +0100 Subject: [PATCH 118/505] Fixed site preview functionality. --- .../qortal/api/resource/WebsiteResource.java | 102 +++++++++++------- 1 file changed, 66 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 4125c7c4..858daac4 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -63,6 +63,11 @@ public class WebsiteResource { private static final Logger LOGGER = LogManager.getLogger(WebsiteResource.class); + public enum ResourceIdType { + SIGNATURE, + FILE_HASH + }; + @Context HttpServletRequest request; @Context HttpServletResponse response; @Context ServletContext context; @@ -199,7 +204,7 @@ public class WebsiteResource { if (dataFile != null) { String digest58 = dataFile.digest58(); if (digest58 != null) { - return "http://localhost:12393/site/" + digest58; + return "http://localhost:12393/site/hash/" + digest58 + "?secret=" + Base58.encode(dataFile.getSecret()); } } return "Unable to generate preview URL"; @@ -277,38 +282,51 @@ public class WebsiteResource { } @GET - @Path("{resource}") - public HttpServletResponse getResourceIndex(@PathParam("resource") String resourceId) { - return this.get(resourceId, "/", true); + @Path("{signature}") + public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) { + return this.get(signature, ResourceIdType.SIGNATURE, "/", null,true); } @GET - @Path("{resource}/{path:.*}") - public HttpServletResponse getResourcePath(@PathParam("resource") String resourceId, @PathParam("path") String inPath) { - return this.get(resourceId, inPath, true); + @Path("{signature}/{path:.*}") + public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) { + return this.get(signature, ResourceIdType.SIGNATURE, inPath,null,true); + } + + @GET + @Path("/hash/{hash}") + public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) { + return this.get(hash58, ResourceIdType.FILE_HASH, "/", secret58,true); + } + + @GET + @Path("/hash/{hash}/{path:.*}") + public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath, + @QueryParam("secret") String secret58) { + return this.get(hash58, ResourceIdType.FILE_HASH, inPath, secret58,true); } @GET @Path("/domainmap") - public HttpServletResponse getDomainMapIndex() { - Map domainMap = Settings.getInstance().getSimpleDomainMap(); - if (domainMap != null && domainMap.containsKey(request.getServerName())) { - return this.get(domainMap.get(request.getServerName()), "/", false); - } - return this.get404Response(); + public HttpServletResponse getIndexByDomainMap() { + return this.getDomainMap("/"); } @GET @Path("/domainmap/{path:.*}") - public HttpServletResponse getDomainMapPath(@PathParam("path") String inPath) { + public HttpServletResponse getPathByDomainMap(@PathParam("path") String inPath) { + return this.getDomainMap(inPath); + } + + private HttpServletResponse getDomainMap(String inPath) { Map domainMap = Settings.getInstance().getSimpleDomainMap(); if (domainMap != null && domainMap.containsKey(request.getServerName())) { - return this.get(domainMap.get(request.getServerName()), inPath, false); + return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, false); } return this.get404Response(); } - private HttpServletResponse get(String resourceId, String inPath, boolean usePrefix) { + private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58, boolean usePrefix) { if (!inPath.startsWith(File.separator)) { inPath = File.separator + inPath; } @@ -322,29 +340,41 @@ public class WebsiteResource { // Load the full transaction data so we can access the file hashes try (final Repository repository = RepositoryManager.getRepository()) { + DataFile dataFile = null; + byte[] digest = null; + byte[] secret = null; - ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); - if (!(transactionData instanceof ArbitraryTransactionData)) { - return this.get404Response(); - } - - // Load hashes - byte[] digest = transactionData.getData(); - byte[] chunkHashes = transactionData.getChunkHashes(); - - // Load secret - byte[] secret = transactionData.getSecret(); - - // Load data file(s) - DataFile dataFile = DataFile.fromHash(digest); - if (!dataFile.exists()) { - if (!dataFile.allChunksExist(chunkHashes)) { - // TODO: fetch them? + if (resourceIdType == ResourceIdType.SIGNATURE) { + ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); + if (!(transactionData instanceof ArbitraryTransactionData)) { return this.get404Response(); } - // We have all the chunks but not the complete file, so join them - dataFile.addChunkHashes(chunkHashes); - dataFile.join(); + + // Load hashes + digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + // Load secret + secret = transactionData.getSecret(); + + // Load data file(s) + dataFile = DataFile.fromHash(digest); + if (!dataFile.exists()) { + if (!dataFile.allChunksExist(chunkHashes)) { + // TODO: fetch them? + return this.get404Response(); + } + // We have all the chunks but not the complete file, so join them + dataFile.addChunkHashes(chunkHashes); + dataFile.join(); + } + + + } + else if (resourceIdType == ResourceIdType.FILE_HASH) { + dataFile = DataFile.fromHash58(resourceId); + digest = Base58.decode(resourceId); + secret = secret58 != null ? Base58.decode(secret58) : null; } // If the complete file still doesn't exist then something went wrong From fe387931a4702751f4ebee7b130fb90456a0973d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Jul 2021 15:48:27 +0100 Subject: [PATCH 119/505] Increased buffer when serving assets from 1kiB to 10kiB This speeds up image loading. Ultimately we will stop serving these via the application memory altogether. --- src/main/java/org/qortal/api/resource/WebsiteResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 858daac4..4761228b 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -436,7 +436,7 @@ public class WebsiteResource { FileInputStream inputStream = new FileInputStream(file); response.setContentType(context.getMimeType(filename)); int bytesRead, length = 0; - byte[] buffer = new byte[1024]; + byte[] buffer = new byte[10240]; while ((bytesRead = inputStream.read(buffer)) != -1) { response.getOutputStream().write(buffer, 0, bytesRead); length += bytesRead; From 7a77b12834a05d6fe1b360f24ce06d10f05f7e92 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Jul 2021 19:18:22 +0100 Subject: [PATCH 120/505] Repalce & with & in HTML documents. This is not a long term solution, but it's a quick fix to solve URL problems when converting to a static site via httrack. --- src/main/java/org/qortal/api/HTMLParser.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index 0c72f5da..ea99afba 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -78,11 +78,17 @@ public class HTMLParser { } } } - return document.html().getBytes(); + String html = document.html(); + html = this.replaceAmpersands(html); + return html.getBytes(); } return data; } + private String replaceAmpersands(String html) { + return html.replace("&", "&"); + } + private boolean shouldReplaceLink(String elementHtml) { List prefixes = new ArrayList<>(); prefixes.add("http"); // Don't modify absolute links From f72374488ef3bd8b2633fcd12b7b9d2fb3c34ae3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 31 Jul 2021 22:54:19 +0100 Subject: [PATCH 121/505] Revert "Removed block 212937" This reverts commit b917da765c9406adacbb5abd261c3b6ed236ec74. --- src/main/java/org/qortal/block/Block.java | 11 ++ .../java/org/qortal/block/Block212937.java | 153 ++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 src/main/java/org/qortal/block/Block212937.java diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 5f6e1641..798a4f91 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1104,6 +1104,10 @@ public class Block { // Create repository savepoint here so we can rollback to it after testing transactions repository.setSavepoint(); + if (this.blockData.getHeight() == 212937) + // Apply fix for block 212937 but fix will be rolled back before we exit method + Block212937.processFix(this); + for (Transaction transaction : this.getTransactions()) { TransactionData transactionData = transaction.getTransactionData(); @@ -1308,6 +1312,9 @@ public class Block { // Distribute block rewards, including transaction fees, before transactions processed processBlockRewards(); + if (this.blockData.getHeight() == 212937) + // Apply fix for block 212937 + Block212937.processFix(this); } // We're about to (test-)process a batch of transactions, @@ -1542,6 +1549,10 @@ public class Block { // Invalidate expandedAccounts as they may have changed due to orphaning TRANSFER_PRIVS transactions, etc. this.cachedExpandedAccounts = null; + if (this.blockData.getHeight() == 212937) + // Revert fix for block 212937 + Block212937.orphanFix(this); + // Block rewards, including transaction fees, removed after transactions undone orphanBlockRewards(); diff --git a/src/main/java/org/qortal/block/Block212937.java b/src/main/java/org/qortal/block/Block212937.java new file mode 100644 index 00000000..a53c9d31 --- /dev/null +++ b/src/main/java/org/qortal/block/Block212937.java @@ -0,0 +1,153 @@ +package org.qortal.block; + +import java.io.InputStream; +import java.util.List; +import java.util.stream.Collectors; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.UnmarshalException; +import javax.xml.bind.Unmarshaller; +import javax.xml.transform.stream.StreamSource; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.persistence.jaxb.JAXBContextFactory; +import org.eclipse.persistence.jaxb.UnmarshallerProperties; +import org.qortal.data.account.AccountBalanceData; +import org.qortal.repository.DataException; + +/** + * Block 212937 + *

    + * Somehow a node minted a version of block 212937 that contained one transaction: + * a PAYMENT transaction that attempted to spend more QORT than that account had as QORT balance. + *

    + * This invalid transaction made block 212937 (rightly) invalid to several nodes, + * which refused to use that block. + * However, it seems there were no other nodes minting an alternative, valid block at that time + * and so the chain stalled for several nodes in the network. + *

    + * Additionally, the invalid block 212937 affected all new installations, regardless of whether + * they synchronized from scratch (block 1) or used an 'official release' bootstrap. + *

    + * After lengthy diagnosis, it was discovered that + * the invalid transaction seemed to rely on incorrect balances in a corrupted database. + * Copies of DB files containing the broken chain were also shared around, exacerbating the problem. + *

    + * There were three options: + *

      + *
    1. roll back the chain to last known valid block 212936 and re-mint empty blocks to current height
      2. + *
    2. keep existing chain, but apply database edits at block 212937 to allow current chain to be valid
      3. + *
    3. attempt to mint an alternative chain, retaining as many valid transactions as possible
      4. + *
    + *

    + * Option 1 was highly undesirable due to knock-on effects from wiping 700+ transactions, some of which + * might have affect cross-chain trades, although there were no cross-chain trade completed during + * the decision period. + *

    + * Option 3 was essentially a slightly better version of option 1 and rejected for similar reasons. + * Attempts at option 3 also rapidly hit cumulative problems with every replacement block due to + * differing block timestamps making some transactions, and then even some blocks themselves, invalid. + *

    + * This class is the implementation of option 2. + *

    + * The change in account balances are relatively small, see block-212937-deltas.json resource + * for actual values. These values were obtained by exporting the AccountBalances table from + * both versions of the database with chain at block 212936, and then comparing. The values were also + * tested by syncing both databases up to block 225500, re-exporting and re-comparing. + *

    + * The invalid block 212937 signature is: 2J3GVJjv...qavh6KkQ. + *

    + * The invalid transaction in block 212937 is: + *

    + * 

    +   {
+      "amount" : "0.10788294",
+      "approvalStatus" : "NOT_REQUIRED",
+      "blockHeight" : 212937,
+      "creatorAddress" : "QLdw5uabviLJgRGkRiydAFmAtZzxHfNXSs",
+      "fee" : "0.00100000",
+      "recipient" : "QZi1mNHDbiLvsytxTgxDr9nhJe4pNZaWpw",
+      "reference" : "J6JukdTVuXZ3JYbHatfZzwxG2vSiZwVCPDzW5K7PsVQKRj8XZeDtqnkGCGGjaSQZ9bQMtV44ky88NnGM4YBQKU6",
+      "senderPublicKey" : "DBFfbD2M3uh4jPE5PaUcZVvNPfrrJzVB7seeEtBn5SPs",
+      "signature" : "qkitxdCEEnKt8w6wRfFixtErbXsxWE6zG2ESNhpqBdScikV1WxeA6WZTTMJVV4tCeZdBFXw3V1X5NVztv6LirWK",
+      "timestamp" : 1607863074904,
+      "txGroupId" : 0,
+      "type" : "PAYMENT"
+   }
+
    + *

    + * Account QLdw5uabviLJgRGkRiydAFmAtZzxHfNXSs attempted to spend 0.10888294 (including fees) + * when their QORT balance was really only 0.10886665. + *

    + * However, on the broken DB nodes, their balance + * seemed to be 0.10890293 which was sufficient to make the transaction valid. + */ +public final class Block212937 { + + private static final Logger LOGGER = LogManager.getLogger(Block212937.class); + private static final String ACCOUNT_DELTAS_SOURCE = "block-212937-deltas.json"; + + private static final List accountDeltas = readAccountDeltas(); + + private Block212937() { + /* Do not instantiate */ + } + + @SuppressWarnings("unchecked") + private static List readAccountDeltas() { + Unmarshaller unmarshaller; + + try { + // Create JAXB context aware of classes we need to unmarshal + JAXBContext jc = JAXBContextFactory.createContext(new Class[] { + AccountBalanceData.class + }, null); + + // Create unmarshaller + unmarshaller = jc.createUnmarshaller(); + + // Set the unmarshaller media type to JSON + unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json"); + + // Tell unmarshaller that there's no JSON root element in the JSON input + unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false); + } catch (JAXBException e) { + String message = "Failed to setup unmarshaller to read block 212937 deltas"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } + + ClassLoader classLoader = BlockChain.class.getClassLoader(); + InputStream in = classLoader.getResourceAsStream(ACCOUNT_DELTAS_SOURCE); + StreamSource jsonSource = new StreamSource(in); + + try { + // Attempt to unmarshal JSON stream to BlockChain config + return (List) unmarshaller.unmarshal(jsonSource, AccountBalanceData.class).getValue(); + } catch (UnmarshalException e) { + String message = "Failed to parse block 212937 deltas"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } catch (JAXBException e) { + String message = "Unexpected JAXB issue while processing block 212937 deltas"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); + } + } + + public static void processFix(Block block) throws DataException { + block.repository.getAccountRepository().modifyAssetBalances(accountDeltas); + } + + public static void orphanFix(Block block) throws DataException { + // Create inverse deltas + List inverseDeltas = accountDeltas.stream() + .map(delta -> new AccountBalanceData(delta.getAddress(), delta.getAssetId(), 0 - delta.getBalance())) + .collect(Collectors.toList()); + + block.repository.getAccountRepository().modifyAssetBalances(inverseDeltas); + } + +} From dc8a402a4aa9391882fb809ebdad138dcd85c00a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 31 Jul 2021 22:55:27 +0100 Subject: [PATCH 122/505] Revert "Removed all cross-chain code. It's not needed for data nodes." This reverts commit cc59510cd05ffb3f4845ee7d5389ac197a8fd0fc. # Conflicts: # src/main/java/org/qortal/controller/Controller.java --- src/main/java/org/qortal/api/ApiService.java | 6 + .../model/CrossChainBitcoinRedeemRequest.java | 34 + .../model/CrossChainBitcoinRefundRequest.java | 31 + .../CrossChainBitcoinTemplateRequest.java | 23 + .../model/CrossChainBitcoinyHTLCStatus.java | 31 + .../api/model/CrossChainBuildRequest.java | 39 + .../api/model/CrossChainCancelRequest.java | 20 + .../model/CrossChainDualSecretRequest.java | 29 + .../api/model/CrossChainOfferSummary.java | 127 ++ .../api/model/CrossChainSecretRequest.java | 26 + .../api/model/CrossChainTradeRequest.java | 23 + .../api/model/CrossChainTradeSummary.java | 54 + .../model/crosschain/BitcoinSendRequest.java | 29 + .../model/crosschain/LitecoinSendRequest.java | 29 + .../crosschain/TradeBotCreateRequest.java | 46 + .../crosschain/TradeBotRespondRequest.java | 29 + .../qortal/api/resource/ApiDefinition.java | 3 +- .../CrossChainBitcoinACCTv1Resource.java | 363 +++++ .../resource/CrossChainBitcoinResource.java | 167 +++ .../api/resource/CrossChainHtlcResource.java | 603 ++++++++ .../CrossChainLitecoinACCTv1Resource.java | 145 ++ .../resource/CrossChainLitecoinResource.java | 167 +++ .../api/resource/CrossChainResource.java | 424 ++++++ .../resource/CrossChainTradeBotResource.java | 286 ++++ .../api/websocket/PresenceWebSocket.java | 244 ++++ .../api/websocket/TradeBotWebSocket.java | 157 ++ .../api/websocket/TradeOffersWebSocket.java | 351 +++++ .../org/qortal/at/QortalFunctionCode.java | 15 + .../org/qortal/controller/Controller.java | 5 + .../controller/tradebot/AcctTradeBot.java | 30 + .../tradebot/BitcoinACCTv1TradeBot.java | 1273 +++++++++++++++++ .../tradebot/LitecoinACCTv1TradeBot.java | 894 ++++++++++++ .../qortal/controller/tradebot/TradeBot.java | 373 +++++ src/main/java/org/qortal/crosschain/ACCT.java | 23 + .../java/org/qortal/crosschain/AcctMode.java | 21 + .../java/org/qortal/crosschain/Bitcoin.java | 190 +++ .../org/qortal/crosschain/BitcoinACCTv1.java | 921 ++++++++++++ .../java/org/qortal/crosschain/Bitcoiny.java | 740 ++++++++++ .../BitcoinyBlockchainProvider.java | 40 + .../org/qortal/crosschain/BitcoinyHTLC.java | 438 ++++++ .../crosschain/BitcoinyTransaction.java | 146 ++ .../java/org/qortal/crosschain/ElectrumX.java | 688 +++++++++ .../qortal/crosschain/ForeignBlockchain.java | 9 + .../ForeignBlockchainException.java | 77 + .../java/org/qortal/crosschain/Litecoin.java | 175 +++ .../org/qortal/crosschain/LitecoinACCTv1.java | 853 +++++++++++ .../qortal/crosschain/SimpleTransaction.java | 32 + .../crosschain/SupportedBlockchain.java | 113 ++ .../qortal/crosschain/TransactionHash.java | 31 + .../org/qortal/crosschain/UnspentOutput.java | 16 + .../data/crosschain/CrossChainTradeData.java | 109 ++ .../qortal/data/crosschain/TradeBotData.java | 268 ++++ .../transaction/PresenceTransactionData.java | 73 + .../data/transaction/TransactionData.java | 2 +- .../repository/CrossChainRepository.java | 21 + .../org/qortal/repository/Repository.java | 2 + .../hsqldb/HSQLDBCrossChainRepository.java | 202 +++ .../hsqldb/HSQLDBDatabaseUpdates.java | 51 + .../repository/hsqldb/HSQLDBRepository.java | 70 +- .../HSQLDBPresenceTransactionRepository.java | 57 + .../java/org/qortal/settings/Settings.java | 12 + .../transaction/PresenceTransaction.java | 256 ++++ .../PresenceTransactionTransformer.java | 108 ++ .../java/org/qortal/test/PresenceTests.java | 133 ++ .../java/org/qortal/test/RepositoryTests.java | 20 + .../qortal/test/api/CrossChainApiTests.java | 42 + .../qortal/test/crosschain/BitcoinTests.java | 115 ++ .../test/crosschain/ElectrumXTests.java | 201 +++ .../org/qortal/test/crosschain/HtlcTests.java | 128 ++ .../qortal/test/crosschain/LitecoinTests.java | 114 ++ .../test/crosschain/apps/BuildHTLC.java | 114 ++ .../test/crosschain/apps/CheckHTLC.java | 135 ++ .../qortal/test/crosschain/apps/Common.java | 158 ++ .../apps/GetNextReceiveAddress.java | 78 + .../test/crosschain/apps/GetTransaction.java | 84 ++ .../apps/GetWalletTransactions.java | 82 ++ .../org/qortal/test/crosschain/apps/Pay.java | 80 ++ .../test/crosschain/apps/RedeemHTLC.java | 166 +++ .../test/crosschain/apps/RefundHTLC.java | 163 +++ .../bitcoinv1/BitcoinACCTv1Tests.java | 795 ++++++++++ .../test/crosschain/bitcoinv1/DeployAT.java | 169 +++ .../test/crosschain/litecoinv1/DeployAT.java | 150 ++ .../litecoinv1/LitecoinACCTv1Tests.java | 770 ++++++++++ .../litecoinv1/SendCancelMessage.java | 90 ++ .../litecoinv1/SendRedeemMessage.java | 101 ++ .../litecoinv1/SendTradeMessage.java | 118 ++ 86 files changed, 15821 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainBuildRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainCancelRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainOfferSummary.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainSecretRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainTradeRequest.java create mode 100644 src/main/java/org/qortal/api/model/CrossChainTradeSummary.java create mode 100644 src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java create mode 100644 src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java create mode 100644 src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java create mode 100644 src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java create mode 100644 src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java create mode 100644 src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java create mode 100644 src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java create mode 100644 src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java create mode 100644 src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java create mode 100644 src/main/java/org/qortal/api/resource/CrossChainResource.java create mode 100644 src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java create mode 100644 src/main/java/org/qortal/api/websocket/PresenceWebSocket.java create mode 100644 src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java create mode 100644 src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java create mode 100644 src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java create mode 100644 src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java create mode 100644 src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java create mode 100644 src/main/java/org/qortal/controller/tradebot/TradeBot.java create mode 100644 src/main/java/org/qortal/crosschain/ACCT.java create mode 100644 src/main/java/org/qortal/crosschain/AcctMode.java create mode 100644 src/main/java/org/qortal/crosschain/Bitcoin.java create mode 100644 src/main/java/org/qortal/crosschain/BitcoinACCTv1.java create mode 100644 src/main/java/org/qortal/crosschain/Bitcoiny.java create mode 100644 src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java create mode 100644 src/main/java/org/qortal/crosschain/BitcoinyHTLC.java create mode 100644 src/main/java/org/qortal/crosschain/BitcoinyTransaction.java create mode 100644 src/main/java/org/qortal/crosschain/ElectrumX.java create mode 100644 src/main/java/org/qortal/crosschain/ForeignBlockchain.java create mode 100644 src/main/java/org/qortal/crosschain/ForeignBlockchainException.java create mode 100644 src/main/java/org/qortal/crosschain/Litecoin.java create mode 100644 src/main/java/org/qortal/crosschain/LitecoinACCTv1.java create mode 100644 src/main/java/org/qortal/crosschain/SimpleTransaction.java create mode 100644 src/main/java/org/qortal/crosschain/SupportedBlockchain.java create mode 100644 src/main/java/org/qortal/crosschain/TransactionHash.java create mode 100644 src/main/java/org/qortal/crosschain/UnspentOutput.java create mode 100644 src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java create mode 100644 src/main/java/org/qortal/data/crosschain/TradeBotData.java create mode 100644 src/main/java/org/qortal/data/transaction/PresenceTransactionData.java create mode 100644 src/main/java/org/qortal/repository/CrossChainRepository.java create mode 100644 src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java create mode 100644 src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java create mode 100644 src/main/java/org/qortal/transaction/PresenceTransaction.java create mode 100644 src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java create mode 100644 src/test/java/org/qortal/test/PresenceTests.java create mode 100644 src/test/java/org/qortal/test/api/CrossChainApiTests.java create mode 100644 src/test/java/org/qortal/test/crosschain/BitcoinTests.java create mode 100644 src/test/java/org/qortal/test/crosschain/ElectrumXTests.java create mode 100644 src/test/java/org/qortal/test/crosschain/HtlcTests.java create mode 100644 src/test/java/org/qortal/test/crosschain/LitecoinTests.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/Common.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/Pay.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java create mode 100644 src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java create mode 100644 src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java create mode 100644 src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java create mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java create mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java create mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java create mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java create mode 100644 src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index b88edb5a..5baf2c5d 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -43,6 +43,9 @@ import org.qortal.api.websocket.ActiveChatsWebSocket; import org.qortal.api.websocket.AdminStatusWebSocket; import org.qortal.api.websocket.BlocksWebSocket; import org.qortal.api.websocket.ChatMessagesWebSocket; +import org.qortal.api.websocket.PresenceWebSocket; +import org.qortal.api.websocket.TradeBotWebSocket; +import org.qortal.api.websocket.TradeOffersWebSocket; import org.qortal.settings.Settings; public class ApiService { @@ -196,6 +199,9 @@ public class ApiService { context.addServlet(BlocksWebSocket.class, "/websockets/blocks"); context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*"); context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages"); + context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers"); + context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot"); + context.addServlet(PresenceWebSocket.class, "/websockets/presence"); // Start server this.server.start(); diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java new file mode 100644 index 00000000..074fd24d --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainBitcoinRedeemRequest.java @@ -0,0 +1,34 @@ +package org.qortal.api.model; + +import java.math.BigDecimal; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainBitcoinRedeemRequest { + + @Schema(description = "Bitcoin HASH160(public key) for refund", example = "2nGDBPPPFS1c9w1h33YwFk4KUJU2") + public byte[] refundPublicKeyHash; + + @Schema(description = "Bitcoin PRIVATE KEY for redeem", example = "cUvGNSnu14q6Hr1X7TESjYVTqBpFjj8GGLGjGdpJwD9NhSQKeYUk") + public byte[] redeemPrivateKey; + + @Schema(description = "Qortal AT address") + public String atAddress; + + @Schema(description = "Bitcoin miner fee", example = "0.00001000") + public BigDecimal bitcoinMinerFee; + + @Schema(description = "32-byte secret", example = "6gVbAXCVzJXAWwtAVGAfgAkkXpeXvPUwSciPmCfSfXJG") + public byte[] secret; + + @Schema(description = "Bitcoin HASH160(public key) for receiving funds, or omit to derive from private key", example = "u17kBVKkKSp12oUzaxFwNnq1JZf") + public byte[] receivingAccountInfo; + + public CrossChainBitcoinRedeemRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java new file mode 100644 index 00000000..f2485389 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainBitcoinRefundRequest.java @@ -0,0 +1,31 @@ +package org.qortal.api.model; + +import java.math.BigDecimal; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainBitcoinRefundRequest { + + @Schema(description = "Bitcoin PRIVATE KEY for refund", example = "cSP3zTb6bfm8GATtAcEJ8LqYtNQmzZ9jE2wQUVnZGiBzojDdrwKV") + public byte[] refundPrivateKey; + + @Schema(description = "Bitcoin HASH160(public key) for redeem", example = "2daMveGc5pdjRyFacbxBzMksCbyC") + public byte[] redeemPublicKeyHash; + + @Schema(description = "Qortal AT address") + public String atAddress; + + @Schema(description = "Bitcoin miner fee", example = "0.00001000") + public BigDecimal bitcoinMinerFee; + + @Schema(description = "Bitcoin HASH160(public key) for receiving funds, or omit to derive from private key", example = "u17kBVKkKSp12oUzaxFwNnq1JZf") + public byte[] receivingAccountInfo; + + public CrossChainBitcoinRefundRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java new file mode 100644 index 00000000..b7510eaa --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainBitcoinTemplateRequest.java @@ -0,0 +1,23 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainBitcoinTemplateRequest { + + @Schema(description = "Bitcoin HASH160(public key) for refund", example = "2nGDBPPPFS1c9w1h33YwFk4KUJU2") + public byte[] refundPublicKeyHash; + + @Schema(description = "Bitcoin HASH160(public key) for redeem", example = "2daMveGc5pdjRyFacbxBzMksCbyC") + public byte[] redeemPublicKeyHash; + + @Schema(description = "Qortal AT address") + public String atAddress; + + public CrossChainBitcoinTemplateRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java b/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java new file mode 100644 index 00000000..2772eae1 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainBitcoinyHTLCStatus.java @@ -0,0 +1,31 @@ +package org.qortal.api.model; + +import java.math.BigDecimal; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainBitcoinyHTLCStatus { + + @Schema(description = "P2SH address", example = "3CdH27kTpV8dcFHVRYjQ8EEV5FJg9X8pSJ (mainnet), 2fMiRRXVsxhZeyfum9ifybZvaMHbQTmwdZw (testnet)") + public String bitcoinP2shAddress; + + @Schema(description = "P2SH balance") + public BigDecimal bitcoinP2shBalance; + + @Schema(description = "Can HTLC redeem yet?") + public boolean canRedeem; + + @Schema(description = "Can HTLC refund yet?") + public boolean canRefund; + + @Schema(description = "Secret used by HTLC redeemer") + public byte[] secret; + + public CrossChainBitcoinyHTLCStatus() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java b/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java new file mode 100644 index 00000000..e8d38703 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainBuildRequest.java @@ -0,0 +1,39 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainBuildRequest { + + @Schema(description = "AT creator's public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] creatorPublicKey; + + @Schema(description = "Final QORT amount paid out on successful trade", example = "80.40200000") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long qortAmount; + + @Schema(description = "QORT amount funding AT, including covering AT execution fees", example = "123.45670000") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long fundingQortAmount; + + @Schema(description = "HASH160 of creator's Bitcoin public key", example = "2daMveGc5pdjRyFacbxBzMksCbyC") + public byte[] bitcoinPublicKeyHash; + + @Schema(description = "HASH160 of secret", example = "43vnftqkjxrhb5kJdkU1ZFQLEnWV") + public byte[] hashOfSecretB; + + @Schema(description = "Bitcoin P2SH BTC balance for release of secret", example = "0.00864200") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long bitcoinAmount; + + @Schema(description = "Trade time window (minutes) from trade agreement to automatic refund", example = "10080") + public Integer tradeTimeout; + + public CrossChainBuildRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java b/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java new file mode 100644 index 00000000..25a18952 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainCancelRequest.java @@ -0,0 +1,20 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainCancelRequest { + + @Schema(description = "AT creator's public key", example = "K6wuddsBV3HzRrXFFezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] creatorPublicKey; + + @Schema(description = "Qortal trade AT address") + public String atAddress; + + public CrossChainCancelRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java b/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java new file mode 100644 index 00000000..b6705d5d --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainDualSecretRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model; + +import io.swagger.v3.oas.annotations.media.Schema; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainDualSecretRequest { + + @Schema(description = "Public key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] partnerPublicKey; + + @Schema(description = "Qortal AT address") + public String atAddress; + + @Schema(description = "secret-A (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1") + public byte[] secretA; + + @Schema(description = "secret-B (32 bytes)", example = "EN2Bgx3BcEMtxFCewmCVSMkfZjVKYhx3KEXC5A21KBGx") + public byte[] secretB; + + @Schema(description = "Qortal address for receiving QORT from AT") + public String receivingAddress; + + public CrossChainDualSecretRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java b/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java new file mode 100644 index 00000000..bf71c2d2 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainOfferSummary.java @@ -0,0 +1,127 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.qortal.crosschain.AcctMode; +import org.qortal.data.crosschain.CrossChainTradeData; + +import io.swagger.v3.oas.annotations.media.Schema; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainOfferSummary { + + // Properties + + @Schema(description = "AT's Qortal address") + private String qortalAtAddress; + + @Schema(description = "AT creator's Qortal address") + private String qortalCreator; + + @Schema(description = "AT creator's ephemeral trading key-pair represented as Qortal address") + private String qortalCreatorTradeAddress; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long qortAmount; + + @Schema(description = "Bitcoin amount - DEPRECATED: use foreignAmount") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + @Deprecated + private long btcAmount; + + @Schema(description = "Foreign blockchain amount") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long foreignAmount; + + @Schema(description = "Suggested trade timeout (minutes)", example = "10080") + private int tradeTimeout; + + @Schema(description = "Current AT execution mode") + private AcctMode mode; + + private long timestamp; + + @Schema(description = "Trade partner's Qortal receiving address") + private String partnerQortalReceivingAddress; + + private String foreignBlockchain; + + private String acctName; + + protected CrossChainOfferSummary() { + /* For JAXB */ + } + + public CrossChainOfferSummary(CrossChainTradeData crossChainTradeData, long timestamp) { + this.qortalAtAddress = crossChainTradeData.qortalAtAddress; + this.qortalCreator = crossChainTradeData.qortalCreator; + this.qortalCreatorTradeAddress = crossChainTradeData.qortalCreatorTradeAddress; + this.qortAmount = crossChainTradeData.qortAmount; + this.foreignAmount = crossChainTradeData.expectedForeignAmount; + this.btcAmount = this.foreignAmount; // Duplicate for deprecated field + this.tradeTimeout = crossChainTradeData.tradeTimeout; + this.mode = crossChainTradeData.mode; + this.timestamp = timestamp; + this.partnerQortalReceivingAddress = crossChainTradeData.qortalPartnerReceivingAddress; + this.foreignBlockchain = crossChainTradeData.foreignBlockchain; + this.acctName = crossChainTradeData.acctName; + } + + public String getQortalAtAddress() { + return this.qortalAtAddress; + } + + public String getQortalCreator() { + return this.qortalCreator; + } + + public String getQortalCreatorTradeAddress() { + return this.qortalCreatorTradeAddress; + } + + public long getQortAmount() { + return this.qortAmount; + } + + public long getBtcAmount() { + return this.btcAmount; + } + + public long getForeignAmount() { + return this.foreignAmount; + } + + public int getTradeTimeout() { + return this.tradeTimeout; + } + + public AcctMode getMode() { + return this.mode; + } + + public long getTimestamp() { + return this.timestamp; + } + + public String getPartnerQortalReceivingAddress() { + return this.partnerQortalReceivingAddress; + } + + public String getForeignBlockchain() { + return this.foreignBlockchain; + } + + public String getAcctName() { + return this.acctName; + } + + // For debugging mostly + + public String toString() { + return String.format("%s: %s", this.qortalAtAddress, this.mode); + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java b/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java new file mode 100644 index 00000000..2db475e5 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainSecretRequest.java @@ -0,0 +1,26 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainSecretRequest { + + @Schema(description = "Private key to match AT's trade 'partner'", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] partnerPrivateKey; + + @Schema(description = "Qortal AT address") + public String atAddress; + + @Schema(description = "Secret (32 bytes)", example = "FHMzten4he9jZ4HGb4297Utj6F5g2w7serjq2EnAg2s1") + public byte[] secret; + + @Schema(description = "Qortal address for receiving QORT from AT") + public String receivingAddress; + + public CrossChainSecretRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java b/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java new file mode 100644 index 00000000..1afd7290 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainTradeRequest.java @@ -0,0 +1,23 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainTradeRequest { + + @Schema(description = "AT creator's 'trade' public key", example = "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry") + public byte[] tradePublicKey; + + @Schema(description = "Qortal AT address") + public String atAddress; + + @Schema(description = "Signature of trading partner's 'offer' MESSAGE transaction") + public byte[] messageTransactionSignature; + + public CrossChainTradeRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java b/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java new file mode 100644 index 00000000..274dd818 --- /dev/null +++ b/src/main/java/org/qortal/api/model/CrossChainTradeSummary.java @@ -0,0 +1,54 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.qortal.data.crosschain.CrossChainTradeData; + +import io.swagger.v3.oas.annotations.media.Schema; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainTradeSummary { + + private long tradeTimestamp; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long qortAmount; + + @Deprecated + @Schema(description = "DEPRECATED: use foreignAmount instead") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long btcAmount; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long foreignAmount; + + protected CrossChainTradeSummary() { + /* For JAXB */ + } + + public CrossChainTradeSummary(CrossChainTradeData crossChainTradeData, long timestamp) { + this.tradeTimestamp = timestamp; + this.qortAmount = crossChainTradeData.qortAmount; + this.foreignAmount = crossChainTradeData.expectedForeignAmount; + this.btcAmount = this.foreignAmount; + } + + public long getTradeTimestamp() { + return this.tradeTimestamp; + } + + public long getQortAmount() { + return this.qortAmount; + } + + public long getBtcAmount() { + return this.btcAmount; + } + + public long getForeignAmount() { + return this.foreignAmount; + } +} diff --git a/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java b/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java new file mode 100644 index 00000000..86d3d7c8 --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/BitcoinSendRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BitcoinSendRequest { + + @Schema(description = "Bitcoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________") + public String xprv58; + + @Schema(description = "Recipient's Bitcoin address ('legacy' P2PKH only)", example = "1BitcoinEaterAddressDontSendf59kuE") + public String receivingAddress; + + @Schema(description = "Amount of BTC to send", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long bitcoinAmount; + + @Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 BTC (100 sats) per byte", example = "0.00000100", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long feePerByte; + + public BitcoinSendRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java b/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java new file mode 100644 index 00000000..5f215740 --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/LitecoinSendRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class LitecoinSendRequest { + + @Schema(description = "Litecoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________") + public String xprv58; + + @Schema(description = "Recipient's Litecoin address ('legacy' P2PKH only)", example = "LiTecoinEaterAddressDontSendhLfzKD") + public String receivingAddress; + + @Schema(description = "Amount of LTC to send", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long litecoinAmount; + + @Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 LTC (100 sats) per byte", example = "0.00000100", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long feePerByte; + + public LitecoinSendRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java b/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java new file mode 100644 index 00000000..1f96488e --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/TradeBotCreateRequest.java @@ -0,0 +1,46 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.qortal.crosschain.SupportedBlockchain; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class TradeBotCreateRequest { + + @Schema(description = "Trade creator's public key", example = "2zR1WFsbM7akHghqSCYKBPk6LDP8aKiQSRS1FrwoLvoB") + public byte[] creatorPublicKey; + + @Schema(description = "QORT amount paid out on successful trade", example = "80.40000000", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long qortAmount; + + @Schema(description = "QORT amount funding AT, including covering AT execution fees", example = "80.50000000", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long fundingQortAmount; + + @Deprecated + @Schema(description = "Bitcoin amount wanted in return. DEPRECATED: use foreignAmount instead", example = "0.00864200", type = "number", hidden = true) + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long bitcoinAmount; + + @Schema(description = "Foreign blockchain. Note: default (BITCOIN) to be removed in the future", example = "BITCOIN", implementation = SupportedBlockchain.class) + public SupportedBlockchain foreignBlockchain; + + @Schema(description = "Foreign blockchain amount wanted in return", example = "0.00864200", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public Long foreignAmount; + + @Schema(description = "Suggested trade timeout (minutes)", example = "10080") + public int tradeTimeout; + + @Schema(description = "Foreign blockchain address for receiving", example = "1BitcoinEaterAddressDontSendf59kuE") + public String receivingAddress; + + public TradeBotCreateRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java b/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java new file mode 100644 index 00000000..ecc8ed6f --- /dev/null +++ b/src/main/java/org/qortal/api/model/crosschain/TradeBotRespondRequest.java @@ -0,0 +1,29 @@ +package org.qortal.api.model.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import io.swagger.v3.oas.annotations.media.Schema; + +@XmlAccessorType(XmlAccessType.FIELD) +public class TradeBotRespondRequest { + + @Schema(description = "Qortal AT address", example = "Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + public String atAddress; + + @Deprecated + @Schema(description = "Bitcoin BIP32 extended private key. DEPRECATED: use foreignKey instead", hidden = true, + example = "xprv___________________________________________________________________________________________________________") + public String xprv58; + + @Schema(description = "Foreign blockchain private key, e.g. BIP32 'm' key for Bitcoin/Litecoin starting with 'xprv'", + example = "xprv___________________________________________________________________________________________________________") + public String foreignKey; + + @Schema(description = "Qortal address for receiving QORT from AT", example = "Qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq") + public String receivingAddress; + + public TradeBotRespondRequest() { + } + +} diff --git a/src/main/java/org/qortal/api/resource/ApiDefinition.java b/src/main/java/org/qortal/api/resource/ApiDefinition.java index fa27bfbb..f9ec7459 100644 --- a/src/main/java/org/qortal/api/resource/ApiDefinition.java +++ b/src/main/java/org/qortal/api/resource/ApiDefinition.java @@ -22,6 +22,7 @@ import org.qortal.api.Security; @Tag(name = "Automated Transactions"), @Tag(name = "Blocks"), @Tag(name = "Chat"), + @Tag(name = "Cross-Chain"), @Tag(name = "Groups"), @Tag(name = "Names"), @Tag(name = "Payments"), @@ -40,4 +41,4 @@ import org.qortal.api.Security; @SecurityScheme(name = "apiKey", type = SecuritySchemeType.APIKEY, in = SecuritySchemeIn.HEADER, paramName = Security.API_KEY_HEADER) }) public class ApiDefinition { -} +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java new file mode 100644 index 00000000..20a27241 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java @@ -0,0 +1,363 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.Arrays; +import java.util.Random; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.CrossChainBuildRequest; +import org.qortal.api.model.CrossChainDualSecretRequest; +import org.qortal.api.model.CrossChainTradeRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.transform.transaction.MessageTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +@Path("/crosschain/BitcoinACCTv1") +@Tag(name = "Cross-Chain (BitcoinACCTv1)") +public class CrossChainBitcoinACCTv1Resource { + + @Context + HttpServletRequest request; + + @POST + @Path("/build") + @Operation( + summary = "Build Bitcoin cross-chain trading AT", + description = "Returns raw, unsigned DEPLOY_AT transaction", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainBuildRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_DATA, ApiError.INVALID_REFERENCE, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE}) + public String buildTrade(CrossChainBuildRequest tradeRequest) { + Security.checkApiCallAllowed(request); + + byte[] creatorPublicKey = tradeRequest.creatorPublicKey; + + if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + if (tradeRequest.hashOfSecretB == null || tradeRequest.hashOfSecretB.length != Bitcoiny.HASH160_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.tradeTimeout == null) + tradeRequest.tradeTimeout = 7 * 24 * 60; // 7 days + else + if (tradeRequest.tradeTimeout < 10 || tradeRequest.tradeTimeout > 50000) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.qortAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.fundingQortAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + // funding amount must exceed initial + final + if (tradeRequest.fundingQortAmount <= tradeRequest.qortAmount) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (tradeRequest.bitcoinAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + try (final Repository repository = RepositoryManager.getRepository()) { + PublicKeyAccount creatorAccount = new PublicKeyAccount(repository, creatorPublicKey); + + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(creatorAccount.getAddress(), tradeRequest.bitcoinPublicKeyHash, tradeRequest.hashOfSecretB, + tradeRequest.qortAmount, tradeRequest.bitcoinAmount, tradeRequest.tradeTimeout); + + long txTimestamp = NTP.getTime(); + byte[] lastReference = creatorAccount.getLastReference(); + if (lastReference == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_REFERENCE); + + long fee = 0; + String name = "QORT-BTC cross-chain trade"; + String description = "Qortal-Bitcoin cross-chain trade"; + String atType = "ACCT"; + String tags = "QORT-BTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, creatorAccount.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, tradeRequest.fundingQortAmount, Asset.QORT); + + Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + ValidationResult result = deployAtTransaction.isValidUnconfirmed(); + if (result != ValidationResult.OK) + throw TransactionsResource.createTransactionInvalidException(request, result); + + byte[] bytes = DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + return Base58.encode(bytes); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/trademessage") + @Operation( + summary = "Builds raw, unsigned 'trade' MESSAGE transaction that sends cross-chain trade recipient address, triggering 'trade' mode", + description = "Specify address of cross-chain AT that needs to be messaged, and signature of 'offer' MESSAGE from trade partner.
    " + + "AT needs to be in 'offer' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
    " + + "You need to sign output with trade private key otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainTradeRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public String buildTradeMessage(CrossChainTradeRequest tradeRequest) { + Security.checkApiCallAllowed(request); + + byte[] tradePublicKey = tradeRequest.tradePublicKey; + + if (tradePublicKey == null || tradePublicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + if (tradeRequest.atAddress == null || !Crypto.isValidAtAddress(tradeRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (tradeRequest.messageTransactionSignature == null || !Crypto.isValidAddress(tradeRequest.messageTransactionSignature)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, tradeRequest.atAddress); + CrossChainTradeData crossChainTradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.OFFERING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Does supplied public key match trade public key? + if (!Crypto.toAddress(tradePublicKey).equals(crossChainTradeData.qortalCreatorTradeAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + TransactionData transactionData = repository.getTransactionRepository().fromSignature(tradeRequest.messageTransactionSignature); + if (transactionData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_UNKNOWN); + + if (transactionData.getType() != TransactionType.MESSAGE) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); + + MessageTransactionData messageTransactionData = (MessageTransactionData) transactionData; + byte[] messageData = messageTransactionData.getData(); + BitcoinACCTv1.OfferMessageData offerMessageData = BitcoinACCTv1.extractOfferMessageData(messageData); + if (offerMessageData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); + + // Good to make MESSAGE + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerBitcoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(messageTransactionData.getTimestamp(), lockTimeA); + + byte[] outgoingMessageData = BitcoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + byte[] messageTransactionBytes = buildAtMessage(repository, tradePublicKey, tradeRequest.atAddress, outgoingMessageData); + + return Base58.encode(messageTransactionBytes); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/redeemmessage") + @Operation( + summary = "Builds raw, unsigned 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner", + description = "Specify address of cross-chain AT that needs to be messaged, both 32-byte secrets and an address for receiving QORT from AT.
    " + + "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
    " + + "You need to sign output with account the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainDualSecretRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public String buildRedeemMessage(CrossChainDualSecretRequest secretRequest) { + Security.checkApiCallAllowed(request); + + byte[] partnerPublicKey = secretRequest.partnerPublicKey; + + if (partnerPublicKey == null || partnerPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (secretRequest.secretA == null || secretRequest.secretA.length != BitcoinACCTv1.SECRET_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (secretRequest.secretB == null || secretRequest.secretB.length != BitcoinACCTv1.SECRET_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress); + CrossChainTradeData crossChainTradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.TRADING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + String partnerAddress = Crypto.toAddress(partnerPublicKey); + + // MESSAGE must come from address that AT considers trade partner + if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + // Good to make MESSAGE + + byte[] messageData = BitcoinACCTv1.buildRedeemMessage(secretRequest.secretA, secretRequest.secretB, secretRequest.receivingAddress); + byte[] messageTransactionBytes = buildAtMessage(repository, partnerPublicKey, secretRequest.atAddress, messageData); + + return Base58.encode(messageTransactionBytes); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // Must be correct AT - check functionality using code hash + if (!Arrays.equals(atData.getCodeHash(), BitcoinACCTv1.CODE_BYTES_HASH)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + + private byte[] buildAtMessage(Repository repository, byte[] senderPublicKey, String atAddress, byte[] messageData) throws DataException { + long txTimestamp = NTP.getTime(); + + // senderPublicKey could be ephemeral trade public key where there is no corresponding account and hence no reference + String senderAddress = Crypto.toAddress(senderPublicKey); + byte[] lastReference = repository.getAccountRepository().getLastReference(senderAddress); + final boolean requiresPoW = lastReference == null; + + if (requiresPoW) { + Random random = new Random(); + lastReference = new byte[Transformer.SIGNATURE_LENGTH]; + random.nextBytes(lastReference); + } + + int version = 4; + int nonce = 0; + long amount = 0L; + Long assetId = null; // no assetId as amount is zero + Long fee = 0L; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, senderPublicKey, fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, atAddress, amount, assetId, messageData, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + if (requiresPoW) { + messageTransaction.computeNonce(); + } else { + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + } + + ValidationResult result = messageTransaction.isValidUnconfirmed(); + if (result != ValidationResult.OK) + throw TransactionsResource.createTransactionInvalidException(request, result); + + try { + return MessageTransactionTransformer.toBytes(messageTransactionData); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java new file mode 100644 index 00000000..2c1c6991 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -0,0 +1,167 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.bitcoinj.core.Transaction; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.crosschain.BitcoinSendRequest; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.SimpleTransaction; + +@Path("/crosschain/btc") +@Tag(name = "Cross-Chain (Bitcoin)") +public class CrossChainBitcoinResource { + + @Context + HttpServletRequest request; + + @POST + @Path("/walletbalance") + @Operation( + summary = "Returns BTC balance for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + public String getBitcoinWalletBalance(String key58) { + Security.checkApiCallAllowed(request); + + Bitcoin bitcoin = Bitcoin.getInstance(); + + if (!bitcoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + Long balance = bitcoin.getWalletBalance(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + + return balance.toString(); + } + + @POST + @Path("/wallettransactions") + @Operation( + summary = "Returns transactions for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + public List getBitcoinWalletTransactions(String key58) { + Security.checkApiCallAllowed(request); + + Bitcoin bitcoin = Bitcoin.getInstance(); + + if (!bitcoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + return bitcoin.getWalletTransactions(key58); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/send") + @Operation( + summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address", + description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = BitcoinSendRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + public String sendBitcoin(BitcoinSendRequest bitcoinSendRequest) { + Security.checkApiCallAllowed(request); + + if (bitcoinSendRequest.bitcoinAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (bitcoinSendRequest.feePerByte != null && bitcoinSendRequest.feePerByte <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Bitcoin bitcoin = Bitcoin.getInstance(); + + if (!bitcoin.isValidAddress(bitcoinSendRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!bitcoin.isValidDeterministicKey(bitcoinSendRequest.xprv58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + Transaction spendTransaction = bitcoin.buildSpend(bitcoinSendRequest.xprv58, + bitcoinSendRequest.receivingAddress, + bitcoinSendRequest.bitcoinAmount, + bitcoinSendRequest.feePerByte); + + if (spendTransaction == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); + + try { + bitcoin.broadcastTransaction(spendTransaction); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + + return spendTransaction.getTxId().toString(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java new file mode 100644 index 00000000..98e9b01d --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java @@ -0,0 +1,603 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.math.BigDecimal; +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.*; +import org.bitcoinj.script.Script; +import org.qortal.api.*; +import org.qortal.api.model.CrossChainBitcoinyHTLCStatus; +import org.qortal.crosschain.*; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +@Path("/crosschain/htlc") +@Tag(name = "Cross-Chain (Hash time-locked contracts)") +public class CrossChainHtlcResource { + + private static final Logger LOGGER = LogManager.getLogger(CrossChainHtlcResource.class); + + @Context + HttpServletRequest request; + + @GET + @Path("/address/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}") + @Operation( + summary = "Returns HTLC address based on trade info", + description = "Blockchain can be BITCOIN or LITECOIN. Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_CRITERIA}) + public String deriveHtlcAddress(@PathParam("blockchain") String blockchainName, + @PathParam("refundPKH") String refundPKH, + @PathParam("locktime") int lockTime, + @PathParam("redeemPKH") String redeemPKH, + @PathParam("hashOfSecret") String hashOfSecret) { + SupportedBlockchain blockchain = SupportedBlockchain.valueOf(blockchainName); + if (blockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] refunderPubKeyHash; + byte[] redeemerPubKeyHash; + byte[] decodedHashOfSecret; + + try { + refunderPubKeyHash = Base58.decode(refundPKH); + redeemerPubKeyHash = Base58.decode(redeemPKH); + + if (refunderPubKeyHash.length != 20 || redeemerPubKeyHash.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } + + try { + decodedHashOfSecret = Base58.decode(hashOfSecret); + if (decodedHashOfSecret.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + byte[] redeemScript = BitcoinyHTLC.buildScript(refunderPubKeyHash, lockTime, redeemerPubKeyHash, decodedHashOfSecret); + + Bitcoiny bitcoiny = (Bitcoiny) blockchain.getInstance(); + + return bitcoiny.deriveP2shAddress(redeemScript); + } + + @GET + @Path("/status/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}") + @Operation( + summary = "Checks HTLC status", + description = "Blockchain can be BITCOIN or LITECOIN. Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = CrossChainBitcoinyHTLCStatus.class)) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + public CrossChainBitcoinyHTLCStatus checkHtlcStatus(@PathParam("blockchain") String blockchainName, + @PathParam("refundPKH") String refundPKH, + @PathParam("locktime") int lockTime, + @PathParam("redeemPKH") String redeemPKH, + @PathParam("hashOfSecret") String hashOfSecret) { + Security.checkApiCallAllowed(request); + + SupportedBlockchain blockchain = SupportedBlockchain.valueOf(blockchainName); + if (blockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] refunderPubKeyHash; + byte[] redeemerPubKeyHash; + byte[] decodedHashOfSecret; + + try { + refunderPubKeyHash = Base58.decode(refundPKH); + redeemerPubKeyHash = Base58.decode(redeemPKH); + + if (refunderPubKeyHash.length != 20 || redeemerPubKeyHash.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + } + + try { + decodedHashOfSecret = Base58.decode(hashOfSecret); + if (decodedHashOfSecret.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } catch (IllegalArgumentException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + byte[] redeemScript = BitcoinyHTLC.buildScript(refunderPubKeyHash, lockTime, redeemerPubKeyHash, decodedHashOfSecret); + + Bitcoiny bitcoiny = (Bitcoiny) blockchain.getInstance(); + + String p2shAddress = bitcoiny.deriveP2shAddress(redeemScript); + + long now = NTP.getTime(); + + try { + int medianBlockTime = bitcoiny.getMedianBlockTime(); + + // Check P2SH is funded + long p2shBalance = bitcoiny.getConfirmedBalance(p2shAddress.toString()); + + CrossChainBitcoinyHTLCStatus htlcStatus = new CrossChainBitcoinyHTLCStatus(); + htlcStatus.bitcoinP2shAddress = p2shAddress; + htlcStatus.bitcoinP2shBalance = BigDecimal.valueOf(p2shBalance, 8); + + List fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddress.toString()); + + if (p2shBalance > 0L && !fundingOutputs.isEmpty()) { + htlcStatus.canRedeem = now >= medianBlockTime * 1000L; + htlcStatus.canRefund = now >= lockTime * 1000L; + } + + if (now >= medianBlockTime * 1000L) { + // See if we can extract secret + htlcStatus.secret = BitcoinyHTLC.findHtlcSecret(bitcoiny, htlcStatus.bitcoinP2shAddress); + } + + return htlcStatus; + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @GET + @Path("/redeem/LITECOIN/{ataddress}/{tradePrivateKey}/{secret}/{receivingAddress}") + @Operation( + summary = "Redeems HTLC associated with supplied AT, using private key, secret, and receiving address", + description = "Secret and private key should be 32 bytes (base58 encoded). Receiving address must be a valid LTC P2PKH address.
    " + + "The secret can be found in Alice's trade bot data or in the message to Bob's AT.
    " + + "The trade private key and receiving address can be found in Bob's trade bot data.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + public boolean redeemHtlc(@PathParam("ataddress") String atAddress, + @PathParam("tradePrivateKey") String tradePrivateKey, + @PathParam("secret") String secret, + @PathParam("receivingAddress") String receivingAddress) { + Security.checkApiCallAllowed(request); + + // base58 decode the trade private key + byte[] decodedTradePrivateKey = null; + if (tradePrivateKey != null) + decodedTradePrivateKey = Base58.decode(tradePrivateKey); + + // base58 decode the secret + byte[] decodedSecret = null; + if (secret != null) + decodedSecret = Base58.decode(secret); + + // Convert supplied Litecoin receiving address into public key hash (we only support P2PKH at this time) + Address litecoinReceivingAddress; + try { + litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), receivingAddress); + } catch (AddressFormatException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + if (litecoinReceivingAddress.getOutputScriptType() != Script.ScriptType.P2PKH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); + + return this.doRedeemHtlc(atAddress, decodedTradePrivateKey, decodedSecret, litecoinReceivingAccountInfo); + } + + @GET + @Path("/redeem/LITECOIN/{ataddress}") + @Operation( + summary = "Redeems HTLC associated with supplied AT", + description = "To be used by a QORT seller (Bob) who needs to redeem LTC proceeds that are stuck in a P2SH.
    " + + "This requires Bob's trade bot data to be present in the database for this AT.
    " + + "It will fail if the buyer has yet to redeem the QORT held in the AT.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + public boolean redeemHtlc(@PathParam("ataddress") String atAddress) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Attempt to find secret from the buyer's message to AT + byte[] decodedSecret = LitecoinACCTv1.findSecretA(repository, crossChainTradeData); + if (decodedSecret == null) { + LOGGER.info(() -> String.format("Unable to find secret-A from redeem message to AT %s", atAddress)); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); + + // Search for the tradePrivateKey in the tradebot data + byte[] decodedPrivateKey = null; + if (tradeBotData != null) + decodedPrivateKey = tradeBotData.getTradePrivateKey(); + + // Search for the litecoin receiving address in the tradebot data + byte[] litecoinReceivingAccountInfo = null; + if (tradeBotData != null) + // Use receiving address PKH from tradebot data + litecoinReceivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + + return this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, litecoinReceivingAccountInfo); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/redeemAll/LITECOIN") + @Operation( + summary = "Redeems HTLC for all applicable ATs in tradebot data", + description = "To be used by a QORT seller (Bob) who needs to redeem LTC proceeds that are stuck in P2SH transactions.
    " + + "This requires Bob's trade bot data to be present in the database for any ATs that need redeeming.
    " + + "Returns true if at least one trade is redeemed. More detail is available in the log.txt.* file.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + public boolean redeemAllHtlc() { + Security.checkApiCallAllowed(request); + boolean success = false; + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + for (TradeBotData tradeBotData : allTradeBotData) { + String atAddress = tradeBotData.getAtAddress(); + if (atAddress == null) { + LOGGER.info("Missing AT address in tradebot data", atAddress); + continue; + } + + String tradeState = tradeBotData.getState(); + if (tradeState == null) { + LOGGER.info("Missing trade state for AT {}", atAddress); + continue; + } + + if (tradeState.startsWith("ALICE")) { + LOGGER.info("AT {} isn't redeemable because it is a buy order", atAddress); + continue; + } + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) { + LOGGER.info("Couldn't find AT with address {}", atAddress); + continue; + } + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) { + continue; + } + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) { + LOGGER.info("Couldn't find crosschain trade data for AT {}", atAddress); + continue; + } + + // Attempt to find secret from the buyer's message to AT + byte[] decodedSecret = LitecoinACCTv1.findSecretA(repository, crossChainTradeData); + if (decodedSecret == null) { + LOGGER.info("Unable to find secret-A from redeem message to AT {}", atAddress); + continue; + } + + // Search for the tradePrivateKey in the tradebot data + byte[] decodedPrivateKey = tradeBotData.getTradePrivateKey(); + + // Search for the litecoin receiving address PKH in the tradebot data + byte[] litecoinReceivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + + try { + LOGGER.info("Attempting to redeem P2SH balance associated with AT {}...", atAddress); + boolean redeemed = this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, litecoinReceivingAccountInfo); + if (redeemed) { + LOGGER.info("Redeemed P2SH balance associated with AT {}", atAddress); + success = true; + } + else { + LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Already redeemed?", atAddress); + } + } catch (ApiException e) { + LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Missing data?", atAddress); + } + } + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + + return success; + } + + private boolean doRedeemHtlc(String atAddress, byte[] decodedTradePrivateKey, byte[] decodedSecret, byte[] litecoinReceivingAccountInfo) { + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Validate trade private key + if (decodedTradePrivateKey == null || decodedTradePrivateKey.length != 32) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Validate secret + if (decodedSecret == null || decodedSecret.length != 32) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Validate receiving address + if (litecoinReceivingAccountInfo == null || litecoinReceivingAccountInfo.length != 20) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Make sure the receiving address isn't a QORT address, given that we can share the same field for both QORT and LTC + if (Crypto.isValidAddress(litecoinReceivingAccountInfo)) + if (Base58.encode(litecoinReceivingAccountInfo).startsWith("Q")) + // This is likely a QORT address, not an LTC + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + + // Use secret-A to redeem P2SH-A + + Litecoin litecoin = Litecoin.getInstance(); + + int lockTime = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTime, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + LOGGER.info(String.format("Redeeming P2SH address: %s", p2shAddressA)); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return false; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + return false; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return false; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(decodedTradePrivateKey); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, decodedSecret, litecoinReceivingAccountInfo); + + litecoin.broadcastTransaction(p2shRedeemTransaction); + return true; // TODO: validate? + } + } + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); + } + + return false; + } + + @GET + @Path("/refund/LITECOIN/{ataddress}") + @Operation( + summary = "Refunds HTLC associated with supplied AT", + description = "To be used by a QORT buyer (Alice) who needs to refund their LTC that is stuck in a P2SH.
    " + + "This requires Alice's trade bot data to be present in the database for this AT.
    " + + "It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + public boolean refundHtlc(@PathParam("ataddress") String atAddress) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); + if (tradeBotData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (tradeBotData.getForeignKey() == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Determine LTC receive address for refund + Litecoin litecoin = Litecoin.getInstance(); + String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + + return this.doRefundHtlc(atAddress, receiveAddress); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); + } + } + + @GET + @Path("/refund/LITECOIN/{ataddress}/{receivingAddress}") + @Operation( + summary = "Refunds HTLC associated with supplied AT, to the specified LTC receiving address", + description = "To be used by a QORT buyer (Alice) who needs to refund their LTC that is stuck in a P2SH.
    " + + "This requires Alice's trade bot data to be present in the database for this AT.
    " + + "It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean")) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN}) + public boolean refundHtlc(@PathParam("ataddress") String atAddress, + @PathParam("receivingAddress") String receivingAddress) { + Security.checkApiCallAllowed(request); + return this.doRefundHtlc(atAddress, receivingAddress); + } + + + private boolean doRefundHtlc(String atAddress, String receiveAddress) { + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null); + if (tradeBotData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + + int lockTime = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTime * 1000L) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); + + Litecoin litecoin = Litecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = litecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTime) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTime, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + LOGGER.info(String.format("Refunding P2SH address: %s", p2shAddressA)); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON); + + case REDEEM_IN_PROGRESS: + case REDEEMED: + case REFUND_IN_PROGRESS: + case REFUNDED: + // Too late! + return false; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + // Validate the destination LTC address + Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); + if (receiving.getOutputScriptType() != Script.ScriptType.P2PKH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTime, receiving.getHash()); + + litecoin.broadcastTransaction(p2shRefundTransaction); + return true; // TODO: validate? + } + } + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, e); + } + + return false; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java new file mode 100644 index 00000000..04923133 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java @@ -0,0 +1,145 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.CrossChainSecretRequest; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.MessageTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import java.util.Arrays; +import java.util.Random; + +@Path("/crosschain/LitecoinACCTv1") +@Tag(name = "Cross-Chain (LitecoinACCTv1)") +public class CrossChainLitecoinACCTv1Resource { + + @Context + HttpServletRequest request; + + @POST + @Path("/redeemmessage") + @Operation( + summary = "Signs and broadcasts a 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner", + description = "Specify address of cross-chain AT that needs to be messaged, Alice's trade private key, the 32-byte secret,
    " + + "and an address for receiving QORT from AT. All of these can be found in Alice's trade bot data.
    " + + "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!
    " + + "You need to use the private key that the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainSecretRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public boolean buildRedeemMessage(CrossChainSecretRequest secretRequest) { + Security.checkApiCallAllowed(request); + + byte[] partnerPrivateKey = secretRequest.partnerPrivateKey; + + if (partnerPrivateKey == null || partnerPrivateKey.length != Transformer.PRIVATE_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (secretRequest.secret == null || secretRequest.secret.length != LitecoinACCTv1.SECRET_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress); + CrossChainTradeData crossChainTradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.TRADING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + byte[] partnerPublicKey = new PrivateKeyAccount(null, partnerPrivateKey).getPublicKey(); + String partnerAddress = Crypto.toAddress(partnerPublicKey); + + // MESSAGE must come from address that AT considers trade partner + if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + // Good to make MESSAGE + + byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secretRequest.secret, secretRequest.receivingAddress); + + PrivateKeyAccount sender = new PrivateKeyAccount(repository, partnerPrivateKey); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, secretRequest.atAddress, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID); + + return true; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // Must be correct AT - check functionality using code hash + if (!Arrays.equals(atData.getCodeHash(), LitecoinACCTv1.CODE_BYTES_HASH)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + +} diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java new file mode 100644 index 00000000..8883f964 --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -0,0 +1,167 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.List; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.bitcoinj.core.Transaction; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.crosschain.LitecoinSendRequest; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.SimpleTransaction; + +@Path("/crosschain/ltc") +@Tag(name = "Cross-Chain (Litecoin)") +public class CrossChainLitecoinResource { + + @Context + HttpServletRequest request; + + @POST + @Path("/walletbalance") + @Operation( + summary = "Returns LTC balance for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + public String getLitecoinWalletBalance(String key58) { + Security.checkApiCallAllowed(request); + + Litecoin litecoin = Litecoin.getInstance(); + + if (!litecoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + Long balance = litecoin.getWalletBalance(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + + return balance.toString(); + } + + @POST + @Path("/wallettransactions") + @Operation( + summary = "Returns transactions for hierarchical, deterministic BIP32 wallet", + description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + description = "BIP32 'm' private/public key in base58", + example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + public List getLitecoinWalletTransactions(String key58) { + Security.checkApiCallAllowed(request); + + Litecoin litecoin = Litecoin.getInstance(); + + if (!litecoin.isValidDeterministicKey(key58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + try { + return litecoin.getWalletTransactions(key58); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + } + + @POST + @Path("/send") + @Operation( + summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address", + description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = LitecoinSendRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE}) + public String sendBitcoin(LitecoinSendRequest litecoinSendRequest) { + Security.checkApiCallAllowed(request); + + if (litecoinSendRequest.litecoinAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (litecoinSendRequest.feePerByte != null && litecoinSendRequest.feePerByte <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + Litecoin litecoin = Litecoin.getInstance(); + + if (!litecoin.isValidAddress(litecoinSendRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!litecoin.isValidDeterministicKey(litecoinSendRequest.xprv58)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + Transaction spendTransaction = litecoin.buildSpend(litecoinSendRequest.xprv58, + litecoinSendRequest.receivingAddress, + litecoinSendRequest.litecoinAmount, + litecoinSendRequest.feePerByte); + + if (spendTransaction == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); + + try { + litecoin.broadcastTransaction(spendTransaction); + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } + + return spendTransaction.getTxId().toString(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java new file mode 100644 index 00000000..fdd74b7d --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -0,0 +1,424 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.Supplier; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.CrossChainCancelRequest; +import org.qortal.api.model.CrossChainTradeSummary; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.MessageTransactionTransformer; +import org.qortal.utils.Amounts; +import org.qortal.utils.Base58; +import org.qortal.utils.ByteArray; +import org.qortal.utils.NTP; + +@Path("/crosschain") +@Tag(name = "Cross-Chain") +public class CrossChainResource { + + @Context + HttpServletRequest request; + + @GET + @Path("/tradeoffers") + @Operation( + summary = "Find cross-chain trade offers", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = CrossChainTradeData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public List getTradeOffers( + @Parameter( + description = "Limit to specific blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain, + @Parameter( ref = "limit") @QueryParam("limit") Integer limit, + @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, + @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { + // Impose a limit on 'limit' + if (limit != null && limit > 100) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + final boolean isExecutable = true; + List crossChainTradesData = new ArrayList<>(); + + try (final Repository repository = RepositoryManager.getRepository()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + List atsData = repository.getATRepository().getATsByFunctionality(codeHash, isExecutable, limit, offset, reverse); + + for (ATData atData : atsData) { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + crossChainTradesData.add(crossChainTradeData); + } + } + + return crossChainTradesData; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/trade/{ataddress}") + @Operation( + summary = "Show detailed trade info", + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + implementation = CrossChainTradeData.class + ) + ) + ) + } + ) + @ApiErrors({ApiError.ADDRESS_UNKNOWN, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public CrossChainTradeData getTrade(@PathParam("ataddress") String atAddress) { + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return acct.populateTradeData(repository, atData); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/trades") + @Operation( + summary = "Find completed cross-chain trades", + description = "Returns summary info about successfully completed cross-chain trades", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = CrossChainTradeSummary.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public List getCompletedTrades( + @Parameter( + description = "Limit to specific blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain, + @Parameter( + description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)", + example = "1597310000000" + ) @QueryParam("minimumTimestamp") Long minimumTimestamp, + @Parameter( ref = "limit") @QueryParam("limit") Integer limit, + @Parameter( ref = "offset" ) @QueryParam("offset") Integer offset, + @Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) { + // Impose a limit on 'limit' + if (limit != null && limit > 100) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // minimumTimestamp (if given) needs to be positive + if (minimumTimestamp != null && minimumTimestamp <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + final Boolean isFinished = Boolean.TRUE; + + try (final Repository repository = RepositoryManager.getRepository()) { + Integer minimumFinalHeight = null; + + if (minimumTimestamp != null) { + minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp); + + if (minimumFinalHeight == 0) + // We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return + return Collections.emptyList(); + + // height returned from repository is for block BEFORE timestamp + // but we want trades AFTER timestamp so bump height accordingly + minimumFinalHeight++; + } + + List crossChainTrades = new ArrayList<>(); + + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight, + limit, offset, reverse); + + for (ATStateData atState : atStates) { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); + + // We also need block timestamp for use as trade timestamp + long timestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); + + CrossChainTradeSummary crossChainTradeSummary = new CrossChainTradeSummary(crossChainTradeData, timestamp); + crossChainTrades.add(crossChainTradeSummary); + } + } + + return crossChainTrades; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @GET + @Path("/price/{blockchain}") + @Operation( + summary = "Request current estimated trading price", + description = "Returns price based on most recent completed trades. Price is expressed in terms of QORT per unit foreign currency.", + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "number" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public long getTradePriceEstimate( + @Parameter( + description = "foreign blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @PathParam("blockchain") SupportedBlockchain foreignBlockchain, + @Parameter( + description = "Maximum number of trades to include in price calculation", + example = "10", + schema = @Schema(type = "integer", defaultValue = "10") + ) @QueryParam("maxtrades") Integer maxtrades) { + // foreignBlockchain is required + if (foreignBlockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // We want both a minimum of 5 trades and enough trades to span at least 4 hours + int minimumCount = 5; + int maximumCount = maxtrades != null ? maxtrades : 10; + long minimumPeriod = 4 * 60 * 60 * 1000L; // ms + Boolean isFinished = Boolean.TRUE; + + try (final Repository repository = RepositoryManager.getRepository()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain); + + long totalForeign = 0; + long totalQort = 0; + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + List atStates = repository.getATRepository().getMatchingFinalATStatesQuorum(codeHash, + isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, maximumCount, minimumPeriod); + + for (ATStateData atState : atStates) { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); + totalForeign += crossChainTradeData.expectedForeignAmount; + totalQort += crossChainTradeData.qortAmount; + } + } + + return Amounts.scaledDivide(totalQort, totalForeign); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @DELETE + @Path("/tradeoffer") + @Operation( + summary = "Builds raw, unsigned 'cancel' MESSAGE transaction that cancels cross-chain trade offer", + description = "Specify address of cross-chain AT that needs to be cancelled.
    " + + "AT needs to be in 'offer' mode. Messages sent to an AT in 'trade' mode will be ignored.
    " + + "Performs MESSAGE proof-of-work.
    " + + "You need to sign output with AT creator's private key otherwise the MESSAGE transaction will be invalid.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = CrossChainCancelRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content( + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + @SecurityRequirement(name = "apiKey") + public String cancelTrade(CrossChainCancelRequest cancelRequest) { + Security.checkApiCallAllowed(request); + + byte[] creatorPublicKey = cancelRequest.creatorPublicKey; + + if (creatorPublicKey == null || creatorPublicKey.length != Transformer.PUBLIC_KEY_LENGTH) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + if (cancelRequest.atAddress == null || !Crypto.isValidAtAddress(cancelRequest.atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, cancelRequest.atAddress); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + + if (crossChainTradeData.mode != AcctMode.OFFERING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + // Does supplied public key match AT creator's public key? + if (!Arrays.equals(creatorPublicKey, atData.getCreatorPublicKey())) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY); + + // Good to make MESSAGE + + String atCreatorAddress = Crypto.toAddress(creatorPublicKey); + byte[] messageData = acct.buildCancelMessage(atCreatorAddress); + + byte[] messageTransactionBytes = buildAtMessage(repository, creatorPublicKey, cancelRequest.atAddress, messageData); + + return Base58.encode(messageTransactionBytes); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + + private byte[] buildAtMessage(Repository repository, byte[] senderPublicKey, String atAddress, byte[] messageData) throws DataException { + long txTimestamp = NTP.getTime(); + + // senderPublicKey could be ephemeral trade public key where there is no corresponding account and hence no reference + String senderAddress = Crypto.toAddress(senderPublicKey); + byte[] lastReference = repository.getAccountRepository().getLastReference(senderAddress); + final boolean requiresPoW = lastReference == null; + + if (requiresPoW) { + Random random = new Random(); + lastReference = new byte[Transformer.SIGNATURE_LENGTH]; + random.nextBytes(lastReference); + } + + int version = 4; + int nonce = 0; + long amount = 0L; + Long assetId = null; // no assetId as amount is zero + Long fee = 0L; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, senderPublicKey, fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, atAddress, amount, assetId, messageData, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + if (requiresPoW) { + messageTransaction.computeNonce(); + } else { + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + } + + ValidationResult result = messageTransaction.isValidUnconfirmed(); + if (result != ValidationResult.OK) + throw TransactionsResource.createTransactionInvalidException(request, result); + + try { + return MessageTransactionTransformer.toBytes(messageTransactionData); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java b/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java new file mode 100644 index 00000000..cd8766ca --- /dev/null +++ b/src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java @@ -0,0 +1,286 @@ +package org.qortal.api.resource; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; + +import java.util.List; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; + +import org.qortal.account.Account; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.ApiError; +import org.qortal.api.ApiErrors; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.Security; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.api.model.crosschain.TradeBotRespondRequest; +import org.qortal.asset.Asset; +import org.qortal.controller.tradebot.AcctTradeBot; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.crosschain.ForeignBlockchain; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; + +@Path("/crosschain/tradebot") +@Tag(name = "Cross-Chain (Trade-Bot)") +public class CrossChainTradeBotResource { + + @Context + HttpServletRequest request; + + @GET + @Operation( + summary = "List current trade-bot states", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = TradeBotData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.REPOSITORY_ISSUE}) + public List getTradeBotStates( + @Parameter( + description = "Limit to specific blockchain", + example = "LITECOIN", + schema = @Schema(implementation = SupportedBlockchain.class) + ) @QueryParam("foreignBlockchain") SupportedBlockchain foreignBlockchain) { + Security.checkApiCallAllowed(request); + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + if (foreignBlockchain == null) + return allTradeBotData; + + return allTradeBotData.stream().filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain.name())).collect(Collectors.toList()); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/create") + @Operation( + summary = "Create a trade offer (trade-bot entry)", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = TradeBotCreateRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.INSUFFICIENT_BALANCE, ApiError.REPOSITORY_ISSUE}) + @SuppressWarnings("deprecation") + public String tradeBotCreator(TradeBotCreateRequest tradeBotCreateRequest) { + Security.checkApiCallAllowed(request); + + if (tradeBotCreateRequest.foreignBlockchain == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + ForeignBlockchain foreignBlockchain = tradeBotCreateRequest.foreignBlockchain.getInstance(); + + // We prefer foreignAmount to deprecated bitcoinAmount + if (tradeBotCreateRequest.foreignAmount == null) + tradeBotCreateRequest.foreignAmount = tradeBotCreateRequest.bitcoinAmount; + + if (!foreignBlockchain.isValidAddress(tradeBotCreateRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (tradeBotCreateRequest.tradeTimeout < 60) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (tradeBotCreateRequest.foreignAmount == null || tradeBotCreateRequest.foreignAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + if (tradeBotCreateRequest.qortAmount <= 0 || tradeBotCreateRequest.fundingQortAmount <= 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + try (final Repository repository = RepositoryManager.getRepository()) { + // Do some simple checking first + Account creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + if (creator.getConfirmedBalance(Asset.QORT) < tradeBotCreateRequest.fundingQortAmount) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INSUFFICIENT_BALANCE); + + byte[] unsignedBytes = TradeBot.getInstance().createTrade(repository, tradeBotCreateRequest); + if (unsignedBytes == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return Base58.encode(unsignedBytes); + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @POST + @Path("/respond") + @Operation( + summary = "Respond to a trade offer. NOTE: WILL SPEND FUNDS!)", + description = "Start a new trade-bot entry to respond to chosen trade offer.", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema( + implementation = TradeBotRespondRequest.class + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE, ApiError.REPOSITORY_ISSUE}) + @SuppressWarnings("deprecation") + public String tradeBotResponder(TradeBotRespondRequest tradeBotRespondRequest) { + Security.checkApiCallAllowed(request); + + final String atAddress = tradeBotRespondRequest.atAddress; + + // We prefer foreignKey to deprecated xprv58 + if (tradeBotRespondRequest.foreignKey == null) + tradeBotRespondRequest.foreignKey = tradeBotRespondRequest.xprv58; + + if (tradeBotRespondRequest.foreignKey == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + if (atAddress == null || !Crypto.isValidAtAddress(atAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (tradeBotRespondRequest.receivingAddress == null || !Crypto.isValidAddress(tradeBotRespondRequest.receivingAddress)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + // Extract data from cross-chain trading AT + try (final Repository repository = RepositoryManager.getRepository()) { + ATData atData = fetchAtDataWithChecking(repository, atAddress); + + // TradeBot uses AT's code hash to map to ACCT + ACCT acct = TradeBot.getInstance().getAcctUsingAtData(atData); + if (acct == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (!acct.getBlockchain().isValidWalletKey(tradeBotRespondRequest.foreignKey)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData); + if (crossChainTradeData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + if (crossChainTradeData.mode != AcctMode.OFFERING) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + AcctTradeBot.ResponseResult result = TradeBot.getInstance().startResponse(repository, atData, acct, crossChainTradeData, + tradeBotRespondRequest.foreignKey, tradeBotRespondRequest.receivingAddress); + + switch (result) { + case OK: + return "true"; + + case BALANCE_ISSUE: + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE); + + case NETWORK_ISSUE: + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + + default: + return "false"; + } + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + @DELETE + @Operation( + summary = "Delete completed trade", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string", + example = "93MB2qRDNVLxbmmPuYpLdAqn3u2x9ZhaVZK5wELHueP8" + ) + ) + ), + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string")) + ) + } + ) + @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE}) + public String tradeBotDelete(String tradePrivateKey58) { + Security.checkApiCallAllowed(request); + + final byte[] tradePrivateKey; + try { + tradePrivateKey = Base58.decode(tradePrivateKey58); + + if (tradePrivateKey.length != 32) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + } catch (NumberFormatException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + // Handed off to TradeBot + return TradeBot.getInstance().deleteEntry(repository, tradePrivateKey) ? "true" : "false"; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + + private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + if (atData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN); + + // No point sending message to AT that's finished + if (atData.getIsFinished()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + + return atData; + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java new file mode 100644 index 00000000..26d131c4 --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/PresenceWebSocket.java @@ -0,0 +1,244 @@ +package org.qortal.api.websocket; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.controller.Controller; +import org.qortal.crypto.Crypto; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +@WebSocket +@SuppressWarnings("serial") +public class PresenceWebSocket extends ApiWebSocket implements Listener { + + @XmlAccessorType(XmlAccessType.FIELD) + @SuppressWarnings("unused") + private static class PresenceInfo { + private final PresenceType presenceType; + private final String publicKey; + private final long timestamp; + private final String address; + + protected PresenceInfo() { + this.presenceType = null; + this.publicKey = null; + this.timestamp = 0L; + this.address = null; + } + + public PresenceInfo(PresenceType presenceType, String pubKey58, long timestamp) { + this.presenceType = presenceType; + this.publicKey = pubKey58; + this.timestamp = timestamp; + this.address = Crypto.toAddress(Base58.decode(this.publicKey)); + } + + public PresenceType getPresenceType() { + return this.presenceType; + } + + public String getPublicKey() { + return this.publicKey; + } + + public long getTimestamp() { + return this.timestamp; + } + + public String getAddress() { + return this.address; + } + } + + /** Outer map key is PresenceType (enum), inner map key is public key in base58, inner map value is timestamp */ + private static final Map> currentEntries = Collections.synchronizedMap(new EnumMap<>(PresenceType.class)); + + /** (Optional) PresenceType used for filtering by that Session. */ + private static final Map sessionPresenceTypes = Collections.synchronizedMap(new HashMap<>()); + + @Override + public void configure(WebSocketServletFactory factory) { + factory.register(PresenceWebSocket.class); + + try (final Repository repository = RepositoryManager.getRepository()) { + populateCurrentInfo(repository); + } catch (DataException e) { + // How to fail properly? + return; + } + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + // We use NewBlockEvent as a proxy for 1-minute timer + if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent)) + return; + + removeOldEntries(); + + if (event instanceof Controller.NewBlockEvent) + // We only wanted a chance to cull old entries + return; + + TransactionData transactionData = ((Controller.NewTransactionEvent) event).getTransactionData(); + + if (transactionData.getType() != TransactionType.PRESENCE) + return; + + PresenceTransactionData presenceData = (PresenceTransactionData) transactionData; + PresenceType presenceType = presenceData.getPresenceType(); + + // Put/replace for this publickey making sure we keep newest timestamp + String pubKey58 = Base58.encode(presenceData.getCreatorPublicKey()); + long ourTimestamp = presenceData.getTimestamp(); + long computedTimestamp = mergePresence(presenceType, pubKey58, ourTimestamp); + + if (computedTimestamp != ourTimestamp) + // nothing changed + return; + + List presenceInfo = Collections.singletonList(new PresenceInfo(presenceType, pubKey58, computedTimestamp)); + + // Notify sessions + for (Session session : getSessions()) { + PresenceType sessionPresenceType = sessionPresenceTypes.get(session); + + if (sessionPresenceType == null || sessionPresenceType == presenceType) + sendPresenceInfo(session, presenceInfo); + } + } + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + Map> queryParams = session.getUpgradeRequest().getParameterMap(); + List presenceTypes = queryParams.get("presenceType"); + + // We only support ONE presenceType + String presenceTypeName = presenceTypes == null || presenceTypes.isEmpty() ? null : presenceTypes.get(0); + + PresenceType presenceType = presenceTypeName == null ? null : PresenceType.fromString(presenceTypeName); + + // Make sure that if caller does give a presenceType, that it is a valid/known one. + if (presenceTypeName != null && presenceType == null) { + session.close(4003, "unknown presenceType: " + presenceTypeName); + return; + } + + // Save session's requested PresenceType, if given + if (presenceType != null) + sessionPresenceTypes.put(session, presenceType); + + List presenceInfo; + + synchronized (currentEntries) { + presenceInfo = currentEntries.entrySet().stream() + .filter(entry -> presenceType == null ? true : entry.getKey() == presenceType) + .flatMap(entry -> entry.getValue().entrySet().stream().map(innerEntry -> new PresenceInfo(entry.getKey(), innerEntry.getKey(), innerEntry.getValue()))) + .collect(Collectors.toList()); + } + + if (!sendPresenceInfo(session, presenceInfo)) { + session.close(4002, "websocket issue"); + return; + } + + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + // clean up + sessionPresenceTypes.remove(session); + + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + /* ignored */ + } + + private boolean sendPresenceInfo(Session session, List presenceInfo) { + try { + StringWriter stringWriter = new StringWriter(); + marshall(stringWriter, presenceInfo); + + String output = stringWriter.toString(); + session.getRemote().sendStringByFuture(output); + } catch (IOException e) { + // No output this time? + return false; + } + + return true; + } + + private static void populateCurrentInfo(Repository repository) throws DataException { + // We want ALL PRESENCE transactions + + List presenceTransactionsData = repository.getTransactionRepository().getUnconfirmedTransactions(TransactionType.PRESENCE, null); + + for (TransactionData transactionData : presenceTransactionsData) { + PresenceTransactionData presenceData = (PresenceTransactionData) transactionData; + + PresenceType presenceType = presenceData.getPresenceType(); + + // Put/replace for this publickey making sure we keep newest timestamp + String pubKey58 = Base58.encode(presenceData.getCreatorPublicKey()); + long ourTimestamp = presenceData.getTimestamp(); + + mergePresence(presenceType, pubKey58, ourTimestamp); + } + } + + private static long mergePresence(PresenceType presenceType, String pubKey58, long ourTimestamp) { + Map typedPubkeyTimestamps = currentEntries.computeIfAbsent(presenceType, someType -> Collections.synchronizedMap(new HashMap<>())); + return typedPubkeyTimestamps.compute(pubKey58, (somePubKey58, currentTimestamp) -> (currentTimestamp == null || currentTimestamp < ourTimestamp) ? ourTimestamp : currentTimestamp); + } + + private static void removeOldEntries() { + long now = NTP.getTime(); + + currentEntries.entrySet().forEach(entry -> { + long expiryThreshold = now - entry.getKey().getLifetime(); + entry.getValue().entrySet().removeIf(pubkeyTimestamp -> pubkeyTimestamp.getValue() < expiryThreshold); + }); + } + +} diff --git a/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java new file mode 100644 index 00000000..55969c6b --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/TradeBotWebSocket.java @@ -0,0 +1,157 @@ +package org.qortal.api.websocket; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; + +@WebSocket +@SuppressWarnings("serial") +public class TradeBotWebSocket extends ApiWebSocket implements Listener { + + /** Cache of trade-bot entry states, keyed by trade-bot entry's "trade private key" (base58) */ + private static final Map PREVIOUS_STATES = new HashMap<>(); + + private static final Map sessionBlockchain = Collections.synchronizedMap(new HashMap<>()); + + @Override + public void configure(WebSocketServletFactory factory) { + factory.register(TradeBotWebSocket.class); + + try (final Repository repository = RepositoryManager.getRepository()) { + List tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); + if (tradeBotEntries == null) + // How do we properly fail here? + return; + + PREVIOUS_STATES.putAll(tradeBotEntries.stream().collect(Collectors.toMap(entry -> Base58.encode(entry.getTradePrivateKey()), TradeBotData::getStateValue))); + } catch (DataException e) { + // No output this time + } + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + if (!(event instanceof TradeBot.StateChangeEvent)) + return; + + TradeBotData tradeBotData = ((TradeBot.StateChangeEvent) event).getTradeBotData(); + String tradePrivateKey58 = Base58.encode(tradeBotData.getTradePrivateKey()); + + synchronized (PREVIOUS_STATES) { + Integer previousStateValue = PREVIOUS_STATES.get(tradePrivateKey58); + if (previousStateValue != null && previousStateValue == tradeBotData.getStateValue()) + // Not changed + return; + + PREVIOUS_STATES.put(tradePrivateKey58, tradeBotData.getStateValue()); + } + + List tradeBotEntries = Collections.singletonList(tradeBotData); + + for (Session session : getSessions()) { + // Only send if this session has this/no preferred blockchain + String preferredBlockchain = sessionBlockchain.get(session); + + if (preferredBlockchain == null || preferredBlockchain.equals(tradeBotData.getForeignBlockchain())) + sendEntries(session, tradeBotEntries); + } + } + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + Map> queryParams = session.getUpgradeRequest().getParameterMap(); + + List foreignBlockchains = queryParams.get("foreignBlockchain"); + final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); + + // Make sure blockchain (if any) is valid + if (foreignBlockchain != null && SupportedBlockchain.fromString(foreignBlockchain) == null) { + session.close(4003, "unknown blockchain: " + foreignBlockchain); + return; + } + + // save session's preferred blockchain (if any) + sessionBlockchain.put(session, foreignBlockchain); + + // Send all known trade-bot entries + try (final Repository repository = RepositoryManager.getRepository()) { + List tradeBotEntries = repository.getCrossChainRepository().getAllTradeBotData(); + + // Optional filtering + if (foreignBlockchain != null) + tradeBotEntries = tradeBotEntries.stream() + .filter(tradeBotData -> tradeBotData.getForeignBlockchain().equals(foreignBlockchain)) + .collect(Collectors.toList()); + + if (!sendEntries(session, tradeBotEntries)) { + session.close(4002, "websocket issue"); + return; + } + } catch (DataException e) { + session.close(4001, "repository issue fetching trade-bot entries"); + return; + } + + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + // clean up + sessionBlockchain.remove(session); + + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + /* ignored */ + } + + private boolean sendEntries(Session session, List tradeBotEntries) { + try { + StringWriter stringWriter = new StringWriter(); + marshall(stringWriter, tradeBotEntries); + + String output = stringWriter.toString(); + session.getRemote().sendStringByFuture(output); + } catch (IOException e) { + // No output this time? + return false; + } + + return true; + } + +} diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java new file mode 100644 index 00000000..186f79e3 --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java @@ -0,0 +1,351 @@ +package org.qortal.api.websocket; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.api.model.CrossChainOfferSummary; +import org.qortal.controller.Controller; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.ByteArray; +import org.qortal.utils.NTP; + +@WebSocket +@SuppressWarnings("serial") +public class TradeOffersWebSocket extends ApiWebSocket implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(TradeOffersWebSocket.class); + + private static class CachedOfferInfo { + public final Map previousAtModes = new HashMap<>(); + + // OFFERING + public final Map currentSummaries = new HashMap<>(); + // REDEEMED/REFUNDED/CANCELLED + public final Map historicSummaries = new HashMap<>(); + } + // Manual synchronization + private static final Map cachedInfoByBlockchain = new HashMap<>(); + + private static final Predicate isHistoric = offerSummary + -> offerSummary.getMode() == AcctMode.REDEEMED + || offerSummary.getMode() == AcctMode.REFUNDED + || offerSummary.getMode() == AcctMode.CANCELLED; + + private static final Map sessionBlockchain = Collections.synchronizedMap(new HashMap<>()); + + @Override + public void configure(WebSocketServletFactory factory) { + factory.register(TradeOffersWebSocket.class); + + try (final Repository repository = RepositoryManager.getRepository()) { + populateCurrentSummaries(repository); + + populateHistoricSummaries(repository); + } catch (DataException e) { + // How to fail properly? + return; + } + + EventBus.INSTANCE.addListener(this::listen); + } + + @Override + public void listen(Event event) { + if (!(event instanceof Controller.NewBlockEvent)) + return; + + BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData(); + + // Process any new info + + try (final Repository repository = RepositoryManager.getRepository()) { + // Find any new/changed trade ATs since this block + final Boolean isFinished = null; + final Integer dataByteOffset = null; + final Long expectedValue = null; + final Integer minimumFinalHeight = blockData.getHeight(); + + for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); + + List crossChainOfferSummaries = new ArrayList<>(); + + synchronized (cachedInfoByBlockchain) { + CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + List atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, dataByteOffset, expectedValue, minimumFinalHeight, + null, null, null); + + crossChainOfferSummaries.addAll(produceSummaries(repository, acct, atStates, blockData.getTimestamp())); + } + + // Remove any entries unchanged from last time + crossChainOfferSummaries.removeIf(offerSummary -> cachedInfo.previousAtModes.get(offerSummary.getQortalAtAddress()) == offerSummary.getMode()); + + // Skip to next blockchain if nothing has changed (for this blockchain) + if (crossChainOfferSummaries.isEmpty()) + continue; + + // Update + for (CrossChainOfferSummary offerSummary : crossChainOfferSummaries) { + String offerAtAddress = offerSummary.getQortalAtAddress(); + + cachedInfo.previousAtModes.put(offerAtAddress, offerSummary.getMode()); + LOGGER.trace(() -> String.format("Block height: %d, AT: %s, mode: %s", blockData.getHeight(), offerAtAddress, offerSummary.getMode().name())); + + switch (offerSummary.getMode()) { + case OFFERING: + cachedInfo.currentSummaries.put(offerAtAddress, offerSummary); + cachedInfo.historicSummaries.remove(offerAtAddress); + break; + + case REDEEMED: + case REFUNDED: + case CANCELLED: + cachedInfo.currentSummaries.remove(offerAtAddress); + cachedInfo.historicSummaries.put(offerAtAddress, offerSummary); + break; + + case TRADING: + cachedInfo.currentSummaries.remove(offerAtAddress); + cachedInfo.historicSummaries.remove(offerAtAddress); + break; + } + } + + // Remove any historic offers that are over 24 hours old + final long tooOldTimestamp = NTP.getTime() - 24 * 60 * 60 * 1000L; + cachedInfo.historicSummaries.values().removeIf(historicSummary -> historicSummary.getTimestamp() < tooOldTimestamp); + } + + // Notify sessions + for (Session session : getSessions()) { + // Only send if this session has this/no preferred blockchain + String preferredBlockchain = sessionBlockchain.get(session); + + if (preferredBlockchain == null || preferredBlockchain.equals(blockchain.name())) + sendOfferSummaries(session, crossChainOfferSummaries); + } + + } + } catch (DataException e) { + // No output this time + } + } + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + Map> queryParams = session.getUpgradeRequest().getParameterMap(); + final boolean includeHistoric = queryParams.get("includeHistoric") != null; + + List foreignBlockchains = queryParams.get("foreignBlockchain"); + final String foreignBlockchain = foreignBlockchains == null ? null : foreignBlockchains.get(0); + + // Make sure blockchain (if any) is valid + if (foreignBlockchain != null && SupportedBlockchain.fromString(foreignBlockchain) == null) { + session.close(4003, "unknown blockchain: " + foreignBlockchain); + return; + } + + // Save session's preferred blockchain, if given + if (foreignBlockchain != null) + sessionBlockchain.put(session, foreignBlockchain); + + List crossChainOfferSummaries = new ArrayList<>(); + + synchronized (cachedInfoByBlockchain) { + Collection cachedInfos; + + if (foreignBlockchain == null) + // No preferred blockchain, so iterate through all of them + cachedInfos = cachedInfoByBlockchain.values(); + else + cachedInfos = Collections.singleton(cachedInfoByBlockchain.computeIfAbsent(foreignBlockchain, k -> new CachedOfferInfo())); + + for (CachedOfferInfo cachedInfo : cachedInfos) { + crossChainOfferSummaries.addAll(cachedInfo.currentSummaries.values()); + + if (includeHistoric) + crossChainOfferSummaries.addAll(cachedInfo.historicSummaries.values()); + } + } + + if (!sendOfferSummaries(session, crossChainOfferSummaries)) { + session.close(4002, "websocket issue"); + return; + } + + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + // clean up + sessionBlockchain.remove(session); + + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* ignored */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + /* ignored */ + } + + private boolean sendOfferSummaries(Session session, List crossChainOfferSummaries) { + try { + StringWriter stringWriter = new StringWriter(); + marshall(stringWriter, crossChainOfferSummaries); + + String output = stringWriter.toString(); + session.getRemote().sendStringByFuture(output); + } catch (IOException e) { + // No output this time? + return false; + } + + return true; + } + + private static void populateCurrentSummaries(Repository repository) throws DataException { + // We want ALL OFFERING trades + Boolean isFinished = Boolean.FALSE; + Long expectedValue = (long) AcctMode.OFFERING.value; + Integer minimumFinalHeight = null; + + for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); + + CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + Integer dataByteOffset = acct.getModeByteOffset(); + List initialAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, dataByteOffset, expectedValue, minimumFinalHeight, + null, null, null); + + if (initialAtStates == null) + throw new DataException("Couldn't fetch current trades from repository"); + + // Save initial AT modes + cachedInfo.previousAtModes.putAll(initialAtStates.stream().collect(Collectors.toMap(ATStateData::getATAddress, atState -> AcctMode.OFFERING))); + + // Convert to offer summaries + cachedInfo.currentSummaries.putAll(produceSummaries(repository, acct, initialAtStates, null).stream() + .collect(Collectors.toMap(CrossChainOfferSummary::getQortalAtAddress, offerSummary -> offerSummary))); + } + } + } + + private static void populateHistoricSummaries(Repository repository) throws DataException { + // We want REDEEMED/REFUNDED/CANCELLED trades over the last 24 hours + long timestamp = System.currentTimeMillis() - 24 * 60 * 60 * 1000L; + int minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(timestamp); + + if (minimumFinalHeight == 0) + throw new DataException("Couldn't fetch block timestamp from repository"); + + Boolean isFinished = Boolean.TRUE; + Integer dataByteOffset = null; + Long expectedValue = null; + ++minimumFinalHeight; // because height is just *before* timestamp + + for (SupportedBlockchain blockchain : SupportedBlockchain.values()) { + Map> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(blockchain); + + CachedOfferInfo cachedInfo = cachedInfoByBlockchain.computeIfAbsent(blockchain.name(), k -> new CachedOfferInfo()); + + for (Map.Entry> acctInfo : acctsByCodeHash.entrySet()) { + byte[] codeHash = acctInfo.getKey().value; + ACCT acct = acctInfo.getValue().get(); + + List historicAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, + isFinished, dataByteOffset, expectedValue, minimumFinalHeight, + null, null, null); + + if (historicAtStates == null) + throw new DataException("Couldn't fetch historic trades from repository"); + + for (ATStateData historicAtState : historicAtStates) { + CrossChainOfferSummary historicOfferSummary = produceSummary(repository, acct, historicAtState, null); + + if (!isHistoric.test(historicOfferSummary)) + continue; + + // Add summary to initial burst + cachedInfo.historicSummaries.put(historicOfferSummary.getQortalAtAddress(), historicOfferSummary); + + // Save initial AT mode + cachedInfo.previousAtModes.put(historicOfferSummary.getQortalAtAddress(), historicOfferSummary.getMode()); + } + } + } + } + + private static CrossChainOfferSummary produceSummary(Repository repository, ACCT acct, ATStateData atState, Long timestamp) throws DataException { + CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); + + long atStateTimestamp; + + if (crossChainTradeData.mode == AcctMode.OFFERING) + // We want when trade was created, not when it was last updated + atStateTimestamp = crossChainTradeData.creationTimestamp; + else + atStateTimestamp = timestamp != null ? timestamp : repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); + + return new CrossChainOfferSummary(crossChainTradeData, atStateTimestamp); + } + + private static List produceSummaries(Repository repository, ACCT acct, List atStates, Long timestamp) throws DataException { + List offerSummaries = new ArrayList<>(); + + for (ATStateData atState : atStates) + offerSummaries.add(produceSummary(repository, acct, atState, timestamp)); + + return offerSummaries; + } + +} diff --git a/src/main/java/org/qortal/at/QortalFunctionCode.java b/src/main/java/org/qortal/at/QortalFunctionCode.java index 8f989e19..0d11e488 100644 --- a/src/main/java/org/qortal/at/QortalFunctionCode.java +++ b/src/main/java/org/qortal/at/QortalFunctionCode.java @@ -10,8 +10,10 @@ import org.ciyam.at.ExecutionException; import org.ciyam.at.FunctionData; import org.ciyam.at.IllegalFunctionCodeException; import org.ciyam.at.MachineState; +import org.qortal.crosschain.Bitcoin; import org.qortal.crypto.Crypto; import org.qortal.data.transaction.TransactionData; +import org.qortal.settings.Settings; /** * Qortal-specific CIYAM-AT Functions. @@ -98,6 +100,19 @@ public enum QortalFunctionCode { setB(state, pkh); } }, + /** + * Convert 20-byte value in LSB of B1, and all of B2 & B3 to P2SH.
    + * 0x0511
    + * P2SH stored in lower 25 bytes of B. + */ + CONVERT_B_TO_P2SH(0x0511, 0, false) { + @Override + protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException { + byte addressPrefix = Settings.getInstance().getBitcoinNet() == Bitcoin.BitcoinNet.MAIN ? 0x05 : (byte) 0xc4; + + convertAddressInB(addressPrefix, state); + } + }, /** * Convert 20-byte value in LSB of B1, and all of B2 & B3 to Qortal address.
    * 0x0512
    diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index df6babd1..3d2bd48e 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -14,6 +14,8 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.crypto.Crypto; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; @@ -441,6 +443,9 @@ public class Controller extends Thread { blockMinter = new BlockMinter(); blockMinter.start(); + LOGGER.info("Starting trade-bot"); + TradeBot.getInstance(); + // Arbitrary transaction data manager LOGGER.info("Starting arbitrary-transaction data manager"); ArbitraryDataManager.getInstance().start(); diff --git a/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java b/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java new file mode 100644 index 00000000..84a0d484 --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java @@ -0,0 +1,30 @@ +package org.qortal.controller.tradebot; + +import java.util.List; + +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +public interface AcctTradeBot { + + public enum ResponseResult { OK, BALANCE_ISSUE, NETWORK_ISSUE, TRADE_ALREADY_EXISTS } + + /** Returns list of state names for trade-bot entries that have ended, e.g. redeemed, refunded or cancelled. */ + public List getEndStates(); + + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException; + + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, + CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException; + + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException; + + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException; + +} diff --git a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java new file mode 100644 index 00000000..ca2e2518 --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java @@ -0,0 +1,1273 @@ +package org.qortal.controller.tradebot; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +/** + * Performing cross-chain trading steps on behalf of user. + *

    + * We deal with three different independent state-spaces here: + *

      + *
    • Qortal blockchain
      • + *
    • Foreign blockchain
      • + *
    • Trade-bot entries
      • + *
    + */ +public class BitcoinACCTv1TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(BitcoinACCTv1TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_P2SH_B(20, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_P2SH_A(80, true, true), + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_WATCH_P2SH_B(90, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_B(100, true, true), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + /** P2SH-B output amount to avoid dust threshold (3000 sats/kB). */ + private static final long P2SH_B_OUTPUT_AMOUNT = 1000L; + + private static BitcoinACCTv1TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDING_B, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private BitcoinACCTv1TradeBot() { + } + + public static synchronized BitcoinACCTv1TradeBot getInstance() { + if (instance == null) + instance = new BitcoinACCTv1TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for BTC. + *

    + * Generates: + *

      + *
    • new 'trade' private key
      • + *
    • secret-B
      • + *
    + * Derives: + *
      + *
    • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
      • + *
    • 'foreign' (as in Bitcoin) public key, public key hash
      • + *
    • HASH160 of secret-B
      • + *
    + * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
      + *
    • 'native'/Qortal 'trade' address - used as a MESSAGE contact
      • + *
    • 'foreign'/Bitcoin public key hash - used by Alice's P2SH scripts to allow redeem
      • + *
    • HASH160 of secret-B - used by AT and P2SH to validate a potential secret-B
      • + *
    • QORT amount on offer by Bob
      • + *
    • BTC amount expected in return by Bob (from Alice)
      • + *
    • trading timeout, in case things go wrong and everyone needs to refund
      • + *
    + * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

    + * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

    + * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretB = TradeBot.generateSecret(); + byte[] hashOfSecretB = Crypto.hash160(secretB); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Bitcoin receiving address into public key hash (we only support P2PKH at this time) + Address bitcoinReceivingAddress; + try { + bitcoinReceivingAddress = Address.fromString(Bitcoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Bitcoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (bitcoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Bitcoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] bitcoinReceivingAccountInfo = bitcoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/BTC ACCT"; + String description = "QORT/BTC cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT BTC"; + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, hashOfSecretB, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, BitcoinACCTv1.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretB, hashOfSecretB, + SupportedBlockchain.BITCOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, bitcoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching BTC to an existing offer. + *

    + * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Bitcoin wallet via xprv58. + *

    + * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

    + * Access to a funded wallet is via a Bitcoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

    + * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
    + * wallet.keystore.xprv
    + * which should result in a base58 string starting with either 'xprv' (for Bitcoin main-net) + * or 'tprv' for (Bitcoin test-net). + *

    + * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

    + * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Bitcoin amount expected by 'Bob'. + *

    + * If the Bitcoin transaction is successfully broadcast to the network then the trade-bot entry + * is saved to the repository and the cross-chain trading process commences. + *

    + * Trade-bot will wait for P2SH-A to confirm before taking next step. + *

    + * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Bitcoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, BitcoinACCTv1.NAME, + State.ALICE_WAITING_FOR_P2SH_A.name(), State.ALICE_WAITING_FOR_P2SH_A.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.BITCOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Check we have enough funds via xprv58 to fund both P2SHs to cover expectedBitcoin + String tradeForeignAddress = Bitcoin.getInstance().pkhToAddress(tradeForeignPublicKeyHash); + + long p2shFee; + try { + p2shFee = Bitcoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Bitcoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long fundsRequiredForP2shA = p2shFee /*funding P2SH-A*/ + crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-A*/; + long fundsRequiredForP2shB = p2shFee /*funding P2SH-B*/ + P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-B*/; + long totalFundsRequired = fundsRequiredForP2shA + fundsRequiredForP2shB; + + // As buildSpend also adds a fee, this is more pessimistic than required + Transaction fundingCheckTransaction = Bitcoin.getInstance().buildSpend(xprv58, tradeForeignAddress, totalFundsRequired); + if (fundingCheckTransaction == null) + return ResponseResult.BALANCE_ISSUE; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Bitcoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Fund P2SH-A + + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFee /*redeeming/refunding P2SH-A*/; + + Transaction p2shFundingTransaction = Bitcoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Bitcoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Waiting for confirmation", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case ALICE_WAITING_FOR_P2SH_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForP2shA(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_P2SH_B: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForP2shB(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WATCH_P2SH_B: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWatchingP2shB(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_B: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shB(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

    + * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for Alice's P2SH-A to confirm. + *

    + * If P2SH-A is confirmed, then trade-bot's next step is to MESSAGE Bob's trade address with Alice's trade info. + *

    + * It is possible between broadcast and confirmation of P2SH-A funding transaction, that Bob has cancelled his trade offer. + * If this is detected then trade-bot's next step is to wait until P2SH-A can refund back to Alice. + *

    + * In normal operation, trade-bot send a zero-fee, PoW MESSAGE on Alice's behalf containing: + *

      + *
    • Alice's 'foreign'/Bitcoin public key hash - so Bob's trade-bot can derive P2SH-A address and check balance
      • + *
    • HASH160 of Alice's secret-A - also used to derive P2SH-A address
      • + *
    • lockTime of P2SH-A - also used to derive P2SH-A address, but also for other use later in the trading process
      • + *
    + * If MESSAGE transaction is successfully broadcast, trade-bot's next step is to wait until Bob's AT has locked trade to Alice only. + * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getLockTimeA(), crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestampA = calcP2shAFeeTimestamp(tradeBotData.getLockTimeA(), crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively check P2SH-B in case we haven't redeemed the AT + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // P2SH-A funding confirmed + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = BitcoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WAITING_FOR_AT_LOCK, + () -> String.format("P2SH-A %s funding confirmed. Messaged %s. Waiting for AT %s to lock to us", + p2shAddressA, messageRecipient, tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

    + * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

    + * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

    + * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

    + * Assuming P2SH-A has at least expected Bitcoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

    + * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

    + * Trade-bot's next step is to wait for P2SH-B, which will allow Bob to reveal his secret-B, + * needed by Alice to progress her side of the trade. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Bitcoin bitcoin = Bitcoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + final byte[] originalLastTransactionSignature = tradeBotData.getLastTransactionSignature(); + + // Skip past previously processed messages + if (originalLastTransactionSignature != null) + for (int i = 0; i < messageTransactionsData.size(); ++i) + if (Arrays.equals(messageTransactionsData.get(i).getSignature(), originalLastTransactionSignature)) { + messageTransactionsData.subList(0, i + 1).clear(); + break; + } + + while (!messageTransactionsData.isEmpty()) { + MessageTransactionData messageTransactionData = messageTransactionsData.remove(0); + tradeBotData.setLastTransactionSignature(messageTransactionData.getSignature()); + + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Bitcoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + BitcoinACCTv1.OfferMessageData offerMessageData = BitcoinACCTv1.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerBitcoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + final long minimumAmountA = tradeBotData.getForeignAmount() - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_P2SH_B, + () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(messageTransactionData.getTimestamp(), lockTimeA); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = BitcoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeB, tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret()); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_P2SH_B, + () -> String.format("Locked AT %s to %s. Waiting for P2SH-B %s", tradeBotData.getAtAddress(), aliceNativeAddress, p2shAddressB)); + + return; + } + + // Don't resave/notify if we don't need to + if (tradeBotData.getLastTransactionSignature() != originalLastTransactionSignature) + TradeBot.updateTradeBotState(repository, tradeBotData, null); + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

    + * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

    + * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

    + * If all is well, trade-bot then uses Bitcoin wallet to (token) fund P2SH-B. + *

    + * If P2SH-B funding transaction is successfully broadcast to the Bitcoin network, trade-bot's next + * step is to watch for Bob revealing secret-B by redeeming P2SH-B. + * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= tradeBotData.getLockTimeA() * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // This shouldn't occur, but defensively revert back to waiting for P2SH-A + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WAITING_FOR_P2SH_A, + () -> String.format("P2SH-A %s no longer funded? Defensively checking P2SH-A next", p2shAddressA)); + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("P2SH-A %s already spent? Defensively checking P2SH-B next", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + case FUNDED: + // Fall-through out of switch... + break; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Alice needs to fund P2SH-B here + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(recipientMessageTimestamp, lockTimeA); + + // Our calculated lockTime-B should match AT's calculated lockTime-B + if (lockTimeB != crossChainTradeData.lockTimeB) { + LOGGER.debug(() -> String.format("Trade AT lockTime-B '%d' doesn't match our lockTime-B '%d'", crossChainTradeData.lockTimeB, lockTimeB)); + // We'll eventually refund + return; + } + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(lockTimeA, lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + + // Have we funded P2SH-B already? + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: { + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB /*redeeming/refunding P2SH-B*/; + + Transaction p2shFundingTransaction = bitcoin.buildSpend(tradeBotData.getForeignKey(), p2shAddressB, amountB); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-B funding transaction - lack of funds?"); + return; + } + + bitcoin.broadcastTransaction(p2shFundingTransaction); + break; + } + + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("P2SH-B %s already spent? Defensively checking P2SH-B next", p2shAddressB)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s already refunded. Refunding P2SH-A next", p2shAddressB)); + return; + } + + // P2SH-B funded, now we wait for Bob to redeem it + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_WATCH_P2SH_B, + () -> String.format("AT %s locked to us (%s). P2SH-B %s funded. Watching P2SH-B for secret-B", + tradeBotData.getAtAddress(), tradeBotData.getTradeNativeAddress(), p2shAddressB)); + } + + /** + * Trade-bot is waiting for P2SH-B to funded. + *

    + * It's possible than Bob's AT has reached it's trading timeout and automatically refunded QORT back to Bob. + * In which case, trade-bot is done with this specific trade and finalizes on refunded state. + *

    + * Assuming P2SH-B is funded, trade-bot 'redeems' this P2SH using secret-B, thus revealing it to Alice. + *

    + * Trade-bot's next step is to wait for Alice to use secret-B, and her secret-A, to redeem Bob's AT. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForP2shB(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If we've passed AT refund timestamp then AT will have finished after auto-refunding + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + // It's possible AT hasn't processed our previous MESSAGE yet and so lockTimeB won't be set + if (crossChainTradeData.lockTimeB == null) + // AT yet to process MESSAGE + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, crossChainTradeData.lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, crossChainTradeData.lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-B to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // This shouldn't occur, but defensively bump to next state + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("P2SH-B %s already spent (exposing secret-B)? Checking AT %s for secret-A", p2shAddressB, tradeBotData.getAtAddress())); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // AT should auto-refund - we don't need to do anything here + return; + + case FUNDED: + break; + } + + // Redeem P2SH-B using secret-B + Coin redeemAmount = Coin.valueOf(P2SH_B_OUTPUT_AMOUNT); // An actual amount to avoid dust filter, remaining used as fees. The real funds are in P2SH-A. + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressB); + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptB, tradeBotData.getSecret(), receivingAccountInfo); + + bitcoin.broadcastTransaction(p2shRedeemTransaction); + + // P2SH-B redeemed, now we wait for Alice to use secret-A to redeem AT + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("P2SH-B %s redeemed (exposing secret-B). Watching AT %s for secret-A", p2shAddressB, tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for Bob to redeem P2SH-B thus revealing secret-B to Alice. + *

    + * It's possible that this process has taken so long that we've reached P2SH-B's locktime. + * In which case, trade-bot switches to begin the refund process. + *

    + * If trade-bot can extract a valid secret-B from the spend of P2SH-B, then it creates a + * zero-fee, PoW MESSAGE to send to Bob's AT, including both secret-B and also Alice's secret-A. + *

    + * Both secrets are needed to release the QORT funds from Bob's AT to Alice's 'native'/Qortal + * trade address. + *

    + * In revealing a valid secret-A, Bob can then redeem the BTC funds from P2SH-A. + *

    + * If trade-bot successfully broadcasts the MESSAGE transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleAliceWatchingP2shB(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), crossChainTradeData.lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, crossChainTradeData.lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + case REDEEM_IN_PROGRESS: + // Still waiting for P2SH-B to be funded/redeemed... + return; + + case REDEEMED: + // Bob has redeemed P2SH-B, so double-check that we have redeemed AT... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // We've refunded P2SH-B? Bump to refunding P2SH-A then + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s already refunded. Refunding P2SH-A next", p2shAddressB)); + return; + } + + byte[] secretB = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddressB); + if (secretB == null) + // Secret not revealed at this time + return; + + // Send 'redeem' MESSAGE to AT using both secrets + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-B %s redeemed, using secrets to redeem AT %s. Funds should arrive at %s", + p2shAddressB, tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the BTC funds from P2SH-A. + *

    + * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

    + * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the BTC funds from P2SH-A + * to Bob's 'foreign'/Bitcoin trade legacy-format address, as derived from trade private key. + *

    + * (This could potentially be 'improved' to send BTC to any address of Bob's choosing by changing the transaction output). + *

    + * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is not REDEEMED then something has gone wrong + if (crossChainTradeData.mode != AcctMode.REDEEMED) { + // Not redeemed so must be refunded/cancelled + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = BitcoinACCTv1.findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Bitcoin bitcoin = Bitcoin.getInstance(); + int lockTimeA = crossChainTradeData.lockTimeA; + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + bitcoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = bitcoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-B. + *

    + * We could potentially skip this step as P2SH-B is only funded with a token amount to cover the mining fee should Bob redeem P2SH-B. + *

    + * Upon successful broadcast of P2SH-B refunding transaction, trade-bot's next step is to begin refunding of P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shB(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeB = crossChainTradeData.lockTimeB; + + // We can't refund P2SH-B until lockTime-B has passed + if (NTP.getTime() <= lockTimeB * 1000L) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + // We can't refund P2SH-B until median block time has passed lockTime-B (see BIP113) + int medianBlockTime = bitcoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeB) + return; + + byte[] redeemScriptB = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeB, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretB); + String p2shAddressB = bitcoin.deriveP2shAddress(redeemScriptB); + + long feeTimestampB = calcP2shBFeeTimestamp(crossChainTradeData.lockTimeA, lockTimeB); + long p2shFeeB = bitcoin.getP2shFee(feeTimestampB); + final long minimumAmountB = P2SH_B_OUTPUT_AMOUNT + p2shFeeB; + + BitcoinyHTLC.Status htlcStatusB = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressB, minimumAmountB); + + switch (htlcStatusB) { + case UNFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s never funded?. Refunding P2SH-A next", p2shAddressB)); + return; + + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-B to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We must be very close to trade timeout. Defensively try to refund P2SH-A + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("P2SH-B %s already spent?. Refunding P2SH-A next", p2shAddressB)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(P2SH_B_OUTPUT_AMOUNT); // An actual amount to avoid dust filter, remaining used as fees. + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressB); + + // Determine receive address for refund + String receiveAddress = bitcoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(bitcoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptB, lockTimeB, receiving.getHash()); + + bitcoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("Refunded P2SH-B %s. Waiting for LockTime-A", p2shAddressB)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Bitcoin bitcoin = Bitcoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = bitcoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = bitcoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestampA = calcP2shAFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFeeA = bitcoin.getP2shFee(feeTimestampA); + long minimumAmountA = crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT + p2shFeeA; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount - P2SH_B_OUTPUT_AMOUNT); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = bitcoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = bitcoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(bitcoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + bitcoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

    + * Will automatically update trade-bot state to ALICE_REFUNDING_B or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING && isAtLockedToUs) + return false; + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_B, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcP2shAFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + + private long calcP2shBFeeTimestamp(int lockTimeA, int lockTimeB) { + // lockTimeB is halfway between offerMessageTimestamp and lockTimeA + return (lockTimeA - (lockTimeA - lockTimeB) * 2) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java new file mode 100644 index 00000000..0bd2972b --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java @@ -0,0 +1,894 @@ +package org.qortal.controller.tradebot; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.asset.Asset; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.DeployAtTransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +/** + * Performing cross-chain trading steps on behalf of user. + *

    + * We deal with three different independent state-spaces here: + *

      + *
    • Qortal blockchain
      • + *
    • Foreign blockchain
      • + *
    • Trade-bot entries
      • + *
    + */ +public class LitecoinACCTv1TradeBot implements AcctTradeBot { + + private static final Logger LOGGER = LogManager.getLogger(LitecoinACCTv1TradeBot.class); + + public enum State implements TradeBot.StateNameAndValueSupplier { + BOB_WAITING_FOR_AT_CONFIRM(10, false, false), + BOB_WAITING_FOR_MESSAGE(15, true, true), + BOB_WAITING_FOR_AT_REDEEM(25, true, true), + BOB_DONE(30, false, false), + BOB_REFUNDED(35, false, false), + + ALICE_WAITING_FOR_AT_LOCK(85, true, true), + ALICE_DONE(95, false, false), + ALICE_REFUNDING_A(105, true, true), + ALICE_REFUNDED(110, false, false); + + private static final Map map = stream(State.values()).collect(toMap(state -> state.value, state -> state)); + + public final int value; + public final boolean requiresAtData; + public final boolean requiresTradeData; + + State(int value, boolean requiresAtData, boolean requiresTradeData) { + this.value = value; + this.requiresAtData = requiresAtData; + this.requiresTradeData = requiresTradeData; + } + + public static State valueOf(int value) { + return map.get(value); + } + + @Override + public String getState() { + return this.name(); + } + + @Override + public int getStateValue() { + return this.value; + } + } + + /** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */ + private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms + + private static LitecoinACCTv1TradeBot instance; + + private final List endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream() + .map(State::name) + .collect(Collectors.toUnmodifiableList()); + + private LitecoinACCTv1TradeBot() { + } + + public static synchronized LitecoinACCTv1TradeBot getInstance() { + if (instance == null) + instance = new LitecoinACCTv1TradeBot(); + + return instance; + } + + @Override + public List getEndStates() { + return this.endStates; + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for LTC. + *

    + * Generates: + *

      + *
    • new 'trade' private key
      • + *
    + * Derives: + *
      + *
    • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
      • + *
    • 'foreign' (as in Litecoin) public key, public key hash
      • + *
    + * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
      + *
    • 'native'/Qortal 'trade' address - used as a MESSAGE contact
      • + *
    • 'foreign'/Litecoin public key hash - used by Alice's P2SH scripts to allow redeem
      • + *
    • QORT amount on offer by Bob
      • + *
    • LTC amount expected in return by Bob (from Alice)
      • + *
    • trading timeout, in case things go wrong and everyone needs to refund
      • + *
    + * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

    + * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

    + * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) + Address litecoinReceivingAddress; + try { + litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + } + if (litecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH) + throw new DataException("Unsupported Litecoin receiving address: " + tradeBotCreateRequest.receivingAddress); + + byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); + + PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey); + + // Deploy AT + long timestamp = NTP.getTime(); + byte[] reference = creator.getLastReference(); + long fee = 0L; + byte[] signature = null; + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature); + + String name = "QORT/LTC ACCT"; + String description = "QORT/LTC cross-chain trade"; + String aTType = "ACCT"; + String tags = "ACCT QORT LTC"; + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount, + tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout); + long amount = tradeBotCreateRequest.fundingQortAmount; + + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, + State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + tradeBotCreateRequest.foreignAmount, null, null, null, litecoinReceivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository); + + // Return to user for signing and broadcast as we don't have their Qortal private key + try { + return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + throw new DataException("Failed to transform DEPLOY_AT transaction?", e); + } + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching LTC to an existing offer. + *

    + * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a Litecoin wallet via xprv58. + *

    + * The crossChainTradeData contains the current trade offer state + * as extracted from the AT's data segment. + *

    + * Access to a funded wallet is via a Litecoin BIP32 hierarchical deterministic key, + * passed via xprv58. + * This key will be stored in your node's database + * to allow trade-bot to create/fund the necessary P2SH transactions! + * However, due to the nature of BIP32 keys, it is possible to give the trade-bot + * only a subset of wallet access (see BIP32 for more details). + *

    + * As an example, the xprv58 can be extract from a legacy, password-less + * Electrum wallet by going to the console tab and entering:
    + * wallet.keystore.xprv
    + * which should result in a base58 string starting with either 'xprv' (for Litecoin main-net) + * or 'tprv' for (Litecoin test-net). + *

    + * It is envisaged that the value in xprv58 will actually come from a Qortal-UI-managed wallet. + *

    + * If sufficient funds are available, this method will actually fund the P2SH-A + * with the Litecoin amount expected by 'Bob'. + *

    + * If the Litecoin transaction is successfully broadcast to the network then + * we also send a MESSAGE to Bob's trade-bot to let them know. + *

    + * The trade-bot entry is saved to the repository and the cross-chain trading process commences. + *

    + * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param xprv58 funded wallet xprv in base58 + * @return true if P2SH-A funding transaction successfully broadcast to Litecoin network, false otherwise + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + byte[] secretA = TradeBot.generateSecret(); + byte[] hashOfSecretA = Crypto.hash160(secretA); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH + + // We need to generate lockTime-A: add tradeTimeout to now + long now = NTP.getTime(); + int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, + State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value, + receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secretA, hashOfSecretA, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository); + + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount + long p2shFee; + try { + p2shFee = Litecoin.getInstance().getP2shFee(now); + } catch (ForeignBlockchainException e) { + LOGGER.debug("Couldn't estimate Litecoin fees?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Fee for redeem/refund is subtracted from P2SH-A balance. + // Do not include fee for funding transaction as this is covered by buildSpend() + long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/; + + // P2SH-A to be funded + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA); + String p2shAddress = Litecoin.getInstance().deriveP2shAddress(redeemScriptBytes); + + // Build transaction for funding P2SH-A + Transaction p2shFundingTransaction = Litecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA); + if (p2shFundingTransaction == null) { + LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?"); + return ResponseResult.BALANCE_ISSUE; + } + + try { + Litecoin.getInstance().broadcastTransaction(p2shFundingTransaction); + } catch (ForeignBlockchainException e) { + // We couldn't fund P2SH-A at this time + LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?"); + return ResponseResult.NETWORK_ISSUE; + } + + // Attempt to send MESSAGE to Bob's Qortal trade address + byte[] messageData = LitecoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA()); + String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress; + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name())); + return ResponseResult.NETWORK_ISSUE; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress)); + + return ResponseResult.OK; + } + + @Override + public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) + return true; + + // If the AT doesn't exist then we might as well let the user tidy up + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) + return true; + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + case ALICE_DONE: + case BOB_DONE: + case ALICE_REFUNDED: + case BOB_REFUNDED: + return true; + + default: + return false; + } + } + + @Override + public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException { + State tradeBotState = State.valueOf(tradeBotData.getStateValue()); + if (tradeBotState == null) { + LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress())); + return; + } + + ATData atData = null; + CrossChainTradeData tradeData = null; + + if (tradeBotState.requiresAtData) { + // Attempt to fetch AT data + atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress()); + if (atData == null) { + LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress())); + return; + } + + if (tradeBotState.requiresTradeData) { + tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + if (tradeData == null) { + LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress())); + return; + } + } + } + + switch (tradeBotState) { + case BOB_WAITING_FOR_AT_CONFIRM: + handleBobWaitingForAtConfirm(repository, tradeBotData); + break; + + case BOB_WAITING_FOR_MESSAGE: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_WAITING_FOR_AT_LOCK: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData); + break; + + case BOB_WAITING_FOR_AT_REDEEM: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_DONE: + case BOB_DONE: + break; + + case ALICE_REFUNDING_A: + TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData); + handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData); + break; + + case ALICE_REFUNDED: + case BOB_REFUNDED: + break; + } + } + + /** + * Trade-bot is waiting for Bob's AT to deploy. + *

    + * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice. + */ + private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException { + if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) { + if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD) + return; + + // We've waited ages for AT to be confirmed into a block but something has gone awry. + // After this long we assume transaction loss so give up with trade-bot entry too. + tradeBotData.setState(State.BOB_REFUNDED.name()); + tradeBotData.setStateValue(State.BOB_REFUNDED.value); + tradeBotData.setTimestamp(NTP.getTime()); + // We delete trade-bot entry here instead of saving, hence not using updateTradeBotState() + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + repository.saveChanges(); + + LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress())); + TradeBot.notifyStateChange(tradeBotData); + return; + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE, + () -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress())); + } + + /** + * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info. + *

    + * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund, + * in which case trade-bot is done with this specific trade and finalizes on refunded state. + *

    + * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot. + *

    + * Details from Alice are used to derive P2SH-A address and this is checked for funding balance. + *

    + * Assuming P2SH-A has at least expected Litecoin balance, + * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details. + *

    + * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice. + *

    + * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to + * extract secret-A needed to redeem Alice's P2SH. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // If AT has finished then Bob likely cancelled his trade offer + if (atData.getIsFinished()) { + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress())); + return; + } + + Litecoin litecoin = Litecoin.getInstance(); + + String address = tradeBotData.getTradeNativeAddress(); + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null); + + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + if (messageTransactionData.isText()) + continue; + + // We're expecting: HASH160(secret-A), Alice's Litecoin pubkeyhash and lockTime-A + byte[] messageData = messageTransactionData.getData(); + LitecoinACCTv1.OfferMessageData offerMessageData = LitecoinACCTv1.extractOfferMessageData(messageData); + if (offerMessageData == null) + continue; + + byte[] aliceForeignPublicKeyHash = offerMessageData.partnerLitecoinPKH; + byte[] hashOfSecretA = offerMessageData.hashOfSecretA; + int lockTimeA = (int) offerMessageData.lockTimeA; + long messageTimestamp = messageTransactionData.getTimestamp(); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(messageTimestamp, lockTimeA); + + // Determine P2SH-A address and confirm funded + byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // There might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // We've already redeemed this? + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A... + continue; + + case FUNDED: + // Fall-through out of switch... + break; + } + + // Good to go - send MESSAGE to AT + + String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey()); + + // Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume + byte[] outgoingMessageData = LitecoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false); + + outgoingMessageTransaction.computeNonce(); + outgoingMessageTransaction.sign(sender); + + // reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM, + () -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress)); + + return; + } + } + + /** + * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only. + *

    + * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow + * this process has taken so long that we've reached P2SH-A's locktime, or that someone else + * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process. + *

    + * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct. + *

    + * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice. + *

    + * In revealing a valid secret-A, Bob can then redeem the LTC funds from P2SH-A. + *

    + * @throws ForeignBlockchainException + */ + private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData)) + return; + + Litecoin litecoin = Litecoin.getInstance(); + int lockTimeA = tradeBotData.getLockTimeA(); + + // Refund P2SH-A if we've passed lockTime-A + if (NTP.getTime() >= lockTimeA * 1000L) { + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + case FUNDED: + break; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Already redeemed? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA)); + return; + + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> atData.getIsFinished() + ? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA) + : String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA)); + + return; + } + + // We're waiting for AT to be in TRADE mode + if (crossChainTradeData.mode != AcctMode.TRADING) + return; + + // AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above + + // Find our MESSAGE to AT from previous state + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(), + crossChainTradeData.qortalCreatorTradeAddress, null, null, null); + if (messageTransactionsData == null || messageTransactionsData.isEmpty()) { + LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress)); + return; + } + + long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp(); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(recipientMessageTimestamp, lockTimeA); + + // Our calculated refundTimeout should match AT's refundTimeout + if (refundTimeout != crossChainTradeData.refundTimeout) { + LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout)); + // We'll eventually refund + return; + } + + // We're good to redeem AT + + // Send 'redeem' MESSAGE to AT using both secret + byte[] secretA = tradeBotData.getSecret(); + String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH + byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secretA, qortalReceivingAddress); + String messageRecipient = tradeBotData.getAtAddress(); + + boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData); + if (!isMessageAlreadySent) { + PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false); + + messageTransaction.computeNonce(); + messageTransaction.sign(sender); + + // Reset repository state to prevent deadlock + repository.discardChanges(); + ValidationResult result = messageTransaction.importAsUnconfirmed(); + + if (result != ValidationResult.OK) { + LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name())); + return; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("Redeeming AT %s. Funds should arrive at %s", + tradeBotData.getAtAddress(), qortalReceivingAddress)); + } + + /** + * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the LTC funds from P2SH-A. + *

    + * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case, + * trade-bot is done with this specific trade and finalizes in refunded state. + *

    + * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the LTC funds from P2SH-A + * to Bob's 'foreign'/Litecoin trade legacy-format address, as derived from trade private key. + *

    + * (This could potentially be 'improved' to send LTC to any address of Bob's choosing by changing the transaction output). + *

    + * If trade-bot successfully broadcasts the transaction, then this specific trade is done. + * @throws ForeignBlockchainException + */ + private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // AT should be 'finished' once Alice has redeemed QORT funds + if (!atData.getIsFinished()) + // Not finished yet + return; + + // If AT is REFUNDED or CANCELLED then something has gone wrong + if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) { + // Alice hasn't redeemed the QORT, so there is no point in trying to redeem the LTC + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED, + () -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress())); + + return; + } + + byte[] secretA = LitecoinACCTv1.findSecretA(repository, crossChainTradeData); + if (secretA == null) { + LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress())); + return; + } + + // Use secret-A to redeem P2SH-A + + Litecoin litecoin = Litecoin.getInstance(); + + byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo(); + int lockTimeA = crossChainTradeData.lockTimeA; + byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Double-check that we have redeemed P2SH-A... + break; + + case REFUND_IN_PROGRESS: + case REFUNDED: + // Wait for AT to auto-refund + return; + + case FUNDED: { + Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey, + fundingOutputs, redeemScriptA, secretA, receivingAccountInfo); + + litecoin.broadcastTransaction(p2shRedeemTransaction); + break; + } + } + + String receivingAddress = litecoin.pkhToAddress(receivingAccountInfo); + + TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE, + () -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress)); + } + + /** + * Trade-bot is attempting to refund P2SH-A. + * @throws ForeignBlockchainException + */ + private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + int lockTimeA = tradeBotData.getLockTimeA(); + + // We can't refund P2SH-A until lockTime-A has passed + if (NTP.getTime() <= lockTimeA * 1000L) + return; + + Litecoin litecoin = Litecoin.getInstance(); + + // We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113) + int medianBlockTime = litecoin.getMedianBlockTime(); + if (medianBlockTime <= lockTimeA) + return; + + byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret()); + String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA); + + // Fee for redeem/refund is subtracted from P2SH-A balance. + long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout); + long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp); + long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee; + BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA); + + switch (htlcStatusA) { + case UNFUNDED: + case FUNDING_IN_PROGRESS: + // Still waiting for P2SH-A to be funded... + return; + + case REDEEM_IN_PROGRESS: + case REDEEMED: + // Too late! + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("P2SH-A %s already spent!", p2shAddressA)); + return; + + case REFUND_IN_PROGRESS: + case REFUNDED: + break; + + case FUNDED:{ + Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount); + ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey()); + List fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA); + + // Determine receive address for refund + String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey()); + Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress); + + Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey, + fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash()); + + litecoin.broadcastTransaction(p2shRefundTransaction); + break; + } + } + + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED, + () -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA)); + } + + /** + * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else. + *

    + * Will automatically update trade-bot state to ALICE_REFUNDING_A or ALICE_DONE as necessary. + * + * @throws DataException + * @throws ForeignBlockchainException + */ + private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData, + ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException { + // This is OK + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING) + return false; + + boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress); + + if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING) + if (isAtLockedToUs) { + // AT is trading with us - OK + return false; + } else { + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress)); + + return true; + } + + if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) { + // We've redeemed already? + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE, + () -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress())); + } else { + // Any other state is not good, so start defensive refund + TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A, + () -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress())); + } + + return true; + } + + private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) { + return (lockTimeA - tradeTimeout * 60) * 1000L; + } + +} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java new file mode 100644 index 00000000..fa3b599e --- /dev/null +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -0,0 +1,373 @@ +package org.qortal.controller.tradebot; + +import java.awt.TrayIcon.MessageType; +import java.security.SecureRandom; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.util.Supplier; +import org.bitcoinj.core.ECKey; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.model.crosschain.TradeBotCreateRequest; +import org.qortal.controller.Controller; +import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.Listener; +import org.qortal.group.Group; +import org.qortal.gui.SysTray; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.transaction.PresenceTransaction; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.NTP; + +import com.google.common.primitives.Longs; + +/** + * Performing cross-chain trading steps on behalf of user. + *

    + * We deal with three different independent state-spaces here: + *

      + *
    • Qortal blockchain
      • + *
    • Foreign blockchain
      • + *
    • Trade-bot entries
      • + *
    + */ +public class TradeBot implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(TradeBot.class); + private static final Random RANDOM = new SecureRandom(); + + public interface StateNameAndValueSupplier { + public String getState(); + public int getStateValue(); + } + + public static class StateChangeEvent implements Event { + private final TradeBotData tradeBotData; + + public StateChangeEvent(TradeBotData tradeBotData) { + this.tradeBotData = tradeBotData; + } + + public TradeBotData getTradeBotData() { + return this.tradeBotData; + } + } + + private static final Map, Supplier> acctTradeBotSuppliers = new HashMap<>(); + static { + acctTradeBotSuppliers.put(BitcoinACCTv1.class, BitcoinACCTv1TradeBot::getInstance); + acctTradeBotSuppliers.put(LitecoinACCTv1.class, LitecoinACCTv1TradeBot::getInstance); + } + + private static TradeBot instance; + + private final Map presenceTimestampsByAtAddress = Collections.synchronizedMap(new HashMap<>()); + + private TradeBot() { + EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event)); + } + + public static synchronized TradeBot getInstance() { + if (instance == null) + instance = new TradeBot(); + + return instance; + } + + public ACCT getAcctUsingAtData(ATData atData) { + byte[] codeHash = atData.getCodeHash(); + if (codeHash == null) + return null; + + return SupportedBlockchain.getAcctByCodeHash(codeHash); + } + + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ACCT acct = this.getAcctUsingAtData(atData); + if (acct == null) + return null; + + return acct.populateTradeData(repository, atData); + } + + /** + * Creates a new trade-bot entry from the "Bob" viewpoint, + * i.e. OFFERing QORT in exchange for foreign blockchain currency. + *

    + * Generates: + *

      + *
    • new 'trade' private key
      • + *
    • secret(s)
      • + *
    + * Derives: + *
      + *
    • 'native' (as in Qortal) public key, public key hash, address (starting with Q)
      • + *
    • 'foreign' public key, public key hash
      • + *
    • hash(es) of secret(s)
      • + *
    + * A Qortal AT is then constructed including the following as constants in the 'data segment': + *
      + *
    • 'native' (Qortal) 'trade' address - used to MESSAGE AT
      • + *
    • 'foreign' public key hash - used by Alice's to allow redeem of currency on foreign blockchain
      • + *
    • hash(es) of secret(s) - used by AT (optional) and foreign blockchain as needed
      • + *
    • QORT amount on offer by Bob
      • + *
    • foreign currency amount expected in return by Bob (from Alice)
      • + *
    • trading timeout, in case things go wrong and everyone needs to refund
      • + *
    + * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network. + *

    + * Trade-bot will wait for Bob's AT to be deployed before taking next step. + *

    + * @param repository + * @param tradeBotCreateRequest + * @return raw, unsigned DEPLOY_AT transaction + * @throws DataException + */ + public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException { + // Fetch latest ACCT version for requested foreign blockchain + ACCT acct = tradeBotCreateRequest.foreignBlockchain.getLatestAcct(); + + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + if (acctTradeBot == null) + return null; + + return acctTradeBot.createTrade(repository, tradeBotCreateRequest); + } + + /** + * Creates a trade-bot entry from the 'Alice' viewpoint, + * i.e. matching foreign blockchain currency to an existing QORT offer. + *

    + * Requires a chosen trade offer from Bob, passed by crossChainTradeData + * and access to a foreign blockchain wallet via foreignKey. + *

    + * @param repository + * @param crossChainTradeData chosen trade OFFER that Alice wants to match + * @param foreignKey foreign blockchain wallet key + * @throws DataException + */ + public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, + CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException { + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + if (acctTradeBot == null) { + LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot for AT %s", atData.getATAddress())); + return ResponseResult.NETWORK_ISSUE; + } + + // Check Alice doesn't already have an existing, on-going trade-bot entry for this AT. + if (repository.getCrossChainRepository().existsTradeWithAtExcludingStates(atData.getATAddress(), acctTradeBot.getEndStates())) + return ResponseResult.TRADE_ALREADY_EXISTS; + + return acctTradeBot.startResponse(repository, atData, acct, crossChainTradeData, foreignKey, receivingAddress); + } + + public boolean deleteEntry(Repository repository, byte[] tradePrivateKey) throws DataException { + TradeBotData tradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + if (tradeBotData == null) + // Can't delete what we don't have! + return false; + + boolean canDelete = false; + + ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName()); + if (acct == null) + // We can't/no longer support this ACCT + canDelete = true; + else { + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + canDelete = acctTradeBot == null || acctTradeBot.canDelete(repository, tradeBotData); + } + + if (canDelete) { + repository.getCrossChainRepository().delete(tradePrivateKey); + repository.saveChanges(); + } + + return canDelete; + } + + @Override + public void listen(Event event) { + if (!(event instanceof Controller.NewBlockEvent)) + return; + + synchronized (this) { + List allTradeBotData; + + try (final Repository repository = RepositoryManager.getRepository()) { + allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + } catch (DataException e) { + LOGGER.error("Couldn't run trade bot due to repository issue", e); + return; + } + + for (TradeBotData tradeBotData : allTradeBotData) + try (final Repository repository = RepositoryManager.getRepository()) { + // Find ACCT-specific trade-bot for this entry + ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName()); + if (acct == null) { + LOGGER.debug(() -> String.format("Couldn't find ACCT matching name %s", tradeBotData.getAcctName())); + continue; + } + + AcctTradeBot acctTradeBot = findTradeBotForAcct(acct); + if (acctTradeBot == null) { + LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot matching name %s", tradeBotData.getAcctName())); + continue; + } + + acctTradeBot.progress(repository, tradeBotData); + } catch (DataException e) { + LOGGER.error("Couldn't run trade bot due to repository issue", e); + } catch (ForeignBlockchainException e) { + LOGGER.warn(() -> String.format("Foreign blockchain issue processing trade-bot entry for AT %s: %s", tradeBotData.getAtAddress(), e.getMessage())); + } + } + } + + /*package*/ static byte[] generateTradePrivateKey() { + // The private key is used for both Curve25519 and secp256k1 so needs to be valid for both. + // Curve25519 accepts any seed, so generate a valid secp256k1 key and use that. + return new ECKey().getPrivKeyBytes(); + } + + /*package*/ static byte[] deriveTradeNativePublicKey(byte[] privateKey) { + return PrivateKeyAccount.toPublicKey(privateKey); + } + + /*package*/ static byte[] deriveTradeForeignPublicKey(byte[] privateKey) { + return ECKey.fromPrivate(privateKey).getPubKey(); + } + + /*package*/ static byte[] generateSecret() { + byte[] secret = new byte[32]; + RANDOM.nextBytes(secret); + return secret; + } + + /*package*/ static void backupTradeBotData(Repository repository) { + // Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure + try { + LOGGER.info("About to backup trade bot data..."); + repository.exportNodeLocalData(); + } catch (DataException e) { + LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage())); + } + } + + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ + /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, + String newState, int newStateValue, Supplier logMessageSupplier) throws DataException { + tradeBotData.setState(newState); + tradeBotData.setStateValue(newStateValue); + tradeBotData.setTimestamp(NTP.getTime()); + repository.getCrossChainRepository().save(tradeBotData); + repository.saveChanges(); + + if (Settings.getInstance().isTradebotSystrayEnabled()) + SysTray.getInstance().showMessage("Trade-Bot", String.format("%s: %s", tradeBotData.getAtAddress(), newState), MessageType.INFO); + + if (logMessageSupplier != null) + LOGGER.info(logMessageSupplier); + + LOGGER.debug(() -> String.format("new state for trade-bot entry based on AT %s: %s", tradeBotData.getAtAddress(), newState)); + + notifyStateChange(tradeBotData); + } + + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ + /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, StateNameAndValueSupplier newStateSupplier, Supplier logMessageSupplier) throws DataException { + updateTradeBotState(repository, tradeBotData, newStateSupplier.getState(), newStateSupplier.getStateValue(), logMessageSupplier); + } + + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ + /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, Supplier logMessageSupplier) throws DataException { + updateTradeBotState(repository, tradeBotData, tradeBotData.getState(), tradeBotData.getStateValue(), logMessageSupplier); + } + + /*package*/ static void notifyStateChange(TradeBotData tradeBotData) { + StateChangeEvent stateChangeEvent = new StateChangeEvent(tradeBotData); + EventBus.INSTANCE.notify(stateChangeEvent); + } + + /*package*/ static AcctTradeBot findTradeBotForAcct(ACCT acct) { + Supplier acctTradeBotSupplier = acctTradeBotSuppliers.get(acct.getClass()); + if (acctTradeBotSupplier == null) + return null; + + return acctTradeBotSupplier.get(); + } + + // PRESENCE-related + /*package*/ void updatePresence(Repository repository, TradeBotData tradeBotData, CrossChainTradeData tradeData) + throws DataException { + String atAddress = tradeBotData.getAtAddress(); + + PrivateKeyAccount tradeNativeAccount = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey()); + String signerAddress = tradeNativeAccount.getAddress(); + + /* + * There's no point in Alice trying to build a PRESENCE transaction + * for an AT that isn't locked to her, as other peers won't be able + * to validate the PRESENCE transaction as signing public key won't + * be visible. + */ + if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) + // Signer is neither Bob, nor Alice, or trade not yet locked to Alice + return; + + long now = NTP.getTime(); + long threshold = now - PresenceType.TRADE_BOT.getLifetime(); + + long timestamp = presenceTimestampsByAtAddress.compute(atAddress, (k, v) -> (v == null || v < threshold) ? now : v); + + // If timestamp hasn't been updated then nothing to do + if (timestamp != now) + return; + + int txGroupId = Group.NO_GROUP; + byte[] reference = new byte[TransactionTransformer.SIGNATURE_LENGTH]; + byte[] creatorPublicKey = tradeNativeAccount.getPublicKey(); + long fee = 0L; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, null); + + int nonce = 0; + byte[] timestampSignature = tradeNativeAccount.sign(Longs.toByteArray(timestamp)); + + PresenceTransactionData transactionData = new PresenceTransactionData(baseTransactionData, nonce, PresenceType.TRADE_BOT, timestampSignature); + + PresenceTransaction presenceTransaction = new PresenceTransaction(repository, transactionData); + presenceTransaction.computeNonce(); + + presenceTransaction.sign(tradeNativeAccount); + + ValidationResult result = presenceTransaction.importAsUnconfirmed(); + if (result != ValidationResult.OK) + LOGGER.debug(() -> String.format("Unable to build trade-bot PRESENCE transaction for %s: %s", tradeBotData.getAtAddress(), result.name())); + } + +} diff --git a/src/main/java/org/qortal/crosschain/ACCT.java b/src/main/java/org/qortal/crosschain/ACCT.java new file mode 100644 index 00000000..e557a3e2 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/ACCT.java @@ -0,0 +1,23 @@ +package org.qortal.crosschain; + +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +public interface ACCT { + + public byte[] getCodeBytesHash(); + + public int getModeByteOffset(); + + public ForeignBlockchain getBlockchain(); + + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException; + + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException; + + public byte[] buildCancelMessage(String creatorQortalAddress); + +} diff --git a/src/main/java/org/qortal/crosschain/AcctMode.java b/src/main/java/org/qortal/crosschain/AcctMode.java new file mode 100644 index 00000000..21496032 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/AcctMode.java @@ -0,0 +1,21 @@ +package org.qortal.crosschain; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Map; + +public enum AcctMode { + OFFERING(0), TRADING(1), CANCELLED(2), REFUNDED(3), REDEEMED(4); + + public final int value; + private static final Map map = stream(AcctMode.values()).collect(toMap(mode -> mode.value, mode -> mode)); + + AcctMode(int value) { + this.value = value; + } + + public static AcctMode valueOf(int value) { + return map.get(value); + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/Bitcoin.java b/src/main/java/org/qortal/crosschain/Bitcoin.java new file mode 100644 index 00000000..28275d6a --- /dev/null +++ b/src/main/java/org/qortal/crosschain/Bitcoin.java @@ -0,0 +1,190 @@ +package org.qortal.crosschain; + +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; + +import org.bitcoinj.core.Context; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.params.MainNetParams; +import org.bitcoinj.params.RegTestParams; +import org.bitcoinj.params.TestNet3Params; +import org.qortal.crosschain.ElectrumX.Server; +import org.qortal.crosschain.ElectrumX.Server.ConnectionType; +import org.qortal.settings.Settings; + +public class Bitcoin extends Bitcoiny { + + public static final String CURRENCY_CODE = "BTC"; + + // Temporary values until a dynamic fee system is written. + private static final long OLD_FEE_AMOUNT = 4_000L; // Not 5000 so that existing P2SH-B can output 1000, avoiding dust issue, leaving 4000 for fees. + private static final long NEW_FEE_TIMESTAMP = 1598280000000L; // milliseconds since epoch + private static final long NEW_FEE_AMOUNT = 10_000L; + + private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST + + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); + static { + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); + } + + public enum BitcoinNet { + MAIN { + @Override + public NetworkParameters getParams() { + return MainNetParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + // Servers chosen on NO BASIS WHATSOEVER from various sources! + new Server("128.0.190.26", Server.ConnectionType.SSL, 50002), + new Server("hodlers.beer", Server.ConnectionType.SSL, 50002), + new Server("electrumx.erbium.eu", Server.ConnectionType.TCP, 50001), + new Server("electrumx.erbium.eu", Server.ConnectionType.SSL, 50002), + new Server("btc.lastingcoin.net", Server.ConnectionType.SSL, 50002), + new Server("electrum.bitaroo.net", Server.ConnectionType.SSL, 50002), + new Server("bitcoin.grey.pw", Server.ConnectionType.SSL, 50002), + new Server("2electrumx.hopto.me", Server.ConnectionType.SSL, 56022), + new Server("185.64.116.15", Server.ConnectionType.SSL, 50002), + new Server("kirsche.emzy.de", Server.ConnectionType.SSL, 50002), + new Server("alviss.coinjoined.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.emzy.de", Server.ConnectionType.SSL, 50002), + new Server("electrum.emzy.de", Server.ConnectionType.TCP, 50001), + new Server("vmd71287.contaboserver.net", Server.ConnectionType.SSL, 50002), + new Server("btc.litepay.ch", Server.ConnectionType.SSL, 50002), + new Server("electrum.stippy.com", Server.ConnectionType.SSL, 50002), + new Server("xtrum.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.acinq.co", Server.ConnectionType.SSL, 50002), + new Server("electrum2.taborsky.cz", Server.ConnectionType.SSL, 50002), + new Server("vmd63185.contaboserver.net", Server.ConnectionType.SSL, 50002), + new Server("electrum2.privateservers.network", Server.ConnectionType.SSL, 50002), + new Server("electrumx.alexridevski.net", Server.ConnectionType.SSL, 50002), + new Server("192.166.219.200", Server.ConnectionType.SSL, 50002), + new Server("2ex.digitaleveryware.com", Server.ConnectionType.SSL, 50002), + new Server("dxm.no-ip.biz", Server.ConnectionType.SSL, 50002), + new Server("caleb.vegas", Server.ConnectionType.SSL, 50002)); + } + + @Override + public String getGenesisHash() { + return "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"; + } + + @Override + public long getP2shFee(Long timestamp) { + // TODO: This will need to be replaced with something better in the near future! + if (timestamp != null && timestamp < NEW_FEE_TIMESTAMP) + return OLD_FEE_AMOUNT; + + return NEW_FEE_AMOUNT; + } + }, + TEST3 { + @Override + public NetworkParameters getParams() { + return TestNet3Params.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("tn.not.fyi", Server.ConnectionType.SSL, 55002), + new Server("electrumx-test.1209k.com", Server.ConnectionType.SSL, 50002), + new Server("testnet.qtornado.com", Server.ConnectionType.SSL, 51002), + new Server("testnet.aranguren.org", Server.ConnectionType.TCP, 51001), + new Server("testnet.aranguren.org", Server.ConnectionType.SSL, 51002), + new Server("testnet.hsmiths.com", Server.ConnectionType.SSL, 53012)); + } + + @Override + public String getGenesisHash() { + return "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }, + REGTEST { + @Override + public NetworkParameters getParams() { + return RegTestParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("localhost", Server.ConnectionType.TCP, 50001), + new Server("localhost", Server.ConnectionType.SSL, 50002)); + } + + @Override + public String getGenesisHash() { + // This is unique to each regtest instance + return null; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }; + + public abstract NetworkParameters getParams(); + public abstract Collection getServers(); + public abstract String getGenesisHash(); + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; + } + + private static Bitcoin instance; + + private final BitcoinNet bitcoinNet; + + // Constructors and instance + + private Bitcoin(BitcoinNet bitcoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + super(blockchain, bitcoinjContext, currencyCode); + this.bitcoinNet = bitcoinNet; + + LOGGER.info(() -> String.format("Starting Bitcoin support using %s", this.bitcoinNet.name())); + } + + public static synchronized Bitcoin getInstance() { + if (instance == null) { + BitcoinNet bitcoinNet = Settings.getInstance().getBitcoinNet(); + + BitcoinyBlockchainProvider electrumX = new ElectrumX("Bitcoin-" + bitcoinNet.name(), bitcoinNet.getGenesisHash(), bitcoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); + Context bitcoinjContext = new Context(bitcoinNet.getParams()); + + instance = new Bitcoin(bitcoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); + } + + return instance; + } + + // Getters & setters + + public static synchronized void resetForTesting() { + instance = null; + } + + // Actual useful methods for use by other classes + + /** + * Returns estimated BTC fee, in sats per 1000bytes, optionally for historic timestamp. + * + * @param timestamp optional milliseconds since epoch, or null for 'now' + * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong + */ + @Override + public long getP2shFee(Long timestamp) throws ForeignBlockchainException { + return this.bitcoinNet.getP2shFee(timestamp); + } + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java new file mode 100644 index 00000000..5118e103 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java @@ -0,0 +1,921 @@ +package org.qortal.crosschain; + +import static org.ciyam.at.OpCode.calcOffset; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.API; +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +/** + * Cross-chain trade AT + * + *

    + *

      + *
    • Bob generates Bitcoin & Qortal 'trade' keys, and secret-b + *
        + *
      • private key required to sign P2SH redeem tx
        • + *
      • private key could be used to create 'secret' (e.g. double-SHA256)
        • + *
      • encrypted private key could be stored in Qortal AT for access by Bob from any node
        • + *
      + *
      • + *
    • Bob deploys Qortal AT + *
        + *
      + *
      • + *
    • Alice finds Qortal AT and wants to trade + *
        + *
      • Alice generates Bitcoin & Qortal 'trade' keys
        • + *
      • Alice funds Bitcoin P2SH-A
        • + *
      • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
          + *
        • hash-of-secret-A
          • + *
        • her 'trade' Bitcoin PKH
          • + *
        + *
        • + *
      + *
      • + *
    • Bob receives "offer" MESSAGE + *
        + *
      • Checks Alice's P2SH-A
        • + *
      • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
          + *
        • Alice's trade Qortal address
          • + *
        • Alice's trade Bitcoin PKH
          • + *
        • hash-of-secret-A
          • + *
        + *
        • + *
      + *
      • + *
    • Alice checks Qortal AT to confirm it's locked to her + *
        + *
      • Alice creates/funds Bitcoin P2SH-B
        • + *
      + *
      • + *
    • Bob checks P2SH-B is funded + *
        + *
      • Bob redeems P2SH-B using his Bitcoin trade key and secret-B
        • + *
      + *
      • + *
    • Alice scans P2SH-B redeem transaction to extract secret-B + *
        + *
      • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
          + *
        • secret-A
          • + *
        • secret-B
          • + *
        • Qortal receiving address of her chosing
          • + *
        + *
        • + *
      • AT's QORT funds are sent to Qortal receiving address
        • + *
      + *
      • + *
    • Bob checks AT, extracts secret-A + *
        + *
      • Bob redeems P2SH-A using his Bitcoin trade key and secret-A
        • + *
      • P2SH-A BTC funds end up at Bitcoin address determined by redeem transaction output(s)
        • + *
      + *
      • + *
    + */ +public class BitcoinACCTv1 implements ACCT { + + public static final String NAME = BitcoinACCTv1.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("f7f419522a9aaa3c671149878f8c1374dfc59d4fd86ca43ff2a4d913cfbc9e89").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 68; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerBitcoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerBitcoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Bitcoin PKH (padded from 20 to 24)*/ + + 8 /*lockTimeB*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret*/ + 32 /*secret*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static BitcoinACCTv1 instance; + + private BitcoinACCTv1() { + } + + public static synchronized BitcoinACCTv1 getInstance() { + if (instance == null) + instance = new BitcoinACCTv1(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Bitcoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

    + * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address, also used for refunds + * @param bitcoinPublicKeyHash 20-byte HASH160 of creator's trade Bitcoin public key + * @param hashOfSecretB 20-byte HASH160 of 32-byte secret-B + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param bitcoinAmount how much BTC the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] bitcoinPublicKeyHash, byte[] hashOfSecretB, long qortAmount, long bitcoinAmount, int tradeTimeout) { + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrBitcoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretB = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrBitcoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrHashOfSecretBPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerBitcoinPKHOffset = addrCounter++; + final int addrPartnerBitcoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageSecretBOffset = addrCounter++; + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrLockTimeB = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerBitcoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : "MODE_VALUE_OFFSET does not match addrMode"; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Bitcoin public key hash + assert dataByteBuffer.position() == addrBitcoinPublicKeyHash * MachineState.VALUE_SIZE : "addrBitcoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(bitcoinPublicKeyHash, 32, 0)); + + // Hash of secret-B + assert dataByteBuffer.position() == addrHashOfSecretB * MachineState.VALUE_SIZE : "addrHashOfSecretB incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(hashOfSecretB, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Bitcoin amount + assert dataByteBuffer.position() == addrBitcoinAmount * MachineState.VALUE_SIZE : "addrBitcoinAmount incorrect"; + dataByteBuffer.putLong(bitcoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of hash of secret B, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretBPointer * MachineState.VALUE_SIZE : "addrHashOfSecretBPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretB); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Bitcoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerBitcoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerBitcoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Bitcoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerBitcoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerBitcoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerBitcoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting secret-B + assert dataByteBuffer.position() == addrRedeemMessageSecretBOffset * MachineState.VALUE_SIZE : "addrRedeemMessageSecretBOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelCheckSecretB = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Bitcoin public key hash (PKH) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerBitcoinPKHOffset)); + // Extract partner's Bitcoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerBitcoinPKHPointer)); + // Also extract lockTimeB + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeB)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-a (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTimeA (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade refund timeout: (lockTimeA - lockTimeB) / 2 / 60 + codeByteBuffer.put(OpCode.SET_DAT.compile(addrRefundTimeout, addrLockTimeA)); // refundTimeout = lockTimeA + codeByteBuffer.put(OpCode.SUB_DAT.compile(addrRefundTimeout, addrLockTimeB)); // refundTimeout -= lockTimeB + codeByteBuffer.put(OpCode.DIV_VAL.compile(addrRefundTimeout, 2L * 60L)); // refundTimeout /= 2 * 60 + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckSecretB))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check 'secret-B' in transaction's message */ + + labelCheckSecretB = codeByteBuffer.position(); + + // Extract secret-B from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageSecretBOffset)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretBPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretBPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile BTC-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), BitcoinACCTv1.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.BITCOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Bitcoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // Hash of secret B + tradeData.hashOfSecretB = new byte[20]; + dataByteBuffer.get(tradeData.hashOfSecretB); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.hashOfSecretB.length); // skip to 32 bytes + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected BTC amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-B + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's bitcoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's bitcoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for secret-B + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // Potential lockTimeB (if in trade mode) + int lockTimeB = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Bitcoin PKH + byte[] partnerBitcoinPKH = new byte[20]; + dataByteBuffer.get(partnerBitcoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerBitcoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode acctMode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (acctMode != null && acctMode != AcctMode.OFFERING) { + tradeData.mode = acctMode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerBitcoinPKH; + tradeData.lockTimeA = lockTimeA; + tradeData.lockTimeB = lockTimeB; + + if (acctMode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerBitcoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int lockTimeB) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] lockTimeBBytes = BitTwiddling.toBEByteArray((long) lockTimeB); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(lockTimeBBytes, 0, data, 56, lockTimeBBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner/ to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, byte[] secretB, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(secretB, 0, data, 32, secretB.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 64, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns P2SH-B lockTime (epoch seconds) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcLockTimeB(long offerMessageTimestamp, int lockTimeA) { + // lockTimeB is halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA + (offerMessageTimestamp / 1000L)) / 2L); + } + + public static byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract both secretA & secretB + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + byte[] secretB = new byte[32]; + System.arraycopy(messageData, 32, secretB, 0, secretB.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + byte[] hashOfSecretB = Crypto.hash160(secretB); + if (!Arrays.equals(hashOfSecretB, crossChainTradeData.hashOfSecretB)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java new file mode 100644 index 00000000..fc98f959 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -0,0 +1,740 @@ +package org.qortal.crosschain; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.Context; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.InsufficientMoneyException; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Sha256Hash; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.core.UTXO; +import org.bitcoinj.core.UTXOProvider; +import org.bitcoinj.core.UTXOProviderException; +import org.bitcoinj.crypto.ChildNumber; +import org.bitcoinj.crypto.DeterministicHierarchy; +import org.bitcoinj.crypto.DeterministicKey; +import org.bitcoinj.script.Script.ScriptType; +import org.bitcoinj.script.ScriptBuilder; +import org.bitcoinj.wallet.DeterministicKeyChain; +import org.bitcoinj.wallet.SendRequest; +import org.bitcoinj.wallet.Wallet; +import org.qortal.api.model.SimpleForeignTransaction; +import org.qortal.crypto.Crypto; +import org.qortal.utils.Amounts; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; + +/** Bitcoin-like (Bitcoin, Litecoin, etc.) support */ +public abstract class Bitcoiny implements ForeignBlockchain { + + protected static final Logger LOGGER = LogManager.getLogger(Bitcoiny.class); + + public static final int HASH160_LENGTH = 20; + + protected final BitcoinyBlockchainProvider blockchain; + protected final Context bitcoinjContext; + protected final String currencyCode; + + protected final NetworkParameters params; + + /** Keys that have been previously marked as fully spent,
    + * i.e. keys with transactions but with no unspent outputs. */ + protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); + + /** How many bitcoinj wallet keys to generate in each batch. */ + private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; + + /** Byte offset into raw block headers to block timestamp. */ + private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; + + // Constructors and instance + + protected Bitcoiny(BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + this.blockchain = blockchain; + this.bitcoinjContext = bitcoinjContext; + this.currencyCode = currencyCode; + + this.params = this.bitcoinjContext.getParams(); + } + + // Getters & setters + + public BitcoinyBlockchainProvider getBlockchainProvider() { + return this.blockchain; + } + + public Context getBitcoinjContext() { + return this.bitcoinjContext; + } + + public String getCurrencyCode() { + return this.currencyCode; + } + + public NetworkParameters getNetworkParameters() { + return this.params; + } + + // Interface obligations + + @Override + public boolean isValidAddress(String address) { + try { + ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType(); + + return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH; + } catch (AddressFormatException e) { + return false; + } + } + + @Override + public boolean isValidWalletKey(String walletKey) { + return this.isValidDeterministicKey(walletKey); + } + + // Actual useful methods for use by other classes + + public String format(Coin amount) { + return this.format(amount.value); + } + + public String format(long amount) { + return Amounts.prettyAmount(amount) + " " + this.currencyCode; + } + + public boolean isValidDeterministicKey(String key58) { + try { + Context.propagate(this.bitcoinjContext); + DeterministicKey.deserializeB58(null, key58, this.params); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + /** Returns P2PKH address using passed public key hash. */ + public String pkhToAddress(byte[] publicKeyHash) { + Context.propagate(this.bitcoinjContext); + return LegacyAddress.fromPubKeyHash(this.params, publicKeyHash).toString(); + } + + /** Returns P2SH address using passed redeem script. */ + public String deriveP2shAddress(byte[] redeemScriptBytes) { + Context.propagate(bitcoinjContext); + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + return LegacyAddress.fromScriptHash(this.params, redeemScriptHash).toString(); + } + + /** + * Returns median timestamp from latest 11 blocks, in seconds. + *

    + * @throws ForeignBlockchainException if error occurs + */ + public int getMedianBlockTime() throws ForeignBlockchainException { + int height = this.blockchain.getCurrentHeight(); + + // Grab latest 11 blocks + List blockHeaders = this.blockchain.getRawBlockHeaders(height - 11, 11); + if (blockHeaders.size() < 11) + throw new ForeignBlockchainException("Not enough blocks to determine median block time"); + + List blockTimestamps = blockHeaders.stream().map(blockHeader -> BitTwiddling.intFromLEBytes(blockHeader, TIMESTAMP_OFFSET)).collect(Collectors.toList()); + + // Descending order + blockTimestamps.sort((a, b) -> Integer.compare(b, a)); + + // Pick median + return blockTimestamps.get(5); + } + + /** Returns fee per transaction KB. To be overridden for testnet/regtest. */ + public Coin getFeePerKb() { + return this.bitcoinjContext.getFeePerKb(); + } + + /** + * Returns fixed P2SH spending fee, in sats per 1000bytes, optionally for historic timestamp. + * + * @param timestamp optional milliseconds since epoch, or null for 'now' + * @return sats per 1000bytes + * @throws ForeignBlockchainException if something went wrong + */ + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; + + /** + * Returns confirmed balance, based on passed payment script. + *

    + * @return confirmed balance, or zero if script unknown + * @throws ForeignBlockchainException if there was an error + */ + public long getConfirmedBalance(String base58Address) throws ForeignBlockchainException { + return this.blockchain.getConfirmedBalance(addressToScriptPubKey(base58Address)); + } + + /** + * Returns list of unspent outputs pertaining to passed address. + *

    + * @return list of unspent outputs, or empty list if address unknown + * @throws ForeignBlockchainException if there was an error. + */ + // TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead + public List getUnspentOutputs(String base58Address) throws ForeignBlockchainException { + List unspentOutputs = this.blockchain.getUnspentOutputs(addressToScriptPubKey(base58Address), false); + + List unspentTransactionOutputs = new ArrayList<>(); + for (UnspentOutput unspentOutput : unspentOutputs) { + List transactionOutputs = this.getOutputs(unspentOutput.hash); + + unspentTransactionOutputs.add(transactionOutputs.get(unspentOutput.index)); + } + + return unspentTransactionOutputs; + } + + /** + * Returns list of outputs pertaining to passed transaction hash. + *

    + * @return list of outputs, or empty list if transaction unknown + * @throws ForeignBlockchainException if there was an error. + */ + // TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead + public List getOutputs(byte[] txHash) throws ForeignBlockchainException { + byte[] rawTransactionBytes = this.blockchain.getRawTransaction(txHash); + + Context.propagate(bitcoinjContext); + Transaction transaction = new Transaction(this.params, rawTransactionBytes); + return transaction.getOutputs(); + } + + /** + * Returns list of transaction hashes pertaining to passed address. + *

    + * @return list of unspent outputs, or empty list if script unknown + * @throws ForeignBlockchainException if there was an error. + */ + public List getAddressTransactions(String base58Address, boolean includeUnconfirmed) throws ForeignBlockchainException { + return this.blockchain.getAddressTransactions(addressToScriptPubKey(base58Address), includeUnconfirmed); + } + + /** + * Returns list of raw, confirmed transactions involving given address. + *

    + * @throws ForeignBlockchainException if there was an error + */ + public List getAddressTransactions(String base58Address) throws ForeignBlockchainException { + List transactionHashes = this.blockchain.getAddressTransactions(addressToScriptPubKey(base58Address), false); + + List rawTransactions = new ArrayList<>(); + for (TransactionHash transactionInfo : transactionHashes) { + byte[] rawTransaction = this.blockchain.getRawTransaction(HashCode.fromString(transactionInfo.txHash).asBytes()); + rawTransactions.add(rawTransaction); + } + + return rawTransactions; + } + + /** + * Returns transaction info for passed transaction hash. + *

    + * @throws ForeignBlockchainException.NotFoundException if transaction unknown + * @throws ForeignBlockchainException if error occurs + */ + public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { + return this.blockchain.getTransaction(txHash); + } + + /** + * Broadcasts raw transaction to network. + *

    + * @throws ForeignBlockchainException if error occurs + */ + public void broadcastTransaction(Transaction transaction) throws ForeignBlockchainException { + this.blockchain.broadcastTransaction(transaction.bitcoinSerialize()); + } + + /** + * Returns bitcoinj transaction sending amount to recipient. + * + * @param xprv58 BIP32 private key + * @param recipient P2PKH address + * @param amount unscaled amount + * @param feePerByte unscaled fee per byte, or null to use default fees + * @return transaction, or null if insufficient funds + */ + public Transaction buildSpend(String xprv58, String recipient, long amount, Long feePerByte) { + Context.propagate(bitcoinjContext); + + Wallet wallet = Wallet.fromSpendingKeyB58(this.params, xprv58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); + wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); + + Address destination = Address.fromString(this.params, recipient); + SendRequest sendRequest = SendRequest.to(destination, Coin.valueOf(amount)); + + if (feePerByte != null) + sendRequest.feePerKb = Coin.valueOf(feePerByte * 1000L); // Note: 1000 not 1024 + else + // Allow override of default for TestNet3, etc. + sendRequest.feePerKb = this.getFeePerKb(); + + try { + wallet.completeTx(sendRequest); + return sendRequest.tx; + } catch (InsufficientMoneyException e) { + return null; + } + } + + /** + * Returns bitcoinj transaction sending amount to recipient using default fees. + * + * @param xprv58 BIP32 private key + * @param recipient P2PKH address + * @param amount unscaled amount + * @return transaction, or null if insufficient funds + */ + public Transaction buildSpend(String xprv58, String recipient, long amount) { + return buildSpend(xprv58, recipient, amount, null); + } + + /** + * Returns unspent Bitcoin balance given 'm' BIP32 key. + * + * @param key58 BIP32/HD extended Bitcoin private/public key + * @return unspent BTC balance, or null if unable to determine balance + */ + public Long getWalletBalance(String key58) { + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); + + Coin balance = wallet.getBalance(); + if (balance == null) + return null; + + return balance.value; + } + + public List getWalletTransactions(String key58) throws ForeignBlockchainException { + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); + + List keys = new ArrayList<>(keyChain.getLeafKeys()); + + Set walletTransactions = new HashSet<>(); + Set keySet = new HashSet<>(); + + int ki = 0; + do { + boolean areAllKeysUnused = true; + + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); + + // Check for transactions + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + keySet.add(address.toString()); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); + + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; + + for (TransactionHash transactionHash : historicTransactionHashes) + walletTransactions.add(this.getTransaction(transactionHash.txHash)); + } + } + + if (areAllKeysUnused) + // No transactions for this batch of keys so assume we're done searching. + break; + + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); + + // Process new keys + } while (true); + + Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); + + return walletTransactions.stream().map(t -> convertToSimpleTransaction(t, keySet)).sorted(newestTimestampFirstComparator).collect(Collectors.toList()); + } + + protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) { + long amount = 0; + long total = 0L; + for (BitcoinyTransaction.Input input : t.inputs) { + try { + BitcoinyTransaction t2 = getTransaction(input.outputTxHash); + List senders = t2.outputs.get(input.outputVout).addresses; + for (String sender : senders) { + if (keySet.contains(sender)) { + total += t2.outputs.get(input.outputVout).value; + } + } + } catch (ForeignBlockchainException e) { + LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash); + } + } + if (t.outputs != null && !t.outputs.isEmpty()) { + for (BitcoinyTransaction.Output output : t.outputs) { + for (String address : output.addresses) { + if (keySet.contains(address)) { + if (total > 0L) { + amount -= (total - output.value); + } else { + amount += output.value; + } + } + } + } + } + return new SimpleTransaction(t.txHash, t.timestamp, amount); + } + + /** + * Returns first unused receive address given 'm' BIP32 key. + * + * @param key58 BIP32/HD extended Bitcoin private/public key + * @return P2PKH address + * @throws ForeignBlockchainException if something went wrong + */ + public String getUnusedReceiveAddress(String key58) throws ForeignBlockchainException { + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); + + final int keyChainPathSize = keyChain.getAccountPath().size(); + List keys = new ArrayList<>(keyChain.getLeafKeys()); + + int ki = 0; + do { + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); + List dKeyPath = dKey.getPath(); + + // If keyChain is based on 'm', then make sure dKey is m/0/ki - i.e. a 'receive' address, not 'change' (m/1/ki) + if (dKeyPath.size() != keyChainPathSize + 2 || dKeyPath.get(dKeyPath.size() - 2) != ChildNumber.ZERO) + continue; + + // Check unspent + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + List unspentOutputs = this.blockchain.getUnspentOutputs(script, false); + + /* + * If there are no unspent outputs then either: + * a) all the outputs have been spent + * b) address has never been used + * + * For case (a) we want to remember not to check this address (key) again. + */ + + if (unspentOutputs.isEmpty()) { + // If this is a known key that has been spent before, then we can skip asking for transaction history + if (this.spentKeys.contains(dKey)) { + wallet.getActiveKeyChain().markKeyAsUsed(dKey); + continue; + } + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); + + if (!historicTransactionHashes.isEmpty()) { + // Fully spent key - case (a) + this.spentKeys.add(dKey); + wallet.getActiveKeyChain().markKeyAsUsed(dKey); + continue; + } + + // Key never been used - case (b) + return address.toString(); + } + + // Key has unspent outputs, hence used, so no good to us + this.spentKeys.remove(dKey); + } + + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); + + // Process new keys + } while (true); + } + + // UTXOProvider support + + static class WalletAwareUTXOProvider implements UTXOProvider { + private final Bitcoiny bitcoiny; + private final Wallet wallet; + + private final DeterministicKeyChain keyChain; + + public WalletAwareUTXOProvider(Bitcoiny bitcoiny, Wallet wallet) { + this.bitcoiny = bitcoiny; + this.wallet = wallet; + this.keyChain = this.wallet.getActiveKeyChain(); + + // Set up wallet's key chain + this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + this.keyChain.maybeLookAhead(); + } + + @Override + public List getOpenTransactionOutputs(List keys) throws UTXOProviderException { + List allUnspentOutputs = new ArrayList<>(); + final boolean coinbase = false; + + int ki = 0; + do { + boolean areAllKeysUnspent = true; + + for (; ki < keys.size(); ++ki) { + ECKey key = keys.get(ki); + + Address address = Address.fromKey(this.bitcoiny.params, key, ScriptType.P2PKH); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + List unspentOutputs; + try { + unspentOutputs = this.bitcoiny.blockchain.getUnspentOutputs(script, false); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException(String.format("Unable to fetch unspent outputs for %s", address)); + } + + /* + * If there are no unspent outputs then either: + * a) all the outputs have been spent + * b) address has never been used + * + * For case (a) we want to remember not to check this address (key) again. + */ + + if (unspentOutputs.isEmpty()) { + // If this is a known key that has been spent before, then we can skip asking for transaction history + if (this.bitcoiny.spentKeys.contains(key)) { + this.wallet.getActiveKeyChain().markKeyAsUsed((DeterministicKey) key); + areAllKeysUnspent = false; + continue; + } + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes; + try { + historicTransactionHashes = this.bitcoiny.blockchain.getAddressTransactions(script, false); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException(String.format("Unable to fetch transaction history for %s", address)); + } + + if (!historicTransactionHashes.isEmpty()) { + // Fully spent key - case (a) + this.bitcoiny.spentKeys.add(key); + this.wallet.getActiveKeyChain().markKeyAsUsed((DeterministicKey) key); + areAllKeysUnspent = false; + } else { + // Key never been used - case (b) + } + + continue; + } + + // If we reach here, then there's definitely at least one unspent key + this.bitcoiny.spentKeys.remove(key); + + for (UnspentOutput unspentOutput : unspentOutputs) { + List transactionOutputs; + try { + transactionOutputs = this.bitcoiny.getOutputs(unspentOutput.hash); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException(String.format("Unable to fetch outputs for TX %s", + HashCode.fromBytes(unspentOutput.hash))); + } + + TransactionOutput transactionOutput = transactionOutputs.get(unspentOutput.index); + + UTXO utxo = new UTXO(Sha256Hash.wrap(unspentOutput.hash), unspentOutput.index, + Coin.valueOf(unspentOutput.value), unspentOutput.height, coinbase, + transactionOutput.getScriptPubKey()); + + allUnspentOutputs.add(utxo); + } + } + + if (areAllKeysUnspent) + // No transactions for this batch of keys so assume we're done searching. + return allUnspentOutputs; + + // Generate some more keys + keys.addAll(Bitcoiny.generateMoreKeys(this.keyChain)); + + // Process new keys + } while (true); + } + + @Override + public int getChainHeadHeight() throws UTXOProviderException { + try { + return this.bitcoiny.blockchain.getCurrentHeight(); + } catch (ForeignBlockchainException e) { + throw new UTXOProviderException("Unable to determine Bitcoiny chain height"); + } + } + + @Override + public NetworkParameters getParams() { + return this.bitcoiny.params; + } + } + + // Utility methods for others + + public static List simplifyWalletTransactions(List transactions) { + // Sort by oldest timestamp first + transactions.sort(Comparator.comparingInt(t -> t.timestamp)); + + // Manual 2nd-level sort same-timestamp transactions so that a transaction's input comes first + int fromIndex = 0; + do { + int timestamp = transactions.get(fromIndex).timestamp; + + int toIndex; + for (toIndex = fromIndex + 1; toIndex < transactions.size(); ++toIndex) + if (transactions.get(toIndex).timestamp != timestamp) + break; + + // Process same-timestamp sub-list + List subList = transactions.subList(fromIndex, toIndex); + + // Only if necessary + if (subList.size() > 1) { + // Quick index lookup + Map indexByTxHash = subList.stream().collect(Collectors.toMap(t -> t.txHash, t -> t.timestamp)); + + int restartIndex = 0; + boolean isSorted; + do { + isSorted = true; + + for (int ourIndex = restartIndex; ourIndex < subList.size(); ++ourIndex) { + BitcoinyTransaction ourTx = subList.get(ourIndex); + + for (BitcoinyTransaction.Input input : ourTx.inputs) { + Integer inputIndex = indexByTxHash.get(input.outputTxHash); + + if (inputIndex != null && inputIndex > ourIndex) { + // Input tx is currently after current tx, so swap + BitcoinyTransaction tmpTx = subList.get(inputIndex); + subList.set(inputIndex, ourTx); + subList.set(ourIndex, tmpTx); + + // Update index lookup too + indexByTxHash.put(ourTx.txHash, inputIndex); + indexByTxHash.put(tmpTx.txHash, ourIndex); + + if (isSorted) + restartIndex = Math.max(restartIndex, ourIndex); + + isSorted = false; + break; + } + } + } + } while (!isSorted); + } + + fromIndex = toIndex; + } while (fromIndex < transactions.size()); + + // Simplify + List simpleTransactions = new ArrayList<>(); + + // Quick lookup of txs in our wallet + Set walletTxHashes = transactions.stream().map(t -> t.txHash).collect(Collectors.toSet()); + + for (BitcoinyTransaction transaction : transactions) { + SimpleForeignTransaction.Builder builder = new SimpleForeignTransaction.Builder(); + builder.txHash(transaction.txHash); + builder.timestamp(transaction.timestamp); + + builder.isSentNotReceived(false); + + for (BitcoinyTransaction.Input input : transaction.inputs) { + // TODO: add input via builder + + if (walletTxHashes.contains(input.outputTxHash)) + builder.isSentNotReceived(true); + } + + for (BitcoinyTransaction.Output output : transaction.outputs) + builder.output(output.addresses, output.value); + + simpleTransactions.add(builder.build()); + } + + return simpleTransactions; + } + + // Utility methods for us + + protected static List generateMoreKeys(DeterministicKeyChain keyChain) { + int existingLeafKeyCount = keyChain.getLeafKeys().size(); + + // Increase lookahead size... + keyChain.setLookaheadSize(keyChain.getLookaheadSize() + Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + // ...and lookahead threshold (minimum number of keys to generate)... + keyChain.setLookaheadThreshold(0); + // ...so that this call will generate more keys + keyChain.maybeLookAhead(); + + // This returns *all* keys + List allLeafKeys = keyChain.getLeafKeys(); + + // Only return newly generated keys + return allLeafKeys.subList(existingLeafKeyCount, allLeafKeys.size()); + } + + protected byte[] addressToScriptPubKey(String base58Address) { + Context.propagate(this.bitcoinjContext); + Address address = Address.fromString(this.params, base58Address); + return ScriptBuilder.createOutputScript(address).getProgram(); + } + + protected Wallet walletFromDeterministicKey58(String key58) { + DeterministicKey dKey = DeterministicKey.deserializeB58(null, key58, this.params); + + if (dKey.hasPrivKey()) + return Wallet.fromSpendingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); + else + return Wallet.fromWatchingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS); + } + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java b/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java new file mode 100644 index 00000000..7691efb1 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java @@ -0,0 +1,40 @@ +package org.qortal.crosschain; + +import java.util.List; + +public abstract class BitcoinyBlockchainProvider { + + public static final boolean INCLUDE_UNCONFIRMED = true; + public static final boolean EXCLUDE_UNCONFIRMED = false; + + /** Returns ID unique to bitcoiny network (e.g. "Litecoin-TEST3") */ + public abstract String getNetId(); + + /** Returns current blockchain height. */ + public abstract int getCurrentHeight() throws ForeignBlockchainException; + + /** Returns a list of raw block headers, starting at startHeight (inclusive), up to count max. */ + public abstract List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException; + + /** Returns balance of address represented by scriptPubKey. */ + public abstract long getConfirmedBalance(byte[] scriptPubKey) throws ForeignBlockchainException; + + /** Returns raw, serialized, transaction bytes given txHash. */ + public abstract byte[] getRawTransaction(String txHash) throws ForeignBlockchainException; + + /** Returns raw, serialized, transaction bytes given txHash. */ + public abstract byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException; + + /** Returns unpacked transaction given txHash. */ + public abstract BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException; + + /** Returns list of transaction hashes (and heights) for address represented by scriptPubKey, optionally including unconfirmed transactions. */ + public abstract List getAddressTransactions(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException; + + /** Returns list of unspent transaction outputs for address represented by scriptPubKey, optionally including unconfirmed transactions. */ + public abstract List getUnspentOutputs(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException; + + /** Broadcasts raw, serialized, transaction bytes to network, returning success/failure. */ + public abstract void broadcastTransaction(byte[] rawTransaction) throws ForeignBlockchainException; + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java b/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java new file mode 100644 index 00000000..8ebfffa2 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinyHTLC.java @@ -0,0 +1,438 @@ +package org.qortal.crosschain; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.Transaction.SigHash; +import org.bitcoinj.core.TransactionInput; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.crypto.TransactionSignature; +import org.bitcoinj.script.Script; +import org.bitcoinj.script.ScriptBuilder; +import org.bitcoinj.script.ScriptChunk; +import org.bitcoinj.script.ScriptOpCodes; +import org.qortal.crypto.Crypto; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +public class BitcoinyHTLC { + + public enum Status { + UNFUNDED, FUNDING_IN_PROGRESS, FUNDED, REDEEM_IN_PROGRESS, REDEEMED, REFUND_IN_PROGRESS, REFUNDED + } + + public static final int SECRET_LENGTH = 32; + public static final int MIN_LOCKTIME = 1500000000; + + public static final long NO_LOCKTIME_NO_RBF_SEQUENCE = 0xFFFFFFFFL; + public static final long LOCKTIME_NO_RBF_SEQUENCE = NO_LOCKTIME_NO_RBF_SEQUENCE - 1; + + // Assuming node's trade-bot has no more than 100 entries? + private static final int MAX_CACHE_ENTRIES = 100; + + // Max time-to-live for cache entries (milliseconds) + private static final long CACHE_TIMEOUT = 30_000L; + + @SuppressWarnings("serial") + private static final Map SECRET_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_CACHE_ENTRIES; + } + }; + private static final byte[] NO_SECRET_CACHE_ENTRY = new byte[0]; + + @SuppressWarnings("serial") + private static final Map STATUS_CACHE = new LinkedHashMap<>(MAX_CACHE_ENTRIES + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_CACHE_ENTRIES; + } + }; + + /* + * OP_TUCK (to copy public key to before signature) + * OP_CHECKSIGVERIFY (sig & pubkey must verify or script fails) + * OP_HASH160 (convert public key to PKH) + * OP_DUP (duplicate PKH) + * OP_EQUAL (does PKH match refund PKH?) + * OP_IF + * OP_DROP (no need for duplicate PKH) + * + * OP_CHECKLOCKTIMEVERIFY (if this passes, leftover stack is so script passes) + * OP_ELSE + * OP_EQUALVERIFY (duplicate PKH must match redeem PKH or script fails) + * OP_HASH160 (hash secret) + * OP_EQUAL (do hashes of secrets match? if true, script passes else script fails) + * OP_ENDIF + */ + + private static final byte[] redeemScript1 = HashCode.fromString("7dada97614").asBytes(); // OP_TUCK OP_CHECKSIGVERIFY OP_HASH160 OP_DUP push(0x14 bytes) + private static final byte[] redeemScript2 = HashCode.fromString("87637504").asBytes(); // OP_EQUAL OP_IF OP_DROP push(0x4 bytes) + private static final byte[] redeemScript3 = HashCode.fromString("b16714").asBytes(); // OP_CHECKLOCKTIMEVERIFY OP_ELSE push(0x14 bytes) + private static final byte[] redeemScript4 = HashCode.fromString("88a914").asBytes(); // OP_EQUALVERIFY OP_HASH160 push(0x14 bytes) + private static final byte[] redeemScript5 = HashCode.fromString("8768").asBytes(); // OP_EQUAL OP_ENDIF + + /** + * Returns redeemScript used for cross-chain trading. + *

    + * See comments in {@link BitcoinyHTLC} for more details. + * + * @param refunderPubKeyHash 20-byte HASH160 of P2SH funder's public key, for refunding purposes + * @param lockTime seconds-since-epoch threshold, after which P2SH funder can claim refund + * @param redeemerPubKeyHash 20-byte HASH160 of P2SH redeemer's public key + * @param hashOfSecret 20-byte HASH160 of secret, used by P2SH redeemer to claim funds + */ + public static byte[] buildScript(byte[] refunderPubKeyHash, int lockTime, byte[] redeemerPubKeyHash, byte[] hashOfSecret) { + return Bytes.concat(redeemScript1, refunderPubKeyHash, redeemScript2, BitTwiddling.toLEByteArray((int) (lockTime & 0xffffffffL)), + redeemScript3, redeemerPubKeyHash, redeemScript4, hashOfSecret, redeemScript5); + } + + /** + * Builds a custom transaction to spend HTLC P2SH. + * + * @param params blockchain network parameters + * @param amount output amount, should be total of input amounts, less miner fees + * @param spendKey key for signing transaction, and also where funds are 'sent' (output) + * @param fundingOutput output from transaction that funded P2SH address + * @param redeemScriptBytes the redeemScript itself, in byte[] form + * @param lockTime (optional) transaction nLockTime, used in refund scenario + * @param scriptSigBuilder function for building scriptSig using transaction input signature + * @param outputPublicKeyHash PKH used to create P2PKH output + * @return Signed transaction for spending P2SH + */ + public static Transaction buildP2shTransaction(NetworkParameters params, Coin amount, ECKey spendKey, + List fundingOutputs, byte[] redeemScriptBytes, + Long lockTime, Function scriptSigBuilder, byte[] outputPublicKeyHash) { + Transaction transaction = new Transaction(params); + transaction.setVersion(2); + + // Output is back to P2SH funder + transaction.addOutput(amount, ScriptBuilder.createP2PKHOutputScript(outputPublicKeyHash)); + + for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { + TransactionOutput fundingOutput = fundingOutputs.get(inputIndex); + + // Input (without scriptSig prior to signing) + TransactionInput input = new TransactionInput(params, null, redeemScriptBytes, fundingOutput.getOutPointFor()); + if (lockTime != null) + input.setSequenceNumber(LOCKTIME_NO_RBF_SEQUENCE); // Use max-value - 1, so lockTime can be used but not RBF + else + input.setSequenceNumber(NO_LOCKTIME_NO_RBF_SEQUENCE); // Use max-value, so no lockTime and no RBF + transaction.addInput(input); + } + + // Set locktime after inputs added but before input signatures are generated + if (lockTime != null) + transaction.setLockTime(lockTime); + + for (int inputIndex = 0; inputIndex < fundingOutputs.size(); ++inputIndex) { + // Generate transaction signature for input + final boolean anyoneCanPay = false; + TransactionSignature txSig = transaction.calculateSignature(inputIndex, spendKey, redeemScriptBytes, SigHash.ALL, anyoneCanPay); + + // Calculate transaction signature + byte[] txSigBytes = txSig.encodeToBitcoin(); + + // Build scriptSig using lambda and tx signature + Script scriptSig = scriptSigBuilder.apply(txSigBytes); + + // Set input scriptSig + transaction.getInput(inputIndex).setScriptSig(scriptSig); + } + + return transaction; + } + + /** + * Returns signed transaction claiming refund from HTLC P2SH. + * + * @param params blockchain network parameters + * @param refundAmount refund amount, should be total of input amounts, less miner fees + * @param refundKey key for signing transaction + * @param fundingOutputs outputs from transaction that funded P2SH address + * @param redeemScriptBytes the redeemScript itself, in byte[] form + * @param lockTime transaction nLockTime - must be at least locktime used in redeemScript + * @param receivingAccountInfo public-key-hash used for P2PKH output + * @return Signed transaction for refunding P2SH + */ + public static Transaction buildRefundTransaction(NetworkParameters params, Coin refundAmount, ECKey refundKey, + List fundingOutputs, byte[] redeemScriptBytes, long lockTime, byte[] receivingAccountInfo) { + Function refundSigScriptBuilder = (txSigBytes) -> { + // Build scriptSig with... + ScriptBuilder scriptBuilder = new ScriptBuilder(); + + // transaction signature + scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); + + // redeem public key + byte[] refundPubKey = refundKey.getPubKey(); + scriptBuilder.addChunk(new ScriptChunk(refundPubKey.length, refundPubKey)); + + // redeem script + scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); + + return scriptBuilder.build(); + }; + + // Send funds back to funding address + return buildP2shTransaction(params, refundAmount, refundKey, fundingOutputs, redeemScriptBytes, lockTime, refundSigScriptBuilder, receivingAccountInfo); + } + + /** + * Returns signed transaction redeeming funds from P2SH address. + * + * @param params blockchain network parameters + * @param redeemAmount redeem amount, should be total of input amounts, less miner fees + * @param redeemKey key for signing transaction + * @param fundingOutputs outputs from transaction that funded P2SH address + * @param redeemScriptBytes the redeemScript itself, in byte[] form + * @param secret actual 32-byte secret used when building redeemScript + * @param receivingAccountInfo Bitcoin PKH used for output + * @return Signed transaction for redeeming P2SH + */ + public static Transaction buildRedeemTransaction(NetworkParameters params, Coin redeemAmount, ECKey redeemKey, + List fundingOutputs, byte[] redeemScriptBytes, byte[] secret, byte[] receivingAccountInfo) { + Function redeemSigScriptBuilder = (txSigBytes) -> { + // Build scriptSig with... + ScriptBuilder scriptBuilder = new ScriptBuilder(); + + // secret + scriptBuilder.addChunk(new ScriptChunk(secret.length, secret)); + + // transaction signature + scriptBuilder.addChunk(new ScriptChunk(txSigBytes.length, txSigBytes)); + + // redeem public key + byte[] redeemPubKey = redeemKey.getPubKey(); + scriptBuilder.addChunk(new ScriptChunk(redeemPubKey.length, redeemPubKey)); + + // redeem script + scriptBuilder.addChunk(new ScriptChunk(ScriptOpCodes.OP_PUSHDATA1, redeemScriptBytes)); + + return scriptBuilder.build(); + }; + + return buildP2shTransaction(params, redeemAmount, redeemKey, fundingOutputs, redeemScriptBytes, null, redeemSigScriptBuilder, receivingAccountInfo); + } + + /** + * Returns 'secret', if any, given HTLC's P2SH address. + *

    + * @throws ForeignBlockchainException + */ + public static byte[] findHtlcSecret(Bitcoiny bitcoiny, String p2shAddress) throws ForeignBlockchainException { + NetworkParameters params = bitcoiny.getNetworkParameters(); + String compoundKey = String.format("%s-%s-%d", params.getId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT); + + byte[] secret = SECRET_CACHE.getOrDefault(compoundKey, NO_SECRET_CACHE_ENTRY); + if (secret != NO_SECRET_CACHE_ENTRY) + return secret; + + List rawTransactions = bitcoiny.getAddressTransactions(p2shAddress); + + for (byte[] rawTransaction : rawTransactions) { + Transaction transaction = new Transaction(params, rawTransaction); + + // Cycle through inputs, looking for one that spends our HTLC + for (TransactionInput input : transaction.getInputs()) { + Script scriptSig = input.getScriptSig(); + List scriptChunks = scriptSig.getChunks(); + + // Expected number of script chunks for redeem. Refund might not have the same number. + int expectedChunkCount = 1 /*secret*/ + 1 /*sig*/ + 1 /*pubkey*/ + 1 /*redeemScript*/; + if (scriptChunks.size() != expectedChunkCount) + continue; + + // We're expecting last chunk to contain the actual redeemScript + ScriptChunk lastChunk = scriptChunks.get(scriptChunks.size() - 1); + byte[] redeemScriptBytes = lastChunk.data; + + // If non-push scripts, redeemScript will be null + if (redeemScriptBytes == null) + continue; + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address inputAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!inputAddress.toString().equals(p2shAddress)) + // Input isn't spending our HTLC + continue; + + secret = scriptChunks.get(0).data; + if (secret.length != BitcoinyHTLC.SECRET_LENGTH) + continue; + + // Cache secret for a while + SECRET_CACHE.put(compoundKey, secret); + + return secret; + } + } + + // Cache negative result + SECRET_CACHE.put(compoundKey, null); + + return null; + } + + /** + * Returns HTLC status, given P2SH address and expected redeem/refund amount + *

    + * @throws ForeignBlockchainException if error occurs + */ + public static Status determineHtlcStatus(BitcoinyBlockchainProvider blockchain, String p2shAddress, long minimumAmount) throws ForeignBlockchainException { + String compoundKey = String.format("%s-%s-%d", blockchain.getNetId(), p2shAddress, System.currentTimeMillis() / CACHE_TIMEOUT); + + Status cachedStatus = STATUS_CACHE.getOrDefault(compoundKey, null); + if (cachedStatus != null) + return cachedStatus; + + byte[] ourScriptPubKey = addressToScriptPubKey(p2shAddress); + List transactionHashes = blockchain.getAddressTransactions(ourScriptPubKey, BitcoinyBlockchainProvider.INCLUDE_UNCONFIRMED); + + // Sort by confirmed first, followed by ascending height + transactionHashes.sort(TransactionHash.CONFIRMED_FIRST.thenComparing(TransactionHash::getHeight)); + + // Transaction cache + Map transactionsByHash = new HashMap<>(); + // HASH160(redeem script) for this p2shAddress + byte[] ourRedeemScriptHash = addressToRedeemScriptHash(p2shAddress); + + // Check for spends first, caching full transaction info as we progress just in case we don't return in this loop + for (TransactionHash transactionInfo : transactionHashes) { + BitcoinyTransaction bitcoinyTransaction = blockchain.getTransaction(transactionInfo.txHash); + + // Cache for possible later reuse + transactionsByHash.put(transactionInfo.txHash, bitcoinyTransaction); + + // Acceptable funding is one transaction output, so we're expecting only one input + if (bitcoinyTransaction.inputs.size() != 1) + // Wrong number of inputs + continue; + + String scriptSig = bitcoinyTransaction.inputs.get(0).scriptSig; + + List scriptSigChunks = extractScriptSigChunks(HashCode.fromString(scriptSig).asBytes()); + if (scriptSigChunks.size() < 3 || scriptSigChunks.size() > 4) + // Not valid chunks for our form of HTLC + continue; + + // Last chunk is redeem script + byte[] redeemScriptBytes = scriptSigChunks.get(scriptSigChunks.size() - 1); + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + if (!Arrays.equals(redeemScriptHash, ourRedeemScriptHash)) + // Not spending our specific HTLC redeem script + continue; + + if (scriptSigChunks.size() == 4) + // If we have 4 chunks, then secret is present, hence redeem + cachedStatus = transactionInfo.height == 0 ? Status.REDEEM_IN_PROGRESS : Status.REDEEMED; + else + cachedStatus = transactionInfo.height == 0 ? Status.REFUND_IN_PROGRESS : Status.REFUNDED; + + STATUS_CACHE.put(compoundKey, cachedStatus); + return cachedStatus; + } + + String ourScriptPubKeyHex = HashCode.fromBytes(ourScriptPubKey).toString(); + + // Check for funding + for (TransactionHash transactionInfo : transactionHashes) { + BitcoinyTransaction bitcoinyTransaction = transactionsByHash.get(transactionInfo.txHash); + if (bitcoinyTransaction == null) + // Should be present in map! + throw new ForeignBlockchainException("Cached Bitcoin transaction now missing?"); + + // Check outputs for our specific P2SH + for (BitcoinyTransaction.Output output : bitcoinyTransaction.outputs) { + // Check amount + if (output.value < minimumAmount) + // Output amount too small (not taking fees into account) + continue; + + String scriptPubKeyHex = output.scriptPubKey; + if (!scriptPubKeyHex.equals(ourScriptPubKeyHex)) + // Not funding our specific P2SH + continue; + + cachedStatus = transactionInfo.height == 0 ? Status.FUNDING_IN_PROGRESS : Status.FUNDED; + STATUS_CACHE.put(compoundKey, cachedStatus); + return cachedStatus; + } + } + + cachedStatus = Status.UNFUNDED; + STATUS_CACHE.put(compoundKey, cachedStatus); + return cachedStatus; + } + + private static List extractScriptSigChunks(byte[] scriptSigBytes) { + List chunks = new ArrayList<>(); + + int offset = 0; + int previousOffset = 0; + while (offset < scriptSigBytes.length) { + byte pushOp = scriptSigBytes[offset++]; + + if (pushOp < 0 || pushOp > 0x4c) + // Unacceptable OP + return Collections.emptyList(); + + // Special treatment for OP_PUSHDATA1 + if (pushOp == 0x4c) { + if (offset >= scriptSigBytes.length) + // Run out of scriptSig bytes? + return Collections.emptyList(); + + pushOp = scriptSigBytes[offset++]; + } + + previousOffset = offset; + offset += Byte.toUnsignedInt(pushOp); + + byte[] chunk = Arrays.copyOfRange(scriptSigBytes, previousOffset, offset); + chunks.add(chunk); + } + + return chunks; + } + + private static byte[] addressToScriptPubKey(String p2shAddress) { + // We want the HASH160 part of the P2SH address + byte[] p2shAddressBytes = Base58.decode(p2shAddress); + + byte[] scriptPubKey = new byte[1 + 1 + 20 + 1]; + scriptPubKey[0x00] = (byte) 0xa9; /* OP_HASH160 */ + scriptPubKey[0x01] = (byte) 0x14; /* PUSH 0x14 bytes */ + System.arraycopy(p2shAddressBytes, 1, scriptPubKey, 2, 0x14); + scriptPubKey[0x16] = (byte) 0x87; /* OP_EQUAL */ + + return scriptPubKey; + } + + private static byte[] addressToRedeemScriptHash(String p2shAddress) { + // We want the HASH160 part of the P2SH address + byte[] p2shAddressBytes = Base58.decode(p2shAddress); + + return Arrays.copyOfRange(p2shAddressBytes, 1, 1 + 20); + } + +} diff --git a/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java b/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java new file mode 100644 index 00000000..caf0b36d --- /dev/null +++ b/src/main/java/org/qortal/crosschain/BitcoinyTransaction.java @@ -0,0 +1,146 @@ +package org.qortal.crosschain; + +import java.util.List; +import java.util.stream.Collectors; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlTransient; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BitcoinyTransaction { + + public final String txHash; + + @XmlTransient + public final int size; + + @XmlTransient + public final int locktime; + + // Not present if transaction is unconfirmed + public final Integer timestamp; + + public static class Input { + @XmlTransient + public final String scriptSig; + + @XmlTransient + public final int sequence; + + public final String outputTxHash; + + public final int outputVout; + + // For JAXB + protected Input() { + this.scriptSig = null; + this.sequence = 0; + this.outputTxHash = null; + this.outputVout = 0; + } + + public Input(String scriptSig, int sequence, String outputTxHash, int outputVout) { + this.scriptSig = scriptSig; + this.sequence = sequence; + this.outputTxHash = outputTxHash; + this.outputVout = outputVout; + } + + public String toString() { + return String.format("{output %s:%d, sequence %d, scriptSig %s}", + this.outputTxHash, this.outputVout, this.sequence, this.scriptSig); + } + } + @XmlTransient + public final List inputs; + + public static class Output { + @XmlTransient + public final String scriptPubKey; + + public final long value; + + public final List addresses; + + // For JAXB + protected Output() { + this.scriptPubKey = null; + this.value = 0; + this.addresses = null; + } + + public Output(String scriptPubKey, long value) { + this.scriptPubKey = scriptPubKey; + this.value = value; + this.addresses = null; + } + + public Output(String scriptPubKey, long value, List addresses) { + this.scriptPubKey = scriptPubKey; + this.value = value; + this.addresses = addresses; + } + + public String toString() { + return String.format("{value %d, scriptPubKey %s}", this.value, this.scriptPubKey); + } + } + public final List outputs; + + public final long totalAmount; + + // For JAXB + protected BitcoinyTransaction() { + this.txHash = null; + this.size = 0; + this.locktime = 0; + this.timestamp = 0; + this.inputs = null; + this.outputs = null; + this.totalAmount = 0; + } + + public BitcoinyTransaction(String txHash, int size, int locktime, Integer timestamp, + List inputs, List outputs) { + this.txHash = txHash; + this.size = size; + this.locktime = locktime; + this.timestamp = timestamp; + this.inputs = inputs; + this.outputs = outputs; + + this.totalAmount = outputs.stream().map(output -> output.value).reduce(0L, Long::sum); + } + + public String toString() { + return String.format("txHash %s, size %d, locktime %d, timestamp %d\n" + + "\tinputs: [%s]\n" + + "\toutputs: [%s]\n", + this.txHash, + this.size, + this.locktime, + this.timestamp, + this.inputs.stream().map(Input::toString).collect(Collectors.joining(",\n\t\t")), + this.outputs.stream().map(Output::toString).collect(Collectors.joining(",\n\t\t"))); + } + + @Override + public boolean equals(Object other) { + if (other == this) + return true; + + if (!(other instanceof BitcoinyTransaction)) + return false; + + BitcoinyTransaction otherTransaction = (BitcoinyTransaction) other; + + return this.txHash.equals(otherTransaction.txHash); + } + + @Override + public int hashCode() { + return this.txHash.hashCode(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java new file mode 100644 index 00000000..b34aa199 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -0,0 +1,688 @@ +package org.qortal.crosschain; + +import java.io.IOException; +import java.math.BigDecimal; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Random; +import java.util.Scanner; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.net.ssl.SSLSocketFactory; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.qortal.crypto.Crypto; +import org.qortal.crypto.TrustlessSSLSocketFactory; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +/** ElectrumX network support for querying Bitcoiny-related info like block headers, transaction outputs, etc. */ +public class ElectrumX extends BitcoinyBlockchainProvider { + + private static final Logger LOGGER = LogManager.getLogger(ElectrumX.class); + private static final Random RANDOM = new Random(); + + private static final double MIN_PROTOCOL_VERSION = 1.2; + private static final int BLOCK_HEADER_LENGTH = 80; + + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" + private static final Pattern DAEMON_ERROR_REGEX = Pattern.compile("DaemonError\\(\\{.*'code': ?(-?[0-9]+).*\\}\\)\\z"); // Capture 'code' inside curly-brace content + + /** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */ + private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported"; + + public static class Server { + String hostname; + + public enum ConnectionType { TCP, SSL } + ConnectionType connectionType; + + int port; + + public Server(String hostname, ConnectionType connectionType, int port) { + this.hostname = hostname; + this.connectionType = connectionType; + this.port = port; + } + + @Override + public boolean equals(Object other) { + if (other == this) + return true; + + if (!(other instanceof Server)) + return false; + + Server otherServer = (Server) other; + + return this.connectionType == otherServer.connectionType + && this.port == otherServer.port + && this.hostname.equals(otherServer.hostname); + } + + @Override + public int hashCode() { + return this.hostname.hashCode() ^ this.port; + } + + @Override + public String toString() { + return String.format("%s:%s:%d", this.connectionType.name(), this.hostname, this.port); + } + } + private Set servers = new HashSet<>(); + private List remainingServers = new ArrayList<>(); + private Set uselessServers = Collections.synchronizedSet(new HashSet<>()); + + private final String netId; + private final String expectedGenesisHash; + private final Map defaultPorts = new EnumMap<>(Server.ConnectionType.class); + + private final Object serverLock = new Object(); + private Server currentServer; + private Socket socket; + private Scanner scanner; + private int nextId = 1; + + private static final int TX_CACHE_SIZE = 200; + @SuppressWarnings("serial") + private final Map transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { + // This method is called just after a new entry has been added + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > TX_CACHE_SIZE; + } + }); + + // Constructors + + public ElectrumX(String netId, String genesisHash, Collection initialServerList, Map defaultPorts) { + this.netId = netId; + this.expectedGenesisHash = genesisHash; + this.servers.addAll(initialServerList); + this.defaultPorts.putAll(defaultPorts); + } + + // Methods for use by other classes + + @Override + public String getNetId() { + return this.netId; + } + + /** + * Returns current blockchain height. + *

    + * @throws ForeignBlockchainException if error occurs + */ + @Override + public int getCurrentHeight() throws ForeignBlockchainException { + Object blockObj = this.rpc("blockchain.headers.subscribe"); + if (!(blockObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.headers.subscribe RPC"); + + JSONObject blockJson = (JSONObject) blockObj; + + Object heightObj = blockJson.get("height"); + + if (!(heightObj instanceof Long)) + throw new ForeignBlockchainException.NetworkException("Missing/invalid 'height' in JSON from ElectrumX blockchain.headers.subscribe RPC"); + + return ((Long) heightObj).intValue(); + } + + /** + * Returns list of raw block headers, starting from startHeight inclusive. + *

    + * @throws ForeignBlockchainException if error occurs + */ + @Override + public List getRawBlockHeaders(int startHeight, int count) throws ForeignBlockchainException { + Object blockObj = this.rpc("blockchain.block.headers", startHeight, count); + if (!(blockObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.block.headers RPC"); + + JSONObject blockJson = (JSONObject) blockObj; + + Object countObj = blockJson.get("count"); + Object hexObj = blockJson.get("hex"); + + if (!(countObj instanceof Long) || !(hexObj instanceof String)) + throw new ForeignBlockchainException.NetworkException("Missing/invalid 'count' or 'hex' entries in JSON from ElectrumX blockchain.block.headers RPC"); + + Long returnedCount = (Long) countObj; + String hex = (String) hexObj; + + byte[] raw = HashCode.fromString(hex).asBytes(); + if (raw.length != returnedCount * BLOCK_HEADER_LENGTH) + throw new ForeignBlockchainException.NetworkException("Unexpected raw header length in JSON from ElectrumX blockchain.block.headers RPC"); + + List rawBlockHeaders = new ArrayList<>(returnedCount.intValue()); + for (int i = 0; i < returnedCount; ++i) + rawBlockHeaders.add(Arrays.copyOfRange(raw, i * BLOCK_HEADER_LENGTH, (i + 1) * BLOCK_HEADER_LENGTH)); + + return rawBlockHeaders; + } + + /** + * Returns confirmed balance, based on passed payment script. + *

    + * @return confirmed balance, or zero if script unknown + * @throws ForeignBlockchainException if there was an error + */ + @Override + public long getConfirmedBalance(byte[] script) throws ForeignBlockchainException { + byte[] scriptHash = Crypto.digest(script); + Bytes.reverse(scriptHash); + + Object balanceObj = this.rpc("blockchain.scripthash.get_balance", HashCode.fromBytes(scriptHash).toString()); + if (!(balanceObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Unexpected output from ElectrumX blockchain.scripthash.get_balance RPC"); + + JSONObject balanceJson = (JSONObject) balanceObj; + + Object confirmedBalanceObj = balanceJson.get("confirmed"); + + if (!(confirmedBalanceObj instanceof Long)) + throw new ForeignBlockchainException.NetworkException("Missing confirmed balance from ElectrumX blockchain.scripthash.get_balance RPC"); + + return (Long) balanceJson.get("confirmed"); + } + + /** + * Returns list of unspent outputs pertaining to passed payment script. + *

    + * @return list of unspent outputs, or empty list if script unknown + * @throws ForeignBlockchainException if there was an error. + */ + @Override + public List getUnspentOutputs(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { + byte[] scriptHash = Crypto.digest(script); + Bytes.reverse(scriptHash); + + Object unspentJson = this.rpc("blockchain.scripthash.listunspent", HashCode.fromBytes(scriptHash).toString()); + if (!(unspentJson instanceof JSONArray)) + throw new ForeignBlockchainException("Expected array output from ElectrumX blockchain.scripthash.listunspent RPC"); + + List unspentOutputs = new ArrayList<>(); + for (Object rawUnspent : (JSONArray) unspentJson) { + JSONObject unspent = (JSONObject) rawUnspent; + + int height = ((Long) unspent.get("height")).intValue(); + // We only want unspent outputs from confirmed transactions (and definitely not mempool duplicates with height 0) + if (!includeUnconfirmed && height <= 0) + continue; + + byte[] txHash = HashCode.fromString((String) unspent.get("tx_hash")).asBytes(); + int outputIndex = ((Long) unspent.get("tx_pos")).intValue(); + long value = (Long) unspent.get("value"); + + unspentOutputs.add(new UnspentOutput(txHash, outputIndex, height, value)); + } + + return unspentOutputs; + } + + /** + * Returns raw transaction for passed transaction hash. + *

    + * NOTE: Do not mutate returned byte[]! + * + * @throws ForeignBlockchainException.NotFoundException if transaction not found + * @throws ForeignBlockchainException if error occurs + */ + @Override + public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException { + Object rawTransactionHex; + try { + rawTransactionHex = this.rpc("blockchain.transaction.get", txHash, false); + } catch (ForeignBlockchainException.NetworkException e) { + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) + if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) + throw new ForeignBlockchainException.NotFoundException(e.getMessage()); + + throw e; + } + + if (!(rawTransactionHex instanceof String)) + throw new ForeignBlockchainException.NetworkException("Expected hex string as raw transaction from ElectrumX blockchain.transaction.get RPC"); + + return HashCode.fromString((String) rawTransactionHex).asBytes(); + } + + /** + * Returns raw transaction for passed transaction hash. + *

    + * NOTE: Do not mutate returned byte[]! + * + * @throws ForeignBlockchainException.NotFoundException if transaction not found + * @throws ForeignBlockchainException if error occurs + */ + @Override + public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException { + return getRawTransaction(HashCode.fromBytes(txHash).toString()); + } + + /** + * Returns transaction info for passed transaction hash. + *

    + * @throws ForeignBlockchainException.NotFoundException if transaction not found + * @throws ForeignBlockchainException if error occurs + */ + @Override + public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { + // Check cache first + BitcoinyTransaction transaction = transactionCache.get(txHash); + if (transaction != null) + return transaction; + + Object transactionObj = null; + + do { + try { + transactionObj = this.rpc("blockchain.transaction.get", txHash, true); + } catch (ForeignBlockchainException.NetworkException e) { + // DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'}) + if (Integer.valueOf(-5).equals(e.getDaemonErrorCode())) + throw new ForeignBlockchainException.NotFoundException(e.getMessage()); + + // Some servers also return non-standard responses like this: + // {"error":"verbose transactions are currently unsupported","id":3,"jsonrpc":"2.0"} + // We should probably not use this server any more + if (e.getServer() != null && e.getMessage() != null && e.getMessage().contains(VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE)) { + Server uselessServer = (Server) e.getServer(); + LOGGER.trace(() -> String.format("Server %s doesn't support verbose transactions - barring use of that server", uselessServer)); + this.uselessServers.add(uselessServer); + this.closeServer(uselessServer); + continue; + } + + throw e; + } + } while (transactionObj == null); + + if (!(transactionObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException("Expected JSONObject as response from ElectrumX blockchain.transaction.get RPC"); + + JSONObject transactionJson = (JSONObject) transactionObj; + + Object inputsObj = transactionJson.get("vin"); + if (!(inputsObj instanceof JSONArray)) + throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vin' from ElectrumX blockchain.transaction.get RPC"); + + Object outputsObj = transactionJson.get("vout"); + if (!(outputsObj instanceof JSONArray)) + throw new ForeignBlockchainException.NetworkException("Expected JSONArray for 'vout' from ElectrumX blockchain.transaction.get RPC"); + + try { + int size = ((Long) transactionJson.get("size")).intValue(); + int locktime = ((Long) transactionJson.get("locktime")).intValue(); + + // Timestamp might not be present, e.g. for unconfirmed transaction + Object timeObj = transactionJson.get("time"); + Integer timestamp = timeObj != null + ? ((Long) timeObj).intValue() + : null; + + List inputs = new ArrayList<>(); + for (Object inputObj : (JSONArray) inputsObj) { + JSONObject inputJson = (JSONObject) inputObj; + + String scriptSig = (String) ((JSONObject) inputJson.get("scriptSig")).get("hex"); + int sequence = ((Long) inputJson.get("sequence")).intValue(); + String outputTxHash = (String) inputJson.get("txid"); + int outputVout = ((Long) inputJson.get("vout")).intValue(); + + inputs.add(new BitcoinyTransaction.Input(scriptSig, sequence, outputTxHash, outputVout)); + } + + List outputs = new ArrayList<>(); + for (Object outputObj : (JSONArray) outputsObj) { + JSONObject outputJson = (JSONObject) outputObj; + + String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex"); + long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue(); + + // address too, if present + List addresses = null; + Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses"); + if (addressesObj instanceof JSONArray) { + addresses = new ArrayList<>(); + for (Object addressObj : (JSONArray) addressesObj) + addresses.add((String) addressObj); + } + + outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses)); + } + + transaction = new BitcoinyTransaction(txHash, size, locktime, timestamp, inputs, outputs); + + // Save into cache + transactionCache.put(txHash, transaction); + + return transaction; + } catch (NullPointerException | ClassCastException e) { + // Unexpected / invalid response from ElectrumX server + } + + throw new ForeignBlockchainException.NetworkException("Unexpected JSON format from ElectrumX blockchain.transaction.get RPC"); + } + + /** + * Returns list of transactions, relating to passed payment script. + *

    + * @return list of related transactions, or empty list if script unknown + * @throws ForeignBlockchainException if error occurs + */ + @Override + public List getAddressTransactions(byte[] script, boolean includeUnconfirmed) throws ForeignBlockchainException { + byte[] scriptHash = Crypto.digest(script); + Bytes.reverse(scriptHash); + + Object transactionsJson = this.rpc("blockchain.scripthash.get_history", HashCode.fromBytes(scriptHash).toString()); + if (!(transactionsJson instanceof JSONArray)) + throw new ForeignBlockchainException.NetworkException("Expected array output from ElectrumX blockchain.scripthash.get_history RPC"); + + List transactionHashes = new ArrayList<>(); + + for (Object rawTransactionInfo : (JSONArray) transactionsJson) { + JSONObject transactionInfo = (JSONObject) rawTransactionInfo; + + Long height = (Long) transactionInfo.get("height"); + if (!includeUnconfirmed && (height == null || height == 0)) + // We only want confirmed transactions + continue; + + String txHash = (String) transactionInfo.get("tx_hash"); + + transactionHashes.add(new TransactionHash(height.intValue(), txHash)); + } + + return transactionHashes; + } + + /** + * Broadcasts raw transaction to network. + *

    + * @throws ForeignBlockchainException if error occurs + */ + @Override + public void broadcastTransaction(byte[] transactionBytes) throws ForeignBlockchainException { + Object rawBroadcastResult = this.rpc("blockchain.transaction.broadcast", HashCode.fromBytes(transactionBytes).toString()); + + // We're expecting a simple string that is the transaction hash + if (!(rawBroadcastResult instanceof String)) + throw new ForeignBlockchainException.NetworkException("Unexpected response from ElectrumX blockchain.transaction.broadcast RPC"); + } + + // Class-private utility methods + + /** + * Query current server for its list of peer servers, and return those we can parse. + *

    + * @throws ForeignBlockchainException + * @throws ClassCastException to be handled by caller + */ + private Set serverPeersSubscribe() throws ForeignBlockchainException { + Set newServers = new HashSet<>(); + + Object peers = this.connectedRpc("server.peers.subscribe"); + + for (Object rawPeer : (JSONArray) peers) { + JSONArray peer = (JSONArray) rawPeer; + if (peer.size() < 3) + // We're expecting at least 3 fields for each peer entry: IP, hostname, features + continue; + + String hostname = (String) peer.get(1); + JSONArray features = (JSONArray) peer.get(2); + + for (Object rawFeature : features) { + String feature = (String) rawFeature; + Server.ConnectionType connectionType = null; + Integer port = null; + + switch (feature.charAt(0)) { + case 's': + connectionType = Server.ConnectionType.SSL; + port = this.defaultPorts.get(connectionType); + break; + + case 't': + connectionType = Server.ConnectionType.TCP; + port = this.defaultPorts.get(connectionType); + break; + + default: + // e.g. could be 'v' for protocol version, or 'p' for pruning limit + break; + } + + if (connectionType == null || port == null) + // We couldn't extract any peer connection info? + continue; + + // Possible non-default port? + if (feature.length() > 1) + try { + port = Integer.parseInt(feature.substring(1)); + } catch (NumberFormatException e) { + // no good + continue; // for-loop above + } + + Server newServer = new Server(hostname, connectionType, port); + newServers.add(newServer); + } + } + + return newServers; + } + + /** + * Performs RPC call, with automatic reconnection to different server if needed. + *

    + * @return "result" object from within JSON output + * @throws ForeignBlockchainException if server returns error or something goes wrong + */ + private Object rpc(String method, Object...params) throws ForeignBlockchainException { + synchronized (this.serverLock) { + if (this.remainingServers.isEmpty()) + this.remainingServers.addAll(this.servers); + + while (haveConnection()) { + Object response = connectedRpc(method, params); + if (response != null) + return response; + + // Didn't work, try another server... + this.closeServer(); + } + + // Failed to perform RPC - maybe lack of servers? + throw new ForeignBlockchainException.NetworkException(String.format("Failed to perform ElectrumX RPC %s", method)); + } + } + + /** Returns true if we have, or create, a connection to an ElectrumX server. */ + private boolean haveConnection() throws ForeignBlockchainException { + if (this.currentServer != null) + return true; + + while (!this.remainingServers.isEmpty()) { + Server server = this.remainingServers.remove(RANDOM.nextInt(this.remainingServers.size())); + LOGGER.trace(() -> String.format("Connecting to %s", server)); + + try { + SocketAddress endpoint = new InetSocketAddress(server.hostname, server.port); + int timeout = 5000; // ms + + this.socket = new Socket(); + this.socket.connect(endpoint, timeout); + this.socket.setTcpNoDelay(true); + + if (server.connectionType == Server.ConnectionType.SSL) { + SSLSocketFactory factory = TrustlessSSLSocketFactory.getSocketFactory(); + this.socket = factory.createSocket(this.socket, server.hostname, server.port, true); + } + + this.scanner = new Scanner(this.socket.getInputStream()); + this.scanner.useDelimiter("\n"); + + // Check connection is suitable by asking for server features, including genesis block hash + JSONObject featuresJson = (JSONObject) this.connectedRpc("server.features"); + + if (featuresJson == null || Double.valueOf((String) featuresJson.get("protocol_min")) < MIN_PROTOCOL_VERSION) + continue; + + if (this.expectedGenesisHash != null && !((String) featuresJson.get("genesis_hash")).equals(this.expectedGenesisHash)) + continue; + + // Ask for more servers + Set moreServers = serverPeersSubscribe(); + // Discard duplicate servers we already know + moreServers.removeAll(this.servers); + // Add to both lists + this.remainingServers.addAll(moreServers); + this.servers.addAll(moreServers); + + LOGGER.debug(() -> String.format("Connected to %s", server)); + this.currentServer = server; + return true; + } catch (IOException | ForeignBlockchainException | ClassCastException | NullPointerException e) { + // Didn't work, try another server... + closeServer(); + } + } + + return false; + } + + /** + * Perform RPC using currently connected server. + *

    + * @param method + * @param params + * @return response Object, or null if server fails to respond + * @throws ForeignBlockchainException if server returns error + */ + @SuppressWarnings("unchecked") + private Object connectedRpc(String method, Object...params) throws ForeignBlockchainException { + JSONObject requestJson = new JSONObject(); + requestJson.put("id", this.nextId++); + requestJson.put("method", method); + requestJson.put("jsonrpc", "2.0"); + + JSONArray requestParams = new JSONArray(); + requestParams.addAll(Arrays.asList(params)); + requestJson.put("params", requestParams); + + String request = requestJson.toJSONString() + "\n"; + LOGGER.trace(() -> String.format("Request: %s", request)); + + final String response; + + try { + this.socket.getOutputStream().write(request.getBytes()); + response = scanner.next(); + } catch (IOException | NoSuchElementException e) { + // Unable to send, or receive -- try another server? + return null; + } + + LOGGER.trace(() -> String.format("Response: %s", response)); + + if (response.isEmpty()) + // Empty response - try another server? + return null; + + Object responseObj = JSONValue.parse(response); + if (!(responseObj instanceof JSONObject)) + // Unexpected response - try another server? + return null; + + JSONObject responseJson = (JSONObject) responseObj; + + Object errorObj = responseJson.get("error"); + if (errorObj != null) { + if (errorObj instanceof String) + throw new ForeignBlockchainException.NetworkException(String.format("Unexpected error message from ElectrumX RPC %s: %s", method, (String) errorObj), this.currentServer); + + if (!(errorObj instanceof JSONObject)) + throw new ForeignBlockchainException.NetworkException(String.format("Unexpected error response from ElectrumX RPC %s", method), this.currentServer); + + JSONObject errorJson = (JSONObject) errorObj; + + Object messageObj = errorJson.get("message"); + + if (!(messageObj instanceof String)) + throw new ForeignBlockchainException.NetworkException(String.format("Missing/invalid message in error response from ElectrumX RPC %s", method), this.currentServer); + + String message = (String) messageObj; + + // Some error 'messages' are actually wrapped upstream bitcoind errors: + // "message": "daemon error: DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})" + // We want to detect these and extract the upstream error code for caller's use + Matcher messageMatcher = DAEMON_ERROR_REGEX.matcher(message); + if (messageMatcher.find()) + try { + int daemonErrorCode = Integer.parseInt(messageMatcher.group(1)); + throw new ForeignBlockchainException.NetworkException(daemonErrorCode, message, this.currentServer); + } catch (NumberFormatException e) { + // We couldn't parse the error code integer? Fall-through to generic exception... + } + + throw new ForeignBlockchainException.NetworkException(message, this.currentServer); + } + + return responseJson.get("result"); + } + + /** + * Closes connection to server if it is currently connected server. + * @param server + */ + private void closeServer(Server server) { + synchronized (this.serverLock) { + if (this.currentServer == null || !this.currentServer.equals(server)) + return; + + if (this.socket != null && !this.socket.isClosed()) + try { + this.socket.close(); + } catch (IOException e) { + // We did try... + } + + this.socket = null; + this.scanner = null; + this.currentServer = null; + } + } + + /** Closes connection to currently connected server (if any). */ + private void closeServer() { + synchronized (this.serverLock) { + this.closeServer(this.currentServer); + } + } + +} diff --git a/src/main/java/org/qortal/crosschain/ForeignBlockchain.java b/src/main/java/org/qortal/crosschain/ForeignBlockchain.java new file mode 100644 index 00000000..0a71e9d9 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/ForeignBlockchain.java @@ -0,0 +1,9 @@ +package org.qortal.crosschain; + +public interface ForeignBlockchain { + + public boolean isValidAddress(String address); + + public boolean isValidWalletKey(String walletKey); + +} diff --git a/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java b/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java new file mode 100644 index 00000000..1e658621 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/ForeignBlockchainException.java @@ -0,0 +1,77 @@ +package org.qortal.crosschain; + +@SuppressWarnings("serial") +public class ForeignBlockchainException extends Exception { + + public ForeignBlockchainException() { + super(); + } + + public ForeignBlockchainException(String message) { + super(message); + } + + public static class NetworkException extends ForeignBlockchainException { + private final Integer daemonErrorCode; + private final transient Object server; + + public NetworkException() { + super(); + this.daemonErrorCode = null; + this.server = null; + } + + public NetworkException(String message) { + super(message); + this.daemonErrorCode = null; + this.server = null; + } + + public NetworkException(int errorCode, String message) { + super(message); + this.daemonErrorCode = errorCode; + this.server = null; + } + + public NetworkException(String message, Object server) { + super(message); + this.daemonErrorCode = null; + this.server = server; + } + + public NetworkException(int errorCode, String message, Object server) { + super(message); + this.daemonErrorCode = errorCode; + this.server = server; + } + + public Integer getDaemonErrorCode() { + return this.daemonErrorCode; + } + + public Object getServer() { + return this.server; + } + } + + public static class NotFoundException extends ForeignBlockchainException { + public NotFoundException() { + super(); + } + + public NotFoundException(String message) { + super(message); + } + } + + public static class InsufficientFundsException extends ForeignBlockchainException { + public InsufficientFundsException() { + super(); + } + + public InsufficientFundsException(String message) { + super(message); + } + } + +} diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java new file mode 100644 index 00000000..5cbe4044 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -0,0 +1,175 @@ +package org.qortal.crosschain; + +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; + +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.Context; +import org.bitcoinj.core.NetworkParameters; +import org.libdohj.params.LitecoinMainNetParams; +import org.libdohj.params.LitecoinRegTestParams; +import org.libdohj.params.LitecoinTestNet3Params; +import org.qortal.crosschain.ElectrumX.Server; +import org.qortal.crosschain.ElectrumX.Server.ConnectionType; +import org.qortal.settings.Settings; + +public class Litecoin extends Bitcoiny { + + public static final String CURRENCY_CODE = "LTC"; + + private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(10000); // 0.0001 LTC per 1000 bytes + + // Temporary values until a dynamic fee system is written. + private static final long MAINNET_FEE = 1000L; + private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST + + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); + static { + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); + } + + public enum LitecoinNet { + MAIN { + @Override + public NetworkParameters getParams() { + return LitecoinMainNetParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + // Servers chosen on NO BASIS WHATSOEVER from various sources! + new Server("electrum-ltc.someguy123.net", Server.ConnectionType.SSL, 50002), + new Server("backup.electrum-ltc.org", Server.ConnectionType.TCP, 50001), + new Server("backup.electrum-ltc.org", Server.ConnectionType.SSL, 443), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 50001), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002), + new Server("ltc.rentonisk.com", Server.ConnectionType.TCP, 50001), + new Server("ltc.rentonisk.com", Server.ConnectionType.SSL, 50002), + new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), + new Server("ltc.litepay.ch", Server.ConnectionType.SSL, 50022)); + } + + @Override + public String getGenesisHash() { + return "12a765e31ffd4059bada1e25190f6e98c99d9714d334efa41a195a7e7e04bfe2"; + } + + @Override + public long getP2shFee(Long timestamp) { + // TODO: This will need to be replaced with something better in the near future! + return MAINNET_FEE; + } + }, + TEST3 { + @Override + public NetworkParameters getParams() { + return LitecoinTestNet3Params.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("electrum-ltc.bysh.me", Server.ConnectionType.TCP, 51001), + new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 51002), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 51001), + new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 51002)); + } + + @Override + public String getGenesisHash() { + return "4966625a4b2851d9fdee139e56211a0d88575f59ed816ff5e6a63deb4e3e29a0"; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }, + REGTEST { + @Override + public NetworkParameters getParams() { + return LitecoinRegTestParams.get(); + } + + @Override + public Collection getServers() { + return Arrays.asList( + new Server("localhost", Server.ConnectionType.TCP, 50001), + new Server("localhost", Server.ConnectionType.SSL, 50002)); + } + + @Override + public String getGenesisHash() { + // This is unique to each regtest instance + return null; + } + + @Override + public long getP2shFee(Long timestamp) { + return NON_MAINNET_FEE; + } + }; + + public abstract NetworkParameters getParams(); + public abstract Collection getServers(); + public abstract String getGenesisHash(); + public abstract long getP2shFee(Long timestamp) throws ForeignBlockchainException; + } + + private static Litecoin instance; + + private final LitecoinNet litecoinNet; + + // Constructors and instance + + private Litecoin(LitecoinNet litecoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { + super(blockchain, bitcoinjContext, currencyCode); + this.litecoinNet = litecoinNet; + + LOGGER.info(() -> String.format("Starting Litecoin support using %s", this.litecoinNet.name())); + } + + public static synchronized Litecoin getInstance() { + if (instance == null) { + LitecoinNet litecoinNet = Settings.getInstance().getLitecoinNet(); + + BitcoinyBlockchainProvider electrumX = new ElectrumX("Litecoin-" + litecoinNet.name(), litecoinNet.getGenesisHash(), litecoinNet.getServers(), DEFAULT_ELECTRUMX_PORTS); + Context bitcoinjContext = new Context(litecoinNet.getParams()); + + instance = new Litecoin(litecoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); + } + + return instance; + } + + // Getters & setters + + public static synchronized void resetForTesting() { + instance = null; + } + + // Actual useful methods for use by other classes + + /** Default Litecoin fee is lower than Bitcoin: only 10sats/byte. */ + @Override + public Coin getFeePerKb() { + return DEFAULT_FEE_PER_KB; + } + + /** + * Returns estimated LTC fee, in sats per 1000bytes, optionally for historic timestamp. + * + * @param timestamp optional milliseconds since epoch, or null for 'now' + * @return sats per 1000bytes, or throws ForeignBlockchainException if something went wrong + */ + @Override + public long getP2shFee(Long timestamp) throws ForeignBlockchainException { + return this.litecoinNet.getP2shFee(timestamp); + } + +} diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java new file mode 100644 index 00000000..454e80c2 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java @@ -0,0 +1,853 @@ +package org.qortal.crosschain; + +import static org.ciyam.at.OpCode.calcOffset; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.API; +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.qortal.account.Account; +import org.qortal.asset.Asset; +import org.qortal.at.QortalFunctionCode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.utils.Base58; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +/** + * Cross-chain trade AT + * + *

    + *

      + *
    • Bob generates Litecoin & Qortal 'trade' keys + *
        + *
      • private key required to sign P2SH redeem tx
        • + *
      • private key could be used to create 'secret' (e.g. double-SHA256)
        • + *
      • encrypted private key could be stored in Qortal AT for access by Bob from any node
        • + *
      + *
      • + *
    • Bob deploys Qortal AT + *
        + *
      + *
      • + *
    • Alice finds Qortal AT and wants to trade + *
        + *
      • Alice generates Litecoin & Qortal 'trade' keys
        • + *
      • Alice funds Litecoin P2SH-A
        • + *
      • Alice sends 'offer' MESSAGE to Bob from her Qortal trade address, containing: + *
          + *
        • hash-of-secret-A
          • + *
        • her 'trade' Litecoin PKH
          • + *
        + *
        • + *
      + *
      • + *
    • Bob receives "offer" MESSAGE + *
        + *
      • Checks Alice's P2SH-A
        • + *
      • Sends 'trade' MESSAGE to Qortal AT from his trade address, containing: + *
          + *
        • Alice's trade Qortal address
          • + *
        • Alice's trade Litecoin PKH
          • + *
        • hash-of-secret-A
          • + *
        + *
        • + *
      + *
      • + *
    • Alice checks Qortal AT to confirm it's locked to her + *
        + *
      • Alice sends 'redeem' MESSAGE to Qortal AT from her trade address, containing: + *
          + *
        • secret-A
          • + *
        • Qortal receiving address of her chosing
          • + *
        + *
        • + *
      • AT's QORT funds are sent to Qortal receiving address
        • + *
      + *
      • + *
    • Bob checks AT, extracts secret-A + *
        + *
      • Bob redeems P2SH-A using his Litecoin trade key and secret-A
        • + *
      • P2SH-A LTC funds end up at Litecoin address determined by redeem transaction output(s)
        • + *
      + *
      • + *
    + */ +public class LitecoinACCTv1 implements ACCT { + + public static final String NAME = LitecoinACCTv1.class.getSimpleName(); + public static final byte[] CODE_BYTES_HASH = HashCode.fromString("0fb15ad9ad1867dfbcafa51155481aa15d984ff9506f2b428eca4e2a2feac2b3").asBytes(); // SHA256 of AT code bytes + + public static final int SECRET_LENGTH = 32; + + /** Value offset into AT segment where 'mode' variable (long) is stored. (Multiply by MachineState.VALUE_SIZE for byte offset). */ + private static final int MODE_VALUE_OFFSET = 61; + /** Byte offset into AT state data where 'mode' variable (long) is stored. */ + public static final int MODE_BYTE_OFFSET = MachineState.HEADER_LENGTH + (MODE_VALUE_OFFSET * MachineState.VALUE_SIZE); + + public static class OfferMessageData { + public byte[] partnerLitecoinPKH; + public byte[] hashOfSecretA; + public long lockTimeA; + } + public static final int OFFER_MESSAGE_LENGTH = 20 /*partnerLitecoinPKH*/ + 20 /*hashOfSecretA*/ + 8 /*lockTimeA*/; + public static final int TRADE_MESSAGE_LENGTH = 32 /*partner's Qortal trade address (padded from 25 to 32)*/ + + 24 /*partner's Litecoin PKH (padded from 20 to 24)*/ + + 8 /*AT trade timeout (minutes)*/ + + 24 /*hash of secret-A (padded from 20 to 24)*/ + + 8 /*lockTimeA*/; + public static final int REDEEM_MESSAGE_LENGTH = 32 /*secret-A*/ + 32 /*partner's Qortal receiving address padded from 25 to 32*/; + public static final int CANCEL_MESSAGE_LENGTH = 32 /*AT creator's Qortal address*/; + + private static LitecoinACCTv1 instance; + + private LitecoinACCTv1() { + } + + public static synchronized LitecoinACCTv1 getInstance() { + if (instance == null) + instance = new LitecoinACCTv1(); + + return instance; + } + + @Override + public byte[] getCodeBytesHash() { + return CODE_BYTES_HASH; + } + + @Override + public int getModeByteOffset() { + return MODE_BYTE_OFFSET; + } + + @Override + public ForeignBlockchain getBlockchain() { + return Litecoin.getInstance(); + } + + /** + * Returns Qortal AT creation bytes for cross-chain trading AT. + *

    + * tradeTimeout (minutes) is the time window for the trade partner to send the + * 32-byte secret to the AT, before the AT automatically refunds the AT's creator. + * + * @param creatorTradeAddress AT creator's trade Qortal address + * @param litecoinPublicKeyHash 20-byte HASH160 of creator's trade Litecoin public key + * @param qortAmount how much QORT to pay trade partner if they send correct 32-byte secrets to AT + * @param litecoinAmount how much LTC the AT creator is expecting to trade + * @param tradeTimeout suggested timeout for entire trade + */ + public static byte[] buildQortalAT(String creatorTradeAddress, byte[] litecoinPublicKeyHash, long qortAmount, long litecoinAmount, int tradeTimeout) { + if (litecoinPublicKeyHash.length != 20) + throw new IllegalArgumentException("Litecoin public key hash should be 20 bytes"); + + // Labels for data segment addresses + int addrCounter = 0; + + // Constants (with corresponding dataByteBuffer.put*() calls below) + + final int addrCreatorTradeAddress1 = addrCounter++; + final int addrCreatorTradeAddress2 = addrCounter++; + final int addrCreatorTradeAddress3 = addrCounter++; + final int addrCreatorTradeAddress4 = addrCounter++; + + final int addrLitecoinPublicKeyHash = addrCounter; + addrCounter += 4; + + final int addrQortAmount = addrCounter++; + final int addrLitecoinAmount = addrCounter++; + final int addrTradeTimeout = addrCounter++; + + final int addrMessageTxnType = addrCounter++; + final int addrExpectedTradeMessageLength = addrCounter++; + final int addrExpectedRedeemMessageLength = addrCounter++; + + final int addrCreatorAddressPointer = addrCounter++; + final int addrQortalPartnerAddressPointer = addrCounter++; + final int addrMessageSenderPointer = addrCounter++; + + final int addrTradeMessagePartnerLitecoinPKHOffset = addrCounter++; + final int addrPartnerLitecoinPKHPointer = addrCounter++; + final int addrTradeMessageHashOfSecretAOffset = addrCounter++; + final int addrHashOfSecretAPointer = addrCounter++; + + final int addrRedeemMessageReceivingAddressOffset = addrCounter++; + + final int addrMessageDataPointer = addrCounter++; + final int addrMessageDataLength = addrCounter++; + + final int addrPartnerReceivingAddressPointer = addrCounter++; + + final int addrEndOfConstants = addrCounter; + + // Variables + + final int addrCreatorAddress1 = addrCounter++; + final int addrCreatorAddress2 = addrCounter++; + final int addrCreatorAddress3 = addrCounter++; + final int addrCreatorAddress4 = addrCounter++; + + final int addrQortalPartnerAddress1 = addrCounter++; + final int addrQortalPartnerAddress2 = addrCounter++; + final int addrQortalPartnerAddress3 = addrCounter++; + final int addrQortalPartnerAddress4 = addrCounter++; + + final int addrLockTimeA = addrCounter++; + final int addrRefundTimeout = addrCounter++; + final int addrRefundTimestamp = addrCounter++; + final int addrLastTxnTimestamp = addrCounter++; + final int addrBlockTimestamp = addrCounter++; + final int addrTxnType = addrCounter++; + final int addrResult = addrCounter++; + + final int addrMessageSender1 = addrCounter++; + final int addrMessageSender2 = addrCounter++; + final int addrMessageSender3 = addrCounter++; + final int addrMessageSender4 = addrCounter++; + + final int addrMessageLength = addrCounter++; + + final int addrMessageData = addrCounter; + addrCounter += 4; + + final int addrHashOfSecretA = addrCounter; + addrCounter += 4; + + final int addrPartnerLitecoinPKH = addrCounter; + addrCounter += 4; + + final int addrPartnerReceivingAddress = addrCounter; + addrCounter += 4; + + final int addrMode = addrCounter++; + assert addrMode == MODE_VALUE_OFFSET : String.format("addrMode %d does not match MODE_VALUE_OFFSET %d", addrMode, MODE_VALUE_OFFSET); + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // AT creator's trade Qortal address, decoded from Base58 + assert dataByteBuffer.position() == addrCreatorTradeAddress1 * MachineState.VALUE_SIZE : "addrCreatorTradeAddress1 incorrect"; + byte[] creatorTradeAddressBytes = Base58.decode(creatorTradeAddress); + dataByteBuffer.put(Bytes.ensureCapacity(creatorTradeAddressBytes, 32, 0)); + + // Litecoin public key hash + assert dataByteBuffer.position() == addrLitecoinPublicKeyHash * MachineState.VALUE_SIZE : "addrLitecoinPublicKeyHash incorrect"; + dataByteBuffer.put(Bytes.ensureCapacity(litecoinPublicKeyHash, 32, 0)); + + // Redeem Qort amount + assert dataByteBuffer.position() == addrQortAmount * MachineState.VALUE_SIZE : "addrQortAmount incorrect"; + dataByteBuffer.putLong(qortAmount); + + // Expected Litecoin amount + assert dataByteBuffer.position() == addrLitecoinAmount * MachineState.VALUE_SIZE : "addrLitecoinAmount incorrect"; + dataByteBuffer.putLong(litecoinAmount); + + // Suggested trade timeout (minutes) + assert dataByteBuffer.position() == addrTradeTimeout * MachineState.VALUE_SIZE : "addrTradeTimeout incorrect"; + dataByteBuffer.putLong(tradeTimeout); + + // We're only interested in MESSAGE transactions + assert dataByteBuffer.position() == addrMessageTxnType * MachineState.VALUE_SIZE : "addrMessageTxnType incorrect"; + dataByteBuffer.putLong(API.ATTransactionType.MESSAGE.value); + + // Expected length of 'trade' MESSAGE data from AT creator + assert dataByteBuffer.position() == addrExpectedTradeMessageLength * MachineState.VALUE_SIZE : "addrExpectedTradeMessageLength incorrect"; + dataByteBuffer.putLong(TRADE_MESSAGE_LENGTH); + + // Expected length of 'redeem' MESSAGE data from trade partner + assert dataByteBuffer.position() == addrExpectedRedeemMessageLength * MachineState.VALUE_SIZE : "addrExpectedRedeemMessageLength incorrect"; + dataByteBuffer.putLong(REDEEM_MESSAGE_LENGTH); + + // Index into data segment of AT creator's address, used by GET_B_IND + assert dataByteBuffer.position() == addrCreatorAddressPointer * MachineState.VALUE_SIZE : "addrCreatorAddressPointer incorrect"; + dataByteBuffer.putLong(addrCreatorAddress1); + + // Index into data segment of partner's Qortal address, used by SET_B_IND + assert dataByteBuffer.position() == addrQortalPartnerAddressPointer * MachineState.VALUE_SIZE : "addrQortalPartnerAddressPointer incorrect"; + dataByteBuffer.putLong(addrQortalPartnerAddress1); + + // Index into data segment of (temporary) transaction's sender's address, used by GET_B_IND + assert dataByteBuffer.position() == addrMessageSenderPointer * MachineState.VALUE_SIZE : "addrMessageSenderPointer incorrect"; + dataByteBuffer.putLong(addrMessageSender1); + + // Offset into 'trade' MESSAGE data payload for extracting partner's Litecoin PKH + assert dataByteBuffer.position() == addrTradeMessagePartnerLitecoinPKHOffset * MachineState.VALUE_SIZE : "addrTradeMessagePartnerLitecoinPKHOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Index into data segment of partner's Litecoin PKH, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerLitecoinPKHPointer * MachineState.VALUE_SIZE : "addrPartnerLitecoinPKHPointer incorrect"; + dataByteBuffer.putLong(addrPartnerLitecoinPKH); + + // Offset into 'trade' MESSAGE data payload for extracting hash-of-secret-A + assert dataByteBuffer.position() == addrTradeMessageHashOfSecretAOffset * MachineState.VALUE_SIZE : "addrTradeMessageHashOfSecretAOffset incorrect"; + dataByteBuffer.putLong(64L); + + // Index into data segment to hash of secret A, used by GET_B_IND + assert dataByteBuffer.position() == addrHashOfSecretAPointer * MachineState.VALUE_SIZE : "addrHashOfSecretAPointer incorrect"; + dataByteBuffer.putLong(addrHashOfSecretA); + + // Offset into 'redeem' MESSAGE data payload for extracting Qortal receiving address + assert dataByteBuffer.position() == addrRedeemMessageReceivingAddressOffset * MachineState.VALUE_SIZE : "addrRedeemMessageReceivingAddressOffset incorrect"; + dataByteBuffer.putLong(32L); + + // Source location and length for hashing any passed secret + assert dataByteBuffer.position() == addrMessageDataPointer * MachineState.VALUE_SIZE : "addrMessageDataPointer incorrect"; + dataByteBuffer.putLong(addrMessageData); + assert dataByteBuffer.position() == addrMessageDataLength * MachineState.VALUE_SIZE : "addrMessageDataLength incorrect"; + dataByteBuffer.putLong(32L); + + // Pointer into data segment of where to save partner's receiving Qortal address, used by GET_B_IND + assert dataByteBuffer.position() == addrPartnerReceivingAddressPointer * MachineState.VALUE_SIZE : "addrPartnerReceivingAddressPointer incorrect"; + dataByteBuffer.putLong(addrPartnerReceivingAddress); + + assert dataByteBuffer.position() == addrEndOfConstants * MachineState.VALUE_SIZE : "dataByteBuffer position not at end of constants"; + + // Code labels + Integer labelRefund = null; + + Integer labelTradeTxnLoop = null; + Integer labelCheckTradeTxn = null; + Integer labelCheckCancelTxn = null; + Integer labelNotTradeNorCancelTxn = null; + Integer labelCheckNonRefundTradeTxn = null; + Integer labelTradeTxnExtract = null; + Integer labelRedeemTxnLoop = null; + Integer labelCheckRedeemTxn = null; + Integer labelCheckRedeemTxnSender = null; + Integer labelPayout = null; + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(768); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxnTimestamp)); + + // Load B register with AT creator's address so we can save it into addrCreatorAddress1-4 + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_CREATOR_INTO_B)); + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrCreatorAddressPointer)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message from AT creator's trade address containing trade partner details, or AT owner's address to cancel offer */ + + /* Transaction processing loop */ + labelTradeTxnLoop = codeByteBuffer.position(); + + // Find next transaction (if any) to this AT since the last one (referenced by addrLastTxnTimestamp) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrResult to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckTradeTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckTradeTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelTradeTxnLoop))); + + /* Check transaction's sender. We're expecting AT creator's trade address for 'trade' message, or AT creator's own address for 'cancel' message. */ + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of message sender's address with AT creator's trade address. If they don't match, check for cancel situation. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorTradeAddress1, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorTradeAddress2, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorTradeAddress3, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorTradeAddress4, calcOffset(codeByteBuffer, labelCheckCancelTxn))); + // Message sender's address matches AT creator's trade address so go process 'trade' message + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelCheckNonRefundTradeTxn == null ? 0 : labelCheckNonRefundTradeTxn)); + + /* Checking message sender for possible cancel message */ + labelCheckCancelTxn = codeByteBuffer.position(); + + // Compare each part of message sender's address with AT creator's address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrCreatorAddress1, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrCreatorAddress2, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrCreatorAddress3, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrCreatorAddress4, calcOffset(codeByteBuffer, labelNotTradeNorCancelTxn))); + // Partner address is AT creator's address, so cancel offer and finish. + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.CANCELLED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + /* Not trade nor cancel message */ + labelNotTradeNorCancelTxn = codeByteBuffer.position(); + + // Loop to find another transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Possible switch-to-trade-mode message */ + labelCheckNonRefundTradeTxn = codeByteBuffer.position(); + + // Check 'trade' message we received has expected number of message bytes + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to info extraction code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedTradeMessageLength, calcOffset(codeByteBuffer, labelTradeTxnExtract))); + // Message length didn't match - go back to finding another 'trade' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelTradeTxnLoop == null ? 0 : labelTradeTxnLoop)); + + /* Extracting info from 'trade' MESSAGE transaction */ + labelTradeTxnExtract = codeByteBuffer.position(); + + // Extract message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrQortalPartnerAddress1 (as pointed to by addrQortalPartnerAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrQortalPartnerAddressPointer)); + + // Extract trade partner's Litecoin public key hash (PKH) from message into B + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessagePartnerLitecoinPKHOffset)); + // Store partner's Litecoin PKH (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerLitecoinPKHPointer)); + // Extract AT trade timeout (minutes) (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrRefundTimeout)); + + // Grab next 32 bytes + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrTradeMessageHashOfSecretAOffset)); + + // Extract hash-of-secret-A (we only really use values from B1-B3) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrHashOfSecretAPointer)); + // Extract lockTime-A (from B4) + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_B4, addrLockTimeA)); + + // Calculate trade timeout refund 'timestamp' by adding addrRefundTimeout minutes to this transaction's 'timestamp', then save into addrRefundTimestamp + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.ADD_MINUTES_TO_TIMESTAMP, addrRefundTimestamp, addrLastTxnTimestamp, addrRefundTimeout)); + + /* We are in 'trade mode' */ + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.TRADING.value)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for trade timeout or 'redeem' MESSAGE from Qortal trade partner */ + + // Fetch current block 'timestamp' + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_BLOCK_TIMESTAMP, addrBlockTimestamp)); + // If we're not past refund 'timestamp' then look for next transaction + codeByteBuffer.put(OpCode.BLT_DAT.compile(addrBlockTimestamp, addrRefundTimestamp, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + // We're past refund 'timestamp' so go refund everything back to AT creator + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRefund == null ? 0 : labelRefund)); + + /* Transaction processing loop */ + labelRedeemTxnLoop = codeByteBuffer.position(); + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxnTimestamp)); + // If no transaction found, A will be zero. If A is zero, set addrComparator to 1, otherwise 0. + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.CHECK_A_IS_ZERO, addrResult)); + // If addrResult is zero (i.e. A is non-zero, transaction was found) then go check transaction + codeByteBuffer.put(OpCode.BZR_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelCheckRedeemTxn))); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + /* Check transaction */ + labelCheckRedeemTxn = codeByteBuffer.position(); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxnTimestamp)); + // Extract transaction type (message/payment) from transaction and save type in addrTxnType + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TYPE_FROM_TX_IN_A, addrTxnType)); + // If transaction type is not MESSAGE type then go look for another transaction + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrTxnType, addrMessageTxnType, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check message payload length */ + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(QortalFunctionCode.GET_MESSAGE_LENGTH_FROM_TX_IN_A.value, addrMessageLength)); + // If message length matches, branch to sender checking code + codeByteBuffer.put(OpCode.BEQ_DAT.compile(addrMessageLength, addrExpectedRedeemMessageLength, calcOffset(codeByteBuffer, labelCheckRedeemTxnSender))); + // Message length didn't match - go back to finding another 'redeem' MESSAGE transaction + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Check transaction's sender */ + labelCheckRedeemTxnSender = codeByteBuffer.position(); + + // Extract sender address from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_ADDRESS_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageSender1 (as pointed to by addrMessageSenderPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageSenderPointer)); + // Compare each part of transaction's sender's address with expected address. If they don't match, look for another transaction. + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender1, addrQortalPartnerAddress1, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender2, addrQortalPartnerAddress2, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender3, addrQortalPartnerAddress3, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + codeByteBuffer.put(OpCode.BNE_DAT.compile(addrMessageSender4, addrQortalPartnerAddress4, calcOffset(codeByteBuffer, labelRedeemTxnLoop))); + + /* Check 'secret-A' in transaction's message */ + + // Extract secret-A from first 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN.compile(FunctionCode.PUT_MESSAGE_FROM_TX_IN_A_INTO_B)); + // Save B register into data segment starting at addrMessageData (as pointed to by addrMessageDataPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrMessageDataPointer)); + // Load B register with expected hash result (as pointed to by addrHashOfSecretAPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.SET_B_IND, addrHashOfSecretAPointer)); + // Perform HASH160 using source data at addrMessageData. (Location and length specified via addrMessageDataPointer and addrMessageDataLength). + // Save the equality result (1 if they match, 0 otherwise) into addrResult. + codeByteBuffer.put(OpCode.EXT_FUN_RET_DAT_2.compile(FunctionCode.CHECK_HASH160_WITH_B, addrResult, addrMessageDataPointer, addrMessageDataLength)); + // If hashes don't match, addrResult will be zero so go find another transaction + codeByteBuffer.put(OpCode.BNZ_DAT.compile(addrResult, calcOffset(codeByteBuffer, labelPayout))); + codeByteBuffer.put(OpCode.JMP_ADR.compile(labelRedeemTxnLoop == null ? 0 : labelRedeemTxnLoop)); + + /* Success! Pay arranged amount to receiving address */ + labelPayout = codeByteBuffer.position(); + + // Extract Qortal receiving address from next 32 bytes of message from transaction into B register + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(QortalFunctionCode.PUT_PARTIAL_MESSAGE_FROM_TX_IN_A_INTO_B.value, addrRedeemMessageReceivingAddressOffset)); + // Save B register into data segment starting at addrPartnerReceivingAddress (as pointed to by addrPartnerReceivingAddressPointer) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_B_IND, addrPartnerReceivingAddressPointer)); + // Pay AT's balance to receiving address + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PAY_TO_ADDRESS_IN_B, addrQortAmount)); + // Set redeemed mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REDEEMED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + + // Fall-through to refunding any remaining balance back to AT creator + + /* Refund balance back to AT creator */ + labelRefund = codeByteBuffer.position(); + + // Set refunded mode + codeByteBuffer.put(OpCode.SET_VAL.compile(addrMode, AcctMode.REFUNDED.value)); + // We're finished forever (finishing auto-refunds remaining balance to AT creator) + codeByteBuffer.put(OpCode.FIN_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile LTC-QORT ACCT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + assert Arrays.equals(Crypto.digest(codeBytes), LitecoinACCTv1.CODE_BYTES_HASH) + : String.format("BTCACCT.CODE_BYTES_HASH mismatch: expected %s, actual %s", HashCode.fromBytes(CODE_BYTES_HASH), HashCode.fromBytes(Crypto.digest(codeBytes))); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { + ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + @Override + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + } + + /** + * Returns CrossChainTradeData with useful info extracted from AT. + */ + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + byte[] addressBytes = new byte[25]; // for general use + String atAddress = atStateData.getATAddress(); + + CrossChainTradeData tradeData = new CrossChainTradeData(); + + tradeData.foreignBlockchain = SupportedBlockchain.LITECOIN.name(); + tradeData.acctName = NAME; + + tradeData.qortalAtAddress = atAddress; + tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); + tradeData.creationTimestamp = creationTimestamp; + + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + + byte[] stateData = atStateData.getStateData(); + ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); + dataByteBuffer.position(MachineState.HEADER_LENGTH); + + /* Constants */ + + // Skip creator's trade address + dataByteBuffer.get(addressBytes); + tradeData.qortalCreatorTradeAddress = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Creator's Litecoin/foreign public key hash + tradeData.creatorForeignPKH = new byte[20]; + dataByteBuffer.get(tradeData.creatorForeignPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - tradeData.creatorForeignPKH.length); // skip to 32 bytes + + // We don't use secret-B + tradeData.hashOfSecretB = null; + + // Redeem payout + tradeData.qortAmount = dataByteBuffer.getLong(); + + // Expected LTC amount + tradeData.expectedForeignAmount = dataByteBuffer.getLong(); + + // Trade timeout + tradeData.tradeTimeout = (int) dataByteBuffer.getLong(); + + // Skip MESSAGE transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'trade' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip expected 'redeem' message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Qortal trade address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message sender + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's Litecoin PKH + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'trade' message data offset for hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to hash-of-secret-A + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip 'redeem' message data offset for partner's Qortal receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to message data + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip message data length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip pointer to partner's receiving address + dataByteBuffer.position(dataByteBuffer.position() + 8); + + /* End of constants / begin variables */ + + // Skip AT creator's address + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Partner's trade address (if present) + dataByteBuffer.get(addressBytes); + String qortalRecipient = Base58.encode(addressBytes); + dataByteBuffer.position(dataByteBuffer.position() + 32 - addressBytes.length); + + // Potential lockTimeA (if in trade mode) + int lockTimeA = (int) dataByteBuffer.getLong(); + + // AT refund timeout (probably only useful for debugging) + int refundTimeout = (int) dataByteBuffer.getLong(); + + // Trade-mode refund timestamp (AT 'timestamp' converted to Qortal block height) + long tradeRefundTimestamp = dataByteBuffer.getLong(); + + // Skip last transaction timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip block timestamp + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip transaction type + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary result + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message sender + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Skip message length + dataByteBuffer.position(dataByteBuffer.position() + 8); + + // Skip temporary message data + dataByteBuffer.position(dataByteBuffer.position() + 8 * 4); + + // Potential hash160 of secret A + byte[] hashOfSecretA = new byte[20]; + dataByteBuffer.get(hashOfSecretA); + dataByteBuffer.position(dataByteBuffer.position() + 32 - hashOfSecretA.length); // skip to 32 bytes + + // Potential partner's Litecoin PKH + byte[] partnerLitecoinPKH = new byte[20]; + dataByteBuffer.get(partnerLitecoinPKH); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerLitecoinPKH.length); // skip to 32 bytes + + // Partner's receiving address (if present) + byte[] partnerReceivingAddress = new byte[25]; + dataByteBuffer.get(partnerReceivingAddress); + dataByteBuffer.position(dataByteBuffer.position() + 32 - partnerReceivingAddress.length); // skip to 32 bytes + + // Trade AT's 'mode' + long modeValue = dataByteBuffer.getLong(); + AcctMode mode = AcctMode.valueOf((int) (modeValue & 0xffL)); + + /* End of variables */ + + if (mode != null && mode != AcctMode.OFFERING) { + tradeData.mode = mode; + tradeData.refundTimeout = refundTimeout; + tradeData.tradeRefundHeight = new Timestamp(tradeRefundTimestamp).blockHeight; + tradeData.qortalPartnerAddress = qortalRecipient; + tradeData.hashOfSecretA = hashOfSecretA; + tradeData.partnerForeignPKH = partnerLitecoinPKH; + tradeData.lockTimeA = lockTimeA; + + if (mode == AcctMode.REDEEMED) + tradeData.qortalPartnerReceivingAddress = Base58.encode(partnerReceivingAddress); + } else { + tradeData.mode = AcctMode.OFFERING; + } + + tradeData.duplicateDeprecated(); + + return tradeData; + } + + /** Returns 'offer' MESSAGE payload for trade partner to send to AT creator's trade address. */ + public static byte[] buildOfferMessage(byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA) { + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes); + } + + /** Returns info extracted from 'offer' MESSAGE payload sent by trade partner to AT creator's trade address, or null if not valid. */ + public static OfferMessageData extractOfferMessageData(byte[] messageData) { + if (messageData == null || messageData.length != OFFER_MESSAGE_LENGTH) + return null; + + OfferMessageData offerMessageData = new OfferMessageData(); + offerMessageData.partnerLitecoinPKH = Arrays.copyOfRange(messageData, 0, 20); + offerMessageData.hashOfSecretA = Arrays.copyOfRange(messageData, 20, 40); + offerMessageData.lockTimeA = BitTwiddling.longFromBEBytes(messageData, 40); + + return offerMessageData; + } + + /** Returns 'trade' MESSAGE payload for AT creator to send to AT. */ + public static byte[] buildTradeMessage(String partnerQortalTradeAddress, byte[] partnerBitcoinPKH, byte[] hashOfSecretA, int lockTimeA, int refundTimeout) { + byte[] data = new byte[TRADE_MESSAGE_LENGTH]; + byte[] partnerQortalAddressBytes = Base58.decode(partnerQortalTradeAddress); + byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA); + byte[] refundTimeoutBytes = BitTwiddling.toBEByteArray((long) refundTimeout); + + System.arraycopy(partnerQortalAddressBytes, 0, data, 0, partnerQortalAddressBytes.length); + System.arraycopy(partnerBitcoinPKH, 0, data, 32, partnerBitcoinPKH.length); + System.arraycopy(refundTimeoutBytes, 0, data, 56, refundTimeoutBytes.length); + System.arraycopy(hashOfSecretA, 0, data, 64, hashOfSecretA.length); + System.arraycopy(lockTimeABytes, 0, data, 88, lockTimeABytes.length); + + return data; + } + + /** Returns 'cancel' MESSAGE payload for AT creator to cancel trade AT. */ + @Override + public byte[] buildCancelMessage(String creatorQortalAddress) { + byte[] data = new byte[CANCEL_MESSAGE_LENGTH]; + byte[] creatorQortalAddressBytes = Base58.decode(creatorQortalAddress); + + System.arraycopy(creatorQortalAddressBytes, 0, data, 0, creatorQortalAddressBytes.length); + + return data; + } + + /** Returns 'redeem' MESSAGE payload for trade partner to send to AT. */ + public static byte[] buildRedeemMessage(byte[] secretA, String qortalReceivingAddress) { + byte[] data = new byte[REDEEM_MESSAGE_LENGTH]; + byte[] qortalReceivingAddressBytes = Base58.decode(qortalReceivingAddress); + + System.arraycopy(secretA, 0, data, 0, secretA.length); + System.arraycopy(qortalReceivingAddressBytes, 0, data, 32, qortalReceivingAddressBytes.length); + + return data; + } + + /** Returns refund timeout (minutes) based on trade partner's 'offer' MESSAGE timestamp and P2SH-A locktime. */ + public static int calcRefundTimeout(long offerMessageTimestamp, int lockTimeA) { + // refund should be triggered halfway between offerMessageTimestamp and lockTimeA + return (int) ((lockTimeA - (offerMessageTimestamp / 1000L)) / 2L / 60L); + } + + public static byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException { + String atAddress = crossChainTradeData.qortalAtAddress; + String redeemerAddress = crossChainTradeData.qortalPartnerAddress; + + // We don't have partner's public key so we check every message to AT + List messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, atAddress, null, null, null); + if (messageTransactionsData == null) + return null; + + // Find 'redeem' message + for (MessageTransactionData messageTransactionData : messageTransactionsData) { + // Check message payload type/encryption + if (messageTransactionData.isText() || messageTransactionData.isEncrypted()) + continue; + + // Check message payload size + byte[] messageData = messageTransactionData.getData(); + if (messageData.length != REDEEM_MESSAGE_LENGTH) + // Wrong payload length + continue; + + // Check sender + if (!Crypto.toAddress(messageTransactionData.getSenderPublicKey()).equals(redeemerAddress)) + // Wrong sender; + continue; + + // Extract secretA + byte[] secretA = new byte[32]; + System.arraycopy(messageData, 0, secretA, 0, secretA.length); + + byte[] hashOfSecretA = Crypto.hash160(secretA); + if (!Arrays.equals(hashOfSecretA, crossChainTradeData.hashOfSecretA)) + continue; + + return secretA; + } + + return null; + } + +} diff --git a/src/main/java/org/qortal/crosschain/SimpleTransaction.java b/src/main/java/org/qortal/crosschain/SimpleTransaction.java new file mode 100644 index 00000000..0fae20a5 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/SimpleTransaction.java @@ -0,0 +1,32 @@ +package org.qortal.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class SimpleTransaction { + private String txHash; + private Integer timestamp; + private long totalAmount; + + public SimpleTransaction() { + } + + public SimpleTransaction(String txHash, Integer timestamp, long totalAmount) { + this.txHash = txHash; + this.timestamp = timestamp; + this.totalAmount = totalAmount; + } + + public String getTxHash() { + return txHash; + } + + public Integer getTimestamp() { + return timestamp; + } + + public long getTotalAmount() { + return totalAmount; + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/SupportedBlockchain.java b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java new file mode 100644 index 00000000..7b6f91f5 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/SupportedBlockchain.java @@ -0,0 +1,113 @@ +package org.qortal.crosschain; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.qortal.utils.ByteArray; +import org.qortal.utils.Triple; + +public enum SupportedBlockchain { + + BITCOIN(Arrays.asList( + Triple.valueOf(BitcoinACCTv1.NAME, BitcoinACCTv1.CODE_BYTES_HASH, BitcoinACCTv1::getInstance) + // Could add improved BitcoinACCTv2 here in the future + )) { + @Override + public ForeignBlockchain getInstance() { + return Bitcoin.getInstance(); + } + + @Override + public ACCT getLatestAcct() { + return BitcoinACCTv1.getInstance(); + } + }, + + LITECOIN(Arrays.asList( + Triple.valueOf(LitecoinACCTv1.NAME, LitecoinACCTv1.CODE_BYTES_HASH, LitecoinACCTv1::getInstance) + )) { + @Override + public ForeignBlockchain getInstance() { + return Litecoin.getInstance(); + } + + @Override + public ACCT getLatestAcct() { + return LitecoinACCTv1.getInstance(); + } + }; + + private static final Map> supportedAcctsByCodeHash = Arrays.stream(SupportedBlockchain.values()) + .map(supportedBlockchain -> supportedBlockchain.supportedAccts) + .flatMap(List::stream) + .collect(Collectors.toUnmodifiableMap(triple -> new ByteArray(triple.getB()), Triple::getC)); + + private static final Map> supportedAcctsByName = Arrays.stream(SupportedBlockchain.values()) + .map(supportedBlockchain -> supportedBlockchain.supportedAccts) + .flatMap(List::stream) + .collect(Collectors.toUnmodifiableMap(Triple::getA, Triple::getC)); + + private static final Map blockchainsByName = Arrays.stream(SupportedBlockchain.values()) + .collect(Collectors.toUnmodifiableMap(Enum::name, blockchain -> blockchain)); + + private final List>> supportedAccts; + + SupportedBlockchain(List>> supportedAccts) { + this.supportedAccts = supportedAccts; + } + + public abstract ForeignBlockchain getInstance(); + public abstract ACCT getLatestAcct(); + + public static Map> getAcctMap() { + return supportedAcctsByCodeHash; + } + + public static SupportedBlockchain fromString(String name) { + return blockchainsByName.get(name); + } + + public static Map> getFilteredAcctMap(SupportedBlockchain blockchain) { + if (blockchain == null) + return getAcctMap(); + + return blockchain.supportedAccts.stream() + .collect(Collectors.toUnmodifiableMap(triple -> new ByteArray(triple.getB()), Triple::getC)); + } + + public static Map> getFilteredAcctMap(String specificBlockchain) { + if (specificBlockchain == null) + return getAcctMap(); + + SupportedBlockchain blockchain = blockchainsByName.get(specificBlockchain); + if (blockchain == null) + return Collections.emptyMap(); + + return getFilteredAcctMap(blockchain); + } + + public static ACCT getAcctByCodeHash(byte[] codeHash) { + ByteArray wrappedCodeHash = new ByteArray(codeHash); + + Supplier acctInstanceSupplier = supportedAcctsByCodeHash.get(wrappedCodeHash); + + if (acctInstanceSupplier == null) + return null; + + return acctInstanceSupplier.get(); + } + + public static ACCT getAcctByName(String acctName) { + Supplier acctInstanceSupplier = supportedAcctsByName.get(acctName); + + if (acctInstanceSupplier == null) + return null; + + return acctInstanceSupplier.get(); + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/TransactionHash.java b/src/main/java/org/qortal/crosschain/TransactionHash.java new file mode 100644 index 00000000..c002ae80 --- /dev/null +++ b/src/main/java/org/qortal/crosschain/TransactionHash.java @@ -0,0 +1,31 @@ +package org.qortal.crosschain; + +import java.util.Comparator; + +public class TransactionHash { + + public static final Comparator CONFIRMED_FIRST = (a, b) -> Boolean.compare(a.height != 0, b.height != 0); + + public final int height; + public final String txHash; + + public TransactionHash(int height, String txHash) { + this.height = height; + this.txHash = txHash; + } + + public int getHeight() { + return this.height; + } + + public String getTxHash() { + return this.txHash; + } + + public String toString() { + return this.height == 0 + ? String.format("txHash %s (unconfirmed)", this.txHash) + : String.format("txHash %s (height %d)", this.txHash, this.height); + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/UnspentOutput.java b/src/main/java/org/qortal/crosschain/UnspentOutput.java new file mode 100644 index 00000000..86aa533d --- /dev/null +++ b/src/main/java/org/qortal/crosschain/UnspentOutput.java @@ -0,0 +1,16 @@ +package org.qortal.crosschain; + +/** Unspent output info as returned by ElectrumX network. */ +public class UnspentOutput { + public final byte[] hash; + public final int index; + public final int height; + public final long value; + + public UnspentOutput(byte[] hash, int index, int height, long value) { + this.hash = hash; + this.index = index; + this.height = height; + this.value = value; + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java b/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java new file mode 100644 index 00000000..69250e54 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java @@ -0,0 +1,109 @@ +package org.qortal.data.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.qortal.crosschain.AcctMode; + +import io.swagger.v3.oas.annotations.media.Schema; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class CrossChainTradeData { + + // Properties + + @Schema(description = "AT's Qortal address") + public String qortalAtAddress; + + @Schema(description = "AT creator's Qortal address") + public String qortalCreator; + + @Schema(description = "AT creator's ephemeral trading key-pair represented as Qortal address") + public String qortalCreatorTradeAddress; + + @Deprecated + @Schema(description = "DEPRECATED: use creatorForeignPKH instead") + public byte[] creatorBitcoinPKH; + + @Schema(description = "AT creator's foreign blockchain trade public-key-hash (PKH)") + public byte[] creatorForeignPKH; + + @Schema(description = "Timestamp when AT was created (milliseconds since epoch)") + public long creationTimestamp; + + @Schema(description = "Suggested trade timeout (minutes)", example = "10080") + public int tradeTimeout; + + @Schema(description = "AT's current QORT balance") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long qortBalance; + + @Schema(description = "HASH160 of 32-byte secret-A") + public byte[] hashOfSecretA; + + @Schema(description = "HASH160 of 32-byte secret-B") + public byte[] hashOfSecretB; + + @Schema(description = "Final QORT payment that will be sent to Qortal trade partner") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long qortAmount; + + @Schema(description = "Trade partner's Qortal address (trade begins when this is set)") + public String qortalPartnerAddress; + + @Schema(description = "Timestamp when AT switched to trade mode") + public Long tradeModeTimestamp; + + @Schema(description = "How long from AT creation until AT triggers automatic refund to AT creator (minutes)") + public Integer refundTimeout; + + @Schema(description = "Actual Qortal block height when AT will automatically refund to AT creator (after trade begins)") + public Integer tradeRefundHeight; + + @Deprecated + @Schema(description = "DEPRECATED: use expectedForeignAmount instread") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long expectedBitcoin; + + @Schema(description = "Amount, in foreign blockchain currency, that AT creator expects trade partner to pay out (excluding miner fees)") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + public long expectedForeignAmount; + + @Schema(description = "Current AT execution mode") + public AcctMode mode; + + @Schema(description = "Suggested P2SH-A nLockTime based on trade timeout") + public Integer lockTimeA; + + @Schema(description = "Suggested P2SH-B nLockTime based on trade timeout") + public Integer lockTimeB; + + @Deprecated + @Schema(description = "DEPRECATED: use partnerForeignPKH instead") + public byte[] partnerBitcoinPKH; + + @Schema(description = "Trade partner's foreign blockchain public-key-hash (PKH)") + public byte[] partnerForeignPKH; + + @Schema(description = "Trade partner's Qortal receiving address") + public String qortalPartnerReceivingAddress; + + public String foreignBlockchain; + + public String acctName; + + // Constructors + + // Necessary for JAXB + public CrossChainTradeData() { + } + + public void duplicateDeprecated() { + this.creatorBitcoinPKH = this.creatorForeignPKH; + this.expectedBitcoin = this.expectedForeignAmount; + this.partnerBitcoinPKH = this.partnerForeignPKH; + } + +} diff --git a/src/main/java/org/qortal/data/crosschain/TradeBotData.java b/src/main/java/org/qortal/data/crosschain/TradeBotData.java new file mode 100644 index 00000000..19481466 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/TradeBotData.java @@ -0,0 +1,268 @@ +package org.qortal.data.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import io.swagger.v3.oas.annotations.media.Schema; +import org.json.JSONObject; + +import org.qortal.utils.Base58; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class TradeBotData { + + private byte[] tradePrivateKey; + + private String acctName; + private String tradeState; + + // Internal use - not shown via API + @XmlTransient + @Schema(hidden = true) + private int tradeStateValue; + + private String creatorAddress; + private String atAddress; + + private long timestamp; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long qortAmount; + + private byte[] tradeNativePublicKey; + private byte[] tradeNativePublicKeyHash; + String tradeNativeAddress; + + private byte[] secret; + private byte[] hashOfSecret; + + private String foreignBlockchain; + private byte[] tradeForeignPublicKey; + private byte[] tradeForeignPublicKeyHash; + + @Deprecated + @Schema(description = "DEPRECATED: use foreignAmount instead", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long bitcoinAmount; + + @Schema(description = "amount in foreign blockchain currency", type = "number") + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long foreignAmount; + + // Never expose this via API + @XmlTransient + @Schema(hidden = true) + private String foreignKey; + + private byte[] lastTransactionSignature; + private Integer lockTimeA; + + // Could be Bitcoin or Qortal... + private byte[] receivingAccountInfo; + + protected TradeBotData() { + /* JAXB */ + } + + public TradeBotData(byte[] tradePrivateKey, String acctName, String tradeState, int tradeStateValue, + String creatorAddress, String atAddress, + long timestamp, long qortAmount, + byte[] tradeNativePublicKey, byte[] tradeNativePublicKeyHash, String tradeNativeAddress, + byte[] secret, byte[] hashOfSecret, + String foreignBlockchain, byte[] tradeForeignPublicKey, byte[] tradeForeignPublicKeyHash, + long foreignAmount, String foreignKey, + byte[] lastTransactionSignature, Integer lockTimeA, byte[] receivingAccountInfo) { + this.tradePrivateKey = tradePrivateKey; + this.acctName = acctName; + this.tradeState = tradeState; + this.tradeStateValue = tradeStateValue; + this.creatorAddress = creatorAddress; + this.atAddress = atAddress; + this.timestamp = timestamp; + this.qortAmount = qortAmount; + this.tradeNativePublicKey = tradeNativePublicKey; + this.tradeNativePublicKeyHash = tradeNativePublicKeyHash; + this.tradeNativeAddress = tradeNativeAddress; + this.secret = secret; + this.hashOfSecret = hashOfSecret; + this.foreignBlockchain = foreignBlockchain; + this.tradeForeignPublicKey = tradeForeignPublicKey; + this.tradeForeignPublicKeyHash = tradeForeignPublicKeyHash; + // deprecated copy + this.bitcoinAmount = foreignAmount; + this.foreignAmount = foreignAmount; + this.foreignKey = foreignKey; + this.lastTransactionSignature = lastTransactionSignature; + this.lockTimeA = lockTimeA; + this.receivingAccountInfo = receivingAccountInfo; + } + + public byte[] getTradePrivateKey() { + return this.tradePrivateKey; + } + + public String getAcctName() { + return this.acctName; + } + + public String getState() { + return this.tradeState; + } + + public void setState(String state) { + this.tradeState = state; + } + + public int getStateValue() { + return this.tradeStateValue; + } + + public void setStateValue(int stateValue) { + this.tradeStateValue = stateValue; + } + + public String getCreatorAddress() { + return this.creatorAddress; + } + + public String getAtAddress() { + return this.atAddress; + } + + public void setAtAddress(String atAddress) { + this.atAddress = atAddress; + } + + public long getTimestamp() { + return this.timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public long getQortAmount() { + return this.qortAmount; + } + + public byte[] getTradeNativePublicKey() { + return this.tradeNativePublicKey; + } + + public byte[] getTradeNativePublicKeyHash() { + return this.tradeNativePublicKeyHash; + } + + public String getTradeNativeAddress() { + return this.tradeNativeAddress; + } + + public byte[] getSecret() { + return this.secret; + } + + public byte[] getHashOfSecret() { + return this.hashOfSecret; + } + + public String getForeignBlockchain() { + return this.foreignBlockchain; + } + + public byte[] getTradeForeignPublicKey() { + return this.tradeForeignPublicKey; + } + + public byte[] getTradeForeignPublicKeyHash() { + return this.tradeForeignPublicKeyHash; + } + + public long getForeignAmount() { + return this.foreignAmount; + } + + public String getForeignKey() { + return this.foreignKey; + } + + public byte[] getLastTransactionSignature() { + return this.lastTransactionSignature; + } + + public void setLastTransactionSignature(byte[] lastTransactionSignature) { + this.lastTransactionSignature = lastTransactionSignature; + } + + public Integer getLockTimeA() { + return this.lockTimeA; + } + + public void setLockTimeA(Integer lockTimeA) { + this.lockTimeA = lockTimeA; + } + + public byte[] getReceivingAccountInfo() { + return this.receivingAccountInfo; + } + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("tradePrivateKey", Base58.encode(this.getTradePrivateKey())); + jsonObject.put("acctName", this.getAcctName()); + jsonObject.put("tradeState", this.getState()); + jsonObject.put("tradeStateValue", this.getStateValue()); + jsonObject.put("creatorAddress", this.getCreatorAddress()); + jsonObject.put("atAddress", this.getAtAddress()); + jsonObject.put("timestamp", this.getTimestamp()); + jsonObject.put("qortAmount", this.getQortAmount()); + if (this.getTradeNativePublicKey() != null) jsonObject.put("tradeNativePublicKey", Base58.encode(this.getTradeNativePublicKey())); + if (this.getTradeNativePublicKeyHash() != null) jsonObject.put("tradeNativePublicKeyHash", Base58.encode(this.getTradeNativePublicKeyHash())); + jsonObject.put("tradeNativeAddress", this.getTradeNativeAddress()); + if (this.getSecret() != null) jsonObject.put("secret", Base58.encode(this.getSecret())); + if (this.getHashOfSecret() != null) jsonObject.put("hashOfSecret", Base58.encode(this.getHashOfSecret())); + jsonObject.put("foreignBlockchain", this.getForeignBlockchain()); + if (this.getTradeForeignPublicKey() != null) jsonObject.put("tradeForeignPublicKey", Base58.encode(this.getTradeForeignPublicKey())); + if (this.getTradeForeignPublicKeyHash() != null) jsonObject.put("tradeForeignPublicKeyHash", Base58.encode(this.getTradeForeignPublicKeyHash())); + jsonObject.put("foreignKey", this.getForeignKey()); + jsonObject.put("foreignAmount", this.getForeignAmount()); + if (this.getLastTransactionSignature() != null) jsonObject.put("lastTransactionSignature", Base58.encode(this.getLastTransactionSignature())); + jsonObject.put("lockTimeA", this.getLockTimeA()); + if (this.getReceivingAccountInfo() != null) jsonObject.put("receivingAccountInfo", Base58.encode(this.getReceivingAccountInfo())); + return jsonObject; + } + + public static TradeBotData fromJson(JSONObject json) { + return new TradeBotData( + json.isNull("tradePrivateKey") ? null : Base58.decode(json.getString("tradePrivateKey")), + json.isNull("acctName") ? null : json.getString("acctName"), + json.isNull("tradeState") ? null : json.getString("tradeState"), + json.isNull("tradeStateValue") ? null : json.getInt("tradeStateValue"), + json.isNull("creatorAddress") ? null : json.getString("creatorAddress"), + json.isNull("atAddress") ? null : json.getString("atAddress"), + json.isNull("timestamp") ? null : json.getLong("timestamp"), + json.isNull("qortAmount") ? null : json.getLong("qortAmount"), + json.isNull("tradeNativePublicKey") ? null : Base58.decode(json.getString("tradeNativePublicKey")), + json.isNull("tradeNativePublicKeyHash") ? null : Base58.decode(json.getString("tradeNativePublicKeyHash")), + json.isNull("tradeNativeAddress") ? null : json.getString("tradeNativeAddress"), + json.isNull("secret") ? null : Base58.decode(json.getString("secret")), + json.isNull("hashOfSecret") ? null : Base58.decode(json.getString("hashOfSecret")), + json.isNull("foreignBlockchain") ? null : json.getString("foreignBlockchain"), + json.isNull("tradeForeignPublicKey") ? null : Base58.decode(json.getString("tradeForeignPublicKey")), + json.isNull("tradeForeignPublicKeyHash") ? null : Base58.decode(json.getString("tradeForeignPublicKeyHash")), + json.isNull("foreignAmount") ? null : json.getLong("foreignAmount"), + json.isNull("foreignKey") ? null : json.getString("foreignKey"), + json.isNull("lastTransactionSignature") ? null : Base58.decode(json.getString("lastTransactionSignature")), + json.isNull("lockTimeA") ? null : json.getInt("lockTimeA"), + json.isNull("receivingAccountInfo") ? null : Base58.decode(json.getString("receivingAccountInfo")) + ); + } + + // Mostly for debugging + public String toString() { + return String.format("%s: %s (%d)", this.atAddress, this.tradeState, this.tradeStateValue); + } + +} diff --git a/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java b/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java new file mode 100644 index 00000000..001bd5b4 --- /dev/null +++ b/src/main/java/org/qortal/data/transaction/PresenceTransactionData.java @@ -0,0 +1,73 @@ +package org.qortal.data.transaction; + +import javax.xml.bind.Unmarshaller; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.TransactionType; + +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.media.Schema.AccessMode; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +@Schema(allOf = { TransactionData.class }) +public class PresenceTransactionData extends TransactionData { + + // Properties + @Schema(description = "sender's public key", example = "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP") + private byte[] senderPublicKey; + + @Schema(accessMode = AccessMode.READ_ONLY) + private int nonce; + + private PresenceType presenceType; + + @Schema(description = "timestamp signature", example = "2yGEbwRFyhPZZckKA") + private byte[] timestampSignature; + + // Constructors + + // For JAXB + protected PresenceTransactionData() { + super(TransactionType.PRESENCE); + } + + public void afterUnmarshal(Unmarshaller u, Object parent) { + this.creatorPublicKey = this.senderPublicKey; + } + + public PresenceTransactionData(BaseTransactionData baseTransactionData, + int nonce, PresenceType presenceType, byte[] timestampSignature) { + super(TransactionType.PRESENCE, baseTransactionData); + + this.senderPublicKey = baseTransactionData.creatorPublicKey; + this.nonce = nonce; + this.presenceType = presenceType; + this.timestampSignature = timestampSignature; + } + + // Getters/Setters + + public byte[] getSenderPublicKey() { + return this.senderPublicKey; + } + + public int getNonce() { + return this.nonce; + } + + public void setNonce(int nonce) { + this.nonce = nonce; + } + + public PresenceType getPresenceType() { + return this.presenceType; + } + + public byte[] getTimestampSignature() { + return this.timestampSignature; + } + +} diff --git a/src/main/java/org/qortal/data/transaction/TransactionData.java b/src/main/java/org/qortal/data/transaction/TransactionData.java index 397693b8..060901f2 100644 --- a/src/main/java/org/qortal/data/transaction/TransactionData.java +++ b/src/main/java/org/qortal/data/transaction/TransactionData.java @@ -40,7 +40,7 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode; GroupApprovalTransactionData.class, SetGroupTransactionData.class, UpdateAssetTransactionData.class, AccountFlagsTransactionData.class, RewardShareTransactionData.class, - AccountLevelTransactionData.class, ChatTransactionData.class + AccountLevelTransactionData.class, ChatTransactionData.class, PresenceTransactionData.class }) //All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) diff --git a/src/main/java/org/qortal/repository/CrossChainRepository.java b/src/main/java/org/qortal/repository/CrossChainRepository.java new file mode 100644 index 00000000..70ebdbf9 --- /dev/null +++ b/src/main/java/org/qortal/repository/CrossChainRepository.java @@ -0,0 +1,21 @@ +package org.qortal.repository; + +import java.util.List; + +import org.qortal.data.crosschain.TradeBotData; + +public interface CrossChainRepository { + + public TradeBotData getTradeBotData(byte[] tradePrivateKey) throws DataException; + + /** Returns true if there is an existing trade-bot entry relating to given AT address, excluding trade-bot entries with given states. */ + public boolean existsTradeWithAtExcludingStates(String atAddress, List excludeStates) throws DataException; + + public List getAllTradeBotData() throws DataException; + + public void save(TradeBotData tradeBotData) throws DataException; + + /** Delete trade-bot states using passed private key. */ + public int delete(byte[] tradePrivateKey) throws DataException; + +} diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index 9cdfe26c..656e6e1e 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -14,6 +14,8 @@ public interface Repository extends AutoCloseable { public ChatRepository getChatRepository(); + public CrossChainRepository getCrossChainRepository(); + public GroupRepository getGroupRepository(); public MessageRepository getMessageRepository(); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java new file mode 100644 index 00000000..29f2994c --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCrossChainRepository.java @@ -0,0 +1,202 @@ +package org.qortal.repository.hsqldb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.CrossChainRepository; +import org.qortal.repository.DataException; + +public class HSQLDBCrossChainRepository implements CrossChainRepository { + + protected HSQLDBRepository repository; + + public HSQLDBCrossChainRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + @Override + public TradeBotData getTradeBotData(byte[] tradePrivateKey) throws DataException { + String sql = "SELECT acct_name, trade_state, trade_state_value, " + + "creator_address, at_address, " + + "updated_when, qort_amount, " + + "trade_native_public_key, trade_native_public_key_hash, " + + "trade_native_address, secret, hash_of_secret, " + + "foreign_blockchain, trade_foreign_public_key, trade_foreign_public_key_hash, " + + "foreign_amount, foreign_key, last_transaction_signature, locktime_a, receiving_account_info " + + "FROM TradeBotStates " + + "WHERE trade_private_key = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, tradePrivateKey)) { + if (resultSet == null) + return null; + + String acctName = resultSet.getString(1); + String tradeState = resultSet.getString(2); + int tradeStateValue = resultSet.getInt(3); + String creatorAddress = resultSet.getString(4); + String atAddress = resultSet.getString(5); + long timestamp = resultSet.getLong(6); + long qortAmount = resultSet.getLong(7); + byte[] tradeNativePublicKey = resultSet.getBytes(8); + byte[] tradeNativePublicKeyHash = resultSet.getBytes(9); + String tradeNativeAddress = resultSet.getString(10); + byte[] secret = resultSet.getBytes(11); + byte[] hashOfSecret = resultSet.getBytes(12); + String foreignBlockchain = resultSet.getString(13); + byte[] tradeForeignPublicKey = resultSet.getBytes(14); + byte[] tradeForeignPublicKeyHash = resultSet.getBytes(15); + long foreignAmount = resultSet.getLong(16); + String foreignKey = resultSet.getString(17); + byte[] lastTransactionSignature = resultSet.getBytes(18); + Integer lockTimeA = resultSet.getInt(19); + if (lockTimeA == 0 && resultSet.wasNull()) + lockTimeA = null; + byte[] receivingAccountInfo = resultSet.getBytes(20); + + return new TradeBotData(tradePrivateKey, acctName, + tradeState, tradeStateValue, + creatorAddress, atAddress, timestamp, qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secret, hashOfSecret, + foreignBlockchain, tradeForeignPublicKey, tradeForeignPublicKeyHash, + foreignAmount, foreignKey, lastTransactionSignature, lockTimeA, receivingAccountInfo); + } catch (SQLException e) { + throw new DataException("Unable to fetch trade-bot trading state from repository", e); + } + } + + @Override + public boolean existsTradeWithAtExcludingStates(String atAddress, List excludeStates) throws DataException { + if (excludeStates == null) + excludeStates = Collections.emptyList(); + + StringBuilder whereClause = new StringBuilder(256); + whereClause.append("at_address = ?"); + + Object[] bindParams = new Object[1 + excludeStates.size()]; + bindParams[0] = atAddress; + + if (!excludeStates.isEmpty()) { + whereClause.append(" AND trade_state NOT IN (?"); + bindParams[1] = excludeStates.get(0); + + for (int i = 1; i < excludeStates.size(); ++i) { + whereClause.append(", ?"); + bindParams[1 + i] = excludeStates.get(i); + } + + whereClause.append(")"); + } + + try { + return this.repository.exists("TradeBotStates", whereClause.toString(), bindParams); + } catch (SQLException e) { + throw new DataException("Unable to check for trade-bot state in repository", e); + } + } + + @Override + public List getAllTradeBotData() throws DataException { + String sql = "SELECT trade_private_key, acct_name, trade_state, trade_state_value, " + + "creator_address, at_address, " + + "updated_when, qort_amount, " + + "trade_native_public_key, trade_native_public_key_hash, " + + "trade_native_address, secret, hash_of_secret, " + + "foreign_blockchain, trade_foreign_public_key, trade_foreign_public_key_hash, " + + "foreign_amount, foreign_key, last_transaction_signature, locktime_a, receiving_account_info " + + "FROM TradeBotStates"; + + List allTradeBotData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return allTradeBotData; + + do { + byte[] tradePrivateKey = resultSet.getBytes(1); + String acctName = resultSet.getString(2); + String tradeState = resultSet.getString(3); + int tradeStateValue = resultSet.getInt(4); + String creatorAddress = resultSet.getString(5); + String atAddress = resultSet.getString(6); + long timestamp = resultSet.getLong(7); + long qortAmount = resultSet.getLong(8); + byte[] tradeNativePublicKey = resultSet.getBytes(9); + byte[] tradeNativePublicKeyHash = resultSet.getBytes(10); + String tradeNativeAddress = resultSet.getString(11); + byte[] secret = resultSet.getBytes(12); + byte[] hashOfSecret = resultSet.getBytes(13); + String foreignBlockchain = resultSet.getString(14); + byte[] tradeForeignPublicKey = resultSet.getBytes(15); + byte[] tradeForeignPublicKeyHash = resultSet.getBytes(16); + long foreignAmount = resultSet.getLong(17); + String foreignKey = resultSet.getString(18); + byte[] lastTransactionSignature = resultSet.getBytes(19); + Integer lockTimeA = resultSet.getInt(20); + if (lockTimeA == 0 && resultSet.wasNull()) + lockTimeA = null; + byte[] receivingAccountInfo = resultSet.getBytes(21); + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, acctName, + tradeState, tradeStateValue, + creatorAddress, atAddress, timestamp, qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + secret, hashOfSecret, + foreignBlockchain, tradeForeignPublicKey, tradeForeignPublicKeyHash, + foreignAmount, foreignKey, lastTransactionSignature, lockTimeA, receivingAccountInfo); + allTradeBotData.add(tradeBotData); + } while (resultSet.next()); + + return allTradeBotData; + } catch (SQLException e) { + throw new DataException("Unable to fetch trade-bot trading states from repository", e); + } + } + + @Override + public void save(TradeBotData tradeBotData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("TradeBotStates"); + + saveHelper.bind("trade_private_key", tradeBotData.getTradePrivateKey()) + .bind("acct_name", tradeBotData.getAcctName()) + .bind("trade_state", tradeBotData.getState()) + .bind("trade_state_value", tradeBotData.getStateValue()) + .bind("creator_address", tradeBotData.getCreatorAddress()) + .bind("at_address", tradeBotData.getAtAddress()) + .bind("updated_when", tradeBotData.getTimestamp()) + .bind("qort_amount", tradeBotData.getQortAmount()) + .bind("trade_native_public_key", tradeBotData.getTradeNativePublicKey()) + .bind("trade_native_public_key_hash", tradeBotData.getTradeNativePublicKeyHash()) + .bind("trade_native_address", tradeBotData.getTradeNativeAddress()) + .bind("secret", tradeBotData.getSecret()) + .bind("hash_of_secret", tradeBotData.getHashOfSecret()) + .bind("foreign_blockchain", tradeBotData.getForeignBlockchain()) + .bind("trade_foreign_public_key", tradeBotData.getTradeForeignPublicKey()) + .bind("trade_foreign_public_key_hash", tradeBotData.getTradeForeignPublicKeyHash()) + .bind("foreign_amount", tradeBotData.getForeignAmount()) + .bind("foreign_key", tradeBotData.getForeignKey()) + .bind("last_transaction_signature", tradeBotData.getLastTransactionSignature()) + .bind("locktime_a", tradeBotData.getLockTimeA()) + .bind("receiving_account_info", tradeBotData.getReceivingAccountInfo()); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save trade bot data into repository", e); + } + } + + @Override + public int delete(byte[] tradePrivateKey) throws DataException { + try { + return this.repository.delete("TradeBotStates", "trade_private_key = ?", tradePrivateKey); + } catch (SQLException e) { + throw new DataException("Unable to delete trade-bot states from repository", e); + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index d6d48acc..9964117b 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -7,6 +7,7 @@ import java.sql.Statement; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.tradebot.BitcoinACCTv1TradeBot; public class HSQLDBDatabaseUpdates { @@ -616,6 +617,17 @@ public class HSQLDBDatabaseUpdates { break; case 20: + // Trade bot + // See case 25 below for changes + stmt.execute("CREATE TABLE TradeBotStates (trade_private_key QortalKeySeed NOT NULL, trade_state TINYINT NOT NULL, " + + "creator_address QortalAddress NOT NULL, at_address QortalAddress, updated_when BIGINT NOT NULL, qort_amount QortalAmount NOT NULL, " + + "trade_native_public_key QortalPublicKey NOT NULL, trade_native_public_key_hash VARBINARY(32) NOT NULL, " + + "trade_native_address QortalAddress NOT NULL, secret VARBINARY(32) NOT NULL, hash_of_secret VARBINARY(32) NOT NULL, " + + "trade_foreign_public_key VARBINARY(33) NOT NULL, trade_foreign_public_key_hash VARBINARY(32) NOT NULL, " + + "bitcoin_amount BIGINT NOT NULL, xprv58 VARCHAR(200), last_transaction_signature Signature, locktime_a BIGINT, " + + "receiving_account_info VARBINARY(32) NOT NULL, PRIMARY KEY (trade_private_key))"); + break; + case 21: // AT functionality index stmt.execute("CREATE INDEX IF NOT EXISTS ATCodeHashIndex ON ATs (code_hash, is_finished)"); @@ -697,6 +709,14 @@ public class HSQLDBDatabaseUpdates { } } + try (ResultSet resultSet = stmt.executeQuery("SELECT COUNT(*) FROM TradeBotStates")) { + int rowCount = resultSet.next() ? resultSet.getInt(1) : 0; + if (rowCount > 0) { + stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'"); + LOGGER.info("Exported sensitive/node-local trade-bot states into TradeBotStates.script"); + } + } + LOGGER.info("If following reshape takes too long, use bootstrap and import node-local data using API's POST /admin/repository/data"); } @@ -761,6 +781,37 @@ public class HSQLDBDatabaseUpdates { break; case 32: + // Multiple blockchains, ACCTs and trade-bots + stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN acct_name VARCHAR(40) BEFORE trade_state"); + stmt.execute("UPDATE TradeBotStates SET acct_name = 'BitcoinACCTv1' WHERE acct_name IS NULL"); + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN acct_name SET NOT NULL"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN trade_state RENAME TO trade_state_value"); + + stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN trade_state VARCHAR(40) BEFORE trade_state_value"); + // Any existing values will be BitcoinACCTv1 + StringBuilder updateTradeBotStatesSql = new StringBuilder(1024); + updateTradeBotStatesSql.append("UPDATE TradeBotStates SET (trade_state) = (") + .append("SELECT state_name FROM (VALUES ") + .append( + Arrays.stream(BitcoinACCTv1TradeBot.State.values()) + .map(state -> String.format("(%d, '%s')", state.value, state.name())) + .collect(Collectors.joining(", "))) + .append(") AS BitcoinACCTv1States (state_value, state_name) ") + .append("WHERE state_value = trade_state_value)"); + stmt.execute(updateTradeBotStatesSql.toString()); + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN trade_state SET NOT NULL"); + + stmt.execute("ALTER TABLE TradeBotStates ADD COLUMN foreign_blockchain VARCHAR(40) BEFORE trade_foreign_public_key"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN bitcoin_amount RENAME TO foreign_amount"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN xprv58 RENAME TO foreign_key"); + + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN secret SET NULL"); + stmt.execute("ALTER TABLE TradeBotStates ALTER COLUMN hash_of_secret SET NULL"); + break; + case 33: // PRESENCE transactions stmt.execute("CREATE TABLE IF NOT EXISTS PresenceTransactions (" diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 02de9f5f..09c6a6d4 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -28,6 +28,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.PrivateKeyAccount; import org.qortal.crypto.Crypto; +import org.qortal.data.crosschain.TradeBotData; import org.qortal.globalization.Translator; import org.qortal.gui.SysTray; import org.qortal.repository.ATRepository; @@ -36,6 +37,7 @@ import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.AssetRepository; import org.qortal.repository.BlockRepository; import org.qortal.repository.ChatRepository; +import org.qortal.repository.CrossChainRepository; import org.qortal.repository.DataException; import org.qortal.repository.GroupRepository; import org.qortal.repository.MessageRepository; @@ -74,6 +76,7 @@ public class HSQLDBRepository implements Repository { private final AssetRepository assetRepository = new HSQLDBAssetRepository(this); private final BlockRepository blockRepository = new HSQLDBBlockRepository(this); private final ChatRepository chatRepository = new HSQLDBChatRepository(this); + private final CrossChainRepository crossChainRepository = new HSQLDBCrossChainRepository(this); private final GroupRepository groupRepository = new HSQLDBGroupRepository(this); private final MessageRepository messageRepository = new HSQLDBMessageRepository(this); private final NameRepository nameRepository = new HSQLDBNameRepository(this); @@ -144,6 +147,11 @@ public class HSQLDBRepository implements Repository { return this.chatRepository; } + @Override + public CrossChainRepository getCrossChainRepository() { + return this.crossChainRepository; + } + @Override public GroupRepository getGroupRepository() { return this.groupRepository; @@ -450,12 +458,68 @@ public class HSQLDBRepository implements Repository { @Override public void exportNodeLocalData() throws DataException { - // TODO + // Create the qortal-backup folder if it doesn't exist + Path backupPath = Paths.get("qortal-backup"); + try { + Files.createDirectories(backupPath); + } catch (IOException e) { + LOGGER.info("Unable to create backup folder"); + throw new DataException("Unable to create backup folder"); + } + + try { + // Load trade bot data + List allTradeBotData = this.getCrossChainRepository().getAllTradeBotData(); + JSONArray allTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + allTradeBotDataJson.put(tradeBotDataJson); + } + + // We need to combine existing TradeBotStates data before overwriting + String fileName = "qortal-backup/TradeBotStates.json"; + File tradeBotStatesBackupFile = new File(fileName); + if (tradeBotStatesBackupFile.exists()) { + String jsonString = new String(Files.readAllBytes(Paths.get(fileName))); + JSONArray allExistingTradeBotData = new JSONArray(jsonString); + Iterator iterator = allExistingTradeBotData.iterator(); + while(iterator.hasNext()) { + JSONObject existingTradeBotData = (JSONObject)iterator.next(); + String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey"); + // Check if we already have an entry for this trade + boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey)); + if (found == false) + // We need to add this to our list + allTradeBotDataJson.put(existingTradeBotData); + } + } + + FileWriter writer = new FileWriter(fileName); + writer.write(allTradeBotDataJson.toString()); + writer.close(); + LOGGER.info("Exported sensitive/node-local data: trade bot states"); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export trade bot states from repository"); + } } @Override public void importDataFromFile(String filename) throws DataException { - // TODO + LOGGER.info(() -> String.format("Importing data into repository from %s", filename)); + try { + String jsonString = new String(Files.readAllBytes(Paths.get(filename))); + JSONArray tradeBotDataToImport = new JSONArray(jsonString); + Iterator iterator = tradeBotDataToImport.iterator(); + while(iterator.hasNext()) { + JSONObject tradeBotDataJson = (JSONObject)iterator.next(); + TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson); + this.getCrossChainRepository().save(tradeBotData); + } + } catch (IOException e) { + throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage()); + } + LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename)); } @Override @@ -992,4 +1056,4 @@ public class HSQLDBRepository implements Repository { return DEADLOCK_ERROR_CODE.equals(e.getErrorCode()); } -} +} \ No newline at end of file diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java new file mode 100644 index 00000000..309ffcad --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBPresenceTransactionRepository.java @@ -0,0 +1,57 @@ +package org.qortal.repository.hsqldb.transaction; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.hsqldb.HSQLDBRepository; +import org.qortal.repository.hsqldb.HSQLDBSaver; +import org.qortal.transaction.PresenceTransaction.PresenceType; + +public class HSQLDBPresenceTransactionRepository extends HSQLDBTransactionRepository { + + public HSQLDBPresenceTransactionRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException { + String sql = "SELECT nonce, presence_type, timestamp_signature FROM PresenceTransactions WHERE signature = ?"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) { + if (resultSet == null) + return null; + + int nonce = resultSet.getInt(1); + int presenceTypeValue = resultSet.getInt(2); + PresenceType presenceType = PresenceType.valueOf(presenceTypeValue); + + byte[] timestampSignature = resultSet.getBytes(3); + + return new PresenceTransactionData(baseTransactionData, nonce, presenceType, timestampSignature); + } catch (SQLException e) { + throw new DataException("Unable to fetch presence transaction from repository", e); + } + } + + @Override + public void save(TransactionData transactionData) throws DataException { + PresenceTransactionData presenceTransactionData = (PresenceTransactionData) transactionData; + + HSQLDBSaver saveHelper = new HSQLDBSaver("PresenceTransactions"); + + saveHelper.bind("signature", presenceTransactionData.getSignature()) + .bind("nonce", presenceTransactionData.getNonce()) + .bind("presence_type", presenceTransactionData.getPresenceType().value) + .bind("timestamp_signature", presenceTransactionData.getTimestampSignature()); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save chat transaction into repository", e); + } + } + +} diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index d0ff70e1..7c26fc22 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -24,6 +24,8 @@ import org.eclipse.persistence.exceptions.XMLMarshalException; import org.eclipse.persistence.jaxb.JAXBContextFactory; import org.eclipse.persistence.jaxb.UnmarshallerProperties; import org.qortal.block.BlockChain; +import org.qortal.crosschain.Bitcoin.BitcoinNet; +import org.qortal.crosschain.Litecoin.LitecoinNet; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @@ -155,6 +157,8 @@ public class Settings { // Which blockchains this node is running private String blockchainConfig = null; // use default from resources + private BitcoinNet bitcoinNet = BitcoinNet.MAIN; + private LitecoinNet litecoinNet = LitecoinNet.MAIN; // Also crosschain-related: /** Whether to show SysTray pop-up notifications when trade-bot entries change state */ private boolean tradebotSystrayEnabled = false; @@ -507,6 +511,14 @@ public class Settings { return this.blockchainConfig; } + public BitcoinNet getBitcoinNet() { + return this.bitcoinNet; + } + + public LitecoinNet getLitecoinNet() { + return this.litecoinNet; + } + public boolean isTradebotSystrayEnabled() { return this.tradebotSystrayEnabled; } diff --git a/src/main/java/org/qortal/transaction/PresenceTransaction.java b/src/main/java/org/qortal/transaction/PresenceTransaction.java new file mode 100644 index 00000000..729270e0 --- /dev/null +++ b/src/main/java/org/qortal/transaction/PresenceTransaction.java @@ -0,0 +1,256 @@ +package org.qortal.transaction; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.account.Account; +import org.qortal.controller.Controller; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crypto.Crypto; +import org.qortal.crypto.MemoryPoW; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.PresenceTransactionTransformer; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.ByteArray; + +import com.google.common.primitives.Longs; + +public class PresenceTransaction extends Transaction { + + private static final Logger LOGGER = LogManager.getLogger(PresenceTransaction.class); + + // Properties + private PresenceTransactionData presenceTransactionData; + + // Other useful constants + public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes + public static final int POW_DIFFICULTY = 8; // leading zero bits + + public enum PresenceType { + REWARD_SHARE(0) { + @Override + public long getLifetime() { + return Controller.ONLINE_TIMESTAMP_MODULUS; + } + }, + TRADE_BOT(1) { + @Override + public long getLifetime() { + return 30 * 60 * 1000L; // 30 minutes in milliseconds + } + }; + + public final int value; + private static final Map map = stream(PresenceType.values()).collect(toMap(type -> type.value, type -> type)); + + PresenceType(int value) { + this.value = value; + } + + public abstract long getLifetime(); + + public static PresenceType valueOf(int value) { + return map.get(value); + } + + /** Returns PresenceType with matching name or null (instead of throwing IllegalArgumentException). */ + public static PresenceType fromString(String name) { + try { + return PresenceType.valueOf(name); + } catch (IllegalArgumentException e) { + return null; + } + } + } + + // Constructors + + public PresenceTransaction(Repository repository, TransactionData transactionData) { + super(repository, transactionData); + + this.presenceTransactionData = (PresenceTransactionData) this.transactionData; + } + + // More information + + @Override + public long getDeadline() { + return this.transactionData.getTimestamp() + this.presenceTransactionData.getPresenceType().getLifetime(); + } + + @Override + public List getRecipientAddresses() throws DataException { + return Collections.emptyList(); + } + + // Navigation + + public Account getSender() { + return this.getCreator(); + } + + // Processing + + public void computeNonce() throws DataException { + byte[] transactionBytes; + + try { + transactionBytes = TransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + // Clear nonce from transactionBytes + PresenceTransactionTransformer.clearNonce(transactionBytes); + + // Calculate nonce + this.presenceTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, POW_DIFFICULTY)); + } + + /** + * Returns whether PRESENCE transaction has valid txGroupId. + *

    + * We insist on NO_GROUP. + */ + @Override + protected boolean isValidTxGroupId() throws DataException { + int txGroupId = this.transactionData.getTxGroupId(); + + return txGroupId == Group.NO_GROUP; + } + + @Override + public ValidationResult isFeeValid() throws DataException { + if (this.transactionData.getFee() < 0) + return ValidationResult.NEGATIVE_FEE; + + return ValidationResult.OK; + } + + @Override + public boolean hasValidReference() throws DataException { + return true; + } + + @Override + public ValidationResult isValid() throws DataException { + // Nonce checking is done via isSignatureValid() as that method is only called once per import + + // If we exist in the repository then we've been imported as unconfirmed, + // but we don't want to make it into a block, so return fake non-OK result. + if (this.repository.getTransactionRepository().exists(this.presenceTransactionData.getSignature())) + return ValidationResult.INVALID_BUT_OK; + + // We only support TRADE_BOT-type PRESENCE at this time + if (PresenceType.TRADE_BOT != this.presenceTransactionData.getPresenceType()) + return ValidationResult.NOT_YET_RELEASED; + + // Check timestamp signature + byte[] timestampSignature = this.presenceTransactionData.getTimestampSignature(); + byte[] timestampBytes = Longs.toByteArray(this.presenceTransactionData.getTimestamp()); + if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), timestampSignature, timestampBytes)) + return ValidationResult.INVALID_TIMESTAMP_SIGNATURE; + + Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + Set codeHashes = acctSuppliersByCodeHash.keySet(); + boolean isExecutable = true; + + List atsData = repository.getATRepository().getAllATsByFunctionality(codeHashes, isExecutable); + + // Convert signer's public key to address form + String signerAddress = Crypto.toAddress(this.transactionData.getCreatorPublicKey()); + + for (ATData atData : atsData) { + ByteArray atCodeHash = new ByteArray(atData.getCodeHash()); + Supplier acctSupplier = acctSuppliersByCodeHash.get(atCodeHash); + if (acctSupplier == null) + continue; + + CrossChainTradeData crossChainTradeData = acctSupplier.get().populateTradeData(repository, atData); + + // OK if signer's public key (in address form) matches Bob's trade public key (in address form) + if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress)) + return ValidationResult.OK; + + // OK if signer's public key (in address form) matches Alice's trade public key (in address form) + if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress)) + return ValidationResult.OK; + } + + return ValidationResult.AT_UNKNOWN; + } + + @Override + public boolean isSignatureValid() { + byte[] signature = this.transactionData.getSignature(); + if (signature == null) + return false; + + byte[] transactionBytes; + + try { + transactionBytes = PresenceTransactionTransformer.toBytesForSigning(this.transactionData); + } catch (TransformationException e) { + throw new RuntimeException("Unable to transform transaction to byte array for verification", e); + } + + if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), signature, transactionBytes)) + return false; + + int nonce = this.presenceTransactionData.getNonce(); + + // Clear nonce from transactionBytes + PresenceTransactionTransformer.clearNonce(transactionBytes); + + // Check nonce + return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, POW_DIFFICULTY, nonce); + } + + /** + * Remove any PRESENCE transactions by the same signer that have older timestamps. + */ + @Override + protected void onImportAsUnconfirmed() throws DataException { + byte[] creatorPublicKey = this.transactionData.getCreatorPublicKey(); + List creatorsPresenceTransactions = this.repository.getTransactionRepository().getUnconfirmedTransactions(TransactionType.PRESENCE, creatorPublicKey); + + if (creatorsPresenceTransactions.isEmpty()) + return; + + for (TransactionData transactionData : creatorsPresenceTransactions) { + if (transactionData.getTimestamp() >= this.transactionData.getTimestamp()) + continue; + + LOGGER.debug(() -> String.format("Deleting older PRESENCE transaction %s", Base58.encode(transactionData.getSignature()))); + this.repository.getTransactionRepository().delete(transactionData); + } + } + + @Override + public void process() throws DataException { + throw new DataException("PRESENCE transactions should never be processed"); + } + + @Override + public void orphan() throws DataException { + throw new DataException("PRESENCE transactions should never be orphaned"); + } + +} diff --git a/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java new file mode 100644 index 00000000..bf69d102 --- /dev/null +++ b/src/main/java/org/qortal/transform/transaction/PresenceTransactionTransformer.java @@ -0,0 +1,108 @@ +package org.qortal.transform.transaction; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.transform.TransformationException; +import org.qortal.utils.Serialization; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; + +public class PresenceTransactionTransformer extends TransactionTransformer { + + // Property lengths + private static final int NONCE_LENGTH = INT_LENGTH; + private static final int PRESENCE_TYPE_LENGTH = BYTE_LENGTH; + private static final int TIMESTAMP_SIGNATURE_LENGTH = SIGNATURE_LENGTH; + + private static final int EXTRAS_LENGTH = NONCE_LENGTH + PRESENCE_TYPE_LENGTH + TIMESTAMP_SIGNATURE_LENGTH; + + protected static final TransactionLayout layout; + + static { + layout = new TransactionLayout(); + layout.add("txType: " + TransactionType.PRESENCE.valueString, TransformationType.INT); + layout.add("timestamp", TransformationType.TIMESTAMP); + layout.add("transaction's groupID", TransformationType.INT); + layout.add("reference", TransformationType.SIGNATURE); + layout.add("sender's public key", TransformationType.PUBLIC_KEY); + layout.add("proof-of-work nonce", TransformationType.INT); + layout.add("presence type (reward-share=0, trade-bot=1)", TransformationType.BYTE); + layout.add("timestamp-signature", TransformationType.SIGNATURE); + layout.add("fee", TransformationType.AMOUNT); + layout.add("signature", TransformationType.SIGNATURE); + } + + public static TransactionData fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { + long timestamp = byteBuffer.getLong(); + + int txGroupId = byteBuffer.getInt(); + + byte[] reference = new byte[REFERENCE_LENGTH]; + byteBuffer.get(reference); + + byte[] senderPublicKey = Serialization.deserializePublicKey(byteBuffer); + + int nonce = byteBuffer.getInt(); + + PresenceType presenceType = PresenceType.valueOf(byteBuffer.get()); + + byte[] timestampSignature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(timestampSignature); + + long fee = byteBuffer.getLong(); + + byte[] signature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(signature); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature); + + return new PresenceTransactionData(baseTransactionData, nonce, presenceType, timestampSignature); + } + + public static int getDataLength(TransactionData transactionData) { + return getBaseLength(transactionData) + EXTRAS_LENGTH; + } + + public static byte[] toBytes(TransactionData transactionData) throws TransformationException { + try { + PresenceTransactionData presenceTransactionData = (PresenceTransactionData) transactionData; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + transformCommonBytes(transactionData, bytes); + + bytes.write(Ints.toByteArray(presenceTransactionData.getNonce())); + + bytes.write(presenceTransactionData.getPresenceType().value); + + bytes.write(presenceTransactionData.getTimestampSignature()); + + bytes.write(Longs.toByteArray(presenceTransactionData.getFee())); + + if (presenceTransactionData.getSignature() != null) + bytes.write(presenceTransactionData.getSignature()); + + return bytes.toByteArray(); + } catch (IOException | ClassCastException e) { + throw new TransformationException(e); + } + } + + public static void clearNonce(byte[] transactionBytes) { + int nonceIndex = TYPE_LENGTH + TIMESTAMP_LENGTH + GROUPID_LENGTH + REFERENCE_LENGTH + PUBLIC_KEY_LENGTH; + + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + transactionBytes[nonceIndex++] = (byte) 0; + } + +} diff --git a/src/test/java/org/qortal/test/PresenceTests.java b/src/test/java/org/qortal/test/PresenceTests.java new file mode 100644 index 00000000..b53b72cb --- /dev/null +++ b/src/test/java/org/qortal/test/PresenceTests.java @@ -0,0 +1,133 @@ +package org.qortal.test; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.PresenceTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.PresenceTransaction; +import org.qortal.transaction.PresenceTransaction.PresenceType; +import org.qortal.transaction.Transaction; +import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.utils.NTP; + +import com.google.common.primitives.Longs; + +import static org.junit.Assert.*; + +public class PresenceTests extends Common { + + private static final byte[] BITCOIN_PKH = new byte[20]; + private static final byte[] HASH_OF_SECRET_B = new byte[32]; + + private PrivateKeyAccount signer; + private Repository repository; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + + this.repository = RepositoryManager.getRepository(); + this.signer = Common.getTestAccount(this.repository, "bob"); + + // We need to create corresponding test trade offer + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(this.signer.getAddress(), BITCOIN_PKH, HASH_OF_SECRET_B, + 0L, 0L, + 7 * 24 * 60 * 60); + + long txTimestamp = NTP.getTime(); + byte[] lastReference = this.signer.getLastReference(); + + long fee = 0; + String name = "QORT-BTC cross-chain trade"; + String description = "Qortal-Bitcoin cross-chain trade"; + String atType = "ACCT"; + String tags = "QORT-BTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, this.signer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, 1L, Asset.QORT); + + Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndImportValid(this.repository, deployAtTransactionData, this.signer); + BlockUtils.mintBlock(this.repository); + } + + @After + public void afterTest() throws DataException { + if (this.repository != null) + this.repository.close(); + + this.repository = null; + } + + @Test + public void validityTests() throws DataException { + long timestamp = System.currentTimeMillis(); + byte[] timestampBytes = Longs.toByteArray(timestamp); + + byte[] timestampSignature = this.signer.sign(timestampBytes); + + assertTrue(isValid(Group.NO_GROUP, this.signer, timestamp, timestampSignature)); + + PrivateKeyAccount nonTrader = Common.getTestAccount(repository, "alice"); + assertFalse(isValid(Group.NO_GROUP, nonTrader, timestamp, timestampSignature)); + } + + @Test + public void newestOnlyTests() throws DataException { + long OLDER_TIMESTAMP = System.currentTimeMillis() - 2000L; + long NEWER_TIMESTAMP = OLDER_TIMESTAMP + 1000L; + + PresenceTransaction older = buildPresenceTransaction(Group.NO_GROUP, this.signer, OLDER_TIMESTAMP, null); + older.computeNonce(); + TransactionUtils.signAndImportValid(repository, older.getTransactionData(), this.signer); + + assertTrue(this.repository.getTransactionRepository().exists(older.getTransactionData().getSignature())); + + PresenceTransaction newer = buildPresenceTransaction(Group.NO_GROUP, this.signer, NEWER_TIMESTAMP, null); + newer.computeNonce(); + TransactionUtils.signAndImportValid(repository, newer.getTransactionData(), this.signer); + + assertTrue(this.repository.getTransactionRepository().exists(newer.getTransactionData().getSignature())); + assertFalse(this.repository.getTransactionRepository().exists(older.getTransactionData().getSignature())); + } + + private boolean isValid(int txGroupId, PrivateKeyAccount signer, long timestamp, byte[] timestampSignature) throws DataException { + Transaction transaction = buildPresenceTransaction(txGroupId, signer, timestamp, timestampSignature); + return transaction.isValidUnconfirmed() == ValidationResult.OK; + } + + private PresenceTransaction buildPresenceTransaction(int txGroupId, PrivateKeyAccount signer, long timestamp, byte[] timestampSignature) throws DataException { + int nonce = 0; + + byte[] reference = signer.getLastReference(); + byte[] creatorPublicKey = signer.getPublicKey(); + long fee = 0L; + + if (timestampSignature == null) + timestampSignature = this.signer.sign(Longs.toByteArray(timestamp)); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, null); + PresenceTransactionData transactionData = new PresenceTransactionData(baseTransactionData, nonce, PresenceType.TRADE_BOT, timestampSignature); + + return new PresenceTransaction(this.repository, transactionData); + } + +} diff --git a/src/test/java/org/qortal/test/RepositoryTests.java b/src/test/java/org/qortal/test/RepositoryTests.java index 269e2aa3..434e03f0 100644 --- a/src/test/java/org/qortal/test/RepositoryTests.java +++ b/src/test/java/org/qortal/test/RepositoryTests.java @@ -4,6 +4,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.crosschain.BitcoinACCTv1; import org.qortal.crypto.Crypto; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -393,6 +394,25 @@ public class RepositoryTests extends Common { } } + /** Specifically test LATERAL() usage in AT repository */ + @Test + public void testAtLateral() { + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + byte[] codeHash = BitcoinACCTv1.CODE_BYTES_HASH; + Boolean isFinished = null; + Integer dataByteOffset = null; + Long expectedValue = null; + Integer minimumFinalHeight = 2; + Integer limit = null; + Integer offset = null; + Boolean reverse = null; + + hsqldb.getATRepository().getMatchingFinalATStates(codeHash, isFinished, dataByteOffset, expectedValue, minimumFinalHeight, limit, offset, reverse); + } catch (DataException e) { + fail("HSQLDB bug #1580"); + } + } + /** Specifically test LATERAL() usage in Chat repository */ @Test public void testChatLateral() { diff --git a/src/test/java/org/qortal/test/api/CrossChainApiTests.java b/src/test/java/org/qortal/test/api/CrossChainApiTests.java new file mode 100644 index 00000000..d4f25bce --- /dev/null +++ b/src/test/java/org/qortal/test/api/CrossChainApiTests.java @@ -0,0 +1,42 @@ +package org.qortal.test.api; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.api.ApiError; +import org.qortal.api.resource.CrossChainResource; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.test.common.ApiCommon; + +public class CrossChainApiTests extends ApiCommon { + + private static final SupportedBlockchain SPECIFIC_BLOCKCHAIN = null; + + private CrossChainResource crossChainResource; + + @Before + public void buildResource() { + this.crossChainResource = (CrossChainResource) ApiCommon.buildResource(CrossChainResource.class); + } + + @Test + public void testGetTradeOffers() { + assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getTradeOffers(SPECIFIC_BLOCKCHAIN, limit, offset, reverse)); + } + + @Test + public void testGetCompletedTrades() { + long minimumTimestamp = System.currentTimeMillis(); + assertNoApiError((limit, offset, reverse) -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, minimumTimestamp, limit, offset, reverse)); + } + + @Test + public void testInvalidGetCompletedTrades() { + Integer limit = null; + Integer offset = null; + Boolean reverse = null; + + assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, -1L /*minimumTimestamp*/, limit, offset, reverse)); + assertApiError(ApiError.INVALID_CRITERIA, () -> this.crossChainResource.getCompletedTrades(SPECIFIC_BLOCKCHAIN, 0L /*minimumTimestamp*/, limit, offset, reverse)); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/BitcoinTests.java b/src/test/java/org/qortal/test/crosschain/BitcoinTests.java new file mode 100644 index 00000000..af879e08 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/BitcoinTests.java @@ -0,0 +1,115 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import java.util.Arrays; + +import org.bitcoinj.core.Transaction; +import org.bitcoinj.store.BlockStoreException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +public class BitcoinTests extends Common { + + private Bitcoin bitcoin; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // TestNet3 + bitcoin = Bitcoin.getInstance(); + } + + @After + public void afterTest() { + Bitcoin.resetForTesting(); + bitcoin = null; + } + + @Test + public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { + System.out.println(String.format("Starting BTC instance...")); + System.out.println(String.format("BTC instance started")); + + long before = System.currentTimeMillis(); + System.out.println(String.format("Bitcoin median blocktime: %d", bitcoin.getMedianBlockTime())); + long afterFirst = System.currentTimeMillis(); + + System.out.println(String.format("Bitcoin median blocktime: %d", bitcoin.getMedianBlockTime())); + long afterSecond = System.currentTimeMillis(); + + long firstPeriod = afterFirst - before; + long secondPeriod = afterSecond - afterFirst; + + System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); + + assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); + assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); + } + + @Test + public void testFindHtlcSecret() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + + assertNotNull(secret); + assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); + } + + @Test + public void testBuildSpend() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String recipient = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + long amount = 1000L; + + Transaction transaction = bitcoin.buildSpend(xprv58, recipient, amount); + assertNotNull(transaction); + + // Check spent key caching doesn't affect outcome + + transaction = bitcoin.buildSpend(xprv58, recipient, amount); + assertNotNull(transaction); + } + + @Test + public void testGetWalletBalance() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + Long balance = bitcoin.getWalletBalance(xprv58); + + assertNotNull(balance); + + System.out.println(bitcoin.format(balance)); + + // Check spent key caching doesn't affect outcome + + Long repeatBalance = bitcoin.getWalletBalance(xprv58); + + assertNotNull(repeatBalance); + + System.out.println(bitcoin.format(repeatBalance)); + + assertEquals(balance, repeatBalance); + } + + @Test + public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String address = bitcoin.getUnusedReceiveAddress(xprv58); + + assertNotNull(address); + + System.out.println(address); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java b/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java new file mode 100644 index 00000000..b7e57cf3 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/ElectrumXTests.java @@ -0,0 +1,201 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import java.security.Security; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import org.bitcoinj.core.Address; +import org.bitcoinj.params.TestNet3Params; +import org.bitcoinj.script.ScriptBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.junit.Test; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.BitcoinyTransaction; +import org.qortal.crosschain.ElectrumX; +import org.qortal.crosschain.TransactionHash; +import org.qortal.crosschain.UnspentOutput; +import org.qortal.crosschain.Bitcoin.BitcoinNet; +import org.qortal.crosschain.ElectrumX.Server.ConnectionType; +import org.qortal.utils.BitTwiddling; + +import com.google.common.hash.HashCode; + +public class ElectrumXTests { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + } + + private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ElectrumX.Server.ConnectionType.class); + static { + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.TCP, 50001); + DEFAULT_ELECTRUMX_PORTS.put(ConnectionType.SSL, 50002); + } + + private ElectrumX getInstance() { + return new ElectrumX("Bitcoin-" + BitcoinNet.TEST3.name(), BitcoinNet.TEST3.getGenesisHash(), BitcoinNet.TEST3.getServers(), DEFAULT_ELECTRUMX_PORTS); + } + + @Test + public void testInstance() { + ElectrumX electrumX = getInstance(); + assertNotNull(electrumX); + } + + @Test + public void testGetCurrentHeight() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + int height = electrumX.getCurrentHeight(); + + assertTrue(height > 10000); + System.out.println("Current TEST3 height: " + height); + } + + @Test + public void testInvalidRequest() { + ElectrumX electrumX = getInstance(); + try { + electrumX.getRawBlockHeaders(-1, -1); + } catch (ForeignBlockchainException e) { + // Should throw due to negative start block height + return; + } + + fail("Negative start block height should cause error"); + } + + @Test + public void testGetRecentBlocks() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + int height = electrumX.getCurrentHeight(); + assertTrue(height > 10000); + + List recentBlockHeaders = electrumX.getRawBlockHeaders(height - 11, 11); + + System.out.println(String.format("Returned %d recent blocks", recentBlockHeaders.size())); + for (int i = 0; i < recentBlockHeaders.size(); ++i) { + byte[] blockHeader = recentBlockHeaders.get(i); + + // Timestamp(int) is at 4 + 32 + 32 = 68 bytes offset + int offset = 4 + 32 + 32; + int timestamp = BitTwiddling.intFromLEBytes(blockHeader, offset); + System.out.println(String.format("Block %d timestamp: %d", height + i, timestamp)); + } + } + + @Test + public void testGetP2PKHBalance() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "n3GNqMveyvaPvUbH469vDRadqpJMPc84JA"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + long balance = electrumX.getConfirmedBalance(script); + + assertTrue(balance > 0L); + + System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); + } + + @Test + public void testGetP2SHBalance() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + long balance = electrumX.getConfirmedBalance(script); + + assertTrue(balance > 0L); + + System.out.println(String.format("TestNet address %s has balance: %d sats / %d.%08d BTC", address, balance, (balance / 100000000L), (balance % 100000000L))); + } + + @Test + public void testGetUnspentOutputs() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "2N4szZUfigj7fSBCEX4PaC8TVbC5EvidaVF"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + List unspentOutputs = electrumX.getUnspentOutputs(script, false); + + assertFalse(unspentOutputs.isEmpty()); + + for (UnspentOutput unspentOutput : unspentOutputs) + System.out.println(String.format("TestNet address %s has unspent output at tx %s, output index %d", address, HashCode.fromBytes(unspentOutput.hash), unspentOutput.index)); + } + + @Test + public void testGetRawTransaction() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + byte[] txHash = HashCode.fromString("7653fea9ffcd829d45ed2672938419a94951b08175982021e77d619b553f29af").asBytes(); + + byte[] rawTransactionBytes = electrumX.getRawTransaction(txHash); + + assertFalse(rawTransactionBytes.length == 0); + } + + @Test + public void testGetUnknownRawTransaction() { + ElectrumX electrumX = getInstance(); + + byte[] txHash = HashCode.fromString("f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0").asBytes(); + + try { + electrumX.getRawTransaction(txHash); + fail("Bitcoin transaction should be unknown and hence throw exception"); + } catch (ForeignBlockchainException e) { + if (!(e instanceof ForeignBlockchainException.NotFoundException)) + fail("Bitcoin transaction should be unknown and hence throw NotFoundException"); + } + } + + @Test + public void testGetTransaction() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + String txHash = "7653fea9ffcd829d45ed2672938419a94951b08175982021e77d619b553f29af"; + + BitcoinyTransaction transaction = electrumX.getTransaction(txHash); + + assertNotNull(transaction); + assertTrue(transaction.txHash.equals(txHash)); + } + + @Test + public void testGetUnknownTransaction() { + ElectrumX electrumX = getInstance(); + + String txHash = "f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0"; + + try { + electrumX.getTransaction(txHash); + fail("Bitcoin transaction should be unknown and hence throw exception"); + } catch (ForeignBlockchainException e) { + if (!(e instanceof ForeignBlockchainException.NotFoundException)) + fail("Bitcoin transaction should be unknown and hence throw NotFoundException"); + } + } + + @Test + public void testGetAddressTransactions() throws ForeignBlockchainException { + ElectrumX electrumX = getInstance(); + + Address address = Address.fromString(TestNet3Params.get(), "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + List transactionHashes = electrumX.getAddressTransactions(script, false); + + assertFalse(transactionHashes.isEmpty()); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/HtlcTests.java b/src/test/java/org/qortal/test/crosschain/HtlcTests.java new file mode 100644 index 00000000..75b290bf --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/HtlcTests.java @@ -0,0 +1,128 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crypto.Crypto; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +import com.google.common.primitives.Longs; + +public class HtlcTests extends Common { + + private Bitcoin bitcoin; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // TestNet3 + bitcoin = Bitcoin.getInstance(); + } + + @After + public void afterTest() { + Bitcoin.resetForTesting(); + bitcoin = null; + } + + @Test + public void testFindHtlcSecret() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + byte[] secret = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + + assertNotNull(secret); + assertArrayEquals("secret incorrect", expectedSecret, secret); + } + + @Test + @Ignore(value = "Doesn't work, to be fixed later") + public void testHtlcSecretCaching() throws ForeignBlockchainException { + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + + do { + // We need to perform fresh setup for 1st test + Bitcoin.resetForTesting(); + bitcoin = Bitcoin.getInstance(); + + long now = System.currentTimeMillis(); + long timestampBoundary = now / 30_000L; + + byte[] secret1 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + long executionPeriod1 = System.currentTimeMillis() - now; + + assertNotNull(secret1); + assertArrayEquals("secret1 incorrect", expectedSecret, secret1); + + assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); + + byte[] secret2 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress); + long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; + + assertNotNull(secret2); + assertArrayEquals("secret2 incorrect", expectedSecret, secret2); + + // Test is only valid if we've called within same timestampBoundary + if (System.currentTimeMillis() / 30_000L != timestampBoundary) + continue; + + assertArrayEquals(secret1, secret2); + + assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10); + } while (false); + } + + @Test + public void testDetermineHtlcStatus() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + BitcoinyHTLC.Status htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + assertNotNull(htlcStatus); + + System.out.println(String.format("HTLC %s status: %s", p2shAddress, htlcStatus.name())); + } + + @Test + public void testHtlcStatusCaching() throws ForeignBlockchainException { + do { + // We need to perform fresh setup for 1st test + Bitcoin.resetForTesting(); + bitcoin = Bitcoin.getInstance(); + + long now = System.currentTimeMillis(); + long timestampBoundary = now / 30_000L; + + // Won't ever exist + String p2shAddress = bitcoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now))); + + BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + long executionPeriod1 = System.currentTimeMillis() - now; + + assertNotNull(htlcStatus1); + assertTrue("1st execution period should not be instant!", executionPeriod1 > 10); + + BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L); + long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1; + + assertNotNull(htlcStatus2); + assertEquals(htlcStatus1, htlcStatus2); + + // Test is only valid if we've called within same timestampBoundary + if (System.currentTimeMillis() / 30_000L != timestampBoundary) + continue; + + assertTrue("2st execution period should be effectively instant!", executionPeriod2 < 10); + } while (false); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/LitecoinTests.java b/src/test/java/org/qortal/test/crosschain/LitecoinTests.java new file mode 100644 index 00000000..64837347 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/LitecoinTests.java @@ -0,0 +1,114 @@ +package org.qortal.test.crosschain; + +import static org.junit.Assert.*; + +import java.util.Arrays; + +import org.bitcoinj.core.Transaction; +import org.bitcoinj.store.BlockStoreException; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +public class LitecoinTests extends Common { + + private Litecoin litecoin; + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // TestNet3 + litecoin = Litecoin.getInstance(); + } + + @After + public void afterTest() { + Litecoin.resetForTesting(); + litecoin = null; + } + + @Test + public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException { + long before = System.currentTimeMillis(); + System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); + long afterFirst = System.currentTimeMillis(); + + System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime())); + long afterSecond = System.currentTimeMillis(); + + long firstPeriod = afterFirst - before; + long secondPeriod = afterSecond - afterFirst; + + System.out.println(String.format("1st call: %d ms, 2nd call: %d ms", firstPeriod, secondPeriod)); + + assertTrue("2nd call should be quicker than 1st", secondPeriod < firstPeriod); + assertTrue("2nd call should take less than 5 seconds", secondPeriod < 5000L); + } + + @Test + @Ignore(value = "Doesn't work, to be fixed later") + public void testFindHtlcSecret() throws ForeignBlockchainException { + // This actually exists on TEST3 but can take a while to fetch + String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes(); + byte[] secret = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress); + + assertNotNull("secret not found", secret); + assertTrue("secret incorrect", Arrays.equals(expectedSecret, secret)); + } + + @Test + public void testBuildSpend() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String recipient = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + long amount = 1000L; + + Transaction transaction = litecoin.buildSpend(xprv58, recipient, amount); + assertNotNull("insufficient funds", transaction); + + // Check spent key caching doesn't affect outcome + + transaction = litecoin.buildSpend(xprv58, recipient, amount); + assertNotNull("insufficient funds", transaction); + } + + @Test + public void testGetWalletBalance() { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + Long balance = litecoin.getWalletBalance(xprv58); + + assertNotNull(balance); + + System.out.println(litecoin.format(balance)); + + // Check spent key caching doesn't affect outcome + + Long repeatBalance = litecoin.getWalletBalance(xprv58); + + assertNotNull(repeatBalance); + + System.out.println(litecoin.format(repeatBalance)); + + assertEquals(balance, repeatBalance); + } + + @Test + public void testGetUnusedReceiveAddress() throws ForeignBlockchainException { + String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ"; + + String address = litecoin.getUnusedReceiveAddress(xprv58); + + assertNotNull(address); + + System.out.println(address); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java new file mode 100644 index 00000000..fa92fde7 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java @@ -0,0 +1,114 @@ +package org.qortal.test.crosschain.apps; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.BitcoinyHTLC; + +import com.google.common.hash.HashCode; + +public class BuildHTLC { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: BuildHTLC (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: BuildHTLC -l " + + "msAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\t0.00008642 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600000000")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 6 || args.length > 6) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address refundAddress = null; + Coin amount = null; + Address redeemAddress = null; + byte[] hashOfSecret = null; + int lockTime = 0; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + refundAddress = Address.fromString(params, args[argIndex++]); + if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Refund address must be in P2PKH form"); + + amount = Coin.parseCoin(args[argIndex++]); + + redeemAddress = Address.fromString(params, args[argIndex++]); + if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Redeem address must be in P2PKH form"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("Hash of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + int refundTimeoutDelay = lockTime - (int) (System.currentTimeMillis() / 1000L); + if (refundTimeoutDelay < 600 || refundTimeoutDelay > 30 * 24 * 60 * 60) + usage("Locktime (seconds) should be at between 10 minutes and 1 month from now"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("Refund address: %s", refundAddress)); + System.out.println(String.format("Amount: %s", amount.toPlainString())); + System.out.println(String.format("Redeem address: %s", redeemAddress)); + System.out.println(String.format("Refund/redeem miner's fee: %s", bitcoiny.format(p2shFee))); + System.out.println(String.format("Script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); + System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(hashOfSecret))); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + System.out.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + + String p2shAddress = bitcoiny.deriveP2shAddress(redeemScriptBytes); + System.out.println(String.format("P2SH address: %s", p2shAddress)); + + amount = amount.add(p2shFee); + + // Fund P2SH + System.out.println(String.format("\nYou need to fund %s with %s (includes redeem/refund fee of %s)", + p2shAddress, bitcoiny.format(amount), bitcoiny.format(p2shFee))); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java new file mode 100644 index 00000000..8b1cc423 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java @@ -0,0 +1,135 @@ +package org.qortal.test.crosschain.apps; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; + +import com.google.common.hash.HashCode; + +public class CheckHTLC { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: CheckHTLC (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: CheckP2SH -l " + + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" + + "msAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\t0.00008642 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600184800")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 7 || args.length > 7) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address p2shAddress = null; + Address refundAddress = null; + Coin amount = null; + Address redeemAddress = null; + byte[] hashOfSecret = null; + int lockTime = 0; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + p2shAddress = Address.fromString(params, args[argIndex++]); + if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) + usage("P2SH address invalid"); + + refundAddress = Address.fromString(params, args[argIndex++]); + if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Refund address must be in P2PKH form"); + + amount = Coin.parseCoin(args[argIndex++]); + + redeemAddress = Address.fromString(params, args[argIndex++]); + if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Redeem address must be in P2PKH form"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("Hash of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("P2SH address: %s", p2shAddress)); + System.out.println(String.format("Refund PKH: %s", refundAddress)); + System.out.println(String.format("Redeem/refund amount: %s", amount.toPlainString())); + System.out.println(String.format("Redeem PKH: %s", redeemAddress)); + System.out.println(String.format("Hash of secret: %s", HashCode.fromBytes(hashOfSecret))); + System.out.println(String.format("Script lockTime: %s (%d)", LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC), lockTime)); + + System.out.println(String.format("Redeem/refund miner's fee: %s", bitcoiny.format(p2shFee))); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + System.out.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!derivedP2shAddress.equals(p2shAddress)) { + System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); + System.exit(2); + } + + amount = amount.add(p2shFee); + + // Check network's median block time + int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, null); + if (medianBlockTime == 0) + return; + + // Check P2SH is funded + Common.getBalance(bitcoiny, p2shAddress.toString()); + + // Grab all unspent outputs + Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); + + Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), amount.value); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/Common.java b/src/test/java/org/qortal/test/crosschain/apps/Common.java new file mode 100644 index 00000000..78066fe7 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/Common.java @@ -0,0 +1,158 @@ +package org.qortal.test.crosschain.apps; + +import java.security.Security; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Collections; +import java.util.List; + +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +import com.google.common.hash.HashCode; + +public abstract class Common { + + public static void init() { + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + Settings.fileInstance("settings-test.json"); + + NTP.setFixedOffset(0L); + } + + public static long getP2shFee(Bitcoiny bitcoiny) { + long p2shFee; + + try { + p2shFee = bitcoiny.getP2shFee(null); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to determine P2SH fee: %s", e.getMessage())); + return 0; + } + + return p2shFee; + } + + public static int checkMedianBlockTime(Bitcoiny bitcoiny, Integer lockTime) { + int medianBlockTime; + + try { + medianBlockTime = bitcoiny.getMedianBlockTime(); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to determine median block time: %s", e.getMessage())); + return 0; + } + + System.out.println(String.format("Median block time: %s", LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); + + long now = System.currentTimeMillis(); + + if (now < medianBlockTime * 1000L) { + System.out.println(String.format("Too soon (%s) based on median block time %s", + LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), + LocalDateTime.ofInstant(Instant.ofEpochSecond(medianBlockTime), ZoneOffset.UTC))); + return 0; + } + + if (lockTime != null && now < lockTime * 1000L) { + System.err.println(String.format("Too soon (%s) based on lockTime %s", + LocalDateTime.ofInstant(Instant.ofEpochMilli(now), ZoneOffset.UTC), + LocalDateTime.ofInstant(Instant.ofEpochSecond(lockTime), ZoneOffset.UTC))); + return 0; + } + + return medianBlockTime; + } + + public static long getBalance(Bitcoiny bitcoiny, String address58) { + long balance; + + try { + balance = bitcoiny.getConfirmedBalance(address58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to check address %s balance: %s", address58, e.getMessage())); + return 0; + } + + System.out.println(String.format("Address %s balance: %s", address58, bitcoiny.format(balance))); + + return balance; + } + + public static List getUnspentOutputs(Bitcoiny bitcoiny, String address58) { + List unspentOutputs = Collections.emptyList(); + + try { + unspentOutputs = bitcoiny.getUnspentOutputs(address58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Can't find unspent outputs for %s: %s", address58, e.getMessage())); + return unspentOutputs; + } + + System.out.println(String.format("Found %d output%s for %s", + unspentOutputs.size(), + (unspentOutputs.size() != 1 ? "s" : ""), + address58)); + + for (TransactionOutput fundingOutput : unspentOutputs) + System.out.println(String.format("Output %s:%d amount %s", + HashCode.fromBytes(fundingOutput.getParentTransactionHash().getBytes()), fundingOutput.getIndex(), + bitcoiny.format(fundingOutput.getValue()))); + + if (unspentOutputs.isEmpty()) + System.err.println(String.format("Can't use spent/unfunded %s", address58)); + + if (unspentOutputs.size() != 1) + System.err.println(String.format("Expecting only one unspent output?")); + + return unspentOutputs; + } + + public static BitcoinyHTLC.Status determineHtlcStatus(Bitcoiny bitcoiny, String address58, long minimumAmount) { + BitcoinyHTLC.Status htlcStatus = null; + + try { + htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), address58, minimumAmount); + + System.out.println(String.format("HTLC status: %s", htlcStatus.name())); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Unable to determine HTLC status: %s", e.getMessage())); + } + + return htlcStatus; + } + + public static void broadcastTransaction(Bitcoiny bitcoiny, Transaction transaction) { + byte[] rawTransactionBytes = transaction.bitcoinSerialize(); + + System.out.println(String.format("%nRaw transaction bytes:%n%s%n", HashCode.fromBytes(rawTransactionBytes).toString())); + + for (int countDown = 5; countDown >= 1; --countDown) { + System.out.print(String.format("\rBroadcasting transaction in %d second%s... use CTRL-C to abort ", countDown, (countDown != 1 ? "s" : ""))); + try { + Thread.sleep(1000L); + } catch (InterruptedException e) { + System.exit(0); + } + } + System.out.println("Broadcasting transaction... "); + + try { + bitcoiny.broadcastTransaction(transaction); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Failed to broadcast transaction: %s", e.getMessage())); + System.exit(1); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java b/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java new file mode 100644 index 00000000..ef22355b --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java @@ -0,0 +1,78 @@ +package org.qortal.test.crosschain.apps; + +import java.security.Security; + +import org.bitcoinj.core.AddressFormatException; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.settings.Settings; + +public class GetNextReceiveAddress { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: GetNextReceiveAddress (-b | -l) ")); + System.err.println(String.format("example (testnet): GetNextReceiveAddress -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 2) + usage(null); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + Settings.fileInstance("settings-test.json"); + + Bitcoiny bitcoiny = null; + String key58 = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + + key58 = args[argIndex++]; + + if (!bitcoiny.isValidDeterministicKey(key58)) + usage("Not valid xprv/xpub/tprv/tpub"); + } catch (NumberFormatException | AddressFormatException e) { + usage(String.format("Argument format exception: %s", e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + String receiveAddress = null; + try { + receiveAddress = bitcoiny.getUnusedReceiveAddress(key58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Failed to determine next receive address: %s", e.getMessage())); + System.exit(1); + } + + System.out.println(String.format("Next receive address: %s", receiveAddress)); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java b/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java new file mode 100644 index 00000000..9d903a56 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java @@ -0,0 +1,84 @@ +package org.qortal.test.crosschain.apps; + +import java.security.Security; +import java.util.List; + +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.TransactionOutput; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.ForeignBlockchainException; +import org.qortal.crosschain.Litecoin; +import org.qortal.settings.Settings; + +import com.google.common.hash.HashCode; + +public class GetTransaction { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: GetTransaction (-b | -l) ")); + System.err.println(String.format("example (mainnet): GetTransaction -b 816407e79568f165f13e09e9912c5f2243e0a23a007cec425acedc2e89284660")); + System.err.println(String.format("example (testnet): GetTransaction -b 3bfd17a492a4e3d6cb7204e17e20aca6c1ab82e1828bd1106eefbaf086fb8a4e")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 2) + usage(null); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + Settings.fileInstance("settings-test.json"); + + Bitcoiny bitcoiny = null; + byte[] transactionId = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + + transactionId = HashCode.fromString(args[argIndex++]).asBytes(); + } catch (NumberFormatException | AddressFormatException e) { + usage(String.format("Argument format exception: %s", e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + // Grab all outputs from transaction + List fundingOutputs; + try { + fundingOutputs = bitcoiny.getOutputs(transactionId); + } catch (ForeignBlockchainException e) { + System.out.println(String.format("Transaction not found (or error occurred)")); + return; + } + + System.out.println(String.format("Found %d output%s", fundingOutputs.size(), (fundingOutputs.size() != 1 ? "s" : ""))); + + for (TransactionOutput fundingOutput : fundingOutputs) + System.out.println(String.format("Output %d: %s", fundingOutput.getIndex(), fundingOutput.getValue().toPlainString())); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java b/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java new file mode 100644 index 00000000..7a880b1a --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java @@ -0,0 +1,82 @@ +package org.qortal.test.crosschain.apps; + +import java.security.Security; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import org.bitcoinj.core.AddressFormatException; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.crosschain.*; +import org.qortal.settings.Settings; + +public class GetWalletTransactions { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: GetWalletTransactions (-b | -l) ")); + System.err.println(String.format("example (testnet): GetWalletTransactions -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 2) + usage(null); + + Security.insertProviderAt(new BouncyCastleProvider(), 0); + Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); + + Settings.fileInstance("settings-test.json"); + + Bitcoiny bitcoiny = null; + String key58 = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + + key58 = args[argIndex++]; + + if (!bitcoiny.isValidDeterministicKey(key58)) + usage("Not valid xprv/xpub/tprv/tpub"); + } catch (NumberFormatException | AddressFormatException e) { + usage(String.format("Argument format exception: %s", e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + // Grab all outputs from transaction + List transactions = null; + try { + transactions = bitcoiny.getWalletTransactions(key58); + } catch (ForeignBlockchainException e) { + System.err.println(String.format("Failed to obtain wallet transactions: %s", e.getMessage())); + System.exit(1); + } + + System.out.println(String.format("Found %d transaction%s", transactions.size(), (transactions.size() != 1 ? "s" : ""))); + + for (SimpleTransaction transaction : transactions.stream().sorted(Comparator.comparingInt(SimpleTransaction::getTimestamp)).collect(Collectors.toList())) + System.out.println(String.format("%s", transaction)); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/Pay.java b/src/test/java/org/qortal/test/crosschain/apps/Pay.java new file mode 100644 index 00000000..93c7aede --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/Pay.java @@ -0,0 +1,80 @@ +package org.qortal.test.crosschain.apps; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.Litecoin; + +public class Pay { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: Pay (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: Pay -l " + + "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ \\\n" + + "\tmsAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\t0.00008642")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 4 || args.length > 4) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + String xprv58 = null; + Address address = null; + Coin amount = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + xprv58 = args[argIndex++]; + if (!bitcoiny.isValidDeterministicKey(xprv58)) + usage("xprv invalid"); + + address = Address.fromString(params, args[argIndex++]); + + amount = Coin.parseCoin(args[argIndex++]); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + System.out.println(String.format("Address: %s", address)); + System.out.println(String.format("Amount: %s", amount.toPlainString())); + + Transaction transaction = bitcoiny.buildSpend(xprv58, address.toString(), amount.value); + if (transaction == null) { + System.err.println("Insufficent funds"); + System.exit(1); + } + + Common.broadcastTransaction(bitcoiny, transaction); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java new file mode 100644 index 00000000..d4f1bcf1 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java @@ -0,0 +1,166 @@ +package org.qortal.test.crosschain.apps; + +import java.util.Arrays; +import java.util.List; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; + +import com.google.common.hash.HashCode; + +public class RedeemHTLC { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: Redeem (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: Redeem -l " + + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" + + "\tmsAfaDaJ8JiprxxFaAXEEPxKK3JaZCYpLv \\\n" + + "\tefdaed23c4bc85c8ccae40d774af3c2a10391c648b6420cdd83cd44c27fcb5955201c64e372d \\\n" + + "\t5468697320737472696e672069732065786163746c7920333220627974657321 \\\n" + + "\t1600184800 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 7 || args.length > 7) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address p2shAddress = null; + Address refundAddress = null; + byte[] redeemPrivateKey = null; + byte[] secret = null; + int lockTime = 0; + Address outputAddress = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + p2shAddress = Address.fromString(params, args[argIndex++]); + if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) + usage("P2SH address invalid"); + + refundAddress = Address.fromString(params, args[argIndex++]); + if (refundAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Refund address must be in P2PKH form"); + + redeemPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + // Auto-trim + if (redeemPrivateKey.length >= 37 && redeemPrivateKey.length <= 38) + redeemPrivateKey = Arrays.copyOfRange(redeemPrivateKey, 1, 33); + if (redeemPrivateKey.length != 32) + usage("Redeem private key must be 32 bytes"); + + secret = HashCode.fromString(args[argIndex++]).asBytes(); + if (secret.length == 0) + usage("Invalid secret bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + + outputAddress = Address.fromString(params, args[argIndex++]); + if (outputAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Output address invalid"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("Attempting to redeem HTLC %s to %s", p2shAddress, outputAddress)); + + byte[] hashOfSecret = Crypto.hash160(secret); + + ECKey redeemKey = ECKey.fromPrivate(redeemPrivateKey); + Address redeemAddress = Address.fromKey(params, redeemKey, ScriptType.P2PKH); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!derivedP2shAddress.equals(p2shAddress)) { + System.err.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); + System.exit(2); + return; + } + + // Actual live processing... + + int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, null); + if (medianBlockTime == 0) + return; + + // Check P2SH is funded + long p2shBalance = Common.getBalance(bitcoiny, p2shAddress.toString()); + if (p2shBalance == 0) + return; + + // Grab all unspent outputs + List unspentOutputs = Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); + if (unspentOutputs.isEmpty()) + return; + + Coin redeemAmount = Coin.valueOf(p2shBalance).subtract(p2shFee); + + BitcoinyHTLC.Status htlcStatus = Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), redeemAmount.value); + if (htlcStatus == null) + return; + + if (htlcStatus != BitcoinyHTLC.Status.FUNDED) { + System.err.println(String.format("Expecting %s HTLC status, but actual status is %s", "FUNDED", htlcStatus.name())); + System.exit(2); + return; + } + + System.out.println(String.format("Spending %s of outputs, with %s as mining fee", bitcoiny.format(redeemAmount), bitcoiny.format(p2shFee))); + + Transaction redeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoiny.getNetworkParameters(), redeemAmount, redeemKey, + unspentOutputs, redeemScriptBytes, secret, outputAddress.getHash()); + + Common.broadcastTransaction(bitcoiny, redeemTransaction); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java b/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java new file mode 100644 index 00000000..723185f0 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java @@ -0,0 +1,163 @@ +package org.qortal.test.crosschain.apps; + +import java.util.Arrays; +import java.util.List; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.Coin; +import org.bitcoinj.core.ECKey; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.bitcoinj.core.Transaction; +import org.bitcoinj.core.TransactionOutput; +import org.bitcoinj.script.Script.ScriptType; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.BitcoinyHTLC; +import org.qortal.crypto.Crypto; + +import com.google.common.hash.HashCode; + +public class RefundHTLC { + + static { + // This must go before any calls to LogManager/Logger + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager"); + } + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: RefundHTLC (-b | -l) ")); + System.err.println("where: -b means use Bitcoin, -l means use Litecoin"); + System.err.println(String.format("example: RefundHTLC -l " + + "2N4378NbEVGjmiUmoUD9g1vCY6kyx9tDUJ6 \\\n" + + "\tef8f31b49c31b4a140aebcd9605fded88cc2dad0844c4b984f9191a5a416f72d3801e16447b0 \\\n" + + "\tmrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600184800 \\\n" + + "\tmoJtbbhs7T4Z5hmBH2iyKhGrCWBzQWS2CL")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length < 7 || args.length > 7) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = null; + NetworkParameters params = null; + + Address p2shAddress = null; + byte[] refundPrivateKey = null; + Address redeemAddress = null; + byte[] hashOfSecret = null; + int lockTime = 0; + Address outputAddress = null; + + int argIndex = 0; + try { + switch (args[argIndex++]) { + case "-b": + bitcoiny = Bitcoin.getInstance(); + break; + + case "-l": + bitcoiny = Litecoin.getInstance(); + break; + + default: + usage("Only Bitcoin (-b) or Litecoin (-l) supported"); + } + params = bitcoiny.getNetworkParameters(); + + p2shAddress = Address.fromString(params, args[argIndex++]); + if (p2shAddress.getOutputScriptType() != ScriptType.P2SH) + usage("P2SH address invalid"); + + refundPrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + // Auto-trim + if (refundPrivateKey.length >= 37 && refundPrivateKey.length <= 38) + refundPrivateKey = Arrays.copyOfRange(refundPrivateKey, 1, 33); + if (refundPrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + redeemAddress = Address.fromString(params, args[argIndex++]); + if (redeemAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Redeem address must be in P2PKH form"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("HASH160 of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + + outputAddress = Address.fromString(params, args[argIndex++]); + if (outputAddress.getOutputScriptType() != ScriptType.P2PKH) + usage("Output address invalid"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId())); + + Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny)); + if (p2shFee.isZero()) + return; + + System.out.println(String.format("Attempting to refund HTLC %s to %s", p2shAddress, outputAddress)); + + ECKey refundKey = ECKey.fromPrivate(refundPrivateKey); + Address refundAddress = Address.fromKey(params, refundKey, ScriptType.P2PKH); + + byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(refundAddress.getHash(), lockTime, redeemAddress.getHash(), hashOfSecret); + + byte[] redeemScriptHash = Crypto.hash160(redeemScriptBytes); + Address derivedP2shAddress = LegacyAddress.fromScriptHash(params, redeemScriptHash); + + if (!derivedP2shAddress.equals(p2shAddress)) { + System.err.println(String.format("Raw script bytes: %s", HashCode.fromBytes(redeemScriptBytes))); + System.err.println(String.format("Derived P2SH address %s does not match given address %s", derivedP2shAddress, p2shAddress)); + System.exit(2); + } + + // Actual live processing... + + int medianBlockTime = Common.checkMedianBlockTime(bitcoiny, lockTime); + if (medianBlockTime == 0) + return; + + // Check P2SH is funded + long p2shBalance = Common.getBalance(bitcoiny, p2shAddress.toString()); + if (p2shBalance == 0) + return; + + // Grab all unspent outputs + List unspentOutputs = Common.getUnspentOutputs(bitcoiny, p2shAddress.toString()); + if (unspentOutputs.isEmpty()) + return; + + Coin refundAmount = Coin.valueOf(p2shBalance).subtract(p2shFee); + + BitcoinyHTLC.Status htlcStatus = Common.determineHtlcStatus(bitcoiny, p2shAddress.toString(), refundAmount.value); + if (htlcStatus == null) + return; + + if (htlcStatus != BitcoinyHTLC.Status.FUNDED) { + System.err.println(String.format("Expecting %s HTLC status, but actual status is %s", "FUNDED", htlcStatus.name())); + System.exit(2); + return; + } + + System.out.println(String.format("Spending %s of outputs, with %s as mining fee", bitcoiny.format(refundAmount), bitcoiny.format(p2shFee))); + + Transaction refundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoiny.getNetworkParameters(), refundAmount, refundKey, + unspentOutputs, redeemScriptBytes, lockTime, outputAddress.getHash()); + + Common.broadcastTransaction(bitcoiny, refundTransaction); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java b/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java new file mode 100644 index 00000000..4487e874 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/bitcoinv1/BitcoinACCTv1Tests.java @@ -0,0 +1,795 @@ +package org.qortal.test.crosschain.bitcoinv1; + +import static org.junit.Assert.*; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.function.Function; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.utils.Amounts; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +public class BitcoinACCTv1Tests extends Common { + + public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); + public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a + public static final byte[] secretB = "This string is roughly 32 bytes?".getBytes(); + public static final byte[] hashOfSecretB = Crypto.hash160(secretB); // 31f0dd71decf59bbc8ef0661f4030479255cfa58 + public static final byte[] bitcoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); + public static final int tradeTimeout = 20; // blocks + public static final long redeemAmount = 80_40200000L; + public static final long fundingAmount = 123_45600000L; + public static final long bitcoinAmount = 864200L; // 0.00864200 BTC + + private static final Random RANDOM = new Random(); + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testCompile() { + PrivateKeyAccount tradeAccount = createTradeAccount(null); + + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeAccount.getAddress(), bitcoinPublicKeyHash, hashOfSecretB, redeemAmount, bitcoinAmount, tradeTimeout); + assertNotNull(creationBytes); + + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + } + + @Test + public void testDeploy() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = fundingAmount; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + expectedBalance = deployersInitialBalance; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = 0; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancel() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Send creator's address to AT, instead of typical partner's address + byte[] messageData = BitcoinACCTv1.getInstance().buildCancelMessage(deployer.getAddress()); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + + // Check balances + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance - messageFee; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancelInvalidLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Instead of sending creator's address to AT, send too-short/invalid message + byte[] messageData = new byte[7]; + RANDOM.nextBytes(messageData); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testTradingInfoProcessing() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should be in TRADE mode + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check hashOfSecretA was extracted correctly + assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); + + // Check trade partner Qortal address was extracted correctly + assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); + + // Check trade partner's Bitcoin PKH was extracted correctly + assertTrue(Arrays.equals(bitcoinPublicKeyHash, tradeData.partnerForeignPKH)); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) + @SuppressWarnings("unused") + @Test + public void testIncorrectTradeSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT BUT NOT FROM AT CREATOR + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + BlockUtils.mintBlock(repository); + + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should still be in OFFER mode + assertEquals(AcctMode.OFFERING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testAutomaticTradeRefund() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + // Check refund + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REFUNDED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REFUNDED, tradeData.mode); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretsCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secrets to AT, from correct account + messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REDEEMED mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REDEEMED, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); + + // Orphan redeem + BlockUtils.orphanLastBlock(repository); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); + + // Check AT state + ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); + + assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretsIncorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secrets to AT, but from wrong account + messageData = BitcoinACCTv1.buildRedeemMessage(secretA, secretB, partner.getAddress()); + messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testIncorrectSecretsCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send incorrect secrets to AT, from correct account + byte[] wrongSecret = new byte[32]; + RANDOM.nextBytes(wrongSecret); + messageData = BitcoinACCTv1.buildRedeemMessage(wrongSecret, secretB, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Send incorrect secrets to AT, from correct account + messageData = BitcoinACCTv1.buildRedeemMessage(secretA, wrongSecret, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() * 2; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretsCorrectSenderInvalidMessageLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int lockTimeB = BitcoinACCTv1.calcLockTimeB(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = BitcoinACCTv1.buildTradeMessage(partner.getAddress(), bitcoinPublicKeyHash, hashOfSecretA, lockTimeA, lockTimeB); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secrets to AT, from correct account, but missing receive address, hence incorrect length + messageData = Bytes.concat(secretA, secretB); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should be in TRADING mode + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testDescribeDeployed() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + List executableAts = repository.getATRepository().getAllExecutableATs(); + + for (ATData atData : executableAts) { + String atAddress = atData.getATAddress(); + byte[] codeBytes = atData.getCodeBytes(); + byte[] codeHash = Crypto.digest(codeBytes); + + System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", + atAddress, + codeBytes.length, + (codeBytes.length != 1 ? "s": ""), + HashCode.fromBytes(codeHash))); + + // Not one of ours? + if (!Arrays.equals(codeHash, BitcoinACCTv1.CODE_BYTES_HASH)) + continue; + + describeAt(repository, atAddress); + } + } + } + + private int calcTestLockTimeA(long messageTimestamp) { + return (int) (messageTimestamp / 1000L + tradeTimeout * 60); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(tradeAddress, bitcoinPublicKeyHash, hashOfSecretB, redeemAmount, bitcoinAmount, tradeTimeout); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-BTC cross-chain trade"; + String description = String.format("Qortal-Bitcoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-BTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + int refundTimeout = tradeTimeout * 3 / 4 + 1; // close enough + + // AT should automatically refund deployer after 'refundTimeout' blocks + for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) + BlockUtils.mintBlock(repository); + + // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + } + + private void describeAt(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = BitcoinACCTv1.getInstance().populateTradeData(repository, atData); + + Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); + int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); + + System.out.print(String.format("%s:\n" + + "\tmode: %s\n" + + "\tcreator: %s,\n" + + "\tcreation timestamp: %s,\n" + + "\tcurrent balance: %s QORT,\n" + + "\tis finished: %b,\n" + + "\tHASH160 of secret-B: %s,\n" + + "\tredeem payout: %s QORT,\n" + + "\texpected bitcoin: %s BTC,\n" + + "\tcurrent block height: %d,\n", + tradeData.qortalAtAddress, + tradeData.mode, + tradeData.qortalCreator, + epochMilliFormatter.apply(tradeData.creationTimestamp), + Amounts.prettyAmount(tradeData.qortBalance), + atData.getIsFinished(), + HashCode.fromBytes(tradeData.hashOfSecretB).toString().substring(0, 40), + Amounts.prettyAmount(tradeData.qortAmount), + Amounts.prettyAmount(tradeData.expectedForeignAmount), + currentBlockHeight)); + + if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { + System.out.println(String.format("\trefund height: block %d,\n" + + "\tHASH160 of secret-A: %s,\n" + + "\tBitcoin P2SH-A nLockTime: %d (%s),\n" + + "\tBitcoin P2SH-B nLockTime: %d (%s),\n" + + "\ttrade partner: %s", + tradeData.tradeRefundHeight, + HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), + tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), + tradeData.lockTimeB, epochMilliFormatter.apply(tradeData.lockTimeB * 1000L), + tradeData.qortalPartnerAddress)); + } + } + + private PrivateKeyAccount createTradeAccount(Repository repository) { + // We actually use a known test account with funds to avoid PoW compute + return Common.getTestAccount(repository, "alice"); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java b/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java new file mode 100644 index 00000000..f27f7a7b --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/bitcoinv1/DeployAT.java @@ -0,0 +1,169 @@ +package org.qortal.test.crosschain.bitcoinv1; + +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.LegacyAddress; +import org.bitcoinj.core.NetworkParameters; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.controller.Controller; +import org.qortal.crosschain.Bitcoin; +import org.qortal.crosschain.BitcoinACCTv1; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Amounts; +import org.qortal.utils.Base58; + +import com.google.common.hash.HashCode; + +public class DeployAT { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: DeployAT ")); + System.err.println(String.format("example: DeployAT " + + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" + + "\t10 \\\n" + + "\t10.1 \\\n" + + "\t0.00864200 \\\n" + + "\t750b06757a2448b8a4abebaa6e4662833fd5ddbb (or mrBpZYYGYMwUa8tRjTiXfP1ySqNXszWN5h) \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t10080")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 7) + usage(null); + + Common.init(); + + Bitcoiny bitcoiny = Bitcoin.getInstance(); + NetworkParameters params = bitcoiny.getNetworkParameters(); + + byte[] refundPrivateKey = null; + long redeemAmount = 0; + long fundingAmount = 0; + long expectedBitcoin = 0; + byte[] bitcoinPublicKeyHash = null; + byte[] hashOfSecret = null; + int tradeTimeout = 0; + + int argIndex = 0; + try { + refundPrivateKey = Base58.decode(args[argIndex++]); + if (refundPrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + redeemAmount = Long.parseLong(args[argIndex++]); + if (redeemAmount <= 0) + usage("QORT amount must be positive"); + + fundingAmount = Long.parseLong(args[argIndex++]); + if (fundingAmount <= redeemAmount) + usage("AT funding amount must be greater than QORT redeem amount"); + + expectedBitcoin = Long.parseLong(args[argIndex++]); + if (expectedBitcoin <= 0) + usage("Expected BTC amount must be positive"); + + String bitcoinPKHish = args[argIndex++]; + // Try P2PKH first + try { + Address bitcoinAddress = LegacyAddress.fromBase58(params, bitcoinPKHish); + bitcoinPublicKeyHash = bitcoinAddress.getHash(); + } catch (AddressFormatException e) { + // Try parsing as PKH hex string instead + bitcoinPublicKeyHash = HashCode.fromString(bitcoinPKHish).asBytes(); + } + if (bitcoinPublicKeyHash.length != 20) + usage("Bitcoin PKH must be 20 bytes"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("Hash of secret must be 20 bytes"); + + tradeTimeout = Integer.parseInt(args[argIndex++]); + if (tradeTimeout < 60 || tradeTimeout > 50000) + usage("Trade timeout (minutes) must be between 60 and 50000"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount refundAccount = new PrivateKeyAccount(repository, refundPrivateKey); + System.out.println(String.format("Refund Qortal address: %s", refundAccount.getAddress())); + + System.out.println(String.format("QORT redeem amount: %s", Amounts.prettyAmount(redeemAmount))); + + System.out.println(String.format("AT funding amount: %s", Amounts.prettyAmount(fundingAmount))); + + System.out.println(String.format("HASH160 of secret: %s", HashCode.fromBytes(hashOfSecret))); + + // Deploy AT + byte[] creationBytes = BitcoinACCTv1.buildQortalAT(refundAccount.getAddress(), bitcoinPublicKeyHash, hashOfSecret, redeemAmount, expectedBitcoin, tradeTimeout); + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = refundAccount.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", refundAccount.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-BTC cross-chain trade"; + String description = String.format("Qortal-Bitcoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-BTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, refundAccount.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + Transaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + deployAtTransaction.sign(refundAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to base58: %s", e.getMessage())); + System.exit(2); + } + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java new file mode 100644 index 00000000..3a1f9208 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/DeployAT.java @@ -0,0 +1,150 @@ +package org.qortal.test.crosschain.litecoinv1; + +import java.math.BigDecimal; + +import org.bitcoinj.core.ECKey; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Amounts; +import org.qortal.utils.Base58; + +import com.google.common.hash.HashCode; + +public class DeployAT { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: DeployAT ")); + System.err.println("A trading key-pair will be generated for you!"); + System.err.println(String.format("example: DeployAT " + + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" + + "\t10 \\\n" + + "\t10.1 \\\n" + + "\t0.00864200 \\\n" + + "\t120")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 5) + usage(null); + + Common.init(); + + byte[] creatorPrivateKey = null; + long redeemAmount = 0; + long fundingAmount = 0; + long expectedLitecoin = 0; + int tradeTimeout = 0; + + int argIndex = 0; + try { + creatorPrivateKey = Base58.decode(args[argIndex++]); + if (creatorPrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + redeemAmount = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); + if (redeemAmount <= 0) + usage("QORT amount must be positive"); + + fundingAmount = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); + if (fundingAmount <= redeemAmount) + usage("AT funding amount must be greater than QORT redeem amount"); + + expectedLitecoin = new BigDecimal(args[argIndex++]).setScale(8).unscaledValue().longValue(); + if (expectedLitecoin <= 0) + usage("Expected LTC amount must be positive"); + + tradeTimeout = Integer.parseInt(args[argIndex++]); + if (tradeTimeout < 60 || tradeTimeout > 50000) + usage("Trade timeout (minutes) must be between 60 and 50000"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount creatorAccount = new PrivateKeyAccount(repository, creatorPrivateKey); + System.out.println(String.format("Creator Qortal address: %s", creatorAccount.getAddress())); + System.out.println(String.format("QORT redeem amount: %s", Amounts.prettyAmount(redeemAmount))); + System.out.println(String.format("AT funding amount: %s", Amounts.prettyAmount(fundingAmount))); + + // Generate trading key-pair + byte[] tradePrivateKey = new ECKey().getPrivKeyBytes(); + PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); + byte[] litecoinPublicKeyHash = ECKey.fromPrivate(tradePrivateKey).getPubKeyHash(); + + System.out.println(String.format("Trade private key: %s", HashCode.fromBytes(tradePrivateKey))); + + // Deploy AT + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAccount.getAddress(), litecoinPublicKeyHash, redeemAmount, expectedLitecoin, tradeTimeout); + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = creatorAccount.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", creatorAccount.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-LTC cross-chain trade"; + String description = String.format("Qortal-Litecoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-LTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, creatorAccount.getPublicKey(), fee, null); + DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + deployAtTransaction.sign(creatorAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(deployAtTransactionData); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to base58: %s", e.getMessage())); + System.exit(2); + } + + DeployAtTransaction.ensureATAddress(deployAtTransactionData); + String atAddress = deployAtTransactionData.getAtAddress(); + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + + System.out.println(String.format("AT address: %s", atAddress)); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java new file mode 100644 index 00000000..609ff5f3 --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/LitecoinACCTv1Tests.java @@ -0,0 +1,770 @@ +package org.qortal.test.crosschain.litecoinv1; + +import static org.junit.Assert.*; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.function.Function; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.block.Block; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crosschain.AcctMode; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.at.ATStateData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.utils.Amounts; + +import com.google.common.hash.HashCode; +import com.google.common.primitives.Bytes; + +public class LitecoinACCTv1Tests extends Common { + + public static final byte[] secretA = "This string is exactly 32 bytes!".getBytes(); + public static final byte[] hashOfSecretA = Crypto.hash160(secretA); // daf59884b4d1aec8c1b17102530909ee43c0151a + public static final byte[] litecoinPublicKeyHash = HashCode.fromString("bb00bb11bb22bb33bb44bb55bb66bb77bb88bb99").asBytes(); + public static final int tradeTimeout = 20; // blocks + public static final long redeemAmount = 80_40200000L; + public static final long fundingAmount = 123_45600000L; + public static final long litecoinAmount = 864200L; // 0.00864200 LTC + + private static final Random RANDOM = new Random(); + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testCompile() { + PrivateKeyAccount tradeAccount = createTradeAccount(null); + + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAccount.getAddress(), litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); + assertNotNull(creationBytes); + + System.out.println("AT creation bytes: " + HashCode.fromBytes(creationBytes).toString()); + } + + @Test + public void testDeploy() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + long expectedBalance = deployersInitialBalance - fundingAmount - deployAtTransaction.getTransactionData().getFee(); + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = fundingAmount; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-deployment balance incorrect", expectedBalance, actualBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + expectedBalance = deployersInitialBalance; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = 0; + actualBalance = deployAtTransaction.getATAccount().getConfirmedBalance(Asset.QORT); + + assertEquals("AT's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + + expectedBalance = partnersInitialBalance; + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-deployment balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancel() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Send creator's address to AT, instead of typical partner's address + byte[] messageData = LitecoinACCTv1.getInstance().buildCancelMessage(deployer.getAddress()); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + + // Check balances + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee - messageFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + + // Test orphaning + BlockUtils.orphanLastBlock(repository); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance - messageFee; + actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testOfferCancelInvalidLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + // Instead of sending creator's address to AT, send too-short/invalid message + byte[] messageData = new byte[7]; + RANDOM.nextBytes(messageData); + MessageTransaction messageTransaction = sendMessage(repository, deployer, messageData, atAddress); + long messageFee = messageTransaction.getTransactionData().getFee(); + + // AT should process 'cancel' message in next block + // As message is too short, it will be padded to 32bytes but cancel code doesn't care about message content, so should be ok + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in CANCELLED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.CANCELLED, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testTradingInfoProcessing() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should be in TRADE mode + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check hashOfSecretA was extracted correctly + assertTrue(Arrays.equals(hashOfSecretA, tradeData.hashOfSecretA)); + + // Check trade partner Qortal address was extracted correctly + assertEquals(partner.getAddress(), tradeData.qortalPartnerAddress); + + // Check trade partner's Litecoin PKH was extracted correctly + assertTrue(Arrays.equals(litecoinPublicKeyHash, tradeData.partnerForeignPKH)); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + // TEST SENDING TRADING INFO BUT NOT FROM AT CREATOR (SHOULD BE IGNORED) + @SuppressWarnings("unused") + @Test + public void testIncorrectTradeSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT BUT NOT FROM AT CREATOR + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + BlockUtils.mintBlock(repository); + + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-initial-payout balance incorrect", expectedBalance, actualBalance); + + describeAt(repository, atAddress); + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + // AT should still be in OFFER mode + assertEquals(AcctMode.OFFERING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testAutomaticTradeRefund() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + Block postDeploymentBlock = BlockUtils.mintBlock(repository); + int postDeploymentBlockHeight = postDeploymentBlock.getBlockData().getHeight(); + + // Check refund + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REFUNDED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REFUNDED, tradeData.mode); + + // Test orphaning + BlockUtils.orphanToBlock(repository, postDeploymentBlockHeight); + + // Check balances + long expectedBalance = deployersPostDeploymentBalance; + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertEquals("Deployer's post-orphan/pre-refund balance incorrect", expectedBalance, actualBalance); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account + messageData = LitecoinACCTv1.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertTrue(atData.getIsFinished()); + + // AT should be in REDEEMED mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.REDEEMED, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee() + redeemAmount; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-redeem balance incorrect", expectedBalance, actualBalance); + + // Orphan redeem + BlockUtils.orphanLastBlock(repository); + + // Check balances + expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's post-orphan/pre-redeem balance incorrect", expectedBalance, actualBalance); + + // Check AT state + ATStateData postOrphanAtStateData = repository.getATRepository().getLatestATState(atAddress); + + assertTrue("AT states mismatch", Arrays.equals(preRedeemAtStateData.getStateData(), postOrphanAtStateData.getStateData())); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretIncorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + PrivateKeyAccount bystander = Common.getTestAccount(repository, "bob"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, but from wrong account + messageData = LitecoinACCTv1.buildRedeemMessage(secretA, partner.getAddress()); + messageTransaction = sendMessage(repository, bystander, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + // Check balances + long expectedBalance = partnersInitialBalance; + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testIncorrectSecretCorrectSender() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + long deployAtFee = deployAtTransaction.getTransactionData().getFee(); + + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send incorrect secret to AT, from correct account + byte[] wrongSecret = new byte[32]; + RANDOM.nextBytes(wrongSecret); + messageData = LitecoinACCTv1.buildRedeemMessage(wrongSecret, partner.getAddress()); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should still be in TRADE mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + + long expectedBalance = partnersInitialBalance - messageTransaction.getTransactionData().getFee(); + long actualBalance = partner.getConfirmedBalance(Asset.QORT); + + assertEquals("Partner's balance incorrect", expectedBalance, actualBalance); + + // Check eventual refund + checkTradeRefund(repository, deployer, deployersInitialBalance, deployAtFee); + } + } + + @SuppressWarnings("unused") + @Test + public void testCorrectSecretCorrectSenderInvalidMessageLength() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + Account at = deployAtTransaction.getATAccount(); + String atAddress = at.getAddress(); + + long partnersOfferMessageTransactionTimestamp = System.currentTimeMillis(); + int lockTimeA = calcTestLockTimeA(partnersOfferMessageTransactionTimestamp); + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(partnersOfferMessageTransactionTimestamp, lockTimeA); + + // Send trade info to AT + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partner.getAddress(), litecoinPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout); + MessageTransaction messageTransaction = sendMessage(repository, tradeAccount, messageData, atAddress); + + // Give AT time to process message + BlockUtils.mintBlock(repository); + + // Send correct secret to AT, from correct account, but missing receive address, hence incorrect length + messageData = Bytes.concat(secretA); + messageTransaction = sendMessage(repository, partner, messageData, atAddress); + + // AT should NOT send funds in the next block + ATStateData preRedeemAtStateData = repository.getATRepository().getLatestATState(atAddress); + BlockUtils.mintBlock(repository); + + describeAt(repository, atAddress); + + // Check AT is NOT finished + ATData atData = repository.getATRepository().fromATAddress(atAddress); + assertFalse(atData.getIsFinished()); + + // AT should be in TRADING mode + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + assertEquals(AcctMode.TRADING, tradeData.mode); + } + } + + @SuppressWarnings("unused") + @Test + public void testDescribeDeployed() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "chloe"); + PrivateKeyAccount tradeAccount = createTradeAccount(repository); + + PrivateKeyAccount partner = Common.getTestAccount(repository, "dilbert"); + + long deployersInitialBalance = deployer.getConfirmedBalance(Asset.QORT); + long partnersInitialBalance = partner.getConfirmedBalance(Asset.QORT); + + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, tradeAccount.getAddress()); + + List executableAts = repository.getATRepository().getAllExecutableATs(); + + for (ATData atData : executableAts) { + String atAddress = atData.getATAddress(); + byte[] codeBytes = atData.getCodeBytes(); + byte[] codeHash = Crypto.digest(codeBytes); + + System.out.println(String.format("%s: code length: %d byte%s, code hash: %s", + atAddress, + codeBytes.length, + (codeBytes.length != 1 ? "s": ""), + HashCode.fromBytes(codeHash))); + + // Not one of ours? + if (!Arrays.equals(codeHash, LitecoinACCTv1.CODE_BYTES_HASH)) + continue; + + describeAt(repository, atAddress); + } + } + } + + private int calcTestLockTimeA(long messageTimestamp) { + return (int) (messageTimestamp / 1000L + tradeTimeout * 60); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, String tradeAddress) throws DataException { + byte[] creationBytes = LitecoinACCTv1.buildQortalAT(tradeAddress, litecoinPublicKeyHash, redeemAmount, litecoinAmount, tradeTimeout); + + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "QORT-LTC cross-chain trade"; + String description = String.format("Qortal-Litecoin cross-chain trade"); + String atType = "ACCT"; + String tags = "QORT-LTC ACCT"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void checkTradeRefund(Repository repository, Account deployer, long deployersInitialBalance, long deployAtFee) throws DataException { + long deployersPostDeploymentBalance = deployersInitialBalance - fundingAmount - deployAtFee; + int refundTimeout = tradeTimeout / 2 + 1; // close enough + + // AT should automatically refund deployer after 'refundTimeout' blocks + for (int blockCount = 0; blockCount <= refundTimeout; ++blockCount) + BlockUtils.mintBlock(repository); + + // We don't bother to exactly calculate QORT spent running AT for several blocks, but we do know the expected range + long expectedMinimumBalance = deployersPostDeploymentBalance; + long expectedMaximumBalance = deployersInitialBalance - deployAtFee; + + long actualBalance = deployer.getConfirmedBalance(Asset.QORT); + + assertTrue(String.format("Deployer's balance %s should be above minimum %s", actualBalance, expectedMinimumBalance), actualBalance > expectedMinimumBalance); + assertTrue(String.format("Deployer's balance %s should be below maximum %s", actualBalance, expectedMaximumBalance), actualBalance < expectedMaximumBalance); + } + + private void describeAt(Repository repository, String atAddress) throws DataException { + ATData atData = repository.getATRepository().fromATAddress(atAddress); + CrossChainTradeData tradeData = LitecoinACCTv1.getInstance().populateTradeData(repository, atData); + + Function epochMilliFormatter = (timestamp) -> LocalDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC).format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM)); + int currentBlockHeight = repository.getBlockRepository().getBlockchainHeight(); + + System.out.print(String.format("%s:\n" + + "\tmode: %s\n" + + "\tcreator: %s,\n" + + "\tcreation timestamp: %s,\n" + + "\tcurrent balance: %s QORT,\n" + + "\tis finished: %b,\n" + + "\tredeem payout: %s QORT,\n" + + "\texpected Litecoin: %s LTC,\n" + + "\tcurrent block height: %d,\n", + tradeData.qortalAtAddress, + tradeData.mode, + tradeData.qortalCreator, + epochMilliFormatter.apply(tradeData.creationTimestamp), + Amounts.prettyAmount(tradeData.qortBalance), + atData.getIsFinished(), + Amounts.prettyAmount(tradeData.qortAmount), + Amounts.prettyAmount(tradeData.expectedForeignAmount), + currentBlockHeight)); + + if (tradeData.mode != AcctMode.OFFERING && tradeData.mode != AcctMode.CANCELLED) { + System.out.println(String.format("\trefund timeout: %d minutes,\n" + + "\trefund height: block %d,\n" + + "\tHASH160 of secret-A: %s,\n" + + "\tLitecoin P2SH-A nLockTime: %d (%s),\n" + + "\ttrade partner: %s\n" + + "\tpartner's receiving address: %s", + tradeData.refundTimeout, + tradeData.tradeRefundHeight, + HashCode.fromBytes(tradeData.hashOfSecretA).toString().substring(0, 40), + tradeData.lockTimeA, epochMilliFormatter.apply(tradeData.lockTimeA * 1000L), + tradeData.qortalPartnerAddress, + tradeData.qortalPartnerReceivingAddress)); + } + } + + private PrivateKeyAccount createTradeAccount(Repository repository) { + // We actually use a known test account with funds to avoid PoW compute + return Common.getTestAccount(repository, "alice"); + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java new file mode 100644 index 00000000..2d04098c --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendCancelMessage.java @@ -0,0 +1,90 @@ +package org.qortal.test.crosschain.litecoinv1; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; + +public class SendCancelMessage { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: SendCancelMessage ")); + System.err.println(String.format("example: SendCancelMessage " + + "7Eztjz2TsxwbrWUYEaSdLbASKQGTfK2rR7ViFc5gaiZw \\\n" + + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 2) + usage(null); + + Common.init(); + + byte[] qortalPrivateKey = null; + String atAddress = null; + + int argIndex = 0; + try { + qortalPrivateKey = Base58.decode(args[argIndex++]); + if (qortalPrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + atAddress = args[argIndex++]; + if (!Crypto.isValidAtAddress(atAddress)) + usage("Invalid AT address"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount qortalAccount = new PrivateKeyAccount(repository, qortalPrivateKey); + + String creatorQortalAddress = qortalAccount.getAddress(); + System.out.println(String.format("Qortal address: %s", creatorQortalAddress)); + + byte[] messageData = LitecoinACCTv1.getInstance().buildCancelMessage(creatorQortalAddress); + MessageTransaction messageTransaction = MessageTransaction.build(repository, qortalAccount, Group.NO_GROUP, atAddress, messageData, false, false); + + System.out.println("Computing nonce..."); + messageTransaction.computeNonce(); + messageTransaction.sign(qortalAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); + System.exit(2); + } + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java new file mode 100644 index 00000000..20386d2a --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendRedeemMessage.java @@ -0,0 +1,101 @@ +package org.qortal.test.crosschain.litecoinv1; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; + +import com.google.common.hash.HashCode; + +public class SendRedeemMessage { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: SendRedeemMessage ")); + System.err.println(String.format("example: SendRedeemMessage " + + "dbfe739f5a3ecf7b0a22cea71f73d86ec71355b740e5972bcdf9e8bb4721ab9d \\\n" + + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \\\n" + + "\t5468697320737472696e672069732065786163746c7920333220627974657321 \\\n" + + "\tQqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 4) + usage(null); + + Common.init(); + + byte[] tradePrivateKey = null; + String atAddress = null; + byte[] secret = null; + String receiveAddress = null; + + int argIndex = 0; + try { + tradePrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + if (tradePrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + atAddress = args[argIndex++]; + if (!Crypto.isValidAtAddress(atAddress)) + usage("Invalid AT address"); + + secret = HashCode.fromString(args[argIndex++]).asBytes(); + if (secret.length != 32) + usage("Secret must be 32 bytes"); + + receiveAddress = args[argIndex++]; + if (!Crypto.isValidAddress(receiveAddress)) + usage("Invalid Qortal receive address"); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); + + byte[] messageData = LitecoinACCTv1.buildRedeemMessage(secret, receiveAddress); + MessageTransaction messageTransaction = MessageTransaction.build(repository, tradeAccount, Group.NO_GROUP, atAddress, messageData, false, false); + + System.out.println("Computing nonce..."); + messageTransaction.computeNonce(); + messageTransaction.sign(tradeAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); + System.exit(2); + } + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} diff --git a/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java new file mode 100644 index 00000000..83e9a20e --- /dev/null +++ b/src/test/java/org/qortal/test/crosschain/litecoinv1/SendTradeMessage.java @@ -0,0 +1,118 @@ +package org.qortal.test.crosschain.litecoinv1; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.Controller; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crypto.Crypto; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryFactory; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; +import org.qortal.test.crosschain.apps.Common; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import com.google.common.hash.HashCode; + +public class SendTradeMessage { + + private static void usage(String error) { + if (error != null) + System.err.println(error); + + System.err.println(String.format("usage: SendTradeMessage ")); + System.err.println(String.format("example: SendTradeMessage " + + "ed77aa2c62d785a9428725fc7f95b907be8a1cc43213239876a62cf70fdb6ecb \\\n" + + "\tAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \\\n" + + "\tQqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq \\\n" + + "\tffffffffffffffffffffffffffffffffffffffff \\\n" + + "\tdaf59884b4d1aec8c1b17102530909ee43c0151a \\\n" + + "\t1600184800")); + System.exit(1); + } + + public static void main(String[] args) { + if (args.length != 6) + usage(null); + + Common.init(); + + byte[] tradePrivateKey = null; + String atAddress = null; + String partnerTradeAddress = null; + byte[] partnerTradePublicKeyHash = null; + byte[] hashOfSecret = null; + int lockTime = 0; + + int argIndex = 0; + try { + tradePrivateKey = HashCode.fromString(args[argIndex++]).asBytes(); + if (tradePrivateKey.length != 32) + usage("Refund private key must be 32 bytes"); + + atAddress = args[argIndex++]; + if (!Crypto.isValidAtAddress(atAddress)) + usage("Invalid AT address"); + + partnerTradeAddress = args[argIndex++]; + if (!Crypto.isValidAddress(partnerTradeAddress)) + usage("Invalid partner trade Qortal address"); + + partnerTradePublicKeyHash = HashCode.fromString(args[argIndex++]).asBytes(); + if (partnerTradePublicKeyHash.length != 20) + usage("Partner trade PKH must be 20 bytes"); + + hashOfSecret = HashCode.fromString(args[argIndex++]).asBytes(); + if (hashOfSecret.length != 20) + usage("HASH160 of secret must be 20 bytes"); + + lockTime = Integer.parseInt(args[argIndex++]); + } catch (IllegalArgumentException e) { + usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage())); + } + + try { + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(Controller.getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); + } catch (DataException e) { + System.err.println(String.format("Repository start-up issue: %s", e.getMessage())); + System.exit(2); + } + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount tradeAccount = new PrivateKeyAccount(repository, tradePrivateKey); + + int refundTimeout = LitecoinACCTv1.calcRefundTimeout(NTP.getTime(), lockTime); + if (refundTimeout < 1) { + System.err.println("Refund timeout too small. Is locktime in the past?"); + System.exit(2); + } + + byte[] messageData = LitecoinACCTv1.buildTradeMessage(partnerTradeAddress, partnerTradePublicKeyHash, hashOfSecret, lockTime, refundTimeout); + MessageTransaction messageTransaction = MessageTransaction.build(repository, tradeAccount, Group.NO_GROUP, atAddress, messageData, false, false); + + System.out.println("Computing nonce..."); + messageTransaction.computeNonce(); + messageTransaction.sign(tradeAccount); + + byte[] signedBytes = null; + try { + signedBytes = TransactionTransformer.toBytes(messageTransaction.getTransactionData()); + } catch (TransformationException e) { + System.err.println(String.format("Unable to convert transaction to bytes: %s", e.getMessage())); + System.exit(2); + } + + System.out.println(String.format("%nSigned transaction in base58, ready for POST /transactions/process:%n%s", Base58.encode(signedBytes))); + } catch (DataException e) { + System.err.println(String.format("Repository issue: %s", e.getMessage())); + System.exit(2); + } + } + +} From e259a09b894014a0983936d81c9e27c1c8d19e86 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 1 Aug 2021 09:54:15 +0100 Subject: [PATCH 123/505] Fixed merge issues. --- src/main/java/org/qortal/api/ApiError.java | 2 +- .../org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/ApiError.java b/src/main/java/org/qortal/api/ApiError.java index 495910e4..659104e7 100644 --- a/src/main/java/org/qortal/api/ApiError.java +++ b/src/main/java/org/qortal/api/ApiError.java @@ -132,7 +132,7 @@ public enum ApiError { FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408), // Trade portal - ORDER_SIZE_TOO_SMALL(1300, 402); + ORDER_SIZE_TOO_SMALL(1300, 402), // Data FILE_NOT_FOUND(1401, 404), diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 9964117b..757efe34 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -4,6 +4,8 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; From c790ea07ddd3f0f747d74719019c59da38af1c17 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 12 Aug 2021 20:53:42 +0100 Subject: [PATCH 124/505] Started abstracting the file processing code away from the API handlers, and making it more modular. --- .../api/resource/ArbitraryResource.java | 25 +- .../qortal/api/resource/WebsiteResource.java | 231 ++++-------------- .../java/org/qortal/storage/DataFile.java | 8 +- .../org/qortal/storage/DataFileReader.java | 213 ++++++++++++++++ .../org/qortal/storage/DataFileWriter.java | 190 ++++++++++++++ 5 files changed, 466 insertions(+), 201 deletions(-) create mode 100644 src/main/java/org/qortal/storage/DataFileReader.java create mode 100644 src/main/java/org/qortal/storage/DataFileWriter.java diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 3b6c2b6d..12deed16 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -9,9 +9,10 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; -import java.io.File; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -29,6 +30,7 @@ import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.DataType; @@ -45,6 +47,7 @@ import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.storage.DataFileChunk; +import org.qortal.storage.DataFileWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; @@ -265,18 +268,20 @@ public class ArbitraryResource { String name = null; byte[] secret = null; - ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; - ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.ARBITRARY_DATA; - ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.NONE; + Method method = Method.PUT; + Service service = Service.ARBITRARY_DATA; + Compression compression = Compression.NONE; - // Check if a file or directory has been supplied - File file = new File(path); - if (!file.isFile()) { - LOGGER.info("Not a file: {}", path); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression); + try { + dataFileWriter.save(); + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } catch (IllegalStateException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - DataFile dataFile = DataFile.fromPath(path); + DataFile dataFile = dataFileWriter.getDataFile(); if (dataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 4761228b..c4773333 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -1,10 +1,5 @@ package org.qortal.api.resource; -import javax.crypto.BadPaddingException; -import javax.crypto.IllegalBlockSizeException; -import javax.crypto.NoSuchPaddingException; -import javax.crypto.SecretKey; -import javax.crypto.spec.SecretKeySpec; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -13,12 +8,9 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.io.*; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Paths; -import java.security.InvalidAlgorithmParameterException; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -36,10 +28,10 @@ import org.qortal.api.ApiExceptionFactory; import org.qortal.api.HTMLParser; import org.qortal.api.Security; import org.qortal.block.BlockChain; -import org.qortal.crypto.AES; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.group.Group; import org.qortal.repository.DataException; @@ -47,14 +39,15 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; +import org.qortal.storage.DataFile.*; +import org.qortal.storage.DataFileReader; +import org.qortal.storage.DataFileWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; -import org.qortal.transform.Transformer; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.utils.Base58; import org.qortal.utils.NTP; -import org.qortal.utils.ZipUtils; @Path("/site") @@ -63,11 +56,6 @@ public class WebsiteResource { private static final Logger LOGGER = LogManager.getLogger(WebsiteResource.class); - public enum ResourceIdType { - SIGNATURE, - FILE_HASH - }; - @Context HttpServletRequest request; @Context HttpServletResponse response; @Context ServletContext context; @@ -115,7 +103,16 @@ public class WebsiteResource { ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; - DataFile dataFile = this.hostWebsite(path); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression); + try { + dataFileWriter.save(); + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } catch (IllegalStateException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + DataFile dataFile = dataFileWriter.getDataFile(); if (dataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } @@ -200,7 +197,19 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } - DataFile dataFile = this.hostWebsite(directoryPath); + Method method = Method.PUT; + Compression compression = Compression.ZIP; + + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), method, compression); + try { + dataFileWriter.save(); + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } catch (IllegalStateException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + DataFile dataFile = dataFileWriter.getDataFile(); if (dataFile != null) { String digest58 = dataFile.digest58(); if (digest58 != null) { @@ -210,77 +219,6 @@ public class WebsiteResource { return "Unable to generate preview URL"; } - private DataFile hostWebsite(String directoryPath) { - - // Check if a file or directory has been supplied - File file = new File(directoryPath); - if (!file.isDirectory()) { - LOGGER.info("Not a directory: {}", directoryPath); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - - // Ensure temp folder exists - java.nio.file.Path tempDir = null; - try { - tempDir = Files.createTempDirectory("qortal-zip"); - } catch (IOException e) { - LOGGER.error("Unable to create temp directory"); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } - - // Firstly zip up the directory - String zipOutputFilePath = tempDir.toString() + File.separator + "zipped.zip"; - try { - ZipUtils.zip(directoryPath, zipOutputFilePath, "data"); - } catch (IOException e) { - LOGGER.info("Unable to zip directory", e); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - - // Next, encrypt the file with AES - String encryptedFilePath = tempDir.toString() + File.separator + "zipped_encrypted.zip"; - SecretKey aesKey; - try { - aesKey = AES.generateKey(256); - AES.encryptFile("AES", aesKey, zipOutputFilePath, encryptedFilePath); - Files.delete(Paths.get(zipOutputFilePath)); - } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException - | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR); - } - - try { - DataFile dataFile = DataFile.fromPath(encryptedFilePath); - dataFile.setSecret(aesKey.getEncoded()); - DataFile.ValidationResult validationResult = dataFile.isValid(); - if (validationResult != DataFile.ValidationResult.OK) { - LOGGER.error("Invalid file: {}", validationResult); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - LOGGER.info("Whole file digest: {}", dataFile.digest58()); - - int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); - if (chunkCount > 0) { - LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s"))); - LOGGER.info("{}", dataFile.printChunks()); - return dataFile; - } - - return null; - } - finally { - // Clean up - File zippedFile = new File(zipOutputFilePath); - if (zippedFile.exists()) { - zippedFile.delete(); - } - File encryptedFile = new File(encryptedFilePath); - if (encryptedFile.exists()) { - encryptedFile.delete(); - } - } - } - @GET @Path("{signature}") public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) { @@ -331,94 +269,21 @@ public class WebsiteResource { inPath = File.separator + inPath; } - String tempDirectory = System.getProperty("java.io.tmpdir"); - String destPath = tempDirectory + File.separator + "qortal-sites" + File.separator + resourceId; - String unencryptedPath = destPath + File.separator + "zipped.zip"; - String unzippedPath = destPath + File.separator + "data"; - - if (!Files.exists(Paths.get(unzippedPath))) { - - // Load the full transaction data so we can access the file hashes - try (final Repository repository = RepositoryManager.getRepository()) { - DataFile dataFile = null; - byte[] digest = null; - byte[] secret = null; - - if (resourceIdType == ResourceIdType.SIGNATURE) { - ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); - if (!(transactionData instanceof ArbitraryTransactionData)) { - return this.get404Response(); - } - - // Load hashes - digest = transactionData.getData(); - byte[] chunkHashes = transactionData.getChunkHashes(); - - // Load secret - secret = transactionData.getSecret(); - - // Load data file(s) - dataFile = DataFile.fromHash(digest); - if (!dataFile.exists()) { - if (!dataFile.allChunksExist(chunkHashes)) { - // TODO: fetch them? - return this.get404Response(); - } - // We have all the chunks but not the complete file, so join them - dataFile.addChunkHashes(chunkHashes); - dataFile.join(); - } - - - } - else if (resourceIdType == ResourceIdType.FILE_HASH) { - dataFile = DataFile.fromHash58(resourceId); - digest = Base58.decode(resourceId); - secret = secret58 != null ? Base58.decode(secret58) : null; - } - - // If the complete file still doesn't exist then something went wrong - if (!dataFile.exists()) { - return this.get404Response(); - } - - if (!Arrays.equals(dataFile.digest(), digest)) { - LOGGER.info("Unable to validate complete file hash"); - return this.get404Response(); - } - - // Decrypt if we have the secret key. - if (secret != null && secret.length == Transformer.AES256_LENGTH) { - try { - SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES"); - AES.decryptFile("AES", aesKey, dataFile.getFilePath(), unencryptedPath); - } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException - | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { - return this.get404Response(); - } - } - else { - // Assume it is unencrypted. We may block this. - unencryptedPath = dataFile.getFilePath(); - } - - // Unzip - try { - // TODO: compression types - //if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) { - ZipUtils.unzip(unencryptedPath, destPath); - //} - } catch (IOException e) { - LOGGER.info("Unable to unzip file"); - } - - } catch (DataException e) { - return this.get500Response(); - } + DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType); + dataFileReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only + try { + dataFileReader.load(false); + } catch (Exception e) { + return this.get404Response(); } + java.nio.file.Path path = dataFileReader.getFilePath(); + if (path == null) { + return this.get404Response(); + } + String unzippedPath = path.toString(); try { - String filename = this.getFilename(unzippedPath, inPath); + String filename = this.getFilename(unzippedPath.toString(), inPath); String filePath = unzippedPath + File.separator + filename; if (HTMLParser.isHtmlFile(filename)) { @@ -445,7 +310,7 @@ public class WebsiteResource { inputStream.close(); } return response; - } catch (FileNotFoundException e) { + } catch (FileNotFoundException | NoSuchFileException e) { LOGGER.info("File not found at path: {}", unzippedPath); if (inPath.equals("/")) { // Delete the unzipped folder if no index file was found @@ -455,7 +320,6 @@ public class WebsiteResource { LOGGER.info("Unable to delete directory: {}", unzippedPath, e); } } - } catch (IOException e) { LOGGER.info("Unable to serve file at path: {}", inPath, e); } @@ -490,19 +354,6 @@ public class WebsiteResource { return response; } - private HttpServletResponse get500Response() { - try { - String responseString = "500: Internal Server Error"; - byte[] responseData = responseString.getBytes(); - response.setStatus(500); - response.setContentLength(responseData.length); - response.getOutputStream().write(responseData); - } catch (IOException e) { - LOGGER.info("Error writing 500 response"); - } - return response; - } - private List indexFiles() { List indexFiles = new ArrayList<>(); indexFiles.add("index.html"); diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index a6a17385..568674d6 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -41,6 +41,12 @@ public class DataFile { } } + // Resource ID types + public enum ResourceIdType { + SIGNATURE, + FILE_HASH + }; + private static final Logger LOGGER = LogManager.getLogger(DataFile.class); public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB @@ -69,7 +75,7 @@ public class DataFile { } this.hash58 = Base58.encode(Crypto.digest(fileContent)); - LOGGER.debug(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); + LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); String outputFilePath = getOutputFilePath(this.hash58, true); File outputFile = new File(outputFilePath); diff --git a/src/main/java/org/qortal/storage/DataFileReader.java b/src/main/java/org/qortal/storage/DataFileReader.java new file mode 100644 index 00000000..a344c997 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileReader.java @@ -0,0 +1,213 @@ +package org.qortal.storage; + +import org.qortal.crypto.AES; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.storage.DataFile.*; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; +import org.qortal.utils.ZipUtils; + +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; + +public class DataFileReader { + + private String resourceId; + private ResourceIdType resourceIdType; + private String secret58; + private Path filePath; + private DataFile dataFile; + + // Intermediate paths + private Path workingPath; + private Path uncompressedPath; + private Path unencryptedPath; + + public DataFileReader(String resourceId, ResourceIdType resourceIdType) { + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + } + + public void load(boolean overwrite) throws IllegalStateException, IOException, DataException { + + try { + this.preExecute(); + + // Do nothing if files already exist and overwrite is set to false + if (Files.exists(this.uncompressedPath) && !overwrite) { + this.filePath = this.uncompressedPath; + return; + } + + this.fetch(); + this.decrypt(); + this.uncompress(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + this.createWorkingDirectory(); + // Initialize unzipped path as it's used in a few places + this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); + } + + private void postExecute() throws IOException { + this.cleanupFilesystem(); + } + + private void createWorkingDirectory() { + // Use the system tmpdir as our base, as it is deterministic + String baseDir = System.getProperty("java.io.tmpdir"); + Path tempDir = Paths.get(baseDir + File.separator + "qortal" + File.separator + this.resourceId); + try { + Files.createDirectories(tempDir); + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory"); + } + this.workingPath = tempDir; + } + + private void fetch() throws IllegalStateException, IOException, DataException { + switch (resourceIdType) { + + case SIGNATURE: + this.fetchFromSignature(); + break; + + case FILE_HASH: + this.fetchFromFileHash(); + break; + + default: + throw new IllegalStateException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString())); + } + } + + private void fetchFromSignature() throws IllegalStateException, IOException, DataException { + + // Load the full transaction data so we can access the file hashes + ArbitraryTransactionData transactionData; + try (final Repository repository = RepositoryManager.getRepository()) { + transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); + } + if (!(transactionData instanceof ArbitraryTransactionData)) { + throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId)); + } + + // Load hashes + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + // Load secret + byte[] secret = transactionData.getSecret(); + if (secret != null) { + this.secret58 = Base58.encode(secret); + } + + // Load data file(s) + this.dataFile = DataFile.fromHash(digest); + if (!this.dataFile.exists()) { + if (!this.dataFile.allChunksExist(chunkHashes)) { + // TODO: fetch them? + throw new IllegalStateException(String.format("Missing chunks for file {}", dataFile)); + } + // We have all the chunks but not the complete file, so join them + this.dataFile.addChunkHashes(chunkHashes); + this.dataFile.join(); + } + + // If the complete file still doesn't exist then something went wrong + if (!this.dataFile.exists()) { + throw new IOException(String.format("File doesn't exist: %s", dataFile)); + } + // Ensure the complete hash matches the joined chunks + if (!Arrays.equals(dataFile.digest(), digest)) { + throw new IllegalStateException("Unable to validate complete file hash"); + } + // Set filePath to the location of the DataFile + this.filePath = Paths.get(dataFile.getFilePath()); + } + + private void fetchFromFileHash() { + // Load data file directly from the hash + this.dataFile = DataFile.fromHash58(resourceId); + // Set filePath to the location of the DataFile + this.filePath = Paths.get(dataFile.getFilePath()); + } + + private void decrypt() { + // Decrypt if we have the secret key. + byte[] secret = this.secret58 != null ? Base58.decode(this.secret58) : null; + if (secret != null && secret.length == Transformer.AES256_LENGTH) { + try { + this.unencryptedPath = Paths.get(this.workingPath.toString() + File.separator + "zipped.zip"); + SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES"); + AES.decryptFile("AES", aesKey, this.filePath.toString(), this.unencryptedPath.toString()); + + // Replace filePath pointer with the encrypted file path + // Don't delete the original DataFile, as this is handled in the cleanup phase + this.filePath = this.unencryptedPath; + + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException + | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { + throw new IllegalStateException(String.format("Unable to decrypt file %s: %s", dataFile, e.getMessage())); + } + } else { + // Assume it is unencrypted. We may block this in the future. + this.filePath = Paths.get(this.dataFile.getFilePath()); + } + } + + private void uncompress() throws IOException { + try { + // TODO: compression types + //if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) { + ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString()); + //} + } catch (IOException e) { + throw new IllegalStateException(String.format("Unable to unzip file: %s", e.getMessage())); + } + + // Replace filePath pointer with the uncompressed file path + Files.delete(this.filePath); + this.filePath = this.uncompressedPath; + } + + private void cleanupFilesystem() throws IOException { + // Clean up + if (this.uncompressedPath != null) { + File unzippedFile = new File(this.uncompressedPath.toString()); + if (unzippedFile.exists()) { + unzippedFile.delete(); + } + } + } + + + public void setSecret58(String secret58) { + this.secret58 = secret58; + } + + public Path getFilePath() { + return this.filePath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileWriter.java b/src/main/java/org/qortal/storage/DataFileWriter.java new file mode 100644 index 00000000..b0ad0a75 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileWriter.java @@ -0,0 +1,190 @@ +package org.qortal.storage; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.crypto.AES; +import org.qortal.storage.DataFile.*; +import org.qortal.utils.ZipUtils; + +import javax.crypto.BadPaddingException; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +public class DataFileWriter { + + private static final Logger LOGGER = LogManager.getLogger(DataFileWriter.class); + + private Path filePath; + private Method method; + private Compression compression; + + private SecretKey aesKey; + private DataFile dataFile; + + // Intermediate paths to cleanup + private Path workingPath; + private Path compressedPath; + private Path encryptedPath; + + public DataFileWriter(Path filePath, Method method, Compression compression) { + this.filePath = filePath; + this.method = method; + this.compression = compression; + } + + public void save() throws IllegalStateException, IOException { + try { + this.preExecute(); + this.compress(); + this.encrypt(); + this.split(); + this.validate(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + // Enforce compression when uploading a directory + File file = new File(this.filePath.toString()); + if (file.isDirectory() && compression == Compression.NONE) { + throw new IllegalStateException("Unable to upload a directory without compression"); + } + + // Create temporary working directory + this.createWorkingDirectory(); + } + + private void postExecute() throws IOException { + this.cleanupFilesystem(); + } + + private void createWorkingDirectory() { + // Ensure temp folder exists + Path tempDir; + try { + tempDir = Files.createTempDirectory("qortal"); + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory"); + } + this.workingPath = tempDir; + } + + private void compress() { + // Compress the data if requested + if (this.compression != Compression.NONE) { + this.compressedPath = Paths.get(this.workingPath.toString() + File.separator + "zipped.zip"); + try { + + if (this.compression == Compression.ZIP) { + ZipUtils.zip(this.filePath.toString(), this.compressedPath.toString(), "data"); + } + else { + throw new IllegalStateException(String.format("Unknown compression type specified: %s", compression.toString())); + } + // FUTURE: other compression types + + // Replace filePath pointer with the zipped file path + // Don't delete the original file/directory, since this may be outside of our directory scope + this.filePath = this.compressedPath; + + } catch (IOException e) { + throw new IllegalStateException("Unable to zip directory", e); + } + } + } + + private void encrypt() { + this.encryptedPath = Paths.get(this.workingPath.toString() + File.separator + "zipped_encrypted.zip"); + try { + // Encrypt the file with AES + this.aesKey = AES.generateKey(256); + AES.encryptFile("AES", this.aesKey, this.filePath.toString(), this.encryptedPath.toString()); + + // Replace filePath pointer with the encrypted file path + Files.delete(this.filePath); + this.filePath = this.encryptedPath; + + } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException + | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { + throw new IllegalStateException(String.format("Unable to encrypt file %s: %s", this.filePath, e.getMessage())); + } + } + + private void validate() throws IOException { + this.dataFile = DataFile.fromPath(this.filePath.toString()); + if (this.dataFile == null) { + throw new IOException("No file available when validating"); + } + this.dataFile.setSecret(this.aesKey.getEncoded()); + + // Validate the file + ValidationResult validationResult = this.dataFile.isValid(); + if (validationResult != ValidationResult.OK) { + throw new IllegalStateException(String.format("File %s failed validation: %s", this.dataFile, validationResult)); + } + LOGGER.info("Whole file hash is valid: {}", this.dataFile.digest58()); + + // Validate each chunk + for (DataFileChunk chunk : this.dataFile.getChunks()) { + validationResult = chunk.isValid(); + if (validationResult != ValidationResult.OK) { + throw new IllegalStateException(String.format("Chunk %s failed validation: %s", chunk, validationResult)); + } + } + LOGGER.info("Chunk hashes are valid"); + + } + + private void split() throws IOException { + this.dataFile = DataFile.fromPath(this.filePath.toString()); + if (this.dataFile == null) { + throw new IOException("No file available when trying to split"); + } + + int chunkCount = this.dataFile.split(DataFile.CHUNK_SIZE); + if (chunkCount > 0) { + LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); + } + else { + throw new IllegalStateException("Unable to split file into chunks"); + } + } + + private void cleanupFilesystem() throws IOException { + // Clean up + if (this.compressedPath != null) { + File zippedFile = new File(this.compressedPath.toString()); + if (zippedFile.exists()) { + zippedFile.delete(); + } + } + if (this.encryptedPath != null) { + File encryptedFile = new File(this.encryptedPath.toString()); + if (encryptedFile.exists()) { + encryptedFile.delete(); + } + } + if (this.workingPath != null) { + FileUtils.deleteDirectory(new File(this.workingPath.toString())); + } + } + + + public DataFile getDataFile() { + return this.dataFile; + } + +} From 743a61bf49abc5a4e451d651783e14744c365822 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Aug 2021 06:55:44 +0100 Subject: [PATCH 125/505] Fixed bug which overwrote the data file, causing it to be invalid. --- src/main/java/org/qortal/storage/DataFileWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/qortal/storage/DataFileWriter.java b/src/main/java/org/qortal/storage/DataFileWriter.java index b0ad0a75..539cb199 100644 --- a/src/main/java/org/qortal/storage/DataFileWriter.java +++ b/src/main/java/org/qortal/storage/DataFileWriter.java @@ -124,7 +124,6 @@ public class DataFileWriter { } private void validate() throws IOException { - this.dataFile = DataFile.fromPath(this.filePath.toString()); if (this.dataFile == null) { throw new IOException("No file available when validating"); } From 5ac9e3e47afe3e94bed52388efbc075c96d64f07 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Aug 2021 19:14:44 +0100 Subject: [PATCH 126/505] Fixed recently introduced bugs in arbitrary transaction transformation. --- .../ArbitraryTransactionTransformer.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java index 149843f8..7b2b2575 100644 --- a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java @@ -36,9 +36,13 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH; private static final int NAME_SIZE_LENGTH = INT_LENGTH; private static final int COMPRESSION_LENGTH = INT_LENGTH; + private static final int METHOD_LENGTH = INT_LENGTH; + private static final int SECRET_LENGTH = INT_LENGTH; - private static final int EXTRAS_LENGTH = SERVICE_LENGTH + NONCE_LENGTH + NAME_SIZE_LENGTH + SERVICE_LENGTH + - COMPRESSION_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH + RAW_DATA_SIZE_LENGTH + CHUNKS_SIZE_LENGTH; + private static final int EXTRAS_LENGTH = SERVICE_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH; + + private static final int EXTRAS_V5_LENGTH = NONCE_LENGTH + NAME_SIZE_LENGTH + METHOD_LENGTH + SECRET_LENGTH + + COMPRESSION_LENGTH + RAW_DATA_SIZE_LENGTH + CHUNKS_SIZE_LENGTH; protected static final TransactionLayout layout; @@ -51,6 +55,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("sender's public key", TransformationType.PUBLIC_KEY); layout.add("nonce", TransformationType.INT); // Version 5+ + layout.add("name length", TransformationType.INT); // Version 5+ layout.add("name", TransformationType.DATA); // Version 5+ layout.add("method", TransformationType.INT); // Version 5+ layout.add("secret length", TransformationType.INT); // Version 5+ @@ -162,10 +167,15 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; int nameLength = Utf8.encodedLength(arbitraryTransactionData.getName()); + int secretLength = (arbitraryTransactionData.getSecret() != null) ? arbitraryTransactionData.getSecret().length : 0; int dataLength = (arbitraryTransactionData.getData() != null) ? arbitraryTransactionData.getData().length : 0; int chunkHashesLength = (arbitraryTransactionData.getChunkHashes() != null) ? arbitraryTransactionData.getChunkHashes().length : 0; - int length = getBaseLength(transactionData) + EXTRAS_LENGTH + nameLength + dataLength + chunkHashesLength; + int length = getBaseLength(transactionData) + EXTRAS_LENGTH + nameLength + secretLength + dataLength + chunkHashesLength; + + if (arbitraryTransactionData.getVersion() >= 5) { + length += EXTRAS_V5_LENGTH; + } // Optional payments length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength(); From e15cf063c6c2ab05ab5a85b30bf22937360ded39 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 13:11:36 +0100 Subject: [PATCH 127/505] Initial implementation of data patches/updates This adds support for the PATCH method in addition to the existing PUT method. Currently, a patch includes only files that have been added or modified, as well as placeholder files to indicate those that have been removed. This is not production ready, as I am hoping to create patches on a more granular level - i.e. just the modified bytes of each file. It would also make sense to track deletions using a metadata/manifest file in a hidden folder. It also adds early support of accessing files using a name rather than a signature or hash. --- src/main/java/org/qortal/api/HTMLParser.java | 4 +- .../api/resource/ArbitraryResource.java | 4 +- .../qortal/api/resource/WebsiteResource.java | 49 ++-- .../controller/ArbitraryDataManager.java | 2 +- .../repository/ArbitraryRepository.java | 7 + .../hsqldb/HSQLDBArbitraryRepository.java | 145 +++++++++++- .../java/org/qortal/storage/DataFile.java | 4 +- .../org/qortal/storage/DataFileBuilder.java | 131 +++++++++++ .../org/qortal/storage/DataFileCombiner.java | 56 +++++ .../qortal/storage/DataFileCreatePatch.java | 58 +++++ .../java/org/qortal/storage/DataFileDiff.java | 218 ++++++++++++++++++ .../org/qortal/storage/DataFileMerge.java | 190 +++++++++++++++ .../org/qortal/storage/DataFilePatches.java | 66 ++++++ .../org/qortal/storage/DataFileReader.java | 169 +++++++++++--- .../org/qortal/storage/DataFileWriter.java | 40 +++- .../transaction/ArbitraryTransaction.java | 3 +- .../org/qortal/utils/FilesystemUtils.java | 34 +++ 17 files changed, 1124 insertions(+), 56 deletions(-) create mode 100644 src/main/java/org/qortal/storage/DataFileBuilder.java create mode 100644 src/main/java/org/qortal/storage/DataFileCombiner.java create mode 100644 src/main/java/org/qortal/storage/DataFileCreatePatch.java create mode 100644 src/main/java/org/qortal/storage/DataFileDiff.java create mode 100644 src/main/java/org/qortal/storage/DataFileMerge.java create mode 100644 src/main/java/org/qortal/storage/DataFilePatches.java create mode 100644 src/main/java/org/qortal/utils/FilesystemUtils.java diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index ea99afba..24c7f30d 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -13,9 +13,9 @@ public class HTMLParser { private String linkPrefix; - public HTMLParser(String resourceId, String inPath, boolean usePrefix) { + public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix) { String inPathWithoutFilename = inPath.substring(0, inPath.lastIndexOf('/')); - this.linkPrefix = usePrefix ? String.format("/site/%s%s", resourceId, inPathWithoutFilename) : ""; + this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : ""; } /** diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 12deed16..63157320 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -272,10 +272,10 @@ public class ArbitraryResource { Service service = Service.ARBITRARY_DATA; Compression compression = Compression.NONE; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression); try { dataFileWriter.save(); - } catch (IOException e) { + } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index c4773333..01232d2d 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -98,15 +98,15 @@ public class WebsiteResource { } byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); - String name = null; - ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; + String name = "CalDescentTest1"; // TODO: dynamic + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PATCH; ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression); try { dataFileWriter.save(); - } catch (IOException e) { + } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -144,6 +144,7 @@ public class WebsiteResource { secret, compression, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + LOGGER.info("Computing nonce..."); transaction.computeNonce(); Transaction.ValidationResult result = transaction.isValidUnconfirmed(); @@ -197,13 +198,15 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } + String name = null; + Service service = Service.WEBSITE; Method method = Method.PUT; Compression compression = Compression.ZIP; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), method, compression); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), name, service, method, compression); try { dataFileWriter.save(); - } catch (IOException e) { + } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -222,26 +225,38 @@ public class WebsiteResource { @GET @Path("{signature}") public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) { - return this.get(signature, ResourceIdType.SIGNATURE, "/", null,true); + return this.get(signature, ResourceIdType.SIGNATURE, "/", null, "/site", true); } @GET @Path("{signature}/{path:.*}") public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) { - return this.get(signature, ResourceIdType.SIGNATURE, inPath,null,true); + return this.get(signature, ResourceIdType.SIGNATURE, inPath,null, "/site", true); } @GET @Path("/hash/{hash}") public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) { - return this.get(hash58, ResourceIdType.FILE_HASH, "/", secret58,true); + return this.get(hash58, ResourceIdType.FILE_HASH, "/", secret58, "/site/hash", true); + } + + @GET + @Path("/name/{name}/{path:.*}") + public HttpServletResponse getPathByName(@PathParam("name") String name, @PathParam("path") String inPath) { + return this.get(name, ResourceIdType.NAME, inPath, null, "/site/name", true); + } + + @GET + @Path("/name/{name}") + public HttpServletResponse getIndexByName(@PathParam("name") String name) { + return this.get(name, ResourceIdType.NAME, "/", null, "/site/name", true); } @GET @Path("/hash/{hash}/{path:.*}") public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath, @QueryParam("secret") String secret58) { - return this.get(hash58, ResourceIdType.FILE_HASH, inPath, secret58,true); + return this.get(hash58, ResourceIdType.FILE_HASH, inPath, secret58, "/site/hash", true); } @GET @@ -259,19 +274,23 @@ public class WebsiteResource { private HttpServletResponse getDomainMap(String inPath) { Map domainMap = Settings.getInstance().getSimpleDomainMap(); if (domainMap != null && domainMap.containsKey(request.getServerName())) { - return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, false); + return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, "", false); } return this.get404Response(); } - private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58, boolean usePrefix) { + private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58, + String prefix, boolean usePrefix) { if (!inPath.startsWith(File.separator)) { inPath = File.separator + inPath; } - DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType); + Service service = Service.WEBSITE; + DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType, service); dataFileReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { + // TODO: overwrite if new transaction arrives, to invalidate cache + // We could store the latest transaction signature in the extracted folder dataFileReader.load(false); } catch (Exception e) { return this.get404Response(); @@ -289,7 +308,7 @@ public class WebsiteResource { if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory - HTMLParser htmlParser = new HTMLParser(resourceId, inPath, usePrefix); + HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix); data = htmlParser.replaceRelativeLinks(filename, data); response.setContentType(context.getMimeType(filename)); response.setContentLength(data.length); @@ -311,7 +330,7 @@ public class WebsiteResource { } return response; } catch (FileNotFoundException | NoSuchFileException e) { - LOGGER.info("File not found at path: {}", unzippedPath); + LOGGER.info("Unable to serve file: {}", e.getMessage()); if (inPath.equals("/")) { // Delete the unzipped folder if no index file was found try { diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 2968db3d..29827e30 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -368,7 +368,7 @@ public class ArbitraryDataManager extends Thread { // Load file(s) and add any that exist to the list of hashes DataFile dataFile = DataFile.fromHash(hash); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); for (DataFileChunk dataFileChunk : dataFile.getChunks()) { if (dataFileChunk.exists()) { diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 80f8c1e3..5e3e657a 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -1,6 +1,9 @@ package org.qortal.repository; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; + +import java.util.List; public interface ArbitraryRepository { @@ -12,4 +15,8 @@ public interface ArbitraryRepository { public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException; + public List getArbitraryTransactions(String name, Service service, long since) throws DataException; + + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException; + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index b3edf41a..5bc174e2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -3,12 +3,20 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.crypto.Crypto; +import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; -import org.qortal.data.transaction.ArbitraryTransactionData.DataType; import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.DataException; import org.qortal.storage.DataFile; +import org.qortal.transaction.Transaction.ApprovalStatus; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; public class HSQLDBArbitraryRepository implements ArbitraryRepository { @@ -48,7 +56,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { // Load data file(s) DataFile dataFile = DataFile.fromHash(digest); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -83,7 +91,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { // Load data file(s) DataFile dataFile = DataFile.fromHash(digest); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -168,7 +176,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { // Load data file(s) DataFile dataFile = DataFile.fromHash(digest); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -176,4 +184,133 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { dataFile.deleteAll(); } + @Override + public List getArbitraryTransactions(String name, Service service, long since) throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + + "name, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE name = ? AND service = ? AND created_when >= ?" + + "ORDER BY created_when ASC"; + List arbitraryTransactionData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, since)) { + if (resultSet == null) + return null; + + do { + //TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + Service serviceResult = Service.valueOf(resultSet.getInt(13)); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] chunkHashes = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + Method method = Method.valueOf(resultSet.getInt(19)); + byte[] secret = resultSet.getBytes(20); + Compression compression = Compression.valueOf(resultSet.getInt(21)); + + List payments = new ArrayList<>(); // TODO: this.getPaymentsFromSignature(baseTransactionData.getSignature()); + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceResult, nonce, size, nameResult, method, secret, compression, data, + dataType, chunkHashes, payments); + + arbitraryTransactionData.add(transactionData); + } while (resultSet.next()); + + return arbitraryTransactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + + @Override + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + + "name, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE name = ? AND service = ? AND update_method = ? " + + "ORDER BY created_when DESC LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, method.value)) { + if (resultSet == null) + return null; + + //TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + Service serviceResult = Service.valueOf(resultSet.getInt(13)); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] chunkHashes = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + Method methodResult = Method.valueOf(resultSet.getInt(19)); + byte[] secret = resultSet.getBytes(20); + Compression compression = Compression.valueOf(resultSet.getInt(21)); + + List payments = new ArrayList<>(); // TODO: this.getPaymentsFromSignature(baseTransactionData.getSignature()); + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceResult, nonce, size, nameResult, methodResult, secret, compression, data, + dataType, chunkHashes, payments); + + return transactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 568674d6..75579e1a 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -44,7 +44,9 @@ public class DataFile { // Resource ID types public enum ResourceIdType { SIGNATURE, - FILE_HASH + FILE_HASH, + TRANSACTION_DATA, + NAME }; private static final Logger LOGGER = LogManager.getLogger(DataFile.class); diff --git a/src/main/java/org/qortal/storage/DataFileBuilder.java b/src/main/java/org/qortal/storage/DataFileBuilder.java new file mode 100644 index 00000000..b4faba02 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileBuilder.java @@ -0,0 +1,131 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.Method; +import org.qortal.data.transaction.ArbitraryTransactionData.Service; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.storage.DataFile.ResourceIdType; +import org.qortal.utils.Base58; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class DataFileBuilder { + + private static final Logger LOGGER = LogManager.getLogger(DataFileBuilder.class); + + private String name; + private Service service; + + private List transactions; + private ArbitraryTransactionData latestPutTransaction; + private List paths; + private Path finalPath; + + public DataFileBuilder(String name, Service service) { + this.name = name; + this.service = service; + this.paths = new ArrayList<>(); + } + + public void build() throws DataException, IOException { + this.fetchTransactions(); + this.validateTransactions(); + this.processTransactions(); + this.buildLatestState(); + } + + private void fetchTransactions() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Get the most recent PUT + ArbitraryTransactionData latestPut = repository.getArbitraryRepository() + .getLatestTransaction(this.name, this.service, Method.PUT); + if (latestPut == null) { + throw new IllegalStateException("Cannot PATCH without existing PUT. Deploy using PUT first."); + } + this.latestPutTransaction = latestPut; + + // Load all transactions since the latest PUT + List transactionDataList = repository.getArbitraryRepository() + .getArbitraryTransactions(this.name, this.service, latestPut.getTimestamp()); + this.transactions = transactionDataList; + } + } + + private void validateTransactions() { + List transactionDataList = new ArrayList<>(this.transactions); + ArbitraryTransactionData latestPut = this.latestPutTransaction; + + if (latestPut == null) { + throw new IllegalStateException("Cannot PATCH without existing PUT. Deploy using PUT first."); + } + if (latestPut.getMethod() != Method.PUT) { + throw new IllegalStateException("Expected PUT but received PATCH"); + } + if (transactionDataList.size() == 0) { + throw new IllegalStateException(String.format("No transactions found for name %s, service %s, since %d", + name, service, latestPut.getTimestamp())); + } + + // Verify that the signature of the first transaction matches the latest PUT + ArbitraryTransactionData firstTransaction = transactionDataList.get(0); + if (!Objects.equals(firstTransaction.getSignature(), latestPut.getSignature())) { + throw new IllegalStateException("First transaction did not match latest PUT transaction"); + } + + // Remove the first transaction, as it should be the only PUT + transactionDataList.remove(0); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + if (!(transactionData instanceof ArbitraryTransactionData)) { + String sig58 = Base58.encode(transactionData.getSignature()); + throw new IllegalStateException(String.format("Received non-arbitrary transaction: %s", sig58)); + } + if (transactionData.getMethod() != Method.PATCH) { + throw new IllegalStateException("Expected PATCH but received PUT"); + } + } + } + + private void processTransactions() throws IOException, DataException { + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature())); + + // Build the data file, overwriting anything that was previously there + String sig58 = Base58.encode(transactionData.getSignature()); + DataFileReader dataFileReader = new DataFileReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); + dataFileReader.setTransactionData(transactionData); + dataFileReader.load(true); + Path path = dataFileReader.getFilePath(); + if (path == null) { + throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58)); + } + if (!Files.exists(path)) { + throw new IllegalStateException(String.format("Path doesn't exist when building data from transaction %s", sig58)); + } + paths.add(path); + } + } + + private void buildLatestState() throws IOException, DataException { + DataFilePatches dataFilePatches = new DataFilePatches(this.paths); + dataFilePatches.applyPatches(); + this.finalPath = dataFilePatches.getFinalPath(); + } + + public Path getFinalPath() { + return this.finalPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileCombiner.java b/src/main/java/org/qortal/storage/DataFileCombiner.java new file mode 100644 index 00000000..edb7b362 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileCombiner.java @@ -0,0 +1,56 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class DataFileCombiner { + + private static final Logger LOGGER = LogManager.getLogger(DataFileCombiner.class); + + private Path pathBefore; + private Path pathAfter; + private Path finalPath; + + public DataFileCombiner(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void combine() throws IOException { + try { + this.preExecute(); + this.process(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + if (this.pathBefore == null || this.pathAfter == null) { + throw new IllegalStateException(String.format("No paths available to build patch")); + } + if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) { + throw new IllegalStateException(String.format("Unable to create patch because at least one path doesn't exist")); + } + } + + private void postExecute() { + + } + + private void process() throws IOException { + DataFileMerge merge = new DataFileMerge(this.pathBefore, this.pathAfter); + merge.compute(); + this.finalPath = merge.getMergePath(); + } + + public Path getFinalPath() { + return this.finalPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileCreatePatch.java b/src/main/java/org/qortal/storage/DataFileCreatePatch.java new file mode 100644 index 00000000..67ecf9cb --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileCreatePatch.java @@ -0,0 +1,58 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.repository.DataException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class DataFileCreatePatch { + + private static final Logger LOGGER = LogManager.getLogger(DataFileCreatePatch.class); + + private Path pathBefore; + private Path pathAfter; + private Path finalPath; + + public DataFileCreatePatch(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void create() throws DataException, IOException { + try { + this.preExecute(); + this.process(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + if (this.pathBefore == null || this.pathAfter == null) { + throw new IllegalStateException(String.format("No paths available to build patch")); + } + if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) { + throw new IllegalStateException(String.format("Unable to create patch because at least one path doesn't exist")); + } + } + + private void postExecute() { + + } + + private void process() { + + DataFileDiff diff = new DataFileDiff(this.pathBefore, this.pathAfter); + diff.compute(); + this.finalPath = diff.getDiffPath(); + } + + public Path getFinalPath() { + return this.finalPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileDiff.java b/src/main/java/org/qortal/storage/DataFileDiff.java new file mode 100644 index 00000000..e6534f79 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileDiff.java @@ -0,0 +1,218 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; + +public class DataFileDiff { + + private static final Logger LOGGER = LogManager.getLogger(DataFileDiff.class); + + private Path pathBefore; + private Path pathAfter; + private Path diffPath; + + public DataFileDiff(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void compute() { + try { + this.preExecute(); + this.findAddedOrModifiedFiles(); + this.findRemovedFiles(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + this.createOutputDirectory(); + } + + private void postExecute() { + + } + + private void createOutputDirectory() { + // Ensure temp folder exists + Path tempDir; + try { + tempDir = Files.createTempDirectory("qortal-diff"); + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory"); + } + this.diffPath = tempDir; + } + + private void findAddedOrModifiedFiles() { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + +// LOGGER.info("this.pathBefore: {}", this.pathBefore); +// LOGGER.info("this.pathAfter: {}", this.pathAfter); +// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute); +// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute); +// LOGGER.info("diffPathAbsolute: {}", diffPathAbsolute); + + + try { + // Check for additions or modifications + Files.walkFileTree(this.pathAfter, new FileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException { + Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath()); + Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter); + + boolean wasAdded = false; + boolean wasModified = false; + + if (!Files.exists(filePathBefore)) { + LOGGER.info("File was added: {}", after.toString()); + wasAdded = true; + } + else if (Files.size(after) != Files.size(filePathBefore)) { + // Check file size first because it's quicker + LOGGER.info("File size was modified: {}", after.toString()); + wasModified = true; + } + else if (!Arrays.equals(DataFileDiff.digestFromPath(after), DataFileDiff.digestFromPath(filePathBefore))) { + // Check hashes as a last resort + LOGGER.info("File contents were modified: {}", after.toString()); + wasModified = true; + } + + if (wasAdded | wasModified) { + DataFileDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter); + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + } + } + + private void findRemovedFiles() { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + try { + // Check for removals + Files.walkFileTree(this.pathBefore, new FileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path before, BasicFileAttributes attrs) throws IOException { + Path directoryPathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); + Path directoryPathAfter = pathAfterAbsolute.resolve(directoryPathBefore); + + if (!Files.exists(directoryPathAfter)) { + LOGGER.info("Directory was removed: {}", directoryPathAfter.toString()); + + DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore); + // TODO: we might need to mark directories differently to files + // TODO: add path to manifest JSON + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path before, BasicFileAttributes attrs) throws IOException { + Path filePathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); + Path filePathAfter = pathAfterAbsolute.resolve(filePathBefore); + + if (!Files.exists(filePathAfter)) { + LOGGER.trace("File was removed: {}", before.toString()); + + DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore); + // TODO: add path to manifest JSON + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + } + } + + + private static byte[] digestFromPath(Path path) { + try { + return Crypto.digest(Files.readAllBytes(path)); + } catch (IOException e) { + return null; + } + } + + private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + + private static void markFilePathAsRemoved(Path base, Path relativePath) throws IOException { + String newFilename = relativePath.toString().concat(".removed"); + Path dest = Paths.get(base.toString(), newFilename); + File file = new File(dest.toString()); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + LOGGER.info("Creating file {}", dest); + file.createNewFile(); + } + + + public Path getDiffPath() { + return this.diffPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileMerge.java b/src/main/java/org/qortal/storage/DataFileMerge.java new file mode 100644 index 00000000..c03018cf --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileMerge.java @@ -0,0 +1,190 @@ +package org.qortal.storage; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; +import org.qortal.utils.FilesystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; + +public class DataFileMerge { + + private static final Logger LOGGER = LogManager.getLogger(DataFileMerge.class); + + private Path pathBefore; + private Path pathAfter; + private Path mergePath; + + public DataFileMerge(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void compute() throws IOException { + try { + this.preExecute(); + this.copyPreviousStateToMergePath(); + this.findDifferences(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + this.createOutputDirectory(); + } + + private void postExecute() { + + } + + private void createOutputDirectory() { + // Ensure temp folder exists + Path tempDir; + try { + tempDir = Files.createTempDirectory("qortal-diff"); + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory"); + } + this.mergePath = tempDir; + } + + private void copyPreviousStateToMergePath() throws IOException { + DataFileMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get("")); + } + + private void findDifferences() { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path mergePathAbsolute = this.mergePath.toAbsolutePath(); + +// LOGGER.info("this.pathBefore: {}", this.pathBefore); +// LOGGER.info("this.pathAfter: {}", this.pathAfter); +// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute); +// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute); +// LOGGER.info("mergePathAbsolute: {}", mergePathAbsolute); + + + try { + // Check for additions or modifications + Files.walkFileTree(this.pathAfter, new FileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException { + Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath()); + Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter); + + boolean wasAdded = false; + boolean wasModified = false; + boolean wasRemoved = false; + + if (after.toString().endsWith(".removed")) { + LOGGER.trace("File was removed: {}", after.toString()); + wasRemoved = true; + } + else if (!Files.exists(filePathBefore)) { + LOGGER.trace("File was added: {}", after.toString()); + wasAdded = true; + } + else if (Files.size(after) != Files.size(filePathBefore)) { + // Check file size first because it's quicker + LOGGER.trace("File size was modified: {}", after.toString()); + wasModified = true; + } + else if (!Arrays.equals(DataFileMerge.digestFromPath(after), DataFileMerge.digestFromPath(filePathBefore))) { + // Check hashes as a last resort + LOGGER.trace("File contents were modified: {}", after.toString()); + wasModified = true; + } + + if (wasAdded | wasModified) { + DataFileMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter); + } + + if (wasRemoved) { + if (filePathAfter.toString().endsWith(".removed")) { + // Trim the ".removed" + Path filePathAfterTrimmed = Paths.get(filePathAfter.toString().substring(0, filePathAfter.toString().length()-8)); + DataFileMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed); + } + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + } + } + + + private static byte[] digestFromPath(Path path) { + try { + return Crypto.digest(Files.readAllBytes(path)); + } catch (IOException e) { + return null; + } + } + + private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + + private static void copyDirPathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + FilesystemUtils.copyDirectory(source.toString(), dest.toString()); + } + + private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException { + Path dest = Paths.get(base.toString(), relativePath.toString()); + File file = new File(dest.toString()); + if (file.exists() && file.isFile()) { + LOGGER.trace("Deleting file {}", dest); + Files.delete(dest); + } + if (file.exists() && file.isDirectory()) { + LOGGER.trace("Deleting directory {}", dest); + FileUtils.deleteDirectory(file); + } + } + + public Path getMergePath() { + return this.mergePath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFilePatches.java b/src/main/java/org/qortal/storage/DataFilePatches.java new file mode 100644 index 00000000..98fe2f07 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFilePatches.java @@ -0,0 +1,66 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.qortal.repository.DataException; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +public class DataFilePatches { + + private static final Logger LOGGER = LogManager.getLogger(DataFilePatches.class); + + private List paths; + private Path finalPath; + + public DataFilePatches(List paths) { + this.paths = paths; + } + + public void applyPatches() throws DataException, IOException { + try { + this.preExecute(); + this.process(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + if (this.paths == null || this.paths.isEmpty()) { + throw new IllegalStateException(String.format("No paths available to build latest state")); + } + } + + private void postExecute() { + + } + + private void process() throws IOException { + if (this.paths.size() == 1) { + // No patching needed + this.finalPath = this.paths.get(0); + return; + } + + Path pathBefore = this.paths.get(0); + + // Loop from the second path onwards + for (int i=1; i() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + // Don't delete the parent directory, as we want to leave an empty folder + if (dir.compareTo(uncompressedPath) == 0) { + return FileVisitResult.CONTINUE; + } + + if (e == null) { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } else { + throw e; + } + } + + }); + } catch (IOException e) { + LOGGER.info("Unable to delete file or directory: {}", e.getMessage()); + } + } + } + } + private void fetch() throws IllegalStateException, IOException, DataException { switch (resourceIdType) { + case FILE_HASH: + this.fetchFromFileHash(); + break; + + case NAME: + this.fetchFromName(); + break; + case SIGNATURE: this.fetchFromSignature(); break; - case FILE_HASH: - this.fetchFromFileHash(); + case TRANSACTION_DATA: + this.fetchFromTransactionData(this.transactionData); break; default: @@ -101,9 +163,30 @@ public class DataFileReader { } } + private void fetchFromFileHash() { + // Load data file directly from the hash + DataFile dataFile = DataFile.fromHash58(resourceId); + // Set filePath to the location of the DataFile + this.filePath = Paths.get(dataFile.getFilePath()); + } + + private void fetchFromName() throws IllegalStateException, IOException, DataException { + + // Build the existing state using past transactions + DataFileBuilder builder = new DataFileBuilder(this.resourceId, this.service); + builder.build(); + Path builtPath = builder.getFinalPath(); + if (builtPath == null) { + throw new IllegalStateException("Unable to build path"); + } + + // Set filePath to the builtPath + this.filePath = builtPath; + } + private void fetchFromSignature() throws IllegalStateException, IOException, DataException { - // Load the full transaction data so we can access the file hashes + // Load the full transaction data from the database so we can access the file hashes ArbitraryTransactionData transactionData; try (final Repository repository = RepositoryManager.getRepository()) { transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); @@ -112,6 +195,14 @@ public class DataFileReader { throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId)); } + this.fetchFromTransactionData(transactionData); + } + + private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws IllegalStateException, IOException, DataException { + if (!(transactionData instanceof ArbitraryTransactionData)) { + throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId)); + } + // Load hashes byte[] digest = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); @@ -123,19 +214,19 @@ public class DataFileReader { } // Load data file(s) - this.dataFile = DataFile.fromHash(digest); - if (!this.dataFile.exists()) { - if (!this.dataFile.allChunksExist(chunkHashes)) { + DataFile dataFile = DataFile.fromHash(digest); + if (!dataFile.exists()) { + if (!dataFile.allChunksExist(chunkHashes)) { // TODO: fetch them? throw new IllegalStateException(String.format("Missing chunks for file {}", dataFile)); } // We have all the chunks but not the complete file, so join them - this.dataFile.addChunkHashes(chunkHashes); - this.dataFile.join(); + dataFile.addChunkHashes(chunkHashes); + dataFile.join(); } // If the complete file still doesn't exist then something went wrong - if (!this.dataFile.exists()) { + if (!dataFile.exists()) { throw new IOException(String.format("File doesn't exist: %s", dataFile)); } // Ensure the complete hash matches the joined chunks @@ -146,13 +237,6 @@ public class DataFileReader { this.filePath = Paths.get(dataFile.getFilePath()); } - private void fetchFromFileHash() { - // Load data file directly from the hash - this.dataFile = DataFile.fromHash58(resourceId); - // Set filePath to the location of the DataFile - this.filePath = Paths.get(dataFile.getFilePath()); - } - private void decrypt() { // Decrypt if we have the secret key. byte[] secret = this.secret58 != null ? Base58.decode(this.secret58) : null; @@ -168,15 +252,26 @@ public class DataFileReader { } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { - throw new IllegalStateException(String.format("Unable to decrypt file %s: %s", dataFile, e.getMessage())); + throw new IllegalStateException(String.format("Unable to decrypt file at path %s: %s", this.filePath, e.getMessage())); } } else { - // Assume it is unencrypted. We may block this in the future. - this.filePath = Paths.get(this.dataFile.getFilePath()); + // Assume it is unencrypted. This will be the case when we have built a custom path by combining + // multiple decrypted archives into a single state. } } private void uncompress() throws IOException { + if (this.filePath == null || !Files.exists(this.filePath)) { + throw new IllegalStateException("Can't uncompress non-existent file path"); + } + File file = new File(this.filePath.toString()); + if (file.isDirectory()) { + // Already a directory - nothing to uncompress + // We still need to copy the directory to its final destination if it's not already there + this.copyFilePathToFinalDestination(); + return; + } + try { // TODO: compression types //if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) { @@ -191,6 +286,20 @@ public class DataFileReader { this.filePath = this.uncompressedPath; } + private void copyFilePathToFinalDestination() throws IOException { + if (this.filePath.compareTo(this.uncompressedPath) != 0) { + File source = new File(this.filePath.toString()); + File dest = new File(this.uncompressedPath.toString()); + if (source == null || !source.exists()) { + throw new IllegalStateException("Source directory doesn't exist"); + } + if (dest == null || !dest.exists()) { + throw new IllegalStateException("Destination directory doesn't exist"); + } + FilesystemUtils.copyDirectory(source.toString(), dest.toString()); + } + } + private void cleanupFilesystem() throws IOException { // Clean up if (this.uncompressedPath != null) { @@ -202,6 +311,10 @@ public class DataFileReader { } + public void setTransactionData(ArbitraryTransactionData transactionData) { + this.transactionData = transactionData; + } + public void setSecret58(String secret58) { this.secret58 = secret58; } diff --git a/src/main/java/org/qortal/storage/DataFileWriter.java b/src/main/java/org/qortal/storage/DataFileWriter.java index 539cb199..a0771f25 100644 --- a/src/main/java/org/qortal/storage/DataFileWriter.java +++ b/src/main/java/org/qortal/storage/DataFileWriter.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.crypto.AES; +import org.qortal.repository.DataException; import org.qortal.storage.DataFile.*; import org.qortal.utils.ZipUtils; @@ -26,6 +27,8 @@ public class DataFileWriter { private static final Logger LOGGER = LogManager.getLogger(DataFileWriter.class); private Path filePath; + private String name; + private Service service; private Method method; private Compression compression; @@ -37,15 +40,18 @@ public class DataFileWriter { private Path compressedPath; private Path encryptedPath; - public DataFileWriter(Path filePath, Method method, Compression compression) { + public DataFileWriter(Path filePath, String name, Service service, Method method, Compression compression) { this.filePath = filePath; + this.name = name; + this.service = service; this.method = method; this.compression = compression; } - public void save() throws IllegalStateException, IOException { + public void save() throws IllegalStateException, IOException, DataException { try { this.preExecute(); + this.process(); this.compress(); this.encrypt(); this.split(); @@ -82,6 +88,36 @@ public class DataFileWriter { this.workingPath = tempDir; } + private void process() throws DataException, IOException { + switch (this.method) { + + case PUT: + // Nothing to do + break; + + case PATCH: + this.processPatch(); + break; + + default: + throw new IllegalStateException(String.format("Unknown method specified: %s", method.toString())); + } + } + + private void processPatch() throws DataException, IOException { + + // Build the existing state using past transactions + DataFileBuilder builder = new DataFileBuilder(this.name, this.service); + builder.build(); + Path builtPath = builder.getFinalPath(); + + // Compute a diff of the latest changes on top of the previous state + // Then use only the differences as our data payload + DataFileCreatePatch patch = new DataFileCreatePatch(builtPath, this.filePath); + patch.create(); + this.filePath = patch.getFinalPath(); + } + private void compress() { // Compress the data if requested if (this.compression != Compression.NONE) { diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 6ee8d5ff..cb9142d8 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -108,7 +108,8 @@ public class ArbitraryTransaction extends Transaction { if (chunkHashes == null && expectedChunkHashesSize > 0) { return ValidationResult.INVALID_DATA_LENGTH; } - if (chunkHashes.length != expectedChunkHashesSize) { + int chunkHashesLength = chunkHashes != null ? chunkHashes.length : 0; + if (chunkHashesLength != expectedChunkHashesSize) { return ValidationResult.INVALID_DATA_LENGTH; } } diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java new file mode 100644 index 00000000..a87e18a0 --- /dev/null +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -0,0 +1,34 @@ +package org.qortal.utils; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class FilesystemUtils { + + public static boolean isDirectoryEmpty(Path path) throws IOException { + if (Files.isDirectory(path)) { + try (DirectoryStream directory = Files.newDirectoryStream(path)) { + return !directory.iterator().hasNext(); + } + } + + return false; + } + + public static void copyDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { + Files.walk(Paths.get(sourceDirectoryLocation)) + .forEach(source -> { + Path destination = Paths.get(destinationDirectoryLocation, source.toString() + .substring(sourceDirectoryLocation.length())); + try { + Files.copy(source, destination); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + +} From 16d93b17752ca73e910c941bebeb4c5587041bb2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 13:18:02 +0100 Subject: [PATCH 128/505] Renamed newly added classes to ArbitraryData*.java instead of DataFile*.java --- .../api/resource/ArbitraryResource.java | 8 +++---- .../qortal/api/resource/WebsiteResource.java | 24 +++++++++---------- ...Builder.java => ArbitraryDataBuilder.java} | 20 ++++++++-------- ...mbiner.java => ArbitraryDataCombiner.java} | 8 +++---- ...tch.java => ArbitraryDataCreatePatch.java} | 8 +++---- ...taFileDiff.java => ArbitraryDataDiff.java} | 14 +++++------ ...FileMerge.java => ArbitraryDataMerge.java} | 14 +++++------ ...Patches.java => ArbitraryDataPatches.java} | 8 +++---- ...leReader.java => ArbitraryDataReader.java} | 8 +++---- ...leWriter.java => ArbitraryDataWriter.java} | 10 ++++---- 10 files changed, 61 insertions(+), 61 deletions(-) rename src/main/java/org/qortal/storage/{DataFileBuilder.java => ArbitraryDataBuilder.java} (87%) rename src/main/java/org/qortal/storage/{DataFileCombiner.java => ArbitraryDataCombiner.java} (80%) rename src/main/java/org/qortal/storage/{DataFileCreatePatch.java => ArbitraryDataCreatePatch.java} (81%) rename src/main/java/org/qortal/storage/{DataFileDiff.java => ArbitraryDataDiff.java} (92%) rename src/main/java/org/qortal/storage/{DataFileMerge.java => ArbitraryDataMerge.java} (91%) rename src/main/java/org/qortal/storage/{DataFilePatches.java => ArbitraryDataPatches.java} (83%) rename src/main/java/org/qortal/storage/{DataFileReader.java => ArbitraryDataReader.java} (97%) rename src/main/java/org/qortal/storage/{DataFileWriter.java => ArbitraryDataWriter.java} (94%) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 63157320..a5a97123 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -47,7 +47,7 @@ import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.storage.DataFileChunk; -import org.qortal.storage.DataFileWriter; +import org.qortal.storage.ArbitraryDataWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; @@ -272,16 +272,16 @@ public class ArbitraryResource { Service service = Service.ARBITRARY_DATA; Compression compression = Compression.NONE; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression); + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); try { - dataFileWriter.save(); + arbitraryDataWriter.save(); } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - DataFile dataFile = dataFileWriter.getDataFile(); + DataFile dataFile = arbitraryDataWriter.getDataFile(); if (dataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 01232d2d..85fbd087 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -40,8 +40,8 @@ import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.storage.DataFile; import org.qortal.storage.DataFile.*; -import org.qortal.storage.DataFileReader; -import org.qortal.storage.DataFileWriter; +import org.qortal.storage.ArbitraryDataReader; +import org.qortal.storage.ArbitraryDataWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; @@ -103,16 +103,16 @@ public class WebsiteResource { ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression); + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); try { - dataFileWriter.save(); + arbitraryDataWriter.save(); } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - DataFile dataFile = dataFileWriter.getDataFile(); + DataFile dataFile = arbitraryDataWriter.getDataFile(); if (dataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } @@ -203,16 +203,16 @@ public class WebsiteResource { Method method = Method.PUT; Compression compression = Compression.ZIP; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), name, service, method, compression); + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression); try { - dataFileWriter.save(); + arbitraryDataWriter.save(); } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - DataFile dataFile = dataFileWriter.getDataFile(); + DataFile dataFile = arbitraryDataWriter.getDataFile(); if (dataFile != null) { String digest58 = dataFile.digest58(); if (digest58 != null) { @@ -286,16 +286,16 @@ public class WebsiteResource { } Service service = Service.WEBSITE; - DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType, service); - dataFileReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service); + arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { // TODO: overwrite if new transaction arrives, to invalidate cache // We could store the latest transaction signature in the extracted folder - dataFileReader.load(false); + arbitraryDataReader.load(false); } catch (Exception e) { return this.get404Response(); } - java.nio.file.Path path = dataFileReader.getFilePath(); + java.nio.file.Path path = arbitraryDataReader.getFilePath(); if (path == null) { return this.get404Response(); } diff --git a/src/main/java/org/qortal/storage/DataFileBuilder.java b/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java similarity index 87% rename from src/main/java/org/qortal/storage/DataFileBuilder.java rename to src/main/java/org/qortal/storage/ArbitraryDataBuilder.java index b4faba02..a04c5689 100644 --- a/src/main/java/org/qortal/storage/DataFileBuilder.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java @@ -18,9 +18,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -public class DataFileBuilder { +public class ArbitraryDataBuilder { - private static final Logger LOGGER = LogManager.getLogger(DataFileBuilder.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilder.class); private String name; private Service service; @@ -30,7 +30,7 @@ public class DataFileBuilder { private List paths; private Path finalPath; - public DataFileBuilder(String name, Service service) { + public ArbitraryDataBuilder(String name, Service service) { this.name = name; this.service = service; this.paths = new ArrayList<>(); @@ -104,10 +104,10 @@ public class DataFileBuilder { // Build the data file, overwriting anything that was previously there String sig58 = Base58.encode(transactionData.getSignature()); - DataFileReader dataFileReader = new DataFileReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); - dataFileReader.setTransactionData(transactionData); - dataFileReader.load(true); - Path path = dataFileReader.getFilePath(); + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); + arbitraryDataReader.setTransactionData(transactionData); + arbitraryDataReader.load(true); + Path path = arbitraryDataReader.getFilePath(); if (path == null) { throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58)); } @@ -119,9 +119,9 @@ public class DataFileBuilder { } private void buildLatestState() throws IOException, DataException { - DataFilePatches dataFilePatches = new DataFilePatches(this.paths); - dataFilePatches.applyPatches(); - this.finalPath = dataFilePatches.getFinalPath(); + ArbitraryDataPatches arbitraryDataPatches = new ArbitraryDataPatches(this.paths); + arbitraryDataPatches.applyPatches(); + this.finalPath = arbitraryDataPatches.getFinalPath(); } public Path getFinalPath() { diff --git a/src/main/java/org/qortal/storage/DataFileCombiner.java b/src/main/java/org/qortal/storage/ArbitraryDataCombiner.java similarity index 80% rename from src/main/java/org/qortal/storage/DataFileCombiner.java rename to src/main/java/org/qortal/storage/ArbitraryDataCombiner.java index edb7b362..11bf4675 100644 --- a/src/main/java/org/qortal/storage/DataFileCombiner.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataCombiner.java @@ -7,15 +7,15 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -public class DataFileCombiner { +public class ArbitraryDataCombiner { - private static final Logger LOGGER = LogManager.getLogger(DataFileCombiner.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCombiner.class); private Path pathBefore; private Path pathAfter; private Path finalPath; - public DataFileCombiner(Path pathBefore, Path pathAfter) { + public ArbitraryDataCombiner(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; } @@ -44,7 +44,7 @@ public class DataFileCombiner { } private void process() throws IOException { - DataFileMerge merge = new DataFileMerge(this.pathBefore, this.pathAfter); + ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter); merge.compute(); this.finalPath = merge.getMergePath(); } diff --git a/src/main/java/org/qortal/storage/DataFileCreatePatch.java b/src/main/java/org/qortal/storage/ArbitraryDataCreatePatch.java similarity index 81% rename from src/main/java/org/qortal/storage/DataFileCreatePatch.java rename to src/main/java/org/qortal/storage/ArbitraryDataCreatePatch.java index 67ecf9cb..ec12f070 100644 --- a/src/main/java/org/qortal/storage/DataFileCreatePatch.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataCreatePatch.java @@ -8,15 +8,15 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -public class DataFileCreatePatch { +public class ArbitraryDataCreatePatch { - private static final Logger LOGGER = LogManager.getLogger(DataFileCreatePatch.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCreatePatch.class); private Path pathBefore; private Path pathAfter; private Path finalPath; - public DataFileCreatePatch(Path pathBefore, Path pathAfter) { + public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; } @@ -46,7 +46,7 @@ public class DataFileCreatePatch { private void process() { - DataFileDiff diff = new DataFileDiff(this.pathBefore, this.pathAfter); + ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter); diff.compute(); this.finalPath = diff.getDiffPath(); } diff --git a/src/main/java/org/qortal/storage/DataFileDiff.java b/src/main/java/org/qortal/storage/ArbitraryDataDiff.java similarity index 92% rename from src/main/java/org/qortal/storage/DataFileDiff.java rename to src/main/java/org/qortal/storage/ArbitraryDataDiff.java index e6534f79..e9461338 100644 --- a/src/main/java/org/qortal/storage/DataFileDiff.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataDiff.java @@ -10,15 +10,15 @@ import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; -public class DataFileDiff { +public class ArbitraryDataDiff { - private static final Logger LOGGER = LogManager.getLogger(DataFileDiff.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDiff.class); private Path pathBefore; private Path pathAfter; private Path diffPath; - public DataFileDiff(Path pathBefore, Path pathAfter) { + public ArbitraryDataDiff(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; } @@ -91,14 +91,14 @@ public class DataFileDiff { LOGGER.info("File size was modified: {}", after.toString()); wasModified = true; } - else if (!Arrays.equals(DataFileDiff.digestFromPath(after), DataFileDiff.digestFromPath(filePathBefore))) { + else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) { // Check hashes as a last resort LOGGER.info("File contents were modified: {}", after.toString()); wasModified = true; } if (wasAdded | wasModified) { - DataFileDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter); + ArbitraryDataDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter); } return FileVisitResult.CONTINUE; @@ -138,7 +138,7 @@ public class DataFileDiff { if (!Files.exists(directoryPathAfter)) { LOGGER.info("Directory was removed: {}", directoryPathAfter.toString()); - DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore); + ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore); // TODO: we might need to mark directories differently to files // TODO: add path to manifest JSON } @@ -154,7 +154,7 @@ public class DataFileDiff { if (!Files.exists(filePathAfter)) { LOGGER.trace("File was removed: {}", before.toString()); - DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore); + ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore); // TODO: add path to manifest JSON } diff --git a/src/main/java/org/qortal/storage/DataFileMerge.java b/src/main/java/org/qortal/storage/ArbitraryDataMerge.java similarity index 91% rename from src/main/java/org/qortal/storage/DataFileMerge.java rename to src/main/java/org/qortal/storage/ArbitraryDataMerge.java index c03018cf..85ad93e9 100644 --- a/src/main/java/org/qortal/storage/DataFileMerge.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataMerge.java @@ -12,15 +12,15 @@ import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; -public class DataFileMerge { +public class ArbitraryDataMerge { - private static final Logger LOGGER = LogManager.getLogger(DataFileMerge.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMerge.class); private Path pathBefore; private Path pathAfter; private Path mergePath; - public DataFileMerge(Path pathBefore, Path pathAfter) { + public ArbitraryDataMerge(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; } @@ -56,7 +56,7 @@ public class DataFileMerge { } private void copyPreviousStateToMergePath() throws IOException { - DataFileMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get("")); + ArbitraryDataMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get("")); } private void findDifferences() { @@ -102,21 +102,21 @@ public class DataFileMerge { LOGGER.trace("File size was modified: {}", after.toString()); wasModified = true; } - else if (!Arrays.equals(DataFileMerge.digestFromPath(after), DataFileMerge.digestFromPath(filePathBefore))) { + else if (!Arrays.equals(ArbitraryDataMerge.digestFromPath(after), ArbitraryDataMerge.digestFromPath(filePathBefore))) { // Check hashes as a last resort LOGGER.trace("File contents were modified: {}", after.toString()); wasModified = true; } if (wasAdded | wasModified) { - DataFileMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter); + ArbitraryDataMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter); } if (wasRemoved) { if (filePathAfter.toString().endsWith(".removed")) { // Trim the ".removed" Path filePathAfterTrimmed = Paths.get(filePathAfter.toString().substring(0, filePathAfter.toString().length()-8)); - DataFileMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed); + ArbitraryDataMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed); } } diff --git a/src/main/java/org/qortal/storage/DataFilePatches.java b/src/main/java/org/qortal/storage/ArbitraryDataPatches.java similarity index 83% rename from src/main/java/org/qortal/storage/DataFilePatches.java rename to src/main/java/org/qortal/storage/ArbitraryDataPatches.java index 98fe2f07..cfa11480 100644 --- a/src/main/java/org/qortal/storage/DataFilePatches.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataPatches.java @@ -9,14 +9,14 @@ import java.io.IOException; import java.nio.file.Path; import java.util.List; -public class DataFilePatches { +public class ArbitraryDataPatches { - private static final Logger LOGGER = LogManager.getLogger(DataFilePatches.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataPatches.class); private List paths; private Path finalPath; - public DataFilePatches(List paths) { + public ArbitraryDataPatches(List paths) { this.paths = paths; } @@ -52,7 +52,7 @@ public class DataFilePatches { // Loop from the second path onwards for (int i=1; i Date: Sat, 14 Aug 2021 13:25:45 +0100 Subject: [PATCH 129/505] Renamed DataFile to ArbitraryDataFile, and DataFileChunk to ArbitraryDataFileChunk --- .../api/resource/ArbitraryResource.java | 62 +++++++++--------- .../qortal/api/resource/WebsiteResource.java | 32 +++++----- .../controller/ArbitraryDataManager.java | 40 ++++++------ .../network/message/DataFileMessage.java | 26 ++++---- .../hsqldb/HSQLDBArbitraryRepository.java | 52 +++++++-------- .../qortal/storage/ArbitraryDataBuilder.java | 2 +- .../{DataFile.java => ArbitraryDataFile.java} | 64 +++++++++---------- ...Chunk.java => ArbitraryDataFileChunk.java} | 16 ++--- .../qortal/storage/ArbitraryDataReader.java | 26 ++++---- .../qortal/storage/ArbitraryDataWriter.java | 26 ++++---- .../transaction/ArbitraryTransaction.java | 10 +-- src/test/java/org/qortal/test/DataTests.java | 58 ++++++++--------- 12 files changed, 207 insertions(+), 207 deletions(-) rename src/main/java/org/qortal/storage/{DataFile.java => ArbitraryDataFile.java} (87%) rename src/main/java/org/qortal/storage/{DataFileChunk.java => ArbitraryDataFileChunk.java} (69%) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a5a97123..78cf9213 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -45,8 +45,8 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; -import org.qortal.storage.DataFile; -import org.qortal.storage.DataFileChunk; +import org.qortal.storage.ArbitraryDataFile; +import org.qortal.storage.ArbitraryDataFileChunk; import org.qortal.storage.ArbitraryDataWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; @@ -281,28 +281,28 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - DataFile dataFile = arbitraryDataWriter.getDataFile(); - if (dataFile == null) { + ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); + if (arbitraryDataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } try (final Repository repository = RepositoryManager.getRepository()) { - DataFile.ValidationResult validationResult = dataFile.isValid(); - if (validationResult != DataFile.ValidationResult.OK) { + ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid(); + if (validationResult != ArbitraryDataFile.ValidationResult.OK) { LOGGER.error("Invalid file: {}", validationResult); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - LOGGER.info("Whole file digest: {}", dataFile.digest58()); + LOGGER.info("Whole file digest: {}", arbitraryDataFile.digest58()); - int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); + int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); if (chunkCount == 0) { LOGGER.error("No chunks created"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - String digest58 = dataFile.digest58(); + String digest58 = arbitraryDataFile.digest58(); if (digest58 == null) { LOGGER.error("Unable to calculate digest"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -313,12 +313,12 @@ public class ArbitraryResource { final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); - final int size = (int)dataFile.size(); + final int size = (int) arbitraryDataFile.size(); final int version = 5; final int nonce = 0; final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; - final byte[] digest = dataFile.digest(); - final byte[] chunkHashes = dataFile.chunkHashes(); + final byte[] digest = arbitraryDataFile.digest(); + final byte[] chunkHashes = arbitraryDataFile.chunkHashes(); final List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, @@ -330,7 +330,7 @@ public class ArbitraryResource { Transaction.ValidationResult result = transaction.isValidUnconfirmed(); if (result != Transaction.ValidationResult.OK) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); throw TransactionsResource.createTransactionInvalidException(request, result); } @@ -338,14 +338,14 @@ public class ArbitraryResource { return Base58.encode(bytes); } catch (DataException e) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); LOGGER.error("Repository issue when uploading data", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } catch (TransformationException e) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); } catch (IllegalStateException e) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); LOGGER.error("Invalid upload data", e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); } @@ -380,8 +380,8 @@ public class ArbitraryResource { public String deleteFile(String hash58) { Security.checkApiCallAllowed(request); - DataFile dataFile = DataFile.fromHash58(hash58); - if (dataFile.delete()) { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(hash58); + if (arbitraryDataFile.delete()) { return "true"; } return "false"; @@ -503,9 +503,9 @@ public class ArbitraryResource { private boolean requestFile(String hash58, Peer targetPeer) { try (final Repository repository = RepositoryManager.getRepository()) { - DataFile dataFile = DataFile.fromHash58(hash58); - if (dataFile.exists()) { - LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(hash58); + if (arbitraryDataFile.exists()) { + LOGGER.info("Data file {} already exists but we'll request it anyway", arbitraryDataFile); } byte[] digest = null; @@ -526,11 +526,11 @@ public class ArbitraryResource { } DataFileMessage dataFileMessage = (DataFileMessage) message; - dataFile = dataFileMessage.getDataFile(); - if (dataFile == null || !dataFile.exists()) { + arbitraryDataFile = dataFileMessage.getArbitraryDataFile(); + if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { return false; } - LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())); + LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getArbitraryDataFile(), dataFileMessage.getArbitraryDataFile().size())); return true; } catch (ApiException e) { throw e; @@ -571,22 +571,22 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } - DataFile dataFile = DataFile.fromHash58(combinedHash); - if (dataFile.exists()) { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(combinedHash); + if (arbitraryDataFile.exists()) { LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash); } String hash58List[] = files.split(","); for (String hash58 : hash58List) { if (hash58 != null) { - DataFileChunk chunk = DataFileChunk.fromHash58(hash58); - dataFile.addChunk(chunk); + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash58(hash58); + arbitraryDataFile.addChunk(chunk); } } - boolean success = dataFile.join(); + boolean success = arbitraryDataFile.join(); if (success) { - if (combinedHash.equals(dataFile.digest58())) { - LOGGER.info("Valid hash {} after joining {} files", dataFile.digest58(), dataFile.chunkCount()); + if (combinedHash.equals(arbitraryDataFile.digest58())) { + LOGGER.info("Valid hash {} after joining {} files", arbitraryDataFile.digest58(), arbitraryDataFile.chunkCount()); return Response.ok("true").build(); } } diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 85fbd087..933452a0 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -38,8 +38,8 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; -import org.qortal.storage.DataFile; -import org.qortal.storage.DataFile.*; +import org.qortal.storage.ArbitraryDataFile; +import org.qortal.storage.ArbitraryDataFile.*; import org.qortal.storage.ArbitraryDataReader; import org.qortal.storage.ArbitraryDataWriter; import org.qortal.transaction.ArbitraryTransaction; @@ -112,12 +112,12 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - DataFile dataFile = arbitraryDataWriter.getDataFile(); - if (dataFile == null) { + ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); + if (arbitraryDataFile == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - String digest58 = dataFile.digest58(); + String digest58 = arbitraryDataFile.digest58(); if (digest58 == null) { LOGGER.error("Unable to calculate digest"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -130,13 +130,13 @@ public class WebsiteResource { final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); - final int size = (int)dataFile.size(); + final int size = (int) arbitraryDataFile.size(); final int version = 5; final int nonce = 0; - byte[] secret = dataFile.getSecret(); + byte[] secret = arbitraryDataFile.getSecret(); final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; - final byte[] digest = dataFile.digest(); - final byte[] chunkHashes = dataFile.chunkHashes(); + final byte[] digest = arbitraryDataFile.digest(); + final byte[] chunkHashes = arbitraryDataFile.chunkHashes(); final List payments = new ArrayList<>(); ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, @@ -149,7 +149,7 @@ public class WebsiteResource { Transaction.ValidationResult result = transaction.isValidUnconfirmed(); if (result != Transaction.ValidationResult.OK) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); throw TransactionsResource.createTransactionInvalidException(request, result); } @@ -157,10 +157,10 @@ public class WebsiteResource { return Base58.encode(bytes); } catch (TransformationException e) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); } catch (DataException e) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } } @@ -212,11 +212,11 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - DataFile dataFile = arbitraryDataWriter.getDataFile(); - if (dataFile != null) { - String digest58 = dataFile.digest58(); + ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); + if (arbitraryDataFile != null) { + String digest58 = arbitraryDataFile.digest58(); if (digest58 != null) { - return "http://localhost:12393/site/hash/" + digest58 + "?secret=" + Base58.encode(dataFile.getSecret()); + return "http://localhost:12393/site/hash/" + digest58 + "?secret=" + Base58.encode(arbitraryDataFile.getSecret()); } } return "Unable to generate preview URL"; diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 29827e30..7028b550 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -13,8 +13,8 @@ import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.storage.DataFile; -import org.qortal.storage.DataFileChunk; +import org.qortal.storage.ArbitraryDataFile; +import org.qortal.storage.ArbitraryDataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.Base58; @@ -166,7 +166,7 @@ public class ArbitraryDataManager extends Thread { return true; } - private DataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException { + private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException { String hash58 = Base58.encode(hash); LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); arbitraryDataFileRequests.put(hash58, NTP.getTime()); @@ -181,7 +181,7 @@ public class ArbitraryDataManager extends Thread { } DataFileMessage dataFileMessage = (DataFileMessage) message; - return dataFileMessage.getDataFile(); + return dataFileMessage.getArbitraryDataFile(); } public void cleanupRequestCache(long now) { @@ -269,13 +269,13 @@ public class ArbitraryDataManager extends Thread { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; // Load data file(s) - DataFile dataFile = DataFile.fromHash(arbitraryTransactionData.getData()); - dataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData()); + arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); // Check all hashes exist for (byte[] hash : hashes) { //LOGGER.info("Received hash {}", Base58.encode(hash)); - if (!dataFile.containsChunk(hash)) { + if (!arbitraryDataFile.containsChunk(hash)) { LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58); return; } @@ -287,14 +287,14 @@ public class ArbitraryDataManager extends Thread { // Now fetch actual data from this peer for (byte[] hash : hashes) { - if (!dataFile.chunkExists(hash)) { + if (!arbitraryDataFile.chunkExists(hash)) { // Only request the file if we aren't already requesting it from someone else if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { - DataFile receivedDataFile = fetchArbitraryDataFile(peer, hash); - LOGGER.info("Received data file {} from peer {}", receivedDataFile, peer); + ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, hash); + LOGGER.info("Received data file {} from peer {}", receivedArbitraryDataFile, peer); } else { - LOGGER.info("Already requesting data file {}", dataFile); + LOGGER.info("Already requesting data file {}", arbitraryDataFile); } } } @@ -318,15 +318,15 @@ public class ArbitraryDataManager extends Thread { byte[] hash = getDataFileMessage.getHash(); Controller.getInstance().stats.getDataFileMessageStats.requests.incrementAndGet(); - DataFile dataFile = DataFile.fromHash(hash); - if (dataFile.exists()) { - DataFileMessage dataFileMessage = new DataFileMessage(dataFile); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); + if (arbitraryDataFile.exists()) { + DataFileMessage dataFileMessage = new DataFileMessage(arbitraryDataFile); dataFileMessage.setId(message.getId()); if (!peer.sendMessage(dataFileMessage)) { LOGGER.info("Couldn't sent file"); peer.disconnect("failed to send file"); } - LOGGER.info("Sent file {}", dataFile); + LOGGER.info("Sent file {}", arbitraryDataFile); } else { @@ -334,7 +334,7 @@ public class ArbitraryDataManager extends Thread { Controller.getInstance().stats.getDataFileMessageStats.unknownFiles.getAndIncrement(); // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout - LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, dataFile)); + LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile)); // We'll send empty block summaries message as it's very short // TODO: use a different message type here @@ -344,7 +344,7 @@ public class ArbitraryDataManager extends Thread { LOGGER.info("Couldn't sent file-unknown response"); peer.disconnect("failed to send file-unknown response"); } - LOGGER.info("Sent file-unknown response for file {}", dataFile); + LOGGER.info("Sent file-unknown response for file {}", arbitraryDataFile); } } @@ -367,10 +367,10 @@ public class ArbitraryDataManager extends Thread { byte[] chunkHashes = transactionData.getChunkHashes(); // Load file(s) and add any that exist to the list of hashes - DataFile dataFile = DataFile.fromHash(hash); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); if (chunkHashes != null && chunkHashes.length > 0) { - dataFile.addChunkHashes(chunkHashes); - for (DataFileChunk dataFileChunk : dataFile.getChunks()) { + arbitraryDataFile.addChunkHashes(chunkHashes); + for (ArbitraryDataFileChunk dataFileChunk : arbitraryDataFile.getChunks()) { if (dataFileChunk.exists()) { hashes.add(dataFileChunk.getHash()); //LOGGER.info("Added hash {}", dataFileChunk.getHash58()); diff --git a/src/main/java/org/qortal/network/message/DataFileMessage.java b/src/main/java/org/qortal/network/message/DataFileMessage.java index e31dde67..a2a144dd 100644 --- a/src/main/java/org/qortal/network/message/DataFileMessage.java +++ b/src/main/java/org/qortal/network/message/DataFileMessage.java @@ -1,7 +1,7 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; -import org.qortal.storage.DataFile; +import org.qortal.storage.ArbitraryDataFile; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -10,22 +10,22 @@ import java.nio.ByteBuffer; public class DataFileMessage extends Message { - private final DataFile dataFile; + private final ArbitraryDataFile arbitraryDataFile; - public DataFileMessage(DataFile dataFile) { + public DataFileMessage(ArbitraryDataFile arbitraryDataFile) { super(MessageType.DATA_FILE); - this.dataFile = dataFile; + this.arbitraryDataFile = arbitraryDataFile; } - public DataFileMessage(int id, DataFile dataFile) { + public DataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) { super(id, MessageType.DATA_FILE); - this.dataFile = dataFile; + this.arbitraryDataFile = arbitraryDataFile; } - public DataFile getDataFile() { - return this.dataFile; + public ArbitraryDataFile getArbitraryDataFile() { + return this.arbitraryDataFile; } public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { @@ -36,18 +36,18 @@ public class DataFileMessage extends Message { byte[] data = new byte[dataLength]; byteBuffer.get(data); - DataFile dataFile = new DataFile(data); + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data); - return new DataFileMessage(id, dataFile); + return new DataFileMessage(id, arbitraryDataFile); } @Override protected byte[] toData() { - if (this.dataFile == null) { + if (this.arbitraryDataFile == null) { return null; } - byte[] data = this.dataFile.getBytes(); + byte[] data = this.arbitraryDataFile.getBytes(); if (data == null) { return null; } @@ -66,7 +66,7 @@ public class DataFileMessage extends Message { } public DataFileMessage cloneWithNewId(int newId) { - DataFileMessage clone = new DataFileMessage(this.dataFile); + DataFileMessage clone = new DataFileMessage(this.arbitraryDataFile); clone.setId(newId); return clone; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 5bc174e2..dc70496f 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -10,7 +10,7 @@ import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.DataException; -import org.qortal.storage.DataFile; +import org.qortal.storage.ArbitraryDataFile; import org.qortal.transaction.Transaction.ApprovalStatus; import java.sql.ResultSet; @@ -55,18 +55,18 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = transactionData.getChunkHashes(); // Load data file(s) - DataFile dataFile = DataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); if (chunkHashes != null && chunkHashes.length > 0) { - dataFile.addChunkHashes(chunkHashes); + arbitraryDataFile.addChunkHashes(chunkHashes); } // Check if we already have the complete data file - if (dataFile.exists()) { + if (arbitraryDataFile.exists()) { return true; } // Alternatively, if we have all the chunks, then it's safe to assume the data is local - if (dataFile.allChunksExist(chunkHashes)) { + if (arbitraryDataFile.allChunksExist(chunkHashes)) { return true; } @@ -90,23 +90,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = transactionData.getChunkHashes(); // Load data file(s) - DataFile dataFile = DataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); if (chunkHashes != null && chunkHashes.length > 0) { - dataFile.addChunkHashes(chunkHashes); + arbitraryDataFile.addChunkHashes(chunkHashes); } // If we have the complete data file, return it - if (dataFile.exists()) { - return dataFile.getBytes(); + if (arbitraryDataFile.exists()) { + return arbitraryDataFile.getBytes(); } // Alternatively, if we have all the chunks, combine them into a single file - if (dataFile.allChunksExist(chunkHashes)) { - dataFile.join(); + if (arbitraryDataFile.allChunksExist(chunkHashes)) { + arbitraryDataFile.join(); // Verify that the combined hash matches the expected hash - if (digest.equals(dataFile.digest())) { - return dataFile.getBytes(); + if (digest.equals(arbitraryDataFile.digest())) { + return arbitraryDataFile.getBytes(); } } @@ -134,29 +134,29 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryTransactionData.setDataType(DataType.DATA_HASH); // Create DataFile - DataFile dataFile = new DataFile(rawData); + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(rawData); // Verify that the data file is valid, and that it matches the expected hash - DataFile.ValidationResult validationResult = dataFile.isValid(); - if (validationResult != DataFile.ValidationResult.OK) { - dataFile.deleteAll(); + ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid(); + if (validationResult != ArbitraryDataFile.ValidationResult.OK) { + arbitraryDataFile.deleteAll(); throw new DataException("Invalid data file when attempting to store arbitrary transaction data"); } - if (!dataHash.equals(dataFile.digest())) { - dataFile.deleteAll(); + if (!dataHash.equals(arbitraryDataFile.digest())) { + arbitraryDataFile.deleteAll(); throw new DataException("Could not verify hash when attempting to store arbitrary transaction data"); } // Now create chunks if needed - int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); + int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); if (chunkCount > 0) { LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s"))); - LOGGER.info("{}", dataFile.printChunks()); + LOGGER.info("{}", arbitraryDataFile.printChunks()); // Verify that the chunk hashes match those in the transaction - byte[] chunkHashes = dataFile.chunkHashes(); + byte[] chunkHashes = arbitraryDataFile.chunkHashes(); if (!chunkHashes.equals(arbitraryTransactionData.getChunkHashes())) { - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); throw new DataException("Could not verify chunk hashes when attempting to store arbitrary transaction data"); } @@ -175,13 +175,13 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); // Load data file(s) - DataFile dataFile = DataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); if (chunkHashes != null && chunkHashes.length > 0) { - dataFile.addChunkHashes(chunkHashes); + arbitraryDataFile.addChunkHashes(chunkHashes); } // Delete file and chunks - dataFile.deleteAll(); + arbitraryDataFile.deleteAll(); } @Override diff --git a/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java b/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java index a04c5689..5da39e51 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java @@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.Service; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.storage.DataFile.ResourceIdType; +import org.qortal.storage.ArbitraryDataFile.ResourceIdType; import org.qortal.utils.Base58; import java.io.IOException; diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/ArbitraryDataFile.java similarity index 87% rename from src/main/java/org/qortal/storage/DataFile.java rename to src/main/java/org/qortal/storage/ArbitraryDataFile.java index 75579e1a..008a44d3 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataFile.java @@ -20,7 +20,7 @@ import static java.util.Arrays.stream; import static java.util.stream.Collectors.toMap; -public class DataFile { +public class ArbitraryDataFile { // Validation results public enum ValidationResult { @@ -30,13 +30,13 @@ public class DataFile { public final int value; - private static final Map map = stream(DataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result)); + private static final Map map = stream(ArbitraryDataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result)); ValidationResult(int value) { this.value = value; } - public static DataFile.ValidationResult valueOf(int value) { + public static ArbitraryDataFile.ValidationResult valueOf(int value) { return map.get(value); } } @@ -49,7 +49,7 @@ public class DataFile { NAME }; - private static final Logger LOGGER = LogManager.getLogger(DataFile.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB @@ -57,20 +57,20 @@ public class DataFile { protected String filePath; protected String hash58; - private ArrayList chunks; + private ArrayList chunks; private byte[] secret; - public DataFile() { + public ArbitraryDataFile() { } - public DataFile(String hash58) { + public ArbitraryDataFile(String hash58) { this.createDataDirectory(); - this.filePath = DataFile.getOutputFilePath(hash58, false); + this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, false); this.chunks = new ArrayList<>(); this.hash58 = hash58; } - public DataFile(byte[] fileContent) { + public ArbitraryDataFile(byte[] fileContent) { if (fileContent == null) { LOGGER.error("fileContent is null"); return; @@ -95,28 +95,28 @@ public class DataFile { } } - public static DataFile fromHash58(String hash58) { - return new DataFile(hash58); + public static ArbitraryDataFile fromHash58(String hash58) { + return new ArbitraryDataFile(hash58); } - public static DataFile fromHash(byte[] hash) { - return DataFile.fromHash58(Base58.encode(hash)); + public static ArbitraryDataFile fromHash(byte[] hash) { + return ArbitraryDataFile.fromHash58(Base58.encode(hash)); } - public static DataFile fromPath(String path) { + public static ArbitraryDataFile fromPath(String path) { File file = new File(path); if (file.exists()) { try { byte[] fileContent = Files.readAllBytes(file.toPath()); byte[] digest = Crypto.digest(fileContent); - DataFile dataFile = DataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); // Copy file to base directory if needed Path filePath = Paths.get(path); - if (Files.exists(filePath) && !dataFile.isInBaseDirectory(path)) { - dataFile.copyToDataDirectory(filePath); + if (Files.exists(filePath) && !arbitraryDataFile.isInBaseDirectory(path)) { + arbitraryDataFile.copyToDataDirectory(filePath); } - return dataFile; + return arbitraryDataFile; } catch (IOException e) { LOGGER.error("Couldn't compute digest for DataFile"); @@ -125,8 +125,8 @@ public class DataFile { return null; } - public static DataFile fromFile(File file) { - return DataFile.fromPath(file.getPath()); + public static ArbitraryDataFile fromFile(File file) { + return ArbitraryDataFile.fromPath(file.getPath()); } private boolean createDataDirectory() { @@ -188,7 +188,7 @@ public class DataFile { long fileSize = Files.size(path); if (fileSize > MAX_FILE_SIZE) { LOGGER.error(String.format("DataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE)); - return DataFile.ValidationResult.FILE_TOO_LARGE; + return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE; } } catch (IOException e) { @@ -198,7 +198,7 @@ public class DataFile { return ValidationResult.OK; } - public void addChunk(DataFileChunk chunk) { + public void addChunk(ArbitraryDataFileChunk chunk) { this.chunks.add(chunk); } @@ -210,7 +210,7 @@ public class DataFile { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; byteBuffer.get(chunkDigest); - DataFileChunk chunk = DataFileChunk.fromHash(chunkDigest); + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest); this.addChunk(chunk); } } @@ -232,7 +232,7 @@ public class DataFile { out.write(buffer, 0, numberOfBytes); out.flush(); - DataFileChunk chunk = new DataFileChunk(out.toByteArray()); + ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray()); ValidationResult validationResult = chunk.isValid(); if (validationResult == ValidationResult.OK) { this.chunks.add(chunk); @@ -266,7 +266,7 @@ public class DataFile { // Join the chunks File outputFile = new File(this.filePath); try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) { - for (DataFileChunk chunk : this.chunks) { + for (ArbitraryDataFileChunk chunk : this.chunks) { File sourceFile = new File(chunk.filePath); BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); byte[] buffer = new byte[2048]; @@ -323,7 +323,7 @@ public class DataFile { if (this.chunks != null && this.chunks.size() > 0) { Iterator iterator = this.chunks.iterator(); while (iterator.hasNext()) { - DataFileChunk chunk = (DataFileChunk) iterator.next(); + ArbitraryDataFileChunk chunk = (ArbitraryDataFileChunk) iterator.next(); chunk.delete(); iterator.remove(); success = true; @@ -376,7 +376,7 @@ public class DataFile { } public boolean chunkExists(byte[] hash) { - for (DataFileChunk chunk : this.chunks) { + for (ArbitraryDataFileChunk chunk : this.chunks) { if (Arrays.equals(hash, chunk.getHash())) { return chunk.exists(); } @@ -389,7 +389,7 @@ public class DataFile { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; byteBuffer.get(chunkHash); - DataFileChunk chunk = DataFileChunk.fromHash(chunkHash); + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash); if (!chunk.exists()) { return false; } @@ -398,7 +398,7 @@ public class DataFile { } public boolean containsChunk(byte[] hash) { - for (DataFileChunk chunk : this.chunks) { + for (ArbitraryDataFileChunk chunk : this.chunks) { if (Arrays.equals(hash, chunk.getHash())) { return true; } @@ -419,7 +419,7 @@ public class DataFile { return this.chunks.size(); } - public List getChunks() { + public List getChunks() { return this.chunks; } @@ -432,7 +432,7 @@ public class DataFile { try { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - for (DataFileChunk chunk : this.chunks) { + for (ArbitraryDataFileChunk chunk : this.chunks) { byte[] chunkHash = chunk.digest(); if (chunkHash.length != 32) { LOGGER.info("Invalid chunk hash length: {}", chunkHash.length); @@ -499,7 +499,7 @@ public class DataFile { public String printChunks() { String outputString = ""; if (this.chunkCount() > 0) { - for (DataFileChunk chunk : this.chunks) { + for (ArbitraryDataFileChunk chunk : this.chunks) { if (outputString.length() > 0) { outputString = outputString.concat(","); } diff --git a/src/main/java/org/qortal/storage/DataFileChunk.java b/src/main/java/org/qortal/storage/ArbitraryDataFileChunk.java similarity index 69% rename from src/main/java/org/qortal/storage/DataFileChunk.java rename to src/main/java/org/qortal/storage/ArbitraryDataFileChunk.java index ebc6641e..ce94bf41 100644 --- a/src/main/java/org/qortal/storage/DataFileChunk.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataFileChunk.java @@ -10,24 +10,24 @@ import java.nio.file.Path; import java.nio.file.Paths; -public class DataFileChunk extends DataFile { +public class ArbitraryDataFileChunk extends ArbitraryDataFile { - private static final Logger LOGGER = LogManager.getLogger(DataFileChunk.class); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class); - public DataFileChunk(String hash58) { + public ArbitraryDataFileChunk(String hash58) { super(hash58); } - public DataFileChunk(byte[] fileContent) { + public ArbitraryDataFileChunk(byte[] fileContent) { super(fileContent); } - public static DataFileChunk fromHash58(String hash58) { - return new DataFileChunk(hash58); + public static ArbitraryDataFileChunk fromHash58(String hash58) { + return new ArbitraryDataFileChunk(hash58); } - public static DataFileChunk fromHash(byte[] hash) { - return DataFileChunk.fromHash58(Base58.encode(hash)); + public static ArbitraryDataFileChunk fromHash(byte[] hash) { + return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash)); } @Override diff --git a/src/main/java/org/qortal/storage/ArbitraryDataReader.java b/src/main/java/org/qortal/storage/ArbitraryDataReader.java index adc89f40..8cd9f262 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataReader.java @@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.storage.DataFile.*; +import org.qortal.storage.ArbitraryDataFile.*; import org.qortal.transform.Transformer; import org.qortal.utils.Base58; import org.qortal.utils.FilesystemUtils; @@ -165,9 +165,9 @@ public class ArbitraryDataReader { private void fetchFromFileHash() { // Load data file directly from the hash - DataFile dataFile = DataFile.fromHash58(resourceId); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId); // Set filePath to the location of the DataFile - this.filePath = Paths.get(dataFile.getFilePath()); + this.filePath = Paths.get(arbitraryDataFile.getFilePath()); } private void fetchFromName() throws IllegalStateException, IOException, DataException { @@ -214,27 +214,27 @@ public class ArbitraryDataReader { } // Load data file(s) - DataFile dataFile = DataFile.fromHash(digest); - if (!dataFile.exists()) { - if (!dataFile.allChunksExist(chunkHashes)) { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + if (!arbitraryDataFile.exists()) { + if (!arbitraryDataFile.allChunksExist(chunkHashes)) { // TODO: fetch them? - throw new IllegalStateException(String.format("Missing chunks for file {}", dataFile)); + throw new IllegalStateException(String.format("Missing chunks for file {}", arbitraryDataFile)); } // We have all the chunks but not the complete file, so join them - dataFile.addChunkHashes(chunkHashes); - dataFile.join(); + arbitraryDataFile.addChunkHashes(chunkHashes); + arbitraryDataFile.join(); } // If the complete file still doesn't exist then something went wrong - if (!dataFile.exists()) { - throw new IOException(String.format("File doesn't exist: %s", dataFile)); + if (!arbitraryDataFile.exists()) { + throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile)); } // Ensure the complete hash matches the joined chunks - if (!Arrays.equals(dataFile.digest(), digest)) { + if (!Arrays.equals(arbitraryDataFile.digest(), digest)) { throw new IllegalStateException("Unable to validate complete file hash"); } // Set filePath to the location of the DataFile - this.filePath = Paths.get(dataFile.getFilePath()); + this.filePath = Paths.get(arbitraryDataFile.getFilePath()); } private void decrypt() { diff --git a/src/main/java/org/qortal/storage/ArbitraryDataWriter.java b/src/main/java/org/qortal/storage/ArbitraryDataWriter.java index dc298259..19a29b45 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/storage/ArbitraryDataWriter.java @@ -6,7 +6,7 @@ import org.apache.logging.log4j.Logger; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.crypto.AES; import org.qortal.repository.DataException; -import org.qortal.storage.DataFile.*; +import org.qortal.storage.ArbitraryDataFile.*; import org.qortal.utils.ZipUtils; import javax.crypto.BadPaddingException; @@ -33,7 +33,7 @@ public class ArbitraryDataWriter { private Compression compression; private SecretKey aesKey; - private DataFile dataFile; + private ArbitraryDataFile arbitraryDataFile; // Intermediate paths to cleanup private Path workingPath; @@ -160,20 +160,20 @@ public class ArbitraryDataWriter { } private void validate() throws IOException { - if (this.dataFile == null) { + if (this.arbitraryDataFile == null) { throw new IOException("No file available when validating"); } - this.dataFile.setSecret(this.aesKey.getEncoded()); + this.arbitraryDataFile.setSecret(this.aesKey.getEncoded()); // Validate the file - ValidationResult validationResult = this.dataFile.isValid(); + ValidationResult validationResult = this.arbitraryDataFile.isValid(); if (validationResult != ValidationResult.OK) { - throw new IllegalStateException(String.format("File %s failed validation: %s", this.dataFile, validationResult)); + throw new IllegalStateException(String.format("File %s failed validation: %s", this.arbitraryDataFile, validationResult)); } - LOGGER.info("Whole file hash is valid: {}", this.dataFile.digest58()); + LOGGER.info("Whole file hash is valid: {}", this.arbitraryDataFile.digest58()); // Validate each chunk - for (DataFileChunk chunk : this.dataFile.getChunks()) { + for (ArbitraryDataFileChunk chunk : this.arbitraryDataFile.getChunks()) { validationResult = chunk.isValid(); if (validationResult != ValidationResult.OK) { throw new IllegalStateException(String.format("Chunk %s failed validation: %s", chunk, validationResult)); @@ -184,12 +184,12 @@ public class ArbitraryDataWriter { } private void split() throws IOException { - this.dataFile = DataFile.fromPath(this.filePath.toString()); - if (this.dataFile == null) { + this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath.toString()); + if (this.arbitraryDataFile == null) { throw new IOException("No file available when trying to split"); } - int chunkCount = this.dataFile.split(DataFile.CHUNK_SIZE); + int chunkCount = this.arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); if (chunkCount > 0) { LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); } @@ -218,8 +218,8 @@ public class ArbitraryDataWriter { } - public DataFile getDataFile() { - return this.dataFile; + public ArbitraryDataFile getArbitraryDataFile() { + return this.arbitraryDataFile; } } diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index cb9142d8..890f8d32 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -13,8 +13,8 @@ import org.qortal.data.transaction.TransactionData; import org.qortal.payment.Payment; import org.qortal.repository.DataException; import org.qortal.repository.Repository; -import org.qortal.storage.DataFile; -import org.qortal.storage.DataFileChunk; +import org.qortal.storage.ArbitraryDataFile; +import org.qortal.storage.ArbitraryDataFileChunk; import org.qortal.transform.TransformationException; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.transform.transaction.TransactionTransformer; @@ -31,7 +31,7 @@ public class ArbitraryTransaction extends Transaction { public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes public static final int POW_MIN_DIFFICULTY = 12; // leading zero bits public static final int POW_MAX_DIFFICULTY = 19; // leading zero bits - public static final long MAX_FILE_SIZE = DataFile.MAX_FILE_SIZE; + public static final long MAX_FILE_SIZE = ArbitraryDataFile.MAX_FILE_SIZE; // Constructors @@ -103,7 +103,7 @@ public class ArbitraryTransaction extends Transaction { } // Check expected length of chunk hashes - int chunkCount = (int)Math.ceil((double)arbitraryTransactionData.getSize() / (double)DataFileChunk.CHUNK_SIZE); + int chunkCount = (int)Math.ceil((double)arbitraryTransactionData.getSize() / (double) ArbitraryDataFileChunk.CHUNK_SIZE); int expectedChunkHashesSize = (chunkCount > 1) ? chunkCount * HASH_LENGTH : 0; if (chunkHashes == null && expectedChunkHashesSize > 0) { return ValidationResult.INVALID_DATA_LENGTH; @@ -121,7 +121,7 @@ public class ArbitraryTransaction extends Transaction { if (arbitraryTransactionData.getVersion() >= 5) { // Check reported length of the raw data // We should not download the raw data, so validation of that will be performed later - if (arbitraryTransactionData.getSize() > DataFile.MAX_FILE_SIZE) { + if (arbitraryTransactionData.getSize() > ArbitraryDataFile.MAX_FILE_SIZE) { return ValidationResult.INVALID_DATA_LENGTH; } } diff --git a/src/test/java/org/qortal/test/DataTests.java b/src/test/java/org/qortal/test/DataTests.java index 415025be..6dd8d66c 100644 --- a/src/test/java/org/qortal/test/DataTests.java +++ b/src/test/java/org/qortal/test/DataTests.java @@ -3,7 +3,7 @@ package org.qortal.test; import org.junit.Before; import org.junit.Test; import org.qortal.repository.DataException; -import org.qortal.storage.DataFile; +import org.qortal.storage.ArbitraryDataFile; import org.qortal.test.common.Common; import java.util.Random; @@ -20,28 +20,28 @@ public class DataTests extends Common { @Test public void testSplitAndJoin() { String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; - DataFile dataFile = new DataFile(dummyDataString.getBytes()); - assertTrue(dataFile.exists()); - assertEquals(62, dataFile.size()); - assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58()); + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes()); + assertTrue(arbitraryDataFile.exists()); + assertEquals(62, arbitraryDataFile.size()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58()); // Split into 7 chunks, each 10 bytes long - dataFile.split(10); - assertEquals(7, dataFile.chunkCount()); + arbitraryDataFile.split(10); + assertEquals(7, arbitraryDataFile.chunkCount()); // Delete the original file - dataFile.delete(); - assertFalse(dataFile.exists()); - assertEquals(0, dataFile.size()); + arbitraryDataFile.delete(); + assertFalse(arbitraryDataFile.exists()); + assertEquals(0, arbitraryDataFile.size()); // Now rebuild the original file from the chunks - assertEquals(7, dataFile.chunkCount()); - dataFile.join(); + assertEquals(7, arbitraryDataFile.chunkCount()); + arbitraryDataFile.join(); // Validate that the original file is intact - assertTrue(dataFile.exists()); - assertEquals(62, dataFile.size()); - assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58()); + assertTrue(arbitraryDataFile.exists()); + assertEquals(62, arbitraryDataFile.size()); + assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58()); } @Test @@ -50,28 +50,28 @@ public class DataTests extends Common { byte[] randomData = new byte[fileSize]; new Random().nextBytes(randomData); // No need for SecureRandom here - DataFile dataFile = new DataFile(randomData); - assertTrue(dataFile.exists()); - assertEquals(fileSize, dataFile.size()); - String originalFileDigest = dataFile.digest58(); + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData); + assertTrue(arbitraryDataFile.exists()); + assertEquals(fileSize, arbitraryDataFile.size()); + String originalFileDigest = arbitraryDataFile.digest58(); // Split into chunks using 1MiB chunk size - dataFile.split(1 * 1024 * 1024); - assertEquals(6, dataFile.chunkCount()); + arbitraryDataFile.split(1 * 1024 * 1024); + assertEquals(6, arbitraryDataFile.chunkCount()); // Delete the original file - dataFile.delete(); - assertFalse(dataFile.exists()); - assertEquals(0, dataFile.size()); + arbitraryDataFile.delete(); + assertFalse(arbitraryDataFile.exists()); + assertEquals(0, arbitraryDataFile.size()); // Now rebuild the original file from the chunks - assertEquals(6, dataFile.chunkCount()); - dataFile.join(); + assertEquals(6, arbitraryDataFile.chunkCount()); + arbitraryDataFile.join(); // Validate that the original file is intact - assertTrue(dataFile.exists()); - assertEquals(fileSize, dataFile.size()); - assertEquals(originalFileDigest, dataFile.digest58()); + assertTrue(arbitraryDataFile.exists()); + assertEquals(fileSize, arbitraryDataFile.size()); + assertEquals(originalFileDigest, arbitraryDataFile.digest58()); } } From 172a37ec8cb8238b75f7505dd22876226407ed62 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 13:49:19 +0100 Subject: [PATCH 130/505] Renamed org.qortal.storage package to org.qortal.arbitrary --- .../java/org/qortal/api/resource/ArbitraryResource.java | 6 +++--- .../java/org/qortal/api/resource/WebsiteResource.java | 8 ++++---- .../{storage => arbitrary}/ArbitraryDataBuilder.java | 4 ++-- .../{storage => arbitrary}/ArbitraryDataCombiner.java | 2 +- .../{storage => arbitrary}/ArbitraryDataCreatePatch.java | 2 +- .../qortal/{storage => arbitrary}/ArbitraryDataDiff.java | 2 +- .../qortal/{storage => arbitrary}/ArbitraryDataFile.java | 2 +- .../{storage => arbitrary}/ArbitraryDataFileChunk.java | 2 +- .../qortal/{storage => arbitrary}/ArbitraryDataMerge.java | 2 +- .../{storage => arbitrary}/ArbitraryDataPatches.java | 2 +- .../{storage => arbitrary}/ArbitraryDataReader.java | 4 ++-- .../{storage => arbitrary}/ArbitraryDataWriter.java | 4 ++-- .../java/org/qortal/controller/ArbitraryDataManager.java | 4 ++-- ...DataFileMessage.java => ArbitraryDataFileMessage.java} | 2 +- .../repository/hsqldb/HSQLDBArbitraryRepository.java | 2 +- .../java/org/qortal/transaction/ArbitraryTransaction.java | 4 ++-- src/test/java/org/qortal/test/DataTests.java | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataBuilder.java (98%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataCombiner.java (98%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataCreatePatch.java (98%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataDiff.java (99%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataFile.java (99%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataFileChunk.java (98%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataMerge.java (99%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataPatches.java (98%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataReader.java (99%) rename src/main/java/org/qortal/{storage => arbitrary}/ArbitraryDataWriter.java (99%) rename src/main/java/org/qortal/network/message/{DataFileMessage.java => ArbitraryDataFileMessage.java} (97%) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 78cf9213..15c41a88 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -45,9 +45,9 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; -import org.qortal.storage.ArbitraryDataFile; -import org.qortal.storage.ArbitraryDataFileChunk; -import org.qortal.storage.ArbitraryDataWriter; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFileChunk; +import org.qortal.arbitrary.ArbitraryDataWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 933452a0..9bd1c142 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -38,10 +38,10 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; -import org.qortal.storage.ArbitraryDataFile; -import org.qortal.storage.ArbitraryDataFile.*; -import org.qortal.storage.ArbitraryDataReader; -import org.qortal.storage.ArbitraryDataWriter; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.arbitrary.ArbitraryDataReader; +import org.qortal.arbitrary.ArbitraryDataWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java similarity index 98% rename from src/main/java/org/qortal/storage/ArbitraryDataBuilder.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 5da39e51..7d9984f1 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.Service; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.storage.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.utils.Base58; import java.io.IOException; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java similarity index 98% rename from src/main/java/org/qortal/storage/ArbitraryDataCombiner.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index 11bf4675..0554cdb5 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataCreatePatch.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java similarity index 98% rename from src/main/java/org/qortal/storage/ArbitraryDataCreatePatch.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java index ec12f070..ed4095a4 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataCreatePatch.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java similarity index 99% rename from src/main/java/org/qortal/storage/ArbitraryDataDiff.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index e9461338..defc4a4b 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java similarity index 99% rename from src/main/java/org/qortal/storage/ArbitraryDataFile.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 008a44d3..9cce4879 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataFileChunk.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java similarity index 98% rename from src/main/java/org/qortal/storage/ArbitraryDataFileChunk.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java index ce94bf41..ccd14eae 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataFileChunk.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java similarity index 99% rename from src/main/java/org/qortal/storage/ArbitraryDataMerge.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index 85ad93e9..f1163da4 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataPatches.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java similarity index 98% rename from src/main/java/org/qortal/storage/ArbitraryDataPatches.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java index cfa11480..c12c6a53 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataPatches.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java similarity index 99% rename from src/main/java/org/qortal/storage/ArbitraryDataReader.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 8cd9f262..3fb2c44d 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.storage.ArbitraryDataFile.*; +import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.transform.Transformer; import org.qortal.utils.Base58; import org.qortal.utils.FilesystemUtils; diff --git a/src/main/java/org/qortal/storage/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java similarity index 99% rename from src/main/java/org/qortal/storage/ArbitraryDataWriter.java rename to src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 19a29b45..5d987b8d 100644 --- a/src/main/java/org/qortal/storage/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -1,4 +1,4 @@ -package org.qortal.storage; +package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; @@ -6,7 +6,7 @@ import org.apache.logging.log4j.Logger; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.crypto.AES; import org.qortal.repository.DataException; -import org.qortal.storage.ArbitraryDataFile.*; +import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.utils.ZipUtils; import javax.crypto.BadPaddingException; diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 7028b550..8f7614e1 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -13,8 +13,8 @@ import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; -import org.qortal.storage.ArbitraryDataFile; -import org.qortal.storage.ArbitraryDataFileChunk; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFileChunk; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.Base58; diff --git a/src/main/java/org/qortal/network/message/DataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java similarity index 97% rename from src/main/java/org/qortal/network/message/DataFileMessage.java rename to src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index a2a144dd..33911871 100644 --- a/src/main/java/org/qortal/network/message/DataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -1,7 +1,7 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; -import org.qortal.storage.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFile; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index dc70496f..29c0326d 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -10,7 +10,7 @@ import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.DataException; -import org.qortal.storage.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.transaction.Transaction.ApprovalStatus; import java.sql.ResultSet; diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 890f8d32..0f5bea60 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -13,8 +13,8 @@ import org.qortal.data.transaction.TransactionData; import org.qortal.payment.Payment; import org.qortal.repository.DataException; import org.qortal.repository.Repository; -import org.qortal.storage.ArbitraryDataFile; -import org.qortal.storage.ArbitraryDataFileChunk; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFileChunk; import org.qortal.transform.TransformationException; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.transform.transaction.TransactionTransformer; diff --git a/src/test/java/org/qortal/test/DataTests.java b/src/test/java/org/qortal/test/DataTests.java index 6dd8d66c..1d515239 100644 --- a/src/test/java/org/qortal/test/DataTests.java +++ b/src/test/java/org/qortal/test/DataTests.java @@ -3,7 +3,7 @@ package org.qortal.test; import org.junit.Before; import org.junit.Test; import org.qortal.repository.DataException; -import org.qortal.storage.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.test.common.Common; import java.util.Random; From e99ea4111767e9ab722b7149b35b6d77b4caa73d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 13:52:41 +0100 Subject: [PATCH 131/505] Renamed DataFileMessage to ArbitraryDataFileMessage and GetDataFileMessage to GetArbitraryDataFileMessage --- .../qortal/api/resource/ArbitraryResource.java | 12 ++++++------ .../qortal/controller/ArbitraryDataManager.java | 16 ++++++++-------- .../message/ArbitraryDataFileMessage.java | 12 ++++++------ ...age.java => GetArbitraryDataFileMessage.java} | 8 ++++---- 4 files changed, 24 insertions(+), 24 deletions(-) rename src/main/java/org/qortal/network/message/{GetDataFileMessage.java => GetArbitraryDataFileMessage.java} (80%) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 15c41a88..473ec6f7 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -38,8 +38,8 @@ import org.qortal.group.Group; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.PeerAddress; -import org.qortal.network.message.DataFileMessage; -import org.qortal.network.message.GetDataFileMessage; +import org.qortal.network.message.ArbitraryDataFileMessage; +import org.qortal.network.message.GetArbitraryDataFileMessage; import org.qortal.network.message.Message; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -515,7 +515,7 @@ public class ArbitraryResource { LOGGER.info("Invalid base58 encoded string"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - Message getDataFileMessage = new GetDataFileMessage(digest); + Message getDataFileMessage = new GetArbitraryDataFileMessage(digest); Message message = targetPeer.getResponse(getDataFileMessage); if (message == null) { @@ -525,12 +525,12 @@ public class ArbitraryResource { return false; } - DataFileMessage dataFileMessage = (DataFileMessage) message; - arbitraryDataFile = dataFileMessage.getArbitraryDataFile(); + ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message; + arbitraryDataFile = arbitraryDataFileMessage.getArbitraryDataFile(); if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { return false; } - LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getArbitraryDataFile(), dataFileMessage.getArbitraryDataFile().size())); + LOGGER.info(String.format("Received file %s, size %d bytes", arbitraryDataFileMessage.getArbitraryDataFile(), arbitraryDataFileMessage.getArbitraryDataFile().size())); return true; } catch (ApiException e) { throw e; diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 8f7614e1..5f611204 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -170,7 +170,7 @@ public class ArbitraryDataManager extends Thread { String hash58 = Base58.encode(hash); LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); arbitraryDataFileRequests.put(hash58, NTP.getTime()); - Message getDataFileMessage = new GetDataFileMessage(hash); + Message getDataFileMessage = new GetArbitraryDataFileMessage(hash); Message message = peer.getResponse(getDataFileMessage); arbitraryDataFileRequests.remove(hash58); @@ -180,8 +180,8 @@ public class ArbitraryDataManager extends Thread { return null; } - DataFileMessage dataFileMessage = (DataFileMessage) message; - return dataFileMessage.getArbitraryDataFile(); + ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message; + return arbitraryDataFileMessage.getArbitraryDataFile(); } public void cleanupRequestCache(long now) { @@ -314,15 +314,15 @@ public class ArbitraryDataManager extends Thread { } public void onNetworkGetDataFileMessage(Peer peer, Message message) { - GetDataFileMessage getDataFileMessage = (GetDataFileMessage) message; - byte[] hash = getDataFileMessage.getHash(); + GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message; + byte[] hash = getArbitraryDataFileMessage.getHash(); Controller.getInstance().stats.getDataFileMessageStats.requests.incrementAndGet(); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); if (arbitraryDataFile.exists()) { - DataFileMessage dataFileMessage = new DataFileMessage(arbitraryDataFile); - dataFileMessage.setId(message.getId()); - if (!peer.sendMessage(dataFileMessage)) { + ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(arbitraryDataFile); + arbitraryDataFileMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileMessage)) { LOGGER.info("Couldn't sent file"); peer.disconnect("failed to send file"); } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index 33911871..9c73520b 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -8,17 +8,17 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -public class DataFileMessage extends Message { +public class ArbitraryDataFileMessage extends Message { private final ArbitraryDataFile arbitraryDataFile; - public DataFileMessage(ArbitraryDataFile arbitraryDataFile) { + public ArbitraryDataFileMessage(ArbitraryDataFile arbitraryDataFile) { super(MessageType.DATA_FILE); this.arbitraryDataFile = arbitraryDataFile; } - public DataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) { + public ArbitraryDataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) { super(id, MessageType.DATA_FILE); this.arbitraryDataFile = arbitraryDataFile; @@ -38,7 +38,7 @@ public class DataFileMessage extends Message { byteBuffer.get(data); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data); - return new DataFileMessage(id, arbitraryDataFile); + return new ArbitraryDataFileMessage(id, arbitraryDataFile); } @Override @@ -65,8 +65,8 @@ public class DataFileMessage extends Message { } } - public DataFileMessage cloneWithNewId(int newId) { - DataFileMessage clone = new DataFileMessage(this.arbitraryDataFile); + public ArbitraryDataFileMessage cloneWithNewId(int newId) { + ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.arbitraryDataFile); clone.setId(newId); return clone; } diff --git a/src/main/java/org/qortal/network/message/GetDataFileMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java similarity index 80% rename from src/main/java/org/qortal/network/message/GetDataFileMessage.java rename to src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java index a7ab8d67..f1ae143a 100644 --- a/src/main/java/org/qortal/network/message/GetDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java @@ -7,17 +7,17 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -public class GetDataFileMessage extends Message { +public class GetArbitraryDataFileMessage extends Message { private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; private final byte[] hash; - public GetDataFileMessage(byte[] hash) { + public GetArbitraryDataFileMessage(byte[] hash) { this(-1, hash); } - private GetDataFileMessage(int id, byte[] hash) { + private GetArbitraryDataFileMessage(int id, byte[] hash) { super(id, MessageType.GET_DATA_FILE); this.hash = hash; @@ -35,7 +35,7 @@ public class GetDataFileMessage extends Message { bytes.get(hash); - return new GetDataFileMessage(id, hash); + return new GetArbitraryDataFileMessage(id, hash); } @Override From b18b686545e8f92a8affd635f798cdbc0e46078c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 13:57:44 +0100 Subject: [PATCH 132/505] Renamed DATA_FILE to ARBITRARY_DATA_FILE and GET_DATA_FILE to GET_ARBITRARY_DATA_FILE. This will break existing data nodes, but this is okay, as nothing is in production yet. --- src/main/java/org/qortal/controller/ArbitraryDataManager.java | 2 +- src/main/java/org/qortal/controller/Controller.java | 2 +- .../org/qortal/network/message/ArbitraryDataFileMessage.java | 4 ++-- .../qortal/network/message/GetArbitraryDataFileMessage.java | 2 +- src/main/java/org/qortal/network/message/Message.java | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 5f611204..3fa84466 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -176,7 +176,7 @@ public class ArbitraryDataManager extends Thread { arbitraryDataFileRequests.remove(hash58); LOGGER.info(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); - if (message == null || message.getType() != Message.MessageType.DATA_FILE) { + if (message == null || message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) { return null; } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 8fd9e9da..910f49c3 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1246,7 +1246,7 @@ public class Controller extends Thread { ArbitraryDataManager.getInstance().onNetworkArbitraryDataFileListMessage(peer, message); break; - case GET_DATA_FILE: + case GET_ARBITRARY_DATA_FILE: ArbitraryDataManager.getInstance().onNetworkGetDataFileMessage(peer, message); break; diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index 9c73520b..289dcb49 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -13,13 +13,13 @@ public class ArbitraryDataFileMessage extends Message { private final ArbitraryDataFile arbitraryDataFile; public ArbitraryDataFileMessage(ArbitraryDataFile arbitraryDataFile) { - super(MessageType.DATA_FILE); + super(MessageType.ARBITRARY_DATA_FILE); this.arbitraryDataFile = arbitraryDataFile; } public ArbitraryDataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) { - super(id, MessageType.DATA_FILE); + super(id, MessageType.ARBITRARY_DATA_FILE); this.arbitraryDataFile = arbitraryDataFile; } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java index f1ae143a..f13951b7 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java @@ -18,7 +18,7 @@ public class GetArbitraryDataFileMessage extends Message { } private GetArbitraryDataFileMessage(int id, byte[] hash) { - super(id, MessageType.GET_DATA_FILE); + super(id, MessageType.GET_ARBITRARY_DATA_FILE); this.hash = hash; } diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index 3ecb151b..662eaeaf 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -85,8 +85,8 @@ public abstract class Message { BLOCKS(100), GET_BLOCKS(101), - DATA_FILE(110), - GET_DATA_FILE(111), + ARBITRARY_DATA_FILE(110), + GET_ARBITRARY_DATA_FILE(111), ARBITRARY_DATA_FILE_LIST(120), GET_ARBITRARY_DATA_FILE_LIST(121); From 09e783fbf6c9b8b0f52adebc77976134e3b55103 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 14:01:30 +0100 Subject: [PATCH 133/505] Renamed a few variables and methods that weren't caught in previous refactors. --- .../org/qortal/api/resource/ArbitraryResource.java | 4 ++-- .../org/qortal/controller/ArbitraryDataManager.java | 10 +++++----- src/main/java/org/qortal/controller/Controller.java | 9 ++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 473ec6f7..41ee0b45 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -515,9 +515,9 @@ public class ArbitraryResource { LOGGER.info("Invalid base58 encoded string"); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - Message getDataFileMessage = new GetArbitraryDataFileMessage(digest); + Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(digest); - Message message = targetPeer.getResponse(getDataFileMessage); + Message message = targetPeer.getResponse(getArbitraryDataFileMessage); if (message == null) { return false; } diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 3fa84466..1d53be7f 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -170,9 +170,9 @@ public class ArbitraryDataManager extends Thread { String hash58 = Base58.encode(hash); LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); arbitraryDataFileRequests.put(hash58, NTP.getTime()); - Message getDataFileMessage = new GetArbitraryDataFileMessage(hash); + Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(hash); - Message message = peer.getResponse(getDataFileMessage); + Message message = peer.getResponse(getArbitraryDataFileMessage); arbitraryDataFileRequests.remove(hash58); LOGGER.info(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); @@ -313,10 +313,10 @@ public class ArbitraryDataManager extends Thread { } } - public void onNetworkGetDataFileMessage(Peer peer, Message message) { + public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) { GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message; byte[] hash = getArbitraryDataFileMessage.getHash(); - Controller.getInstance().stats.getDataFileMessageStats.requests.incrementAndGet(); + Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet(); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); if (arbitraryDataFile.exists()) { @@ -331,7 +331,7 @@ public class ArbitraryDataManager extends Thread { else { // We don't have this file - Controller.getInstance().stats.getDataFileMessageStats.unknownFiles.getAndIncrement(); + Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement(); // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile)); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 910f49c3..44eb7f92 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -15,7 +15,6 @@ import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; import org.qortal.controller.tradebot.TradeBot; -import org.qortal.crypto.Crypto; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; @@ -200,14 +199,14 @@ public class Controller extends Thread { } public GetBlockSignaturesV2Stats getBlockSignaturesV2Stats = new GetBlockSignaturesV2Stats(); - public static class GetDataFileMessageStats { + public static class GetArbitraryDataFileMessageStats { public AtomicLong requests = new AtomicLong(); public AtomicLong unknownFiles = new AtomicLong(); - public GetDataFileMessageStats() { + public GetArbitraryDataFileMessageStats() { } } - public GetDataFileMessageStats getDataFileMessageStats = new GetDataFileMessageStats(); + public GetArbitraryDataFileMessageStats getArbitraryDataFileMessageStats = new GetArbitraryDataFileMessageStats(); public static class GetArbitraryDataFileListMessageStats { public AtomicLong requests = new AtomicLong(); @@ -1247,7 +1246,7 @@ public class Controller extends Thread { break; case GET_ARBITRARY_DATA_FILE: - ArbitraryDataManager.getInstance().onNetworkGetDataFileMessage(peer, message); + ArbitraryDataManager.getInstance().onNetworkGetArbitraryDataFileMessage(peer, message); break; case GET_ARBITRARY_DATA_FILE_LIST: From 8dac3ebf96235363481086011bcef96012a38525 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 14:05:33 +0100 Subject: [PATCH 134/505] More miscellaneous refactoring --- .../java/org/qortal/arbitrary/ArbitraryDataFile.java | 6 +++--- .../java/org/qortal/arbitrary/ArbitraryDataReader.java | 6 +++--- .../org/qortal/controller/ArbitraryDataManager.java | 10 +++++----- .../repository/hsqldb/HSQLDBArbitraryRepository.java | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 9cce4879..4b91219e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -119,7 +119,7 @@ public class ArbitraryDataFile { return arbitraryDataFile; } catch (IOException e) { - LOGGER.error("Couldn't compute digest for DataFile"); + LOGGER.error("Couldn't compute digest for ArbitraryDataFile"); } } return null; @@ -187,7 +187,7 @@ public class ArbitraryDataFile { // Validate the file size long fileSize = Files.size(path); if (fileSize > MAX_FILE_SIZE) { - LOGGER.error(String.format("DataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE)); + LOGGER.error(String.format("ArbitraryDataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE)); return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE; } @@ -468,7 +468,7 @@ public class ArbitraryDataFile { return Crypto.digest(fileContent); } catch (IOException e) { - LOGGER.error("Couldn't compute digest for DataFile"); + LOGGER.error("Couldn't compute digest for ArbitraryDataFile"); } } return null; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 3fb2c44d..219d2293 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -166,7 +166,7 @@ public class ArbitraryDataReader { private void fetchFromFileHash() { // Load data file directly from the hash ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId); - // Set filePath to the location of the DataFile + // Set filePath to the location of the ArbitraryDataFile this.filePath = Paths.get(arbitraryDataFile.getFilePath()); } @@ -233,7 +233,7 @@ public class ArbitraryDataReader { if (!Arrays.equals(arbitraryDataFile.digest(), digest)) { throw new IllegalStateException("Unable to validate complete file hash"); } - // Set filePath to the location of the DataFile + // Set filePath to the location of the ArbitraryDataFile this.filePath = Paths.get(arbitraryDataFile.getFilePath()); } @@ -247,7 +247,7 @@ public class ArbitraryDataReader { AES.decryptFile("AES", aesKey, this.filePath.toString(), this.unencryptedPath.toString()); // Replace filePath pointer with the encrypted file path - // Don't delete the original DataFile, as this is handled in the cleanup phase + // Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase this.filePath = this.unencryptedPath; } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 1d53be7f..bf7c4632 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -370,13 +370,13 @@ public class ArbitraryDataManager extends Thread { ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); - for (ArbitraryDataFileChunk dataFileChunk : arbitraryDataFile.getChunks()) { - if (dataFileChunk.exists()) { - hashes.add(dataFileChunk.getHash()); - //LOGGER.info("Added hash {}", dataFileChunk.getHash58()); + for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) { + if (chunk.exists()) { + hashes.add(chunk.getHash()); + //LOGGER.info("Added hash {}", chunk.getHash58()); } else { - LOGGER.info("Couldn't add hash {} because it doesn't exist", dataFileChunk.getHash58()); + LOGGER.info("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); } } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 29c0326d..42321584 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -133,7 +133,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { arbitraryTransactionData.setData(dataHash); arbitraryTransactionData.setDataType(DataType.DATA_HASH); - // Create DataFile + // Create ArbitraryDataFile ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(rawData); // Verify that the data file is valid, and that it matches the expected hash From 63f5946527bc4d7bcc56d0ed474cce9c1eca8c0f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 14:30:06 +0100 Subject: [PATCH 135/505] Switched all system-generated temp paths to a user specified "tempDataPath". This ensures that the temporary files are being kept with the rest of the data, rather than somewhere inappropriate such as on flash storage. It also allows the user to locate them somewhere else, such as on a dedicated drive. --- .../org/qortal/arbitrary/ArbitraryDataDiff.java | 15 ++++++++++++--- .../org/qortal/arbitrary/ArbitraryDataFile.java | 12 +++++++----- .../org/qortal/arbitrary/ArbitraryDataMerge.java | 15 ++++++++++++--- .../org/qortal/arbitrary/ArbitraryDataReader.java | 8 ++++---- .../org/qortal/arbitrary/ArbitraryDataWriter.java | 11 ++++++++--- src/main/java/org/qortal/settings/Settings.java | 6 ++++++ 6 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index defc4a4b..17b67960 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -3,12 +3,14 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.crypto.Crypto; +import org.qortal.settings.Settings; import java.io.File; import java.io.IOException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; +import java.util.UUID; public class ArbitraryDataDiff { @@ -17,6 +19,7 @@ public class ArbitraryDataDiff { private Path pathBefore; private Path pathAfter; private Path diffPath; + private String identifier; public ArbitraryDataDiff(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; @@ -35,6 +38,7 @@ public class ArbitraryDataDiff { } private void preExecute() { + this.createRandomIdentifier(); this.createOutputDirectory(); } @@ -42,11 +46,16 @@ public class ArbitraryDataDiff { } + private void createRandomIdentifier() { + this.identifier = UUID.randomUUID().toString(); + } + private void createOutputDirectory() { - // Ensure temp folder exists - Path tempDir; + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "diff", this.identifier); try { - tempDir = Files.createTempDirectory("qortal-diff"); + Files.createDirectories(tempDir); } catch (IOException e) { throw new IllegalStateException("Unable to create temp directory"); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 4b91219e..9bb6fae9 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -255,13 +255,15 @@ public class ArbitraryDataFile { if (this.chunks != null && this.chunks.size() > 0) { // Create temporary path for joined file - Path tempPath; + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "join", this.chunks.get(0).digest58()); try { - tempPath = Files.createTempFile(this.chunks.get(0).digest58(), ".tmp"); + Files.createDirectories(tempDir); } catch (IOException e) { return false; } - this.filePath = tempPath.toString(); + this.filePath = tempDir.toString(); // Join the chunks File outputFile = new File(this.filePath); @@ -279,8 +281,8 @@ public class ArbitraryDataFile { out.close(); // Copy temporary file to data directory - this.filePath = this.copyToDataDirectory(tempPath); - Files.delete(tempPath); + this.filePath = this.copyToDataDirectory(tempDir); + Files.delete(tempDir); return true; } catch (FileNotFoundException e) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index f1163da4..8ac19db5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.crypto.Crypto; +import org.qortal.settings.Settings; import org.qortal.utils.FilesystemUtils; import java.io.File; @@ -11,6 +12,7 @@ import java.io.IOException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; +import java.util.UUID; public class ArbitraryDataMerge { @@ -19,6 +21,7 @@ public class ArbitraryDataMerge { private Path pathBefore; private Path pathAfter; private Path mergePath; + private String identifier; public ArbitraryDataMerge(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; @@ -37,6 +40,7 @@ public class ArbitraryDataMerge { } private void preExecute() { + this.createRandomIdentifier(); this.createOutputDirectory(); } @@ -44,11 +48,16 @@ public class ArbitraryDataMerge { } + private void createRandomIdentifier() { + this.identifier = UUID.randomUUID().toString(); + } + private void createOutputDirectory() { - // Ensure temp folder exists - Path tempDir; + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "merge", this.identifier); try { - tempDir = Files.createTempDirectory("qortal-diff"); + Files.createDirectories(tempDir); } catch (IOException e) { throw new IllegalStateException("Unable to create temp directory"); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 219d2293..4cf2486e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -9,6 +9,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.settings.Settings; import org.qortal.transform.Transformer; import org.qortal.utils.Base58; import org.qortal.utils.FilesystemUtils; @@ -81,9 +82,9 @@ public class ArbitraryDataReader { } private void createWorkingDirectory() { - // Use the system tmpdir as our base, as it is deterministic - String baseDir = System.getProperty("java.io.tmpdir"); - Path tempDir = Paths.get(baseDir + File.separator + "qortal" + File.separator + this.resourceId); + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "reader", this.resourceId); try { Files.createDirectories(tempDir); } catch (IOException e) { @@ -93,7 +94,6 @@ public class ArbitraryDataReader { } private void createUncompressedDirectory() { - // Use the system tmpdir as our base, as it is deterministic this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); try { Files.createDirectories(this.uncompressedPath); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 5d987b8d..d87256d0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -3,10 +3,13 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.crypto.AES; import org.qortal.repository.DataException; import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; import org.qortal.utils.ZipUtils; import javax.crypto.BadPaddingException; @@ -78,10 +81,12 @@ public class ArbitraryDataWriter { } private void createWorkingDirectory() { - // Ensure temp folder exists - Path tempDir; + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + String identifier = Crypto.digest(this.filePath.toString().getBytes()).toString(); + Path tempDir = Paths.get(baseDir, "writer", identifier); try { - tempDir = Files.createTempDirectory("qortal"); + Files.createDirectories(tempDir); } catch (IOException e) { throw new IllegalStateException("Unable to create temp directory"); } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 7cd0f941..49a8a42c 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -206,6 +206,8 @@ public class Settings { /** Data storage path. */ private String dataPath = "data"; + /** Data storage path (for temporary data). */ + private String tempDataPath = "data/_temp"; // Domain mapping @@ -627,4 +629,8 @@ public class Settings { public String getDataPath() { return this.dataPath; } + + public String getTempDataPath() { + return this.tempDataPath; + } } From 4b5bd6eed7b105e1b7e32bcba5a66f10216f8ade Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 15:43:08 +0100 Subject: [PATCH 136/505] Cleanup temporary files used when building data directories. Also added a safety net to only delete files that are without our data or temp directories. --- .../arbitrary/ArbitraryDataCombiner.java | 32 ++++++++++++++ .../qortal/arbitrary/ArbitraryDataFile.java | 35 ++++++++------- .../qortal/arbitrary/ArbitraryDataMerge.java | 14 +++--- .../arbitrary/ArbitraryDataPatches.java | 3 +- .../qortal/arbitrary/ArbitraryDataReader.java | 44 ++++++++++++------- .../qortal/arbitrary/ArbitraryDataWriter.java | 12 ++--- .../org/qortal/utils/FilesystemUtils.java | 24 +++++++--- 7 files changed, 115 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index 0554cdb5..73d18c22 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -1,8 +1,11 @@ package org.qortal.arbitrary; +import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.utils.FilesystemUtils; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -30,6 +33,35 @@ public class ArbitraryDataCombiner { } } + public void cleanup() { + this.cleanupPath(this.pathBefore); + this.cleanupPath(this.pathAfter); + } + + private void cleanupPath(Path path) { + // Delete pathBefore, if it exists in our data/temp directory + if (FilesystemUtils.pathInsideDataOrTempPath(path)) { + File directory = new File(path.toString()); + try { + FileUtils.deleteDirectory(directory); + } catch (IOException e) { + // This will eventually be cleaned up by a maintenance process, so log the error and continue + LOGGER.info("Unable to cleanup directory {}", directory.toString()); + } + } + + // Delete the parent directory of pathBefore if it is empty (and exists in our data/temp directory) + Path parentDirectory = path.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + try { + Files.deleteIfExists(parentDirectory); + } catch (IOException e) { + // This will eventually be cleaned up by a maintenance process, so log the error and continue + LOGGER.info("Unable to cleanup parent directory {}", parentDirectory.toString()); + } + } + } + private void preExecute() { if (this.pathBefore == null || this.pathAfter == null) { throw new IllegalStateException(String.format("No paths available to build patch")); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 9bb6fae9..75417546 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -6,6 +6,7 @@ import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; import java.io.*; import java.nio.ByteBuffer; @@ -282,7 +283,9 @@ public class ArbitraryDataFile { // Copy temporary file to data directory this.filePath = this.copyToDataDirectory(tempDir); - Files.delete(tempDir); + if (FilesystemUtils.pathInsideDataOrTempPath(tempDir)) { + Files.delete(tempDir); + } return true; } catch (FileNotFoundException e) { @@ -296,22 +299,18 @@ public class ArbitraryDataFile { public boolean delete() { // Delete the complete file - // ... but only if it's inside the Qortal data directory + // ... but only if it's inside the Qortal data or temp directory Path path = Paths.get(this.filePath); - String dataPath = Settings.getInstance().getDataPath(); - Path dataDirectory = Paths.get(dataPath); - if (!path.toAbsolutePath().startsWith(dataDirectory.toAbsolutePath())) { - return false; - } - - if (Files.exists(path)) { - try { - Files.delete(path); - this.cleanupFilesystem(); - LOGGER.debug("Deleted file {}", path.toString()); - return true; - } catch (IOException e) { - LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath); + if (FilesystemUtils.pathInsideDataOrTempPath(path)) { + if (Files.exists(path)) { + try { + Files.delete(path); + this.cleanupFilesystem(); + LOGGER.debug("Deleted file {}", path.toString()); + return true; + } catch (IOException e) { + LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath); + } } } return false; @@ -343,7 +342,9 @@ public class ArbitraryDataFile { try (Stream files = Files.list(directory)) { final long count = files.count(); if (count == 0) { - Files.delete(directory); + if (FilesystemUtils.pathInsideDataOrTempPath(directory)) { + Files.delete(directory); + } } } catch (IOException e) { LOGGER.warn("Unable to count files in directory", e); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index 8ac19db5..da399242 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -176,19 +176,23 @@ public class ArbitraryDataMerge { Path dest = Paths.get(base.toString(), relativePath.toString()); LOGGER.trace("Copying {} to {}", source, dest); - FilesystemUtils.copyDirectory(source.toString(), dest.toString()); + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); } private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException { Path dest = Paths.get(base.toString(), relativePath.toString()); File file = new File(dest.toString()); if (file.exists() && file.isFile()) { - LOGGER.trace("Deleting file {}", dest); - Files.delete(dest); + if (FilesystemUtils.pathInsideDataOrTempPath(dest)) { + LOGGER.trace("Deleting file {}", dest); + Files.delete(dest); + } } if (file.exists() && file.isDirectory()) { - LOGGER.trace("Deleting directory {}", dest); - FileUtils.deleteDirectory(file); + if (FilesystemUtils.pathInsideDataOrTempPath(dest)) { + LOGGER.trace("Deleting directory {}", dest); + FileUtils.deleteDirectory(file); + } } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java index c12c6a53..f8882e78 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java @@ -54,7 +54,8 @@ public class ArbitraryDataPatches { Path pathAfter = this.paths.get(i); ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(pathBefore, pathAfter); combiner.combine(); - pathBefore = combiner.getFinalPath(); // TODO: cleanup + combiner.cleanup(); + pathBefore = combiner.getFinalPath(); } this.finalPath = pathBefore; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 4cf2486e..9a9d034d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -1,5 +1,6 @@ package org.qortal.arbitrary; +import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.crypto.AES; @@ -77,8 +78,8 @@ public class ArbitraryDataReader { this.createUncompressedDirectory(); } - private void postExecute() throws IOException { - this.cleanupFilesystem(); + private void postExecute() { + } private void createWorkingDirectory() { @@ -104,7 +105,7 @@ public class ArbitraryDataReader { private void deleteExistingFiles() { final Path uncompressedPath = this.uncompressedPath; - if (uncompressedPath != null) { + if (FilesystemUtils.pathInsideDataOrTempPath(uncompressedPath)) { if (Files.exists(uncompressedPath)) { LOGGER.trace("Attempting to delete path {}", this.uncompressedPath); try { @@ -268,7 +269,7 @@ public class ArbitraryDataReader { if (file.isDirectory()) { // Already a directory - nothing to uncompress // We still need to copy the directory to its final destination if it's not already there - this.copyFilePathToFinalDestination(); + this.moveFilePathToFinalDestination(); return; } @@ -282,11 +283,13 @@ public class ArbitraryDataReader { } // Replace filePath pointer with the uncompressed file path - Files.delete(this.filePath); + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + Files.delete(this.filePath); + } this.filePath = this.uncompressedPath; } - private void copyFilePathToFinalDestination() throws IOException { + private void moveFilePathToFinalDestination() throws IOException { if (this.filePath.compareTo(this.uncompressedPath) != 0) { File source = new File(this.filePath.toString()); File dest = new File(this.uncompressedPath.toString()); @@ -296,17 +299,28 @@ public class ArbitraryDataReader { if (dest == null || !dest.exists()) { throw new IllegalStateException("Destination directory doesn't exist"); } - FilesystemUtils.copyDirectory(source.toString(), dest.toString()); - } - } + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); - private void cleanupFilesystem() throws IOException { - // Clean up - if (this.uncompressedPath != null) { - File unzippedFile = new File(this.uncompressedPath.toString()); - if (unzippedFile.exists()) { - unzippedFile.delete(); + try { + // Delete existing + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + File directory = new File(this.filePath.toString()); + FileUtils.deleteDirectory(directory); + } + + // ... and its parent directory if empty + Path parentDirectory = this.filePath.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + Files.deleteIfExists(parentDirectory); + } + + } catch (IOException e) { + // This will eventually be cleaned up by a maintenance process, so log the error and continue + LOGGER.info("Unable to cleanup directories: {}", e.getMessage()); } + + // Finally, update filePath to point to uncompressedPath + this.filePath = this.uncompressedPath; } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index d87256d0..221366f0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -9,7 +9,7 @@ import org.qortal.crypto.AES; import org.qortal.repository.DataException; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.settings.Settings; -import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; import org.qortal.utils.ZipUtils; import javax.crypto.BadPaddingException; @@ -155,7 +155,9 @@ public class ArbitraryDataWriter { AES.encryptFile("AES", this.aesKey, this.filePath.toString(), this.encryptedPath.toString()); // Replace filePath pointer with the encrypted file path - Files.delete(this.filePath); + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + Files.delete(this.filePath); + } this.filePath = this.encryptedPath; } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException @@ -205,19 +207,19 @@ public class ArbitraryDataWriter { private void cleanupFilesystem() throws IOException { // Clean up - if (this.compressedPath != null) { + if (FilesystemUtils.pathInsideDataOrTempPath(this.compressedPath)) { File zippedFile = new File(this.compressedPath.toString()); if (zippedFile.exists()) { zippedFile.delete(); } } - if (this.encryptedPath != null) { + if (FilesystemUtils.pathInsideDataOrTempPath(this.encryptedPath)) { File encryptedFile = new File(this.encryptedPath.toString()); if (encryptedFile.exists()) { encryptedFile.delete(); } } - if (this.workingPath != null) { + if (FilesystemUtils.pathInsideDataOrTempPath(this.workingPath)) { FileUtils.deleteDirectory(new File(this.workingPath.toString())); } } diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index a87e18a0..54054c2d 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -1,10 +1,9 @@ package org.qortal.utils; +import org.qortal.settings.Settings; + import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; public class FilesystemUtils { @@ -18,17 +17,30 @@ public class FilesystemUtils { return false; } - public static void copyDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { + public static void copyAndReplaceDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { Files.walk(Paths.get(sourceDirectoryLocation)) .forEach(source -> { Path destination = Paths.get(destinationDirectoryLocation, source.toString() .substring(sourceDirectoryLocation.length())); try { - Files.copy(source, destination); + Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { e.printStackTrace(); } }); } + public static boolean pathInsideDataOrTempPath(Path path) { + if (path == null) { + return false; + } + Path dataPath = Paths.get(Settings.getInstance().getDataPath()).toAbsolutePath(); + Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath()).toAbsolutePath(); + Path absolutePath = path.toAbsolutePath(); + if (absolutePath.startsWith(dataPath) || absolutePath.startsWith(tempDataPath)) { + return true; + } + return false; + } + } From fa684eababf09ddb35f564390e6862c051bcad53 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 15:51:16 +0100 Subject: [PATCH 137/505] Removed ArbitraryDataPatches class This was essentially just a wrapper for a single method. --- .../arbitrary/ArbitraryDataBuilder.java | 29 ++++++-- .../arbitrary/ArbitraryDataPatches.java | 67 ------------------- 2 files changed, 25 insertions(+), 71 deletions(-) delete mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 7d9984f1..d90ef43c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -40,6 +40,7 @@ public class ArbitraryDataBuilder { this.fetchTransactions(); this.validateTransactions(); this.processTransactions(); + this.validatePaths(); this.buildLatestState(); } @@ -118,10 +119,30 @@ public class ArbitraryDataBuilder { } } - private void buildLatestState() throws IOException, DataException { - ArbitraryDataPatches arbitraryDataPatches = new ArbitraryDataPatches(this.paths); - arbitraryDataPatches.applyPatches(); - this.finalPath = arbitraryDataPatches.getFinalPath(); + private void validatePaths() { + if (this.paths == null || this.paths.isEmpty()) { + throw new IllegalStateException(String.format("No paths available from which to build latest state")); + } + } + + private void buildLatestState() throws IOException { + if (this.paths.size() == 1) { + // No patching needed + this.finalPath = this.paths.get(0); + return; + } + + Path pathBefore = this.paths.get(0); + + // Loop from the second path onwards + for (int i=1; i paths; - private Path finalPath; - - public ArbitraryDataPatches(List paths) { - this.paths = paths; - } - - public void applyPatches() throws DataException, IOException { - try { - this.preExecute(); - this.process(); - - } finally { - this.postExecute(); - } - } - - private void preExecute() { - if (this.paths == null || this.paths.isEmpty()) { - throw new IllegalStateException(String.format("No paths available to build latest state")); - } - } - - private void postExecute() { - - } - - private void process() throws IOException { - if (this.paths.size() == 1) { - // No patching needed - this.finalPath = this.paths.get(0); - return; - } - - Path pathBefore = this.paths.get(0); - - // Loop from the second path onwards - for (int i=1; i Date: Sat, 14 Aug 2021 16:53:57 +0100 Subject: [PATCH 138/505] Fixed bug with identifier in ArbitraryDataWriter --- src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 221366f0..27e1d13c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -9,6 +9,7 @@ import org.qortal.crypto.AES; import org.qortal.repository.DataException; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.settings.Settings; +import org.qortal.utils.Base58; import org.qortal.utils.FilesystemUtils; import org.qortal.utils.ZipUtils; @@ -83,7 +84,7 @@ public class ArbitraryDataWriter { private void createWorkingDirectory() { // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware String baseDir = Settings.getInstance().getTempDataPath(); - String identifier = Crypto.digest(this.filePath.toString().getBytes()).toString(); + String identifier = Base58.encode(Crypto.digest(this.filePath.toString().getBytes())); Path tempDir = Paths.get(baseDir, "writer", identifier); try { Files.createDirectories(tempDir); From 179318f1d89606fb770e744335c3d0b9828df1be Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 16:54:35 +0100 Subject: [PATCH 139/505] Ensure a patch exists after creating it. --- .../java/org/qortal/arbitrary/ArbitraryDataWriter.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 27e1d13c..dd278ec5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -122,6 +122,14 @@ public class ArbitraryDataWriter { ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath); patch.create(); this.filePath = patch.getFinalPath(); + + // Validate the patch + if (this.filePath == null) { + throw new IllegalStateException("Null path after creating patch"); + } + if (FilesystemUtils.isDirectoryEmpty(this.filePath)) { + throw new IllegalStateException("Patch has no content. Either no files have changed, or something went wrong"); + } } private void compress() { From d9f5753f5896ad64dec1b99249b95cc6f4319b7c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 16:55:19 +0100 Subject: [PATCH 140/505] Ensure parent directories exist when creating a diff. --- src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 17b67960..1952ddb0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -127,6 +127,7 @@ public class ArbitraryDataDiff { }); } catch (IOException e) { + // TODO: throw exception? LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); } } @@ -202,7 +203,14 @@ public class ArbitraryDataDiff { throw new IOException(String.format("File not found: %s", source.toString())); } + // Ensure parent folders exist in the destination Path dest = Paths.get(base.toString(), relativePath.toString()); + File file = new File(dest.toString()); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + LOGGER.trace("Copying {} to {}", source, dest); Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); } From efaf31342214c869287bac6ebdef23b8b4eaa0ee Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 16:58:10 +0100 Subject: [PATCH 141/505] When creating an ArbitraryDataFile from a path, make sure to move the file to the correct location within the data directory. This bug was introduced now that the temp directory is contained within the data directory. Without this, it would leave it in the temp folder and then fail at a later stage. --- .../qortal/arbitrary/ArbitraryDataFile.java | 8 +- .../org/qortal/utils/FilesystemUtils.java | 103 ++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 75417546..ecc3e147 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -112,11 +112,17 @@ public class ArbitraryDataFile { byte[] digest = Crypto.digest(fileContent); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); - // Copy file to base directory if needed + // Copy file to data directory if needed Path filePath = Paths.get(path); if (Files.exists(filePath) && !arbitraryDataFile.isInBaseDirectory(path)) { arbitraryDataFile.copyToDataDirectory(filePath); } + // Or, if it's already in the data directory, we may need to move it + else if (!filePath.equals(arbitraryDataFile.getFilePath())) { + // Wrong path, so relocate + Path dest = Paths.get(arbitraryDataFile.getFilePath()); + FilesystemUtils.moveFile(filePath, dest, true); + } return arbitraryDataFile; } catch (IOException e) { diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index 54054c2d..0325cb60 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -1,7 +1,9 @@ package org.qortal.utils; +import org.apache.commons.io.FileUtils; import org.qortal.settings.Settings; +import java.io.File; import java.io.IOException; import java.nio.file.*; @@ -30,6 +32,107 @@ public class FilesystemUtils { }); } + + /** + * moveFile + * Allows files to be moved between filesystems + * + * @param source + * @param dest + * @param cleanup + * @throws IOException + */ + public static void moveFile(Path source, Path dest, boolean cleanup) throws IOException { + if (source.compareTo(dest) == 0) { + // Source path matches destination path already + return; + } + + File sourceFile = new File(source.toString()); + File destFile = new File(dest.toString()); + if (sourceFile == null || !sourceFile.exists()) { + throw new IOException("Source file doesn't exist"); + } + if (!sourceFile.isFile()) { + throw new IOException("Source isn't a file"); + } + + // Ensure parent folders exist in the destination + File destParentFile = destFile.getParentFile(); + if (destParentFile != null) { + destParentFile.mkdirs(); + } + if (destParentFile == null || !destParentFile.exists()) { + throw new IOException("Destination directory doesn't exist"); + } + + // Copy to destination + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + + // Delete existing + if (FilesystemUtils.pathInsideDataOrTempPath(source)) { + System.out.println(String.format("Deleting file %s", source.toString())); + Files.delete(source); + } + + if (cleanup) { + // ... and delete its parent directory if empty + Path parentDirectory = source.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + Files.deleteIfExists(parentDirectory); + } + } + } + + /** + * moveDirectory + * Allows directories to be moved between filesystems + * + * @param source + * @param dest + * @param cleanup + * @throws IOException + */ + public static void moveDirectory(Path source, Path dest, boolean cleanup) throws IOException { + if (source.compareTo(dest) == 0) { + // Source path matches destination path already + return; + } + + File sourceFile = new File(source.toString()); + File destFile = new File(dest.toString()); + if (sourceFile == null || !sourceFile.exists()) { + throw new IOException("Source directory doesn't exist"); + } + if (!sourceFile.isDirectory()) { + throw new IOException("Source isn't a directory"); + } + + // Ensure parent folders exist in the destination + destFile.mkdirs(); + if (destFile == null || !destFile.exists()) { + throw new IOException("Destination directory doesn't exist"); + } + + // Copy to destination + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); + + // Delete existing + if (FilesystemUtils.pathInsideDataOrTempPath(source)) { + File directory = new File(source.toString()); + System.out.println(String.format("Deleting directory %s", directory.toString())); + FileUtils.deleteDirectory(directory); + } + + if (cleanup) { + // ... and delete its parent directory if empty + Path parentDirectory = source.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + Files.deleteIfExists(parentDirectory); + } + } + } + public static boolean pathInsideDataOrTempPath(Path path) { if (path == null) { return false; From a7282a5794b8af67850d113dea58354be171d30d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 17:31:58 +0100 Subject: [PATCH 142/505] Renamed encrypted and zipped files. --- src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index dd278ec5..770057ee 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -135,7 +135,7 @@ public class ArbitraryDataWriter { private void compress() { // Compress the data if requested if (this.compression != Compression.NONE) { - this.compressedPath = Paths.get(this.workingPath.toString() + File.separator + "zipped.zip"); + this.compressedPath = Paths.get(this.workingPath.toString() + File.separator + "data.zip"); try { if (this.compression == Compression.ZIP) { @@ -157,7 +157,7 @@ public class ArbitraryDataWriter { } private void encrypt() { - this.encryptedPath = Paths.get(this.workingPath.toString() + File.separator + "zipped_encrypted.zip"); + this.encryptedPath = Paths.get(this.workingPath.toString() + File.separator + "data.zip.encrypted"); try { // Encrypt the file with AES this.aesKey = AES.generateKey(256); From c50a11e58a2e0ef8032d41b5133c2411eeae030b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 17:32:25 +0100 Subject: [PATCH 143/505] Cleanup intermediate paths in ArbitraryDataWriter. --- .../org/qortal/arbitrary/ArbitraryDataWriter.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 770057ee..4fe21bb6 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -123,6 +123,12 @@ public class ArbitraryDataWriter { patch.create(); this.filePath = patch.getFinalPath(); + // Delete the input directory + if (FilesystemUtils.pathInsideDataOrTempPath(builtPath)) { + File directory = new File(builtPath.toString()); + FileUtils.deleteDirectory(directory); + } + // Validate the patch if (this.filePath == null) { throw new IllegalStateException("Null path after creating patch"); @@ -146,8 +152,12 @@ public class ArbitraryDataWriter { } // FUTURE: other compression types + // Delete the input directory + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + File directory = new File(this.filePath.toString()); + FileUtils.deleteDirectory(directory); + } // Replace filePath pointer with the zipped file path - // Don't delete the original file/directory, since this may be outside of our directory scope this.filePath = this.compressedPath; } catch (IOException e) { @@ -163,10 +173,11 @@ public class ArbitraryDataWriter { this.aesKey = AES.generateKey(256); AES.encryptFile("AES", this.aesKey, this.filePath.toString(), this.encryptedPath.toString()); - // Replace filePath pointer with the encrypted file path + // Delete the input file if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { Files.delete(this.filePath); } + // Replace filePath pointer with the encrypted file path this.filePath = this.encryptedPath; } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException From cfba793fcfa09d59fb6d5effbd8b4ee3739f219a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 18:10:59 +0100 Subject: [PATCH 144/505] Fixed bugs in ArbitraryDataFile, introduced when switching to the new temporary path. --- .../java/org/qortal/arbitrary/ArbitraryDataFile.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index ecc3e147..55aafc60 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -264,16 +264,16 @@ public class ArbitraryDataFile { // Create temporary path for joined file // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware String baseDir = Settings.getInstance().getTempDataPath(); - Path tempDir = Paths.get(baseDir, "join", this.chunks.get(0).digest58()); + Path tempDir = Paths.get(baseDir, "join"); try { Files.createDirectories(tempDir); } catch (IOException e) { return false; } - this.filePath = tempDir.toString(); // Join the chunks - File outputFile = new File(this.filePath); + Path outputPath = Paths.get(tempDir.toString(), this.chunks.get(0).digest58()); + File outputFile = new File(outputPath.toString()); try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) { for (ArbitraryDataFileChunk chunk : this.chunks) { File sourceFile = new File(chunk.filePath); @@ -288,9 +288,9 @@ public class ArbitraryDataFile { out.close(); // Copy temporary file to data directory - this.filePath = this.copyToDataDirectory(tempDir); - if (FilesystemUtils.pathInsideDataOrTempPath(tempDir)) { - Files.delete(tempDir); + this.filePath = this.copyToDataDirectory(outputPath); + if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) { + Files.delete(outputPath); } return true; From f095964f7b2949ed92a608955400646c604bb48e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Aug 2021 18:28:22 +0100 Subject: [PATCH 145/505] Fixed edge case. --- .../org/qortal/arbitrary/ArbitraryDataReader.java | 3 +++ src/main/java/org/qortal/utils/FilesystemUtils.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 9a9d034d..45c50afe 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -299,6 +299,9 @@ public class ArbitraryDataReader { if (dest == null || !dest.exists()) { throw new IllegalStateException("Destination directory doesn't exist"); } + // Ensure destination directory doesn't exist + FileUtils.deleteDirectory(dest); + // Move files to destination FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); try { diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index 0325cb60..b3f32c91 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -20,6 +20,15 @@ public class FilesystemUtils { } public static void copyAndReplaceDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { + // Ensure parent folders exist in the destination + File destFile = new File(destinationDirectoryLocation); + if (destFile != null) { + destFile.mkdirs(); + } + if (destFile == null || !destFile.exists()) { + throw new IOException("Destination directory doesn't exist"); + } + Files.walk(Paths.get(sourceDirectoryLocation)) .forEach(source -> { Path destination = Paths.get(destinationDirectoryLocation, source.toString() @@ -49,7 +58,6 @@ public class FilesystemUtils { } File sourceFile = new File(source.toString()); - File destFile = new File(dest.toString()); if (sourceFile == null || !sourceFile.exists()) { throw new IOException("Source file doesn't exist"); } @@ -58,6 +66,7 @@ public class FilesystemUtils { } // Ensure parent folders exist in the destination + File destFile = new File(dest.toString()); File destParentFile = destFile.getParentFile(); if (destParentFile != null) { destParentFile.mkdirs(); From 16ac92b2efc5c0e6eeb92a255dd6bab1fddedf05 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 10:13:32 +0100 Subject: [PATCH 146/505] Write patch metadata to a file inside a hidden ".qortal" folder which is included with each patch. This can be used in place of the existing ".removed" placeholder files to track removals. --- .../arbitrary/ArbitraryDataBuilder.java | 20 +++++ .../arbitrary/ArbitraryDataCreatePatch.java | 23 +++++- .../qortal/arbitrary/ArbitraryDataDiff.java | 61 +++++++++----- .../arbitrary/ArbitraryDataMetadata.java | 82 +++++++++++++++++++ .../qortal/arbitrary/ArbitraryDataWriter.java | 5 +- .../org/qortal/utils/FilesystemUtils.java | 21 ++++- 6 files changed, 185 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index d90ef43c..7bbf962d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -28,6 +28,7 @@ public class ArbitraryDataBuilder { private List transactions; private ArbitraryTransactionData latestPutTransaction; private List paths; + private byte[] latestSignature; private Path finalPath; public ArbitraryDataBuilder(String name, Service service) { @@ -41,6 +42,7 @@ public class ArbitraryDataBuilder { this.validateTransactions(); this.processTransactions(); this.validatePaths(); + this.findLatestSignature(); this.buildLatestState(); } @@ -119,6 +121,20 @@ public class ArbitraryDataBuilder { } } + private void findLatestSignature() { + if (this.transactions.size() == 0) { + throw new IllegalStateException("Unable to find latest signature from empty transaction list"); + } + + // Find the latest signature + ArbitraryTransactionData latestTransaction = this.transactions.get(this.transactions.size() - 1); + if (latestTransaction == null) { + throw new IllegalStateException("Unable to find latest signature from null transaction"); + } + + this.latestSignature = latestTransaction.getSignature(); + } + private void validatePaths() { if (this.paths == null || this.paths.isEmpty()) { throw new IllegalStateException(String.format("No paths available from which to build latest state")); @@ -149,4 +165,8 @@ public class ArbitraryDataBuilder { return this.finalPath; } + public byte[] getLatestSignature() { + return this.latestSignature; + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java index ed4095a4..6d28c455 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.repository.DataException; +import org.qortal.utils.FilesystemUtils; import java.io.IOException; import java.nio.file.Files; @@ -14,11 +15,13 @@ public class ArbitraryDataCreatePatch { private Path pathBefore; private Path pathAfter; + private byte[] previousSignature; private Path finalPath; - public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter) { + public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter, byte[] previousSignature) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; + this.previousSignature = previousSignature; } public void create() throws DataException, IOException { @@ -26,6 +29,10 @@ public class ArbitraryDataCreatePatch { this.preExecute(); this.process(); + } catch (Exception e) { + this.cleanupOnFailure(); + throw e; + } finally { this.postExecute(); } @@ -44,11 +51,19 @@ public class ArbitraryDataCreatePatch { } - private void process() { + private void cleanupOnFailure() { + try { + FilesystemUtils.safeDeleteDirectory(this.finalPath, true); + } catch (IOException e) { + LOGGER.info("Unable to cleanup diff directory on failure"); + } + } - ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter); - diff.compute(); + private void process() throws IOException { + + ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature); this.finalPath = diff.getDiffPath(); + diff.compute(); } public Path getFinalPath() { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 1952ddb0..b59db487 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -9,7 +9,9 @@ import java.io.File; import java.io.IOException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.UUID; public class ArbitraryDataDiff { @@ -18,19 +20,33 @@ public class ArbitraryDataDiff { private Path pathBefore; private Path pathAfter; + private byte[] previousSignature; private Path diffPath; private String identifier; - public ArbitraryDataDiff(Path pathBefore, Path pathAfter) { + private List addedPaths; + private List modifiedPaths; + private List removedPaths; + + public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; + this.previousSignature = previousSignature; + + this.addedPaths = new ArrayList<>(); + this.modifiedPaths = new ArrayList<>(); + this.removedPaths = new ArrayList<>(); + + this.createRandomIdentifier(); + this.createOutputDirectory(); } - public void compute() { + public void compute() throws IOException { try { this.preExecute(); this.findAddedOrModifiedFiles(); this.findRemovedFiles(); + this.writeMetadata(); } finally { this.postExecute(); @@ -38,8 +54,7 @@ public class ArbitraryDataDiff { } private void preExecute() { - this.createRandomIdentifier(); - this.createOutputDirectory(); + } private void postExecute() { @@ -63,18 +78,12 @@ public class ArbitraryDataDiff { } private void findAddedOrModifiedFiles() { - final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); - final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); - final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); - -// LOGGER.info("this.pathBefore: {}", this.pathBefore); -// LOGGER.info("this.pathAfter: {}", this.pathAfter); -// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute); -// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute); -// LOGGER.info("diffPathAbsolute: {}", diffPathAbsolute); - - try { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + final ArbitraryDataDiff diff = this; + // Check for additions or modifications Files.walkFileTree(this.pathAfter, new FileVisitor() { @@ -93,16 +102,19 @@ public class ArbitraryDataDiff { if (!Files.exists(filePathBefore)) { LOGGER.info("File was added: {}", after.toString()); + diff.addedPaths.add(filePathAfter); wasAdded = true; } else if (Files.size(after) != Files.size(filePathBefore)) { // Check file size first because it's quicker LOGGER.info("File size was modified: {}", after.toString()); + diff.modifiedPaths.add(filePathAfter); wasModified = true; } else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) { // Check hashes as a last resort LOGGER.info("File contents were modified: {}", after.toString()); + diff.modifiedPaths.add(filePathAfter); wasModified = true; } @@ -133,10 +145,12 @@ public class ArbitraryDataDiff { } private void findRemovedFiles() { - final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); - final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); - final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); try { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + final ArbitraryDataDiff diff = this; + // Check for removals Files.walkFileTree(this.pathBefore, new FileVisitor() { @@ -147,10 +161,9 @@ public class ArbitraryDataDiff { if (!Files.exists(directoryPathAfter)) { LOGGER.info("Directory was removed: {}", directoryPathAfter.toString()); - + diff.removedPaths.add(directoryPathBefore); ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore); // TODO: we might need to mark directories differently to files - // TODO: add path to manifest JSON } return FileVisitResult.CONTINUE; @@ -163,9 +176,9 @@ public class ArbitraryDataDiff { if (!Files.exists(filePathAfter)) { LOGGER.trace("File was removed: {}", before.toString()); + diff.removedPaths.add(filePathBefore); ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore); - // TODO: add path to manifest JSON } return FileVisitResult.CONTINUE; @@ -189,6 +202,12 @@ public class ArbitraryDataDiff { } } + private void writeMetadata() throws IOException { + ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.addedPaths, this.modifiedPaths, + this.removedPaths, this.diffPath, this.previousSignature); + metadata.write(); + } + private static byte[] digestFromPath(Path path) { try { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java new file mode 100644 index 00000000..7b5f7c22 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java @@ -0,0 +1,82 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.qortal.utils.Base58; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +public class ArbitraryDataMetadata { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class); + + private List addedPaths; + private List modifiedPaths; + private List removedPaths; + private Path filePath; + private Path qortalDirectoryPath; + private byte[] previousSignature; + + private String jsonString; + + public ArbitraryDataMetadata(List addedPaths, List modifiedPaths, List removedPaths, + Path filePath, byte[] previousSignature) { + this.addedPaths = addedPaths; + this.modifiedPaths = modifiedPaths; + this.removedPaths = removedPaths; + this.filePath = filePath; + this.previousSignature = previousSignature; + } + + public void write() throws IOException { + this.buildJson(); + this.createQortalDirectory(); + this.writeToQortalPath(); + } + + private void buildJson() { + JSONArray addedPathsJson = new JSONArray(this.addedPaths); + JSONArray modifiedPathsJson = new JSONArray(this.modifiedPaths); + JSONArray removedPathsJson = new JSONArray(this.removedPaths); + String previousSignature58 = Base58.encode(this.previousSignature); + + JSONObject jsonObject = new JSONObject(); + jsonObject.put("prevSig", previousSignature58); + jsonObject.put("added", addedPathsJson); + jsonObject.put("modified", modifiedPathsJson); + jsonObject.put("removed", removedPathsJson); + + this.jsonString = jsonObject.toString(4); + } + + private void createQortalDirectory() { + Path qortalDir = Paths.get(this.filePath.toString(), ".qortal"); + try { + Files.createDirectories(qortalDir); + } catch (IOException e) { + throw new IllegalStateException("Unable to create .qortal directory"); + } + this.qortalDirectoryPath = qortalDir; + } + + private void writeToQortalPath() throws IOException { + Path statePath = Paths.get(this.qortalDirectoryPath.toString(), "patch"); + BufferedWriter writer = new BufferedWriter(new FileWriter(statePath.toString())); + writer.write(this.jsonString); + writer.close(); + } + + + public String getJsonString() { + return this.jsonString; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 4fe21bb6..61b5c8eb 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -117,9 +117,12 @@ public class ArbitraryDataWriter { builder.build(); Path builtPath = builder.getFinalPath(); + // Obtain the latest signature, so this can be included in the patch + byte[] latestSignature = builder.getLatestSignature(); + // Compute a diff of the latest changes on top of the previous state // Then use only the differences as our data payload - ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath); + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath, latestSignature); patch.create(); this.filePath = patch.getFinalPath(); diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index b3f32c91..1bfdea78 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -80,7 +80,6 @@ public class FilesystemUtils { // Delete existing if (FilesystemUtils.pathInsideDataOrTempPath(source)) { - System.out.println(String.format("Deleting file %s", source.toString())); Files.delete(source); } @@ -142,6 +141,26 @@ public class FilesystemUtils { } } + public static void safeDeleteDirectory(Path path, boolean cleanup) throws IOException { + // Delete path, if it exists in our data/temp directory + if (FilesystemUtils.pathInsideDataOrTempPath(path)) { + File directory = new File(path.toString()); + FileUtils.deleteDirectory(directory); + } + + if (cleanup) { + // Delete the parent directory if it is empty (and exists in our data/temp directory) + Path parentDirectory = path.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + try { + Files.deleteIfExists(parentDirectory); + } catch (IOException e) { + // This part is optional, so ignore failures + } + } + } + } + public static boolean pathInsideDataOrTempPath(Path path) { if (path == null) { return false; From 95044d27cef548f801eddba1f2b7fe74399fc130 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 10:32:37 +0100 Subject: [PATCH 147/505] Fixed NPE caused by having an arbitrary transaction with no chunks (which is expected if the total data size is less than the chunk size). --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 55aafc60..434c626b 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -394,6 +394,9 @@ public class ArbitraryDataFile { } public boolean allChunksExist(byte[] chunks) { + if (chunks == null) { + return true; + } ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; From 3019bb5c979d0dbe215bc7c18b0bed4a457b06de Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 10:39:00 +0100 Subject: [PATCH 148/505] Enforce minBlockchainPeers in ArbitraryDataManager, as there is no point in trying to request data files when we don't have the minimum number of peers. --- .../org/qortal/controller/ArbitraryDataManager.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index bf7c4632..991565bd 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -15,6 +15,7 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFileChunk; +import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.Base58; @@ -75,6 +76,16 @@ public class ArbitraryDataManager extends Thread { while (!isStopping) { Thread.sleep(2000); + List peers = Network.getInstance().getHandshakedPeers(); + + // Disregard peers that have "misbehaved" recently + peers.removeIf(Controller.hasMisbehaved); + + // Don't fetch data if we don't have enough up-to-date peers + if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) { + continue; + } + // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true); From 8929f320689372209ba0f7a9a568bef1a7ae8b1a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 10:43:00 +0100 Subject: [PATCH 149/505] No longer creating the ".removed" files. --- .../org/qortal/arbitrary/ArbitraryDataDiff.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index b59db487..6432eab7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -162,7 +162,6 @@ public class ArbitraryDataDiff { if (!Files.exists(directoryPathAfter)) { LOGGER.info("Directory was removed: {}", directoryPathAfter.toString()); diff.removedPaths.add(directoryPathBefore); - ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore); // TODO: we might need to mark directories differently to files } @@ -177,8 +176,6 @@ public class ArbitraryDataDiff { if (!Files.exists(filePathAfter)) { LOGGER.trace("File was removed: {}", before.toString()); diff.removedPaths.add(filePathBefore); - - ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore); } return FileVisitResult.CONTINUE; @@ -233,19 +230,7 @@ public class ArbitraryDataDiff { LOGGER.trace("Copying {} to {}", source, dest); Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); } - - private static void markFilePathAsRemoved(Path base, Path relativePath) throws IOException { - String newFilename = relativePath.toString().concat(".removed"); - Path dest = Paths.get(base.toString(), newFilename); - File file = new File(dest.toString()); - File parent = file.getParentFile(); - if (parent != null) { - parent.mkdirs(); - } - LOGGER.info("Creating file {}", dest); - file.createNewFile(); - } - + public Path getDiffPath() { return this.diffPath; From 9850c294d133e6feaa7e7e51af0a47f0a7649ee3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 11:28:26 +0100 Subject: [PATCH 150/505] Fixed various issues relating to syncing data for transactions without any chunks. --- .../org/qortal/arbitrary/ArbitraryDataFile.java | 3 +++ .../qortal/controller/ArbitraryDataManager.java | 15 ++++++++++++--- .../hsqldb/HSQLDBArbitraryRepository.java | 5 +++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 434c626b..98b0c667 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -390,6 +390,9 @@ public class ArbitraryDataFile { return chunk.exists(); } } + if (Arrays.equals(this.getHash(), hash)) { + return this.exists(); + } return false; } diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 991565bd..10c5d448 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -287,8 +287,11 @@ public class ArbitraryDataManager extends Thread { for (byte[] hash : hashes) { //LOGGER.info("Received hash {}", Base58.encode(hash)); if (!arbitraryDataFile.containsChunk(hash)) { - LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58); - return; + // Check the hash against the complete file + if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) { + LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58); + return; + } } } @@ -391,6 +394,12 @@ public class ArbitraryDataManager extends Thread { } } } + else { + // This transaction has no chunks, so include the complete file if we have it + if (arbitraryDataFile.exists()) { + hashes.add(arbitraryDataFile.getHash()); + } + } } } catch (DataException e) { @@ -403,7 +412,7 @@ public class ArbitraryDataManager extends Thread { LOGGER.info("Couldn't send list of hashes"); peer.disconnect("failed to send list of hashes"); } - LOGGER.info("Sent list of hashes", hashes); + LOGGER.info("Sent list of hashes (count: {})", hashes.size()); } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 42321584..b6a920af 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -65,6 +65,11 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return true; } + // If this transaction doesn't have any chunks, then we require the complete file + if (chunkHashes == null) { + return false; + } + // Alternatively, if we have all the chunks, then it's safe to assume the data is local if (arbitraryDataFile.allChunksExist(chunkHashes)) { return true; From f5615b1c548c6eb0c6eb17c6be7914db95f7a54b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 17:45:37 +0100 Subject: [PATCH 151/505] Don't exclude hidden files in the zips, as the .qortal folder needs to be included. --- src/main/java/org/qortal/utils/ZipUtils.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index 29da90da..2ccffc63 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -48,9 +48,6 @@ public class ZipUtils { } public static void zip(final File fileToZip, final String fileName, final ZipOutputStream zipOut) throws IOException { - if (fileToZip.isHidden()) { - return; - } if (fileToZip.isDirectory()) { if (fileName.endsWith("/")) { zipOut.putNextEntry(new ZipEntry(fileName)); From fa696a29010e85563bdba3adead82e279994ce42 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 18:37:27 +0100 Subject: [PATCH 152/505] Log exceptions when publishing data updates. --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 2 ++ src/main/java/org/qortal/api/resource/WebsiteResource.java | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 41ee0b45..233780fb 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -276,8 +276,10 @@ public class ArbitraryResource { try { arbitraryDataWriter.save(); } catch (IOException | DataException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 9bd1c142..d9538cc1 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -107,8 +107,10 @@ public class WebsiteResource { try { arbitraryDataWriter.save(); } catch (IOException | DataException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } @@ -207,8 +209,10 @@ public class WebsiteResource { try { arbitraryDataWriter.save(); } catch (IOException | DataException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } From 8fac0a02e5928217d7cf1b9c02825955ca9e82d7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 18:49:30 +0100 Subject: [PATCH 153/505] Use the /.qortal/patch file to apply differences when merging together two layers, rather than walking through the file tree like we did before. --- .../qortal/arbitrary/ArbitraryDataDiff.java | 7 +- .../qortal/arbitrary/ArbitraryDataMerge.java | 112 ++++------------ .../arbitrary/ArbitraryDataMetadata.java | 125 +++++++++++++++--- .../qortal/arbitrary/ArbitraryDataWriter.java | 21 ++- 4 files changed, 157 insertions(+), 108 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 6432eab7..b9d6e6e5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -200,8 +200,11 @@ public class ArbitraryDataDiff { } private void writeMetadata() throws IOException { - ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.addedPaths, this.modifiedPaths, - this.removedPaths, this.diffPath, this.previousSignature); + ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.diffPath); + metadata.setAddedPaths(this.addedPaths); + metadata.setModifiedPaths(this.modifiedPaths); + metadata.setRemovedPaths(this.removedPaths); + metadata.setPreviousSignature(this.previousSignature); metadata.write(); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index da399242..55f487e3 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -3,15 +3,13 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; import org.qortal.utils.FilesystemUtils; import java.io.File; import java.io.IOException; import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; +import java.util.List; import java.util.UUID; public class ArbitraryDataMerge { @@ -22,6 +20,7 @@ public class ArbitraryDataMerge { private Path pathAfter; private Path mergePath; private String identifier; + private ArbitraryDataMetadata metadata; public ArbitraryDataMerge(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; @@ -32,7 +31,8 @@ public class ArbitraryDataMerge { try { this.preExecute(); this.copyPreviousStateToMergePath(); - this.findDifferences(); + this.loadMetadata(); + this.applyDifferences(); } finally { this.postExecute(); @@ -68,97 +68,35 @@ public class ArbitraryDataMerge { ArbitraryDataMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get("")); } - private void findDifferences() { - final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); - final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); - final Path mergePathAbsolute = this.mergePath.toAbsolutePath(); + private void loadMetadata() throws IOException { + this.metadata = new ArbitraryDataMetadata(this.pathAfter); + this.metadata.read(); + } -// LOGGER.info("this.pathBefore: {}", this.pathBefore); -// LOGGER.info("this.pathAfter: {}", this.pathAfter); -// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute); -// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute); -// LOGGER.info("mergePathAbsolute: {}", mergePathAbsolute); + private void applyDifferences() throws IOException { + List addedPaths = this.metadata.getAddedPaths(); + for (Path path : addedPaths) { + LOGGER.info("File was added: {}", path.toString()); + Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); + ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path); + } - try { - // Check for additions or modifications - Files.walkFileTree(this.pathAfter, new FileVisitor() { + List modifiedPaths = this.metadata.getModifiedPaths(); + for (Path path : modifiedPaths) { + LOGGER.info("File was modified: {}", path.toString()); + Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); + ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path); + } - @Override - public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException { - Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath()); - Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter); - - boolean wasAdded = false; - boolean wasModified = false; - boolean wasRemoved = false; - - if (after.toString().endsWith(".removed")) { - LOGGER.trace("File was removed: {}", after.toString()); - wasRemoved = true; - } - else if (!Files.exists(filePathBefore)) { - LOGGER.trace("File was added: {}", after.toString()); - wasAdded = true; - } - else if (Files.size(after) != Files.size(filePathBefore)) { - // Check file size first because it's quicker - LOGGER.trace("File size was modified: {}", after.toString()); - wasModified = true; - } - else if (!Arrays.equals(ArbitraryDataMerge.digestFromPath(after), ArbitraryDataMerge.digestFromPath(filePathBefore))) { - // Check hashes as a last resort - LOGGER.trace("File contents were modified: {}", after.toString()); - wasModified = true; - } - - if (wasAdded | wasModified) { - ArbitraryDataMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter); - } - - if (wasRemoved) { - if (filePathAfter.toString().endsWith(".removed")) { - // Trim the ".removed" - Path filePathAfterTrimmed = Paths.get(filePathAfter.toString().substring(0, filePathAfter.toString().length()-8)); - ArbitraryDataMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed); - } - } - - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException e){ - LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); - // TODO: throw exception? - return FileVisitResult.TERMINATE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException e) { - return FileVisitResult.CONTINUE; - } - - }); - } catch (IOException e) { - LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + List removedPaths = this.metadata.getRemovedPaths(); + for (Path path : removedPaths) { + LOGGER.info("File was removed: {}", path.toString()); + ArbitraryDataMerge.deletePathInBaseDir(this.mergePath, path); } } - private static byte[] digestFromPath(Path path) { - try { - return Crypto.digest(Files.readAllBytes(path)); - } catch (IOException e) { - return null; - } - } - private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { if (!Files.exists(source)) { throw new IOException(String.format("File not found: %s", source.toString())); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java index 7b5f7c22..d333d5ab 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java @@ -7,11 +7,13 @@ import org.json.JSONObject; import org.qortal.utils.Base58; import java.io.BufferedWriter; +import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; public class ArbitraryDataMetadata { @@ -27,13 +29,18 @@ public class ArbitraryDataMetadata { private String jsonString; - public ArbitraryDataMetadata(List addedPaths, List modifiedPaths, List removedPaths, - Path filePath, byte[] previousSignature) { - this.addedPaths = addedPaths; - this.modifiedPaths = modifiedPaths; - this.removedPaths = removedPaths; + public ArbitraryDataMetadata(Path filePath) { this.filePath = filePath; - this.previousSignature = previousSignature; + this.qortalDirectoryPath = Paths.get(filePath.toString(), ".qortal"); + + this.addedPaths = new ArrayList<>(); + this.modifiedPaths = new ArrayList<>(); + this.removedPaths = new ArrayList<>(); + } + + public void read() throws IOException { + this.loadJson(); + this.readJson(); } public void write() throws IOException { @@ -42,39 +49,123 @@ public class ArbitraryDataMetadata { this.writeToQortalPath(); } + + private void loadJson() throws IOException { + Path path = Paths.get(this.qortalDirectoryPath.toString(), "patch"); + File patchFile = new File(path.toString()); + if (!patchFile.exists()) { + throw new IOException(String.format("Patch file doesn't exist: %s", path.toString())); + } + + this.jsonString = new String(Files.readAllBytes(path)); + } + + private void readJson() { + if (this.jsonString == null) { + throw new IllegalStateException("Patch JSON string is null"); + } + + JSONObject patch = new JSONObject(this.jsonString); + if (patch.has("prevSig")) { + String prevSig = (String)patch.get("prevSig"); + if (prevSig != null) { + this.previousSignature = Base58.decode(prevSig); + } + } + if (patch.has("added")) { + JSONArray added = (JSONArray) patch.get("added"); + if (added != null) { + for (int i=0; i addedPaths) { + this.addedPaths = addedPaths; + } + + public List getAddedPaths() { + return this.addedPaths; + } + + public void setModifiedPaths(List modifiedPaths) { + this.modifiedPaths = modifiedPaths; + } + + public List getModifiedPaths() { + return this.modifiedPaths; + } + + public void setRemovedPaths(List removedPaths) { + this.removedPaths = removedPaths; + } + + public List getRemovedPaths() { + return this.removedPaths; + } + + public void setPreviousSignature(byte[] previousSignature) { + this.previousSignature = previousSignature; + } + + public byte[] getPreviousSignature() { + return this.getPreviousSignature(); + } + public String getJsonString() { return this.jsonString; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 61b5c8eb..d9f284e7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -133,11 +133,28 @@ public class ArbitraryDataWriter { } // Validate the patch + this.validatePatch(); + } + + private void validatePatch() throws IOException { if (this.filePath == null) { throw new IllegalStateException("Null path after creating patch"); } - if (FilesystemUtils.isDirectoryEmpty(this.filePath)) { - throw new IllegalStateException("Patch has no content. Either no files have changed, or something went wrong"); + + File qortalMetadataDirectoryFile = Paths.get(this.filePath.toString(), ".qortal").toFile(); + if (!qortalMetadataDirectoryFile.exists()) { + throw new IllegalStateException("Qortal metadata folder doesn't exist in patch"); + } + if (!qortalMetadataDirectoryFile.isDirectory()) { + throw new IllegalStateException("Qortal metadata folder isn't a directory"); + } + + File qortalPatchMetadataFile = Paths.get(this.filePath.toString(), ".qortal", "patch").toFile(); + if (!qortalPatchMetadataFile.exists()) { + throw new IllegalStateException("Qortal patch metadata file doesn't exist in patch"); + } + if (!qortalPatchMetadataFile.isFile()) { + throw new IllegalStateException("Qortal patch metadata file isn't a file"); } } From be0426d9a2a71a2dd62a096186a18c89aaabff39 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 19:19:55 +0100 Subject: [PATCH 154/505] Improved logging --- src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java | 1 + .../java/org/qortal/arbitrary/ArbitraryDataCombiner.java | 3 +++ src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java | 5 ++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 7bbf962d..7ef29d03 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -152,6 +152,7 @@ public class ArbitraryDataBuilder { // Loop from the second path onwards for (int i=1; i Date: Sun, 15 Aug 2021 19:55:56 +0100 Subject: [PATCH 155/505] Include the .qortal folder in the merge output, since it will be needed for validation. We may choose to exclude it from the final output path, but for now it is left there to make debugging easier. --- .../qortal/arbitrary/ArbitraryDataMerge.java | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index 55f487e3..88b910f7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -33,6 +33,7 @@ public class ArbitraryDataMerge { this.copyPreviousStateToMergePath(); this.loadMetadata(); this.applyDifferences(); + this.copyMetadata(); } finally { this.postExecute(); @@ -79,14 +80,14 @@ public class ArbitraryDataMerge { for (Path path : addedPaths) { LOGGER.info("File was added: {}", path.toString()); Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); - ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path); } List modifiedPaths = this.metadata.getModifiedPaths(); for (Path path : modifiedPaths) { LOGGER.info("File was modified: {}", path.toString()); Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); - ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path); } List removedPaths = this.metadata.getRemovedPaths(); @@ -96,15 +97,30 @@ public class ArbitraryDataMerge { } } + private void copyMetadata() throws IOException { + Path filePath = Paths.get(this.pathAfter.toString(), ".qortal"); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, Paths.get(".qortal")); + } - private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + + private static void copyPathToBaseDir(Path source, Path base, Path relativePath) throws IOException { if (!Files.exists(source)) { throw new IOException(String.format("File not found: %s", source.toString())); } + File sourceFile = source.toFile(); Path dest = Paths.get(base.toString(), relativePath.toString()); LOGGER.trace("Copying {} to {}", source, dest); - Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + + if (sourceFile.isFile()) { + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + else if (sourceFile.isDirectory()) { + FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString()); + } + else { + throw new IOException(String.format("Invalid file: %s", source.toString())); + } } private static void copyDirPathToBaseDir(Path source, Path base, Path relativePath) throws IOException { From a5cfedcae953d55576c1e8f1ae9e5d6166c00094 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 20:02:14 +0100 Subject: [PATCH 156/505] Exclude the .qortal metadata directory in all diffs. Also improved logging. --- .../qortal/arbitrary/ArbitraryDataDiff.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index b9d6e6e5..06976e72 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -54,7 +54,7 @@ public class ArbitraryDataDiff { } private void preExecute() { - + LOGGER.info("Generating diff..."); } private void postExecute() { @@ -97,23 +97,28 @@ public class ArbitraryDataDiff { Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath()); Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter); + if (filePathAfter.startsWith(".qortal")) { + // Ignore the .qortal metadata folder + return FileVisitResult.CONTINUE; + } + boolean wasAdded = false; boolean wasModified = false; if (!Files.exists(filePathBefore)) { - LOGGER.info("File was added: {}", after.toString()); + LOGGER.info("File was added: {}", filePathAfter.toString()); diff.addedPaths.add(filePathAfter); wasAdded = true; } else if (Files.size(after) != Files.size(filePathBefore)) { // Check file size first because it's quicker - LOGGER.info("File size was modified: {}", after.toString()); + LOGGER.info("File size was modified: {}", filePathAfter.toString()); diff.modifiedPaths.add(filePathAfter); wasModified = true; } else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) { // Check hashes as a last resort - LOGGER.info("File contents were modified: {}", after.toString()); + LOGGER.info("File contents were modified: {}", filePathAfter.toString()); diff.modifiedPaths.add(filePathAfter); wasModified = true; } @@ -148,7 +153,6 @@ public class ArbitraryDataDiff { try { final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); - final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); final ArbitraryDataDiff diff = this; // Check for removals @@ -159,6 +163,11 @@ public class ArbitraryDataDiff { Path directoryPathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); Path directoryPathAfter = pathAfterAbsolute.resolve(directoryPathBefore); + if (directoryPathBefore.startsWith(".qortal")) { + // Ignore the .qortal metadata folder + return FileVisitResult.CONTINUE; + } + if (!Files.exists(directoryPathAfter)) { LOGGER.info("Directory was removed: {}", directoryPathAfter.toString()); diff.removedPaths.add(directoryPathBefore); @@ -173,8 +182,13 @@ public class ArbitraryDataDiff { Path filePathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); Path filePathAfter = pathAfterAbsolute.resolve(filePathBefore); + if (filePathBefore.startsWith(".qortal")) { + // Ignore the .qortal metadata folder + return FileVisitResult.CONTINUE; + } + if (!Files.exists(filePathAfter)) { - LOGGER.trace("File was removed: {}", before.toString()); + LOGGER.trace("File was removed: {}", filePathBefore.toString()); diff.removedPaths.add(filePathBefore); } From 219a5db60cb731b241d561e9b4bec2433f4a5e21 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 20:21:47 +0100 Subject: [PATCH 157/505] Fixed circular bug in ArbitraryDataMetadata.getPreviousSignature() --- src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java index d333d5ab..7de9cab2 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java @@ -163,7 +163,7 @@ public class ArbitraryDataMetadata { } public byte[] getPreviousSignature() { - return this.getPreviousSignature(); + return this.previousSignature; } public String getJsonString() { From 94da1a30dcd1c0592740001bb9339b7e6bbbc37b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 20:27:16 +0100 Subject: [PATCH 158/505] When merging two states, validate that the transaction signature we are using for the "before" layer matches the "previous transaction signature" that is baked into the "after" layer. This defends against a missing or out-of-order transaction. If this ever fails validation, we may need to rethink the way we are requesting transactions. But in theory this shouldn't happen, given that the "last reference" field of a transaction ensures that out-of-order transactions are invalid already. --- .../arbitrary/ArbitraryDataBuilder.java | 3 ++- .../arbitrary/ArbitraryDataCombiner.java | 24 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 7ef29d03..f8c95596 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -154,7 +154,8 @@ public class ArbitraryDataBuilder { for (int i=1; i Date: Sun, 15 Aug 2021 21:49:45 +0100 Subject: [PATCH 159/505] Added "cache" file to the .qortal metadata folder. This is used to store the transaction signature and build timestamp for each built data resource. It involved a refactor of the ArbitraryDataMetadata class to introduce a subclass for each file ("patch" and "cache"). This allows more files to be easily added later. --- .../arbitrary/ArbitraryDataBuilder.java | 14 ++ .../arbitrary/ArbitraryDataCombiner.java | 2 +- .../qortal/arbitrary/ArbitraryDataDiff.java | 2 +- .../qortal/arbitrary/ArbitraryDataMerge.java | 4 +- .../arbitrary/ArbitraryDataMetadata.java | 132 +++--------------- .../arbitrary/ArbitraryDataMetadataCache.java | 69 +++++++++ .../arbitrary/ArbitraryDataMetadataPatch.java | 119 ++++++++++++++++ 7 files changed, 229 insertions(+), 113 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataCache.java create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataPatch.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index f8c95596..4db89688 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -10,6 +10,7 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.utils.Base58; +import org.qortal.utils.NTP; import java.io.IOException; import java.nio.file.Files; @@ -44,6 +45,7 @@ public class ArbitraryDataBuilder { this.validatePaths(); this.findLatestSignature(); this.buildLatestState(); + this.cacheLatestSignature(); } private void fetchTransactions() throws DataException { @@ -163,6 +165,18 @@ public class ArbitraryDataBuilder { this.finalPath = pathBefore; } + private void cacheLatestSignature() throws IOException { + byte[] latestTransactionSignature = this.transactions.get(this.transactions.size()-1).getSignature(); + if (latestTransactionSignature == null) { + throw new IllegalStateException("Missing latest transaction signature"); + } + + ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.finalPath); + cache.setSignature(latestTransactionSignature); + cache.setTimestamp(NTP.getTime()); + cache.write(); + } + public Path getFinalPath() { return this.finalPath; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index 3d3fbe9f..bf49ae99 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -87,7 +87,7 @@ public class ArbitraryDataCombiner { throw new IllegalStateException("No previous signature passed to the combiner"); } - ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.pathAfter); + ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.pathAfter); metadata.read(); byte[] previousSignature = metadata.getPreviousSignature(); if (previousSignature == null) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 06976e72..84280738 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -214,7 +214,7 @@ public class ArbitraryDataDiff { } private void writeMetadata() throws IOException { - ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.diffPath); + ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath); metadata.setAddedPaths(this.addedPaths); metadata.setModifiedPaths(this.modifiedPaths); metadata.setRemovedPaths(this.removedPaths); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index 88b910f7..79fc5eaa 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -20,7 +20,7 @@ public class ArbitraryDataMerge { private Path pathAfter; private Path mergePath; private String identifier; - private ArbitraryDataMetadata metadata; + private ArbitraryDataMetadataPatch metadata; public ArbitraryDataMerge(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; @@ -70,7 +70,7 @@ public class ArbitraryDataMerge { } private void loadMetadata() throws IOException { - this.metadata = new ArbitraryDataMetadata(this.pathAfter); + this.metadata = new ArbitraryDataMetadataPatch(this.pathAfter); this.metadata.read(); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java index 7de9cab2..5cf04b5d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java @@ -13,31 +13,35 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; public class ArbitraryDataMetadata { - private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class); + protected static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class); - private List addedPaths; - private List modifiedPaths; - private List removedPaths; - private Path filePath; - private Path qortalDirectoryPath; - private byte[] previousSignature; + protected Path filePath; + protected Path qortalDirectoryPath; - private String jsonString; + protected String jsonString; public ArbitraryDataMetadata(Path filePath) { this.filePath = filePath; this.qortalDirectoryPath = Paths.get(filePath.toString(), ".qortal"); - - this.addedPaths = new ArrayList<>(); - this.modifiedPaths = new ArrayList<>(); - this.removedPaths = new ArrayList<>(); } + protected String fileName() { + // To be overridden + return null; + } + + protected void readJson() { + // To be overridden + } + + protected void buildJson() { + // To be overridden + } + + public void read() throws IOException { this.loadJson(); this.readJson(); @@ -50,8 +54,8 @@ public class ArbitraryDataMetadata { } - private void loadJson() throws IOException { - Path path = Paths.get(this.qortalDirectoryPath.toString(), "patch"); + protected void loadJson() throws IOException { + Path path = Paths.get(this.qortalDirectoryPath.toString(), this.fileName()); File patchFile = new File(path.toString()); if (!patchFile.exists()) { throw new IOException(String.format("Patch file doesn't exist: %s", path.toString())); @@ -60,65 +64,7 @@ public class ArbitraryDataMetadata { this.jsonString = new String(Files.readAllBytes(path)); } - private void readJson() { - if (this.jsonString == null) { - throw new IllegalStateException("Patch JSON string is null"); - } - - JSONObject patch = new JSONObject(this.jsonString); - if (patch.has("prevSig")) { - String prevSig = (String)patch.get("prevSig"); - if (prevSig != null) { - this.previousSignature = Base58.decode(prevSig); - } - } - if (patch.has("added")) { - JSONArray added = (JSONArray) patch.get("added"); - if (added != null) { - for (int i=0; i addedPaths) { - this.addedPaths = addedPaths; - } - - public List getAddedPaths() { - return this.addedPaths; - } - - public void setModifiedPaths(List modifiedPaths) { - this.modifiedPaths = modifiedPaths; - } - - public List getModifiedPaths() { - return this.modifiedPaths; - } - - public void setRemovedPaths(List removedPaths) { - this.removedPaths = removedPaths; - } - - public List getRemovedPaths() { - return this.removedPaths; - } - - public void setPreviousSignature(byte[] previousSignature) { - this.previousSignature = previousSignature; - } - - public byte[] getPreviousSignature() { - return this.previousSignature; - } - public String getJsonString() { return this.jsonString; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataCache.java new file mode 100644 index 00000000..cfe56593 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataCache.java @@ -0,0 +1,69 @@ +package org.qortal.arbitrary; + +import org.json.JSONObject; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.nio.file.Path; + +public class ArbitraryDataMetadataCache extends ArbitraryDataMetadata { + + private byte[] signature; + private long timestamp; + + public ArbitraryDataMetadataCache(Path filePath) { + super(filePath); + + } + + @Override + protected String fileName() { + return "cache"; + } + + @Override + protected void readJson() { + if (this.jsonString == null) { + throw new IllegalStateException("Patch JSON string is null"); + } + + JSONObject cache = new JSONObject(this.jsonString); + if (cache.has("signature")) { + String sig = cache.getString("signature"); + if (sig != null) { + this.signature = Base58.decode(sig); + } + } + if (cache.has("timestamp")) { + this.timestamp = cache.getLong("timestamp"); + } + } + + @Override + protected void buildJson() { + JSONObject patch = new JSONObject(); + patch.put("signature", Base58.encode(this.signature)); + patch.put("timestamp", this.timestamp); + + this.jsonString = patch.toString(2); + LOGGER.info("Cache metadata: {}", this.jsonString); + } + + + public void setSignature(byte[] signature) { + this.signature = signature; + } + + public byte[] getSignature() { + return this.signature; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public long getTimestamp() { + return this.timestamp; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataPatch.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataPatch.java new file mode 100644 index 00000000..2c1f4a7c --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataPatch.java @@ -0,0 +1,119 @@ +package org.qortal.arbitrary; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.qortal.utils.Base58; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { + + private List addedPaths; + private List modifiedPaths; + private List removedPaths; + private byte[] previousSignature; + + public ArbitraryDataMetadataPatch(Path filePath) { + super(filePath); + + this.addedPaths = new ArrayList<>(); + this.modifiedPaths = new ArrayList<>(); + this.removedPaths = new ArrayList<>(); + } + + @Override + protected String fileName() { + return "patch"; + } + + @Override + protected void readJson() { + if (this.jsonString == null) { + throw new IllegalStateException("Patch JSON string is null"); + } + + JSONObject patch = new JSONObject(this.jsonString); + if (patch.has("prevSig")) { + String prevSig = patch.getString("prevSig"); + if (prevSig != null) { + this.previousSignature = Base58.decode(prevSig); + } + } + if (patch.has("added")) { + JSONArray added = (JSONArray) patch.get("added"); + if (added != null) { + for (int i=0; i addedPaths) { + this.addedPaths = addedPaths; + } + + public List getAddedPaths() { + return this.addedPaths; + } + + public void setModifiedPaths(List modifiedPaths) { + this.modifiedPaths = modifiedPaths; + } + + public List getModifiedPaths() { + return this.modifiedPaths; + } + + public void setRemovedPaths(List removedPaths) { + this.removedPaths = removedPaths; + } + + public List getRemovedPaths() { + return this.removedPaths; + } + + public void setPreviousSignature(byte[] previousSignature) { + this.previousSignature = previousSignature; + } + + public byte[] getPreviousSignature() { + return this.previousSignature; + } + +} From b46c328811bfb0a055c599c7b888710a7b6930b9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 21:50:19 +0100 Subject: [PATCH 160/505] Fixed bug in FilesystemUtils.copyAndReplaceDirectory() --- src/main/java/org/qortal/utils/FilesystemUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index 1bfdea78..bbf54e74 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -29,6 +29,12 @@ public class FilesystemUtils { throw new IOException("Destination directory doesn't exist"); } + // If the destination directory isn't empty, delete its contents + if (!FilesystemUtils.isDirectoryEmpty(destFile.toPath())) { + FileUtils.deleteDirectory(destFile); + destFile.mkdirs(); + } + Files.walk(Paths.get(sourceDirectoryLocation)) .forEach(source -> { Path destination = Paths.get(destinationDirectoryLocation, source.toString() From 0ed8e04233ec3db7f74721895060b4f7eebeb463 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 21:54:21 +0100 Subject: [PATCH 161/505] Arbitrary data metadata classes moved to their own package. --- src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java | 1 + .../java/org/qortal/arbitrary/ArbitraryDataCombiner.java | 1 + src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java | 1 + src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java | 1 + .../arbitrary/{ => metadata}/ArbitraryDataMetadata.java | 5 +---- .../arbitrary/{ => metadata}/ArbitraryDataMetadataCache.java | 3 +-- .../arbitrary/{ => metadata}/ArbitraryDataMetadataPatch.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename src/main/java/org/qortal/arbitrary/{ => metadata}/ArbitraryDataMetadata.java (94%) rename src/main/java/org/qortal/arbitrary/{ => metadata}/ArbitraryDataMetadataCache.java (96%) rename src/main/java/org/qortal/arbitrary/{ => metadata}/ArbitraryDataMetadataPatch.java (98%) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 4db89688..c8da187f 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -2,6 +2,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.Method; import org.qortal.data.transaction.ArbitraryTransactionData.Service; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index bf49ae99..ccbfb85c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; import org.qortal.utils.FilesystemUtils; import java.io.File; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 84280738..8db1af09 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -2,6 +2,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index 79fc5eaa..d4faf124 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; import org.qortal.settings.Settings; import org.qortal.utils.FilesystemUtils; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java similarity index 94% rename from src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java rename to src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java index 5cf04b5d..7dcfd1b5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java @@ -1,10 +1,7 @@ -package org.qortal.arbitrary; +package org.qortal.arbitrary.metadata; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONArray; -import org.json.JSONObject; -import org.qortal.utils.Base58; import java.io.BufferedWriter; import java.io.File; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataCache.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java similarity index 96% rename from src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataCache.java rename to src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java index cfe56593..d6d7a2b4 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataCache.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java @@ -1,8 +1,7 @@ -package org.qortal.arbitrary; +package org.qortal.arbitrary.metadata; import org.json.JSONObject; import org.qortal.utils.Base58; -import org.qortal.utils.NTP; import java.nio.file.Path; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataPatch.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java similarity index 98% rename from src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataPatch.java rename to src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java index 2c1f4a7c..fd68bec3 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadataPatch.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java @@ -1,4 +1,4 @@ -package org.qortal.arbitrary; +package org.qortal.arbitrary.metadata; import org.json.JSONArray; import org.json.JSONObject; From 95c9cc7f99a56a919c7e27e7c509aca81c24e59e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 16 Aug 2021 22:40:05 +0100 Subject: [PATCH 162/505] Added ArbitraryDataCache This decides whether to build a new state or use an existing cached state when serving a data resource. It will cache a built resource until a new transaction (i.e. layer) arrives. This drastically reduces load, and still allows for almost instant propagation of new layers. --- .../qortal/api/resource/WebsiteResource.java | 2 +- .../qortal/arbitrary/ArbitraryDataCache.java | 158 ++++++++++++++++++ .../qortal/arbitrary/ArbitraryDataReader.java | 22 +-- .../controller/ArbitraryDataManager.java | 64 ++++++- .../hsqldb/HSQLDBArbitraryRepository.java | 16 +- 5 files changed, 245 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index d9538cc1..45b8097e 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -293,12 +293,12 @@ public class WebsiteResource { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service); arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { - // TODO: overwrite if new transaction arrives, to invalidate cache // We could store the latest transaction signature in the extracted folder arbitraryDataReader.load(false); } catch (Exception e) { return this.get404Response(); } + java.nio.file.Path path = arbitraryDataReader.getFilePath(); if (path == null) { return this.get404Response(); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java new file mode 100644 index 00000000..1cb45610 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java @@ -0,0 +1,158 @@ +package org.qortal.arbitrary; + +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; +import org.qortal.controller.ArbitraryDataManager; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.FilesystemUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +public class ArbitraryDataCache { + + private boolean overwrite; + private Path filePath; + private String resourceId; + private ResourceIdType resourceIdType; + private Service service; + + public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId, + ResourceIdType resourceIdType, Service service) { + this.filePath = filePath; + this.overwrite = overwrite; + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + this.service = service; + } + + public boolean shouldInvalidate() { + try { + // If the user has requested an overwrite, always invalidate the cache + if (this.overwrite) { + return true; + } + + // Overwrite is false, but we still need to invalidate if no files exist + if (!Files.exists(this.filePath) || FilesystemUtils.isDirectoryEmpty(this.filePath)) { + return true; + } + + // We might want to overwrite anyway, if an updated version is available + if (this.shouldInvalidateResource()) { + return true; + } + + } catch (IOException e) { + // Something went wrong, so invalidate the cache just in case + return true; + } + + // No need to invalidate the cache + return false; + } + + private boolean shouldInvalidateResource() { + switch (this.resourceIdType) { + + case NAME: + return this.shouldInvalidateName(); + + default: + // Other resource ID types remain constant, so no need to invalidate + return false; + } + } + + private boolean shouldInvalidateName() { + // To avoid spamming the database too often, we shouldn't check sigs or invalidate when rate limited + if (this.rateLimitInEffect()) { + return false; + } + + // If the state's sig doesn't match the latest transaction's sig, we need to invalidate + // This means that an updated layer is available + if (this.shouldInvalidateDueToSignatureMismatch()) { + + // Add to the in-memory cache first, so that we won't check again for a while + ArbitraryDataManager.getInstance().addResourceToCache(this.resourceId); + return true; + } + + return false; + } + + /** + * rateLimitInEffect() + * + * When loading a website, we need to check the cache for every static asset loaded by the page. + * This would involve asking the database for the latest transaction every time. + * To reduce database load and page load times, we maintain an in-memory list to "rate limit" lookups. + * Once a resource ID is in this in-memory list, we will avoid cache invalidations until it + * has been present in the list for a certain amount of time. + * Items are automatically removed from the list when a new arbitrary transaction arrives, so this + * should not prevent updates from taking effect immediately. + * + * @return whether to avoid lookups for this resource due to the in-memory cache + */ + private boolean rateLimitInEffect() { + return ArbitraryDataManager.getInstance().isResourceCached(this.resourceId); + } + + private boolean shouldInvalidateDueToSignatureMismatch() { + + // Fetch the latest transaction for this name and service + byte[] latestTransactionSig = this.fetchLatestTransactionSignature(); + + // Now fetch the transaction signature stored in the cache metadata + byte[] cachedSig = this.fetchCachedSignature(); + + // If either are null, we should invalidate + if (latestTransactionSig == null || cachedSig == null) { + return true; + } + + // Check if they match + if (!Arrays.equals(latestTransactionSig, cachedSig)) { + return true; + } + return false; + } + + private byte[] fetchLatestTransactionSignature() { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Find latest transaction for name and service, with any method + ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository() + .getLatestTransaction(this.resourceId, this.service, Method.PUT); + + if (latestTransaction != null) { + return latestTransaction.getSignature(); + } + + } catch (DataException e) { + return null; + } + + return null; + } + + private byte[] fetchCachedSignature() { + try { + // Fetch the transaction signature stored in the cache metadata + ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.filePath); + cache.read(); + return cache.getSignature(); + + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index a15c7f0e..deb17cf7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; import org.qortal.crypto.AES; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -50,19 +51,23 @@ public class ArbitraryDataReader { this.resourceId = resourceId; this.resourceIdType = resourceIdType; this.service = service; + + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + this.workingPath = Paths.get(baseDir, "reader", this.resourceId); + this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); } public void load(boolean overwrite) throws IllegalStateException, IOException, DataException { try { - this.preExecute(); - - // Do nothing if files already exist and overwrite is set to false - if (!overwrite && Files.exists(this.uncompressedPath) - && !FilesystemUtils.isDirectoryEmpty(this.uncompressedPath)) { + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, + this.resourceId, this.resourceIdType, this.service); + if (!cache.shouldInvalidate()) { this.filePath = this.uncompressedPath; return; } + this.preExecute(); this.deleteExistingFiles(); this.fetch(); this.decrypt(); @@ -83,19 +88,14 @@ public class ArbitraryDataReader { } private void createWorkingDirectory() { - // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware - String baseDir = Settings.getInstance().getTempDataPath(); - Path tempDir = Paths.get(baseDir, "reader", this.resourceId); try { - Files.createDirectories(tempDir); + Files.createDirectories(this.workingPath); } catch (IOException e) { throw new IllegalStateException("Unable to create temp directory"); } - this.workingPath = tempDir; } private void createUncompressedDirectory() { - this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); try { Files.createDirectories(this.uncompressedPath); } catch (IOException e) { diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 10c5d448..df0c670d 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -58,6 +58,19 @@ public class ArbitraryDataManager extends Thread { */ private Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); + /** + * Map to keep track of cached arbitrary transaction resources. + * When an item is present in this list with a timestamp in the future, we won't invalidate + * its cache when serving that data. This reduces the amount of database lookups that are needed. + */ + private Map arbitraryDataCachedResources = Collections.synchronizedMap(new HashMap<>()); + + /** + * The amount of time to cache a data resource before it is invalidated + */ + private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes + + private ArbitraryDataManager() { } @@ -197,11 +210,49 @@ public class ArbitraryDataManager extends Thread { public void cleanupRequestCache(long now) { final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); // TODO: fix NPE arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); } + // Arbitrary data resource cache + public boolean isResourceCached(String resourceId) { + + // We don't have an entry for this resource ID, it is not cached + if (this.arbitraryDataCachedResources == null) { + return false; + } + if (!this.arbitraryDataCachedResources.containsKey(resourceId)) { + return false; + } + Long timestamp = this.arbitraryDataCachedResources.get(resourceId); + if (timestamp == null) { + return false; + } + + // If the timestamp has reached the timeout, we should remove it from the cache + long now = NTP.getTime(); + if (now > timestamp) { + this.arbitraryDataCachedResources.remove(resourceId); + return false; + } + + // Current time hasn't reached the timeout, so treat it as cached + return true; + } + + public void addResourceToCache(String resourceId) { + // Just in case + if (this.arbitraryDataCachedResources == null) { + this.arbitraryDataCachedResources = new HashMap<>(); + } + + // Set the timestamp to now + the timeout + Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT; + this.arbitraryDataCachedResources.put(resourceId, timestamp); + } + + // Network handlers public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { @@ -313,6 +364,17 @@ public class ArbitraryDataManager extends Thread { } } + // If we have all the chunks for this transaction's name, we should invalidate the data cache + // so that it is rebuilt the next time we serve it + if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(arbitraryTransactionData.getChunkHashes())) { + if (arbitraryTransactionData.getName() != null) { + String resourceId = arbitraryTransactionData.getName(); + if (this.arbitraryDataCachedResources.containsKey(resourceId)) { + this.arbitraryDataCachedResources.remove(resourceId); + } + } + } + } catch (DataException | InterruptedException e) { LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index b6a920af..16841cc1 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -258,15 +258,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException { - String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + StringBuilder sql = new StringBuilder(1024); + + sql.append("SELECT type, reference, signature, creator, created_when, fee, " + "tx_group_id, block_height, approval_status, approval_height, " + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + "name, update_method, secret, compression FROM ArbitraryTransactions " + "JOIN Transactions USING (signature) " + - "WHERE name = ? AND service = ? AND update_method = ? " + - "ORDER BY created_when DESC LIMIT 1"; + "WHERE name = ? AND service = ?"); - try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, method.value)) { + if (method != null) { + sql.append(" AND update_method = "); + sql.append(method.value); + } + + sql.append("ORDER BY created_when DESC LIMIT 1"); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name, service.value)) { if (resultSet == null) return null; From 9baccc0784a6b0c290d38664650b616a0e0aa0fc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 16 Aug 2021 22:40:55 +0100 Subject: [PATCH 163/505] Improved HTTP response generation when serving websites. --- .../org/qortal/api/resource/WebsiteResource.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 45b8097e..ea1fc6c5 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -280,7 +280,7 @@ public class WebsiteResource { if (domainMap != null && domainMap.containsKey(request.getServerName())) { return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, "", false); } - return this.get404Response(); + return this.getResponse(404, "Error 404: File Not Found"); } private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58, @@ -296,12 +296,13 @@ public class WebsiteResource { // We could store the latest transaction signature in the extracted folder arbitraryDataReader.load(false); } catch (Exception e) { - return this.get404Response(); + LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage())); + return this.getResponse(500, "Error 500: Internal Server Error"); } java.nio.file.Path path = arbitraryDataReader.getFilePath(); if (path == null) { - return this.get404Response(); + return this.getResponse(404, "Error 404: File Not Found"); } String unzippedPath = path.toString(); @@ -347,7 +348,7 @@ public class WebsiteResource { LOGGER.info("Unable to serve file at path: {}", inPath, e); } - return this.get404Response(); + return this.getResponse(404, "Error 404: File Not Found"); } private String getFilename(String directory, String userPath) { @@ -364,15 +365,14 @@ public class WebsiteResource { return userPath; } - private HttpServletResponse get404Response() { + private HttpServletResponse getResponse(int responseCode, String responseString) { try { - String responseString = "404: File Not Found"; byte[] responseData = responseString.getBytes(); - response.setStatus(404); + response.setStatus(responseCode); response.setContentLength(responseData.length); response.getOutputStream().write(responseData); } catch (IOException e) { - LOGGER.info("Error writing 404 response"); + LOGGER.info("Error writing {} response", responseCode); } return response; } From 0f1927b4b1dd1a82afd54f7fd5cd533dabb1b210 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 17 Aug 2021 08:43:08 +0100 Subject: [PATCH 164/505] Take a hash of the previous state's directory structure and file contents, and then include that hash in the patch metadata (when creating a new patch). This allows the integrity of the layers to be validated as each one is applied. --- .../qortal/arbitrary/ArbitraryDataDiff.java | 9 +++ .../qortal/arbitrary/ArbitraryDataDigest.java | 77 +++++++++++++++++++ .../metadata/ArbitraryDataMetadataPatch.java | 31 ++++++++ .../arbitrary/ArbitraryDataDigestTests.java | 58 ++++++++++++++ .../ArbitraryDataFileTests.java} | 4 +- .../resources/arbitrary/demo1/.qortal/cache | 1 + .../arbitrary/demo1/dir1/dir2/lorem5.txt | 1 + .../resources/arbitrary/demo1/dir1/lorem4.txt | 1 + src/test/resources/arbitrary/demo1/lorem1.txt | 1 + src/test/resources/arbitrary/demo1/lorem2.txt | 1 + src/test/resources/arbitrary/demo1/lorem3.txt | 1 + .../arbitrary/demo2/dir1/dir2/lorem5.txt | 1 + .../resources/arbitrary/demo2/dir1/lorem4.txt | 1 + src/test/resources/arbitrary/demo2/lorem1.txt | 1 + src/test/resources/arbitrary/demo2/lorem2.txt | 1 + src/test/resources/arbitrary/demo2/lorem3.txt | 1 + 16 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java create mode 100644 src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java rename src/test/java/org/qortal/test/{DataTests.java => arbitrary/ArbitraryDataFileTests.java} (96%) create mode 100644 src/test/resources/arbitrary/demo1/.qortal/cache create mode 100644 src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt create mode 100644 src/test/resources/arbitrary/demo1/dir1/lorem4.txt create mode 100644 src/test/resources/arbitrary/demo1/lorem1.txt create mode 100644 src/test/resources/arbitrary/demo1/lorem2.txt create mode 100644 src/test/resources/arbitrary/demo1/lorem3.txt create mode 100644 src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt create mode 100644 src/test/resources/arbitrary/demo2/dir1/lorem4.txt create mode 100644 src/test/resources/arbitrary/demo2/lorem1.txt create mode 100644 src/test/resources/arbitrary/demo2/lorem2.txt create mode 100644 src/test/resources/arbitrary/demo2/lorem3.txt diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 8db1af09..c8f2c2b6 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -22,6 +22,7 @@ public class ArbitraryDataDiff { private Path pathBefore; private Path pathAfter; private byte[] previousSignature; + private byte[] previousHash; private Path diffPath; private String identifier; @@ -45,6 +46,7 @@ public class ArbitraryDataDiff { public void compute() throws IOException { try { this.preExecute(); + this.hashPreviousState(); this.findAddedOrModifiedFiles(); this.findRemovedFiles(); this.writeMetadata(); @@ -78,6 +80,12 @@ public class ArbitraryDataDiff { this.diffPath = tempDir; } + private void hashPreviousState() throws IOException { + ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore); + digest.compute(); + this.previousHash = digest.getHash(); + } + private void findAddedOrModifiedFiles() { try { final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); @@ -220,6 +228,7 @@ public class ArbitraryDataDiff { metadata.setModifiedPaths(this.modifiedPaths); metadata.setRemovedPaths(this.removedPaths); metadata.setPreviousSignature(this.previousSignature); + metadata.setPreviousHash(this.previousHash); metadata.write(); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java new file mode 100644 index 00000000..315d79a7 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java @@ -0,0 +1,77 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.utils.Base58; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ArbitraryDataDigest { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDigest.class); + + private Path path; + private byte[] hash; + + public ArbitraryDataDigest(Path path) { + this.path = path; + } + + public void compute() throws IOException { + List allPaths = new ArrayList<>(); + Files.walk(path).filter(Files::isRegularFile).forEachOrdered(p -> allPaths.add(p)); + Path basePathAbsolute = this.path.toAbsolutePath(); + + MessageDigest sha256 = null; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 hashing algorithm unavailable"); + } + + for (Path path : allPaths) { + // We need to work with paths relative to the base path, to ensure the same hash + // is generated on different systems + Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath()); + + // Exclude Qortal folder since it can be different each time + // We only care about hashing the actual user data + if (relativePath.startsWith(".qortal/")) { + continue; + } + + // Hash path + byte[] filePathBytes = relativePath.toString().toLowerCase().getBytes(StandardCharsets.UTF_8); + sha256.update(filePathBytes); + + // Hash contents + byte[] fileContent = Files.readAllBytes(path); + sha256.update(fileContent); + } + this.hash = sha256.digest(); + } + + public boolean isHashValid(byte[] hash) { + return Arrays.equals(hash, this.hash); + } + + public byte[] getHash() { + return this.hash; + } + + public String getHash58() { + if (this.hash == null) { + return null; + } + return Base58.encode(this.hash); + } + +} diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java index fd68bec3..e6f392cb 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java @@ -1,20 +1,27 @@ package org.qortal.arbitrary.metadata; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.qortal.utils.Base58; +import java.lang.reflect.Field; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class); + private List addedPaths; private List modifiedPaths; private List removedPaths; private byte[] previousSignature; + private byte[] previousHash; public ArbitraryDataMetadataPatch(Path filePath) { super(filePath); @@ -42,6 +49,12 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { this.previousSignature = Base58.decode(prevSig); } } + if (patch.has("prevHash")) { + String prevHash = patch.getString("prevHash"); + if (prevHash != null) { + this.previousHash = Base58.decode(prevHash); + } + } if (patch.has("added")) { JSONArray added = (JSONArray) patch.get("added"); if (added != null) { @@ -74,7 +87,17 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { @Override protected void buildJson() { JSONObject patch = new JSONObject(); + // Attempt to use a LinkedHashMap so that the order of fields is maintained + try { + Field changeMap = patch.getClass().getDeclaredField("map"); + changeMap.setAccessible(true); + changeMap.set(patch, new LinkedHashMap<>()); + changeMap.setAccessible(false); + } catch (IllegalAccessException | NoSuchFieldException e) { + // Don't worry about failures as this is for ordering only + } patch.put("prevSig", Base58.encode(this.previousSignature)); + patch.put("prevHash", Base58.encode(this.previousHash)); patch.put("added", new JSONArray(this.addedPaths)); patch.put("modified", new JSONArray(this.modifiedPaths)); patch.put("removed", new JSONArray(this.removedPaths)); @@ -116,4 +139,12 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { return this.previousSignature; } + public void setPreviousHash(byte[] previousHash) { + this.previousHash = previousHash; + } + + public byte[] getPreviousHash() { + return this.previousHash; + } + } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java new file mode 100644 index 00000000..b2772a28 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java @@ -0,0 +1,58 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +import static org.junit.Assert.*; + +public class ArbitraryDataDigestTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testDirectoryDigest() throws IOException { + Path dataPath = Paths.get("src/test/resources/arbitrary/demo1"); + String expectedHash58 = "59dw8CgVybcHAUL5GYgYUUfFffVVhiMKZLCnULPKT6oC"; + + // Ensure directory exists + assertTrue(dataPath.toFile().exists()); + assertTrue(dataPath.toFile().isDirectory()); + + // Compute a hash + ArbitraryDataDigest digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + + // Write a random file to .qortal/cache to ensure it isn't being included in the digest function + // We exclude all .qortal files from the digest since they can be different with each build, and + // we only care about the actual user files + FileWriter fileWriter = new FileWriter(Paths.get(dataPath.toString(), ".qortal", "cache").toString()); + fileWriter.append(UUID.randomUUID().toString()); + fileWriter.close(); + + // Recompute the hash + digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + + // Now compute the hash 100 more times to ensure it's always the same + for (int i=0; i<100; i++) { + digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + } + } + +} diff --git a/src/test/java/org/qortal/test/DataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java similarity index 96% rename from src/test/java/org/qortal/test/DataTests.java rename to src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java index 1d515239..bb016920 100644 --- a/src/test/java/org/qortal/test/DataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java @@ -1,4 +1,4 @@ -package org.qortal.test; +package org.qortal.test.arbitrary; import org.junit.Before; import org.junit.Test; @@ -10,7 +10,7 @@ import java.util.Random; import static org.junit.Assert.*; -public class DataTests extends Common { +public class ArbitraryDataFileTests extends Common { @Before public void beforeTest() throws DataException { diff --git a/src/test/resources/arbitrary/demo1/.qortal/cache b/src/test/resources/arbitrary/demo1/.qortal/cache new file mode 100644 index 00000000..d10d5a03 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/.qortal/cache @@ -0,0 +1 @@ +2ea5a99a-3b85-4f1f-a259-436787f90bd1 \ No newline at end of file diff --git a/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt b/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt new file mode 100644 index 00000000..ef07da1f --- /dev/null +++ b/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt @@ -0,0 +1 @@ +Pellentesque laoreet laoreet dui ut volutpat. diff --git a/src/test/resources/arbitrary/demo1/dir1/lorem4.txt b/src/test/resources/arbitrary/demo1/dir1/lorem4.txt new file mode 100644 index 00000000..80d1dda7 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/dir1/lorem4.txt @@ -0,0 +1 @@ +Sed non lacus ante. diff --git a/src/test/resources/arbitrary/demo1/lorem1.txt b/src/test/resources/arbitrary/demo1/lorem1.txt new file mode 100644 index 00000000..4f006a88 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem1.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. diff --git a/src/test/resources/arbitrary/demo1/lorem2.txt b/src/test/resources/arbitrary/demo1/lorem2.txt new file mode 100644 index 00000000..8a9c4367 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem2.txt @@ -0,0 +1 @@ +Quisque viverra neque quis eros dapibus diff --git a/src/test/resources/arbitrary/demo1/lorem3.txt b/src/test/resources/arbitrary/demo1/lorem3.txt new file mode 100644 index 00000000..5db7e985 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem3.txt @@ -0,0 +1 @@ +Sed ac magna pretium, suscipit mauris sed, ultrices nunc. diff --git a/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt b/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt new file mode 100644 index 00000000..ef07da1f --- /dev/null +++ b/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt @@ -0,0 +1 @@ +Pellentesque laoreet laoreet dui ut volutpat. diff --git a/src/test/resources/arbitrary/demo2/dir1/lorem4.txt b/src/test/resources/arbitrary/demo2/dir1/lorem4.txt new file mode 100644 index 00000000..80d1dda7 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/dir1/lorem4.txt @@ -0,0 +1 @@ +Sed non lacus ante. diff --git a/src/test/resources/arbitrary/demo2/lorem1.txt b/src/test/resources/arbitrary/demo2/lorem1.txt new file mode 100644 index 00000000..4f006a88 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem1.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. diff --git a/src/test/resources/arbitrary/demo2/lorem2.txt b/src/test/resources/arbitrary/demo2/lorem2.txt new file mode 100644 index 00000000..8a9c4367 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem2.txt @@ -0,0 +1 @@ +Quisque viverra neque quis eros dapibus diff --git a/src/test/resources/arbitrary/demo2/lorem3.txt b/src/test/resources/arbitrary/demo2/lorem3.txt new file mode 100644 index 00000000..5db7e985 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem3.txt @@ -0,0 +1 @@ +Sed ac magna pretium, suscipit mauris sed, ultrices nunc. From f51a08204972157505e72dd8050df7c61a6c61ba Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 17 Aug 2021 09:07:46 +0100 Subject: [PATCH 165/505] Validate the previous state's hash each time a new layer is applied. It's possible that this concept will struggle in the real world if operating systems, virus scanners, etc start interfering with our file stucture. Right now it is using a zero tolerance approach when checking the validity of each layer. We may choose to loosen this slightly if we encounter problems, e.g. by excluding hidden files. But for now it is best to be as strict as possible. --- .../arbitrary/ArbitraryDataCombiner.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index ccbfb85c..681a5949 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; +import org.qortal.utils.Base58; import org.qortal.utils.FilesystemUtils; import java.io.File; @@ -32,6 +33,7 @@ public class ArbitraryDataCombiner { try { this.preExecute(); this.validatePreviousSignature(); + this.validatePreviousHash(); this.process(); } finally { @@ -101,6 +103,24 @@ public class ArbitraryDataCombiner { } } + private void validatePreviousHash() throws IOException { + ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.pathAfter); + metadata.read(); + byte[] previousHash = metadata.getPreviousHash(); + if (previousHash == null) { + throw new IllegalStateException("Unable to extract previous hash from patch metadata"); + } + + ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore); + digest.compute(); + boolean valid = digest.isHashValid(previousHash); + if (!valid) { + String previousHash58 = Base58.encode(previousHash); + throw new IllegalStateException(String.format("Previous state hash mismatch. " + + "Patch prevHash: %s, actual: %s", previousHash58, digest.getHash58())); + } + } + private void process() throws IOException { ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter); merge.compute(); From e1feb46de95eb9c9352f6af8c660f0d1ce1bd870 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 17 Aug 2021 09:08:03 +0100 Subject: [PATCH 166/505] Put test data in a separate folder to real data. --- src/test/resources/test-settings-v2.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/resources/test-settings-v2.json b/src/test/resources/test-settings-v2.json index a8983d3d..f4052647 100644 --- a/src/test/resources/test-settings-v2.json +++ b/src/test/resources/test-settings-v2.json @@ -5,5 +5,6 @@ "blockchainConfig": "src/test/resources/test-chain-v2.json", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "dataPath": "data-test" } From 1968496ce16dfa1846123a1e8ab4dfe34188d555 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 17 Aug 2021 09:30:27 +0100 Subject: [PATCH 167/505] Invalidate the cache if hash validation fails, so that it can be rebuilt next time. --- .../arbitrary/ArbitraryDataCombiner.java | 3 +- .../qortal/arbitrary/ArbitraryDataReader.java | 29 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index 681a5949..b4cc9296 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -9,6 +9,7 @@ import org.qortal.utils.FilesystemUtils; import java.io.File; import java.io.IOException; +import java.io.InvalidObjectException; import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; import java.nio.file.Path; @@ -116,7 +117,7 @@ public class ArbitraryDataCombiner { boolean valid = digest.isHashValid(previousHash); if (!valid) { String previousHash58 = Base58.encode(previousHash); - throw new IllegalStateException(String.format("Previous state hash mismatch. " + + throw new InvalidObjectException(String.format("Previous state hash mismatch. " + "Patch prevHash: %s, actual: %s", previousHash58, digest.getHash58())); } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index deb17cf7..148ace8e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -3,7 +3,7 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; + import org.qortal.crypto.AES; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -24,6 +24,7 @@ import javax.crypto.SecretKey; import javax.crypto.spec.SecretKeySpec; import java.io.File; import java.io.IOException; +import java.io.InvalidObjectException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.security.InvalidAlgorithmParameterException; @@ -172,17 +173,25 @@ public class ArbitraryDataReader { } private void fetchFromName() throws IllegalStateException, IOException, DataException { + try { - // Build the existing state using past transactions - ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service); - builder.build(); - Path builtPath = builder.getFinalPath(); - if (builtPath == null) { - throw new IllegalStateException("Unable to build path"); + // Build the existing state using past transactions + ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service); + builder.build(); + Path builtPath = builder.getFinalPath(); + if (builtPath == null) { + throw new IllegalStateException("Unable to build path"); + } + + // Set filePath to the builtPath + this.filePath = builtPath; + + } catch (InvalidObjectException e) { + // Hash validation failed. Invalidate the cache for this name, so it can be rebuilt + LOGGER.info("Deleting {}", this.workingPath.toString()); + FilesystemUtils.safeDeleteDirectory(this.workingPath, false); + throw(e); } - - // Set filePath to the builtPath - this.filePath = builtPath; } private void fetchFromSignature() throws IllegalStateException, IOException, DataException { From c3b44cee94726acfac8315a3f7c9b13fa44852db Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 07:38:08 +0100 Subject: [PATCH 168/505] Fixed bugs when computing directory digest. The order was being lost when adding to the second List, which was producing different hashes on different systems. It also doesn't make sense to convert paths to lowercase, given that we want our validation to be case sensitive. --- .../java/org/qortal/arbitrary/ArbitraryDataDigest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java index 315d79a7..58b5e444 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java @@ -10,9 +10,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; public class ArbitraryDataDigest { @@ -26,8 +27,8 @@ public class ArbitraryDataDigest { } public void compute() throws IOException { - List allPaths = new ArrayList<>(); - Files.walk(path).filter(Files::isRegularFile).forEachOrdered(p -> allPaths.add(p)); + List allPaths = Files.walk(path).filter(Files::isRegularFile).collect(Collectors.toList()); + Collections.sort(allPaths); Path basePathAbsolute = this.path.toAbsolutePath(); MessageDigest sha256 = null; @@ -49,7 +50,7 @@ public class ArbitraryDataDigest { } // Hash path - byte[] filePathBytes = relativePath.toString().toLowerCase().getBytes(StandardCharsets.UTF_8); + byte[] filePathBytes = relativePath.toString().getBytes(StandardCharsets.UTF_8); sha256.update(filePathBytes); // Hash contents From 79bbadad2f18bdfd090fe3496ac4739904198729 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 07:50:45 +0100 Subject: [PATCH 169/505] Initial implementation of arbitrary data build queue This adds the loadAsynchronously() method to ArbitraryDataReader, in addition to the existing loadSynchronously() method. When requesting a website in a browser, previously the building of the resource's layers would be done synchronously in the API handler. This understandably caused many issues, so the building is now done asynchronously by a dedicated thread. A loading screen is shown in its place which auto refreshes every second until the build has completed. --- .../qortal/api/resource/WebsiteResource.java | 21 ++++- .../ArbitraryDataBuildQueueItem.java | 69 ++++++++++++++++ .../arbitrary/ArbitraryDataBuilder.java | 2 +- .../qortal/arbitrary/ArbitraryDataCache.java | 2 +- .../qortal/arbitrary/ArbitraryDataReader.java | 50 +++++++++++- .../metadata/ArbitraryDataMetadataCache.java | 2 +- .../org/qortal/controller/Controller.java | 3 +- .../arbitrary/ArbitraryDataBuildManager.java | 78 +++++++++++++++++++ .../{ => arbitrary}/ArbitraryDataManager.java | 67 +++++++++++++++- src/main/resources/loading/index.html | 23 ++++++ 10 files changed, 309 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java create mode 100644 src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java rename src/main/java/org/qortal/controller/{ => arbitrary}/ArbitraryDataManager.java (90%) create mode 100644 src/main/resources/loading/index.html diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index ea1fc6c5..87d439c5 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -7,6 +7,8 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.io.*; +import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Paths; @@ -14,6 +16,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import com.google.common.io.Resources; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; @@ -293,8 +296,11 @@ public class WebsiteResource { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service); arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { - // We could store the latest transaction signature in the extracted folder - arbitraryDataReader.load(false); + if (!arbitraryDataReader.isCachedDataAvailable()) { + arbitraryDataReader.loadAsynchronously(); + return this.getLoadingResponse(); + } + } catch (Exception e) { LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage())); return this.getResponse(500, "Error 500: Internal Server Error"); @@ -365,6 +371,17 @@ public class WebsiteResource { return userPath; } + private HttpServletResponse getLoadingResponse() { + String responseString = null; + URL url = Resources.getResource("loading/index.html"); + try { + responseString = Resources.toString(url, StandardCharsets.UTF_8); + } catch (IOException e) { + LOGGER.info("Unable to show loading screen: {}", e.getMessage()); + } + return this.getResponse(503, responseString); + } + private HttpServletResponse getResponse(int responseCode, String responseString) { try { byte[] responseData = responseString.getBytes(); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java new file mode 100644 index 00000000..2cabd3ec --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -0,0 +1,69 @@ +package org.qortal.arbitrary; + +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.repository.DataException; +import org.qortal.utils.NTP; + +import java.io.IOException; + +public class ArbitraryDataBuildQueueItem { + + private String resourceId; + private ResourceIdType resourceIdType; + private Service service; + private Long buildStartTimestamp = null; + + private static long BUILD_TIMEOUT = 60*1000L; // 60 seconds + + public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + this.service = service; + } + + public void build() throws IOException, DataException { + Long now = NTP.getTime(); + if (now == null) { + throw new IllegalStateException("NTP time hasn't synced yet"); + } + + this.buildStartTimestamp = now; + ArbitraryDataReader arbitraryDataReader = + new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service); + + // We do not want to overwrite the existing cache, as this will be invalidated + // automatically if new data has arrived + arbitraryDataReader.loadSynchronously(false); + } + + public boolean isBuilding() { + return this.buildStartTimestamp != null; + } + + public boolean isQueued() { + return this.buildStartTimestamp == null; + } + + public boolean hasReachedBuildTimeout(Long now) { + if (now == null || this.buildStartTimestamp == null) { + return true; + } + return now - this.buildStartTimestamp > BUILD_TIMEOUT; + } + + + public String getResourceId() { + return this.resourceId; + } + + public Long getBuildStartTimestamp() { + return this.buildStartTimestamp; + } + + @Override + public String toString() { + return String.format("%s %s", this.service, this.resourceId); + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index c8da187f..87795a18 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -112,7 +112,7 @@ public class ArbitraryDataBuilder { String sig58 = Base58.encode(transactionData.getSignature()); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); arbitraryDataReader.setTransactionData(transactionData); - arbitraryDataReader.load(true); + arbitraryDataReader.loadSynchronously(true); Path path = arbitraryDataReader.getFilePath(); if (path == null) { throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58)); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java index 1cb45610..e15bbf37 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java @@ -2,7 +2,7 @@ package org.qortal.arbitrary; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; -import org.qortal.controller.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.repository.DataException; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 148ace8e..d35fa322 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.AES; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -59,7 +60,54 @@ public class ArbitraryDataReader { this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); } - public void load(boolean overwrite) throws IllegalStateException, IOException, DataException { + public boolean isCachedDataAvailable() { + // If this resource is in the build queue then we shouldn't attempt to serve + // cached data, as it may not be fully built + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); + if (ArbitraryDataManager.getInstance().isInBuildQueue(queueItem)) { + return false; + } + + // Not in the build queue - so check the cache itself + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false, + this.resourceId, this.resourceIdType, this.service); + if (!cache.shouldInvalidate()) { + this.filePath = this.uncompressedPath; + return true; + } + return false; + } + + /** + * loadAsynchronously + * + * Attempts to load the resource asynchronously + * This adds the build task to a queue, and the result will be cached when complete + * To check the status of the build, periodically call isCachedDataAvailable() + * Once it returns true, you can then use getFilePath() to access the data itself. + * TODO: create API to check the status + * @return + */ + public boolean loadAsynchronously() { + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); + return ArbitraryDataManager.getInstance().addToBuildQueue(queueItem); + } + + /** + * loadSynchronously + * + * Attempts to load the resource synchronously + * Warning: this can block for a long time when building or fetching complex data + * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning + * + * @param overwrite - set to true to force rebuild an existing cache + * @throws IllegalStateException + * @throws IOException + * @throws DataException + */ + public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException { try { ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, this.resourceId, this.resourceIdType, this.service); diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java index d6d7a2b4..bedbddee 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java @@ -45,7 +45,7 @@ public class ArbitraryDataMetadataCache extends ArbitraryDataMetadata { patch.put("timestamp", this.timestamp); this.jsonString = patch.toString(2); - LOGGER.info("Cache metadata: {}", this.jsonString); + LOGGER.trace("Cache metadata: {}", this.jsonString); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 44eb7f92..9d0914b7 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -14,6 +14,7 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; @@ -332,7 +333,7 @@ public class Controller extends Thread { return this.savedArgs; } - /* package */ static boolean isStopping() { + /* package */ public static boolean isStopping() { return isStopping; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java new file mode 100644 index 00000000..a5325c03 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -0,0 +1,78 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.controller.Controller; +import org.qortal.repository.DataException; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.util.Map; + + +public class ArbitraryDataBuildManager implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class); + + public ArbitraryDataBuildManager() { + + } + + public void run() { + Thread.currentThread().setName("Arbitrary Data Build Manager"); + ArbitraryDataManager arbitraryDataManager = ArbitraryDataManager.getInstance(); + + while (!Controller.isStopping()) { + try { + Thread.sleep(1000); + + if (arbitraryDataManager.arbitraryDataBuildQueue == null) { + continue; + } + if (arbitraryDataManager.arbitraryDataBuildQueue.isEmpty()) { + continue; + } + + // Find resources that are queued for building + Map.Entry next = arbitraryDataManager.arbitraryDataBuildQueue + .entrySet().stream().filter(e -> e.getValue().isQueued()).findFirst().get(); + + if (next == null) { + continue; + } + + Long now = NTP.getTime(); + if (now == null) { + continue; + } + + String resourceId = next.getKey(); + ArbitraryDataBuildQueueItem queueItem = next.getValue(); + if (queueItem == null || queueItem.hasReachedBuildTimeout(now)) { + this.removeFromQueue(resourceId); + } + + try { + // Perform the build + LOGGER.info("Building {}...", queueItem); + queueItem.build(); + this.removeFromQueue(resourceId); + LOGGER.info("Finished building {}", queueItem); + + } catch (IOException | DataException e) { + // Something went wrong - so remove it from the queue + // TODO: we may want to keep track of this in a "cooloff" list to prevent frequent re-attempts + this.removeFromQueue(resourceId); + } + + } catch (InterruptedException e) { + // Time to exit + } + } + } + + private void removeFromQueue(String resourceId) { + ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId); + } +} diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java similarity index 90% rename from src/main/java/org/qortal/controller/ArbitraryDataManager.java rename to src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index df0c670d..4a99f55e 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -1,10 +1,14 @@ -package org.qortal.controller; +package org.qortal.controller.arbitrary; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.controller.Controller; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -70,6 +74,11 @@ public class ArbitraryDataManager extends Thread { */ private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes + /** + * Map to keep track of arbitrary transaction resources currently being built (or queued). + */ + public Map arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>()); + private ArbitraryDataManager() { } @@ -85,6 +94,11 @@ public class ArbitraryDataManager extends Thread { public void run() { Thread.currentThread().setName("Arbitrary Data Manager"); + // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) + // This can be expanded to have multiple threads processing the build queue when needed + ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); + arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager()); + try { while (!isStopping) { Thread.sleep(2000); @@ -247,11 +261,62 @@ public class ArbitraryDataManager extends Thread { this.arbitraryDataCachedResources = new HashMap<>(); } + Long now = NTP.getTime(); + if (now == null) { + return; + } + // Set the timestamp to now + the timeout Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT; this.arbitraryDataCachedResources.put(resourceId, timestamp); } + // Build queue + public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) { + // Already in queue + return true; + } + + LOGGER.info("Added {} to build queue", resourceId); + + // Added to queue + return true; + } + + public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (this.arbitraryDataBuildQueue.containsKey(resourceId)) { + // Already in queue + return true; + } + + // Not in queue + return false; + } + // Network handlers diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html new file mode 100644 index 00000000..eaf7ee1b --- /dev/null +++ b/src/main/resources/loading/index.html @@ -0,0 +1,23 @@ + + + Loading... + + + + + +

    Loading... please wait...

    +

    This page will refresh automatically when the content becomes available

    +

    (We can show a Qortal branded loading screen here)

    + + From 9c20967d248c4bcacf57cfdb0b2efe6ab744d4b2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 07:51:26 +0100 Subject: [PATCH 170/505] Catch NPE seen a couple of times in Systray.setTrayIcon() --- src/main/java/org/qortal/gui/SysTray.java | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/gui/SysTray.java b/src/main/java/org/qortal/gui/SysTray.java index 6fc994bf..42e12ab7 100644 --- a/src/main/java/org/qortal/gui/SysTray.java +++ b/src/main/java/org/qortal/gui/SysTray.java @@ -290,21 +290,25 @@ public class SysTray { } public void setTrayIcon(int iconid) { - if (trayIcon != null) { - switch (iconid) { - case 1: - this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_syncing_time-alt.png")); - break; - case 2: - this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_minting.png")); - break; - case 3: - this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_syncing.png")); - break; - case 4: - this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_synced.png")); - break; + try { + if (trayIcon != null) { + switch (iconid) { + case 1: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_syncing_time-alt.png")); + break; + case 2: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_minting.png")); + break; + case 3: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_syncing.png")); + break; + case 4: + this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_synced.png")); + break; + } } + } catch (Exception e) { + LOGGER.info("Unable to set tray icon: {}", e.getMessage()); } } From a6154cbb43a5a02c851d38683e213241cfc71b6a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 07:51:50 +0100 Subject: [PATCH 171/505] Don't allow a new layer to be created if it matches the existing state. --- src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index c8f2c2b6..70c8c361 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -49,6 +49,7 @@ public class ArbitraryDataDiff { this.hashPreviousState(); this.findAddedOrModifiedFiles(); this.findRemovedFiles(); + this.validate(); this.writeMetadata(); } finally { @@ -222,6 +223,12 @@ public class ArbitraryDataDiff { } } + private void validate() { + if (this.addedPaths.isEmpty() && this.modifiedPaths.isEmpty() && this.removedPaths.isEmpty()) { + throw new IllegalStateException("Current state matches previous state. Nothing to do."); + } + } + private void writeMetadata() throws IOException { ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath); metadata.setAddedPaths(this.addedPaths); From cb6fc466d1602c77ab4e91fa661a074227391420 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 09:57:44 +0100 Subject: [PATCH 172/505] Create a "unified-diff" patch using java-diff-utils instead of including the entire modified file. I still need to test how well this works with binary files. --- .../qortal/arbitrary/ArbitraryDataDiff.java | 64 +++++++++++++++++-- .../metadata/ArbitraryDataMetadataPatch.java | 18 +++++- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 70c8c361..0b874d47 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -1,13 +1,17 @@ package org.qortal.arbitrary; +import com.github.difflib.DiffUtils; +import com.github.difflib.UnifiedDiffUtils; +import com.github.difflib.patch.Patch; +import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; -import java.io.File; -import java.io.IOException; +import java.io.*; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; @@ -133,9 +137,14 @@ public class ArbitraryDataDiff { wasModified = true; } - if (wasAdded | wasModified) { + if (wasAdded) { ArbitraryDataDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter); } + if (wasModified) { + // Create patch using java-diff-utils + Path destination = Paths.get(diffPathAbsolute.toString(), filePathAfter.toString()); + ArbitraryDataDiff.createAndCopyDiffUtilsPatch(filePathBefore, after, destination); + } return FileVisitResult.CONTINUE; } @@ -159,7 +168,7 @@ public class ArbitraryDataDiff { } } - private void findRemovedFiles() { + private void findRemovedFiles() throws IOException { try { final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); @@ -219,7 +228,7 @@ public class ArbitraryDataDiff { }); } catch (IOException e) { - LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + throw new IOException(String.format("IOException when walking through file tree: %s", e.getMessage())); } } @@ -231,6 +240,7 @@ public class ArbitraryDataDiff { private void writeMetadata() throws IOException { ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath); + metadata.setPatchType("unified-diff"); metadata.setAddedPaths(this.addedPaths); metadata.setModifiedPaths(this.modifiedPaths); metadata.setRemovedPaths(this.removedPaths); @@ -264,6 +274,50 @@ public class ArbitraryDataDiff { LOGGER.trace("Copying {} to {}", source, dest); Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); } + + private static void createAndCopyDiffUtilsPatch(Path before, Path after, Path destination) throws IOException { + if (!Files.exists(before)) { + throw new IOException(String.format("File not found (before): %s", before.toString())); + } + if (!Files.exists(after)) { + throw new IOException(String.format("File not found (after): %s", after.toString())); + } + + // Ensure parent folders exist in the destination + File file = new File(destination.toString()); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + + // Delete an existing file if it exists + File destFile = destination.toFile(); + if (destFile.exists() && destFile.isFile()) { + Files.delete(destination); + } + + // Load the two files into memory + List original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8); + List revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8); + + // Generate diff information + Patch diff = DiffUtils.diff(original, revised); + + // Generate unified diff format + String originalFileName = before.getFileName().toString(); + String revisedFileName = after.getFileName().toString(); + List unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0); + + // Write the diff to the destination directory + FileWriter fileWriter = new FileWriter(destination.toString(), true); + BufferedWriter writer = new BufferedWriter(fileWriter); + for (String line : unifiedDiff) { + writer.append(line); + writer.newLine(); + } + writer.flush(); + writer.close(); + } public Path getDiffPath() { diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java index e6f392cb..0f7b6b56 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java @@ -17,6 +17,7 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class); + private String patchType; private List addedPaths; private List modifiedPaths; private List removedPaths; @@ -43,6 +44,12 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { } JSONObject patch = new JSONObject(this.jsonString); + if (patch.has("patchType")) { + String patchType = patch.getString("patchType"); + if (patchType != null) { + this.patchType = patchType; + } + } if (patch.has("prevSig")) { String prevSig = patch.getString("prevSig"); if (prevSig != null) { @@ -94,8 +101,9 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { changeMap.set(patch, new LinkedHashMap<>()); changeMap.setAccessible(false); } catch (IllegalAccessException | NoSuchFieldException e) { - // Don't worry about failures as this is for ordering only + // Don't worry about failures as this is for optional ordering only } + patch.put("patchType", this.patchType); patch.put("prevSig", Base58.encode(this.previousSignature)); patch.put("prevHash", Base58.encode(this.previousHash)); patch.put("added", new JSONArray(this.addedPaths)); @@ -107,6 +115,14 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { } + public void setPatchType(String patchType) { + this.patchType = patchType; + } + + public String getPatchType() { + return this.patchType; + } + public void setAddedPaths(List addedPaths) { this.addedPaths = addedPaths; } From 9a88c0d579e2ebee388a5d729a3149da16412775 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 09:58:30 +0100 Subject: [PATCH 173/505] Apply the unified-diff patch when combining layers. --- .../arbitrary/ArbitraryDataCombiner.java | 20 +++--- .../qortal/arbitrary/ArbitraryDataMerge.java | 70 ++++++++++++++++++- 2 files changed, 79 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index b4cc9296..d181a53c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -23,6 +23,7 @@ public class ArbitraryDataCombiner { private Path pathAfter; private byte[] signatureBefore; private Path finalPath; + ArbitraryDataMetadataPatch metadata; public ArbitraryDataCombiner(Path pathBefore, Path pathAfter, byte[] signatureBefore) { this.pathBefore = pathBefore; @@ -33,6 +34,7 @@ public class ArbitraryDataCombiner { public void combine() throws IOException { try { this.preExecute(); + this.readMetadata(); this.validatePreviousSignature(); this.validatePreviousHash(); this.process(); @@ -86,14 +88,17 @@ public class ArbitraryDataCombiner { } - private void validatePreviousSignature() throws IOException { + private void readMetadata() throws IOException { + this.metadata = new ArbitraryDataMetadataPatch(this.pathAfter); + this.metadata.read(); + } + + private void validatePreviousSignature() { if (this.signatureBefore == null) { throw new IllegalStateException("No previous signature passed to the combiner"); } - ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.pathAfter); - metadata.read(); - byte[] previousSignature = metadata.getPreviousSignature(); + byte[] previousSignature = this.metadata.getPreviousSignature(); if (previousSignature == null) { throw new IllegalStateException("Unable to extract previous signature from patch metadata"); } @@ -105,9 +110,7 @@ public class ArbitraryDataCombiner { } private void validatePreviousHash() throws IOException { - ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.pathAfter); - metadata.read(); - byte[] previousHash = metadata.getPreviousHash(); + byte[] previousHash = this.metadata.getPreviousHash(); if (previousHash == null) { throw new IllegalStateException("Unable to extract previous hash from patch metadata"); } @@ -123,7 +126,8 @@ public class ArbitraryDataCombiner { } private void process() throws IOException { - ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter); + String patchType = metadata.getPatchType(); + ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter, metadata.getPatchType()); merge.compute(); this.finalPath = merge.getMergePath(); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index d4faf124..6a1e6b20 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -1,16 +1,25 @@ package org.qortal.arbitrary; +import com.github.difflib.DiffUtils; +import com.github.difflib.UnifiedDiffUtils; +import com.github.difflib.patch.Patch; +import com.github.difflib.patch.PatchFailedException; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.util.IO; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; import org.qortal.settings.Settings; import org.qortal.utils.FilesystemUtils; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.List; +import java.util.Objects; import java.util.UUID; public class ArbitraryDataMerge { @@ -19,13 +28,15 @@ public class ArbitraryDataMerge { private Path pathBefore; private Path pathAfter; + private String patchType; private Path mergePath; private String identifier; private ArbitraryDataMetadataPatch metadata; - public ArbitraryDataMerge(Path pathBefore, Path pathAfter) { + public ArbitraryDataMerge(Path pathBefore, Path pathAfter, String patchType) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; + this.patchType = patchType; } public void compute() throws IOException { @@ -87,8 +98,7 @@ public class ArbitraryDataMerge { List modifiedPaths = this.metadata.getModifiedPaths(); for (Path path : modifiedPaths) { LOGGER.info("File was modified: {}", path.toString()); - Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); - ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path); + this.applyPatch(path); } List removedPaths = this.metadata.getRemovedPaths(); @@ -98,6 +108,60 @@ public class ArbitraryDataMerge { } } + private void applyPatch(Path path) throws IOException { + if (Objects.equals(this.patchType, "unified-diff")) { + // Create destination file from patch + this.applyUnifiedDiffPatch(path); + } + else { + // Copy complete file + Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path); + } + } + + private void applyUnifiedDiffPatch(Path path) throws IOException { + Path originalPath = Paths.get(this.pathBefore.toString(), path.toString()); + Path patchPath = Paths.get(this.pathAfter.toString(), path.toString()); + Path mergePath = Paths.get(this.mergePath.toString(), path.toString()); + + if (!patchPath.toFile().exists()) { + // Patch file doesn't exist, but its path was included in modifiedPaths + // TODO: We ought to throw an exception here, but skipping for now + return; + } + + // Delete an existing file, as we are starting from a duplicate of pathBefore + File destFile = mergePath.toFile(); + if (destFile.exists() && destFile.isFile()) { + Files.delete(mergePath); + } + + List originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8); + List patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8); + + // At first, parse the unified diff file and get the patch + Patch patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents); + + // Then apply the computed patch to the given text + try { + List patchedContents = DiffUtils.patch(originalContents, patch); + + // Write the patched file to the merge directory + FileWriter fileWriter = new FileWriter(mergePath.toString(), true); + BufferedWriter writer = new BufferedWriter(fileWriter); + for (String line : patchedContents) { + writer.append(line); + writer.newLine(); + } + writer.flush(); + writer.close(); + + } catch (PatchFailedException e) { + throw new IllegalStateException(String.format("Failed to apply patch for path %s: %s", path, e.getMessage())); + } + } + private void copyMetadata() throws IOException { Path filePath = Paths.get(this.pathAfter.toString(), ".qortal"); ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, Paths.get(".qortal")); From 2eedafd506773e0b370e2d635087e9ff9c2326a9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 09:58:59 +0100 Subject: [PATCH 174/505] Log the error if a build fails. --- .../qortal/controller/arbitrary/ArbitraryDataBuildManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index a5325c03..9fa383c0 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -61,6 +61,7 @@ public class ArbitraryDataBuildManager implements Runnable { LOGGER.info("Finished building {}", queueItem); } catch (IOException | DataException e) { + LOGGER.info("Error building {}: {}", queueItem, e.getMessage()); // Something went wrong - so remove it from the queue // TODO: we may want to keep track of this in a "cooloff" list to prevent frequent re-attempts this.removeFromQueue(resourceId); From cd958398af077021a336e1ad58b147eb6ea454c8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 09:59:12 +0100 Subject: [PATCH 175/505] Exclude all data* paths from gitignore. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8db57002..cf1e7ed2 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,4 @@ /run.pid /run.log /WindowsInstaller/Install Files/qortal.jar -/data/* +/data* From 8a8ec32f2cbcd1c921cf008a4db4ac48c5b4ae12 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 09:59:56 +0100 Subject: [PATCH 176/505] Added java-diff-utils to pom.xml This was accidentally missing from commit cb6fc46. --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 2c0afe0e..f91d4cfa 100644 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,7 @@ 3.23.8 1.1.0 1.13.1 + 4.10 src/main/java @@ -660,5 +661,10 @@ jsoup ${jsoup.version} + + io.github.java-diff-utils + java-diff-utils + ${java-diff-utils.version} + From 95e905a5ae44986983f91a58dd7ac27fc39eab78 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 16:04:57 +0100 Subject: [PATCH 177/505] If a build fails, prevent any rebuilds for 5 minutes. This prevents a resource with build problems from getting into a loop due to the browser requesting a rebuild as soon as it fails. --- .../ArbitraryDataBuildQueueItem.java | 30 +++++++- .../arbitrary/ArbitraryDataBuildManager.java | 18 ++++- .../arbitrary/ArbitraryDataManager.java | 74 ++++++++++++++++++- 3 files changed, 113 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index 2cabd3ec..75769b27 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -12,14 +12,22 @@ public class ArbitraryDataBuildQueueItem { private String resourceId; private ResourceIdType resourceIdType; private Service service; + private Long creationTimestamp = null; private Long buildStartTimestamp = null; + private Long buildEndTimestamp = null; + private boolean failed = false; - private static long BUILD_TIMEOUT = 60*1000L; // 60 seconds + /* The maximum amount of time to spend on a single build */ + // TODO: interrupt an in-progress build + public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds + /* The amount of time to remember that a build has failed, to avoid retries */ + public static long FAILURE_TIMEOUT = 1*60*1000L; // 5 minutes public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { this.resourceId = resourceId; this.resourceIdType = resourceIdType; this.service = service; + this.creationTimestamp = NTP.getTime(); } public void build() throws IOException, DataException { @@ -34,7 +42,11 @@ public class ArbitraryDataBuildQueueItem { // We do not want to overwrite the existing cache, as this will be invalidated // automatically if new data has arrived - arbitraryDataReader.loadSynchronously(false); + try { + arbitraryDataReader.loadSynchronously(false); + } finally { + this.buildEndTimestamp = NTP.getTime(); + } } public boolean isBuilding() { @@ -46,10 +58,17 @@ public class ArbitraryDataBuildQueueItem { } public boolean hasReachedBuildTimeout(Long now) { + if (now == null || this.creationTimestamp == null) { + return true; + } + return now - this.creationTimestamp > BUILD_TIMEOUT; + } + + public boolean hasReachedFailureTimeout(Long now) { if (now == null || this.buildStartTimestamp == null) { return true; } - return now - this.buildStartTimestamp > BUILD_TIMEOUT; + return now - this.buildStartTimestamp > FAILURE_TIMEOUT; } @@ -61,6 +80,11 @@ public class ArbitraryDataBuildQueueItem { return this.buildStartTimestamp; } + public void setFailed(boolean failed) { + this.failed = failed; + } + + @Override public String toString() { return String.format("%s %s", this.service, this.resourceId); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index 9fa383c0..52eaf5b5 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -36,7 +36,9 @@ public class ArbitraryDataBuildManager implements Runnable { // Find resources that are queued for building Map.Entry next = arbitraryDataManager.arbitraryDataBuildQueue - .entrySet().stream().filter(e -> e.getValue().isQueued()).findFirst().get(); + .entrySet().stream() + .filter(e -> e.getValue().isQueued()) + .findFirst().get(); if (next == null) { continue; @@ -49,10 +51,17 @@ public class ArbitraryDataBuildManager implements Runnable { String resourceId = next.getKey(); ArbitraryDataBuildQueueItem queueItem = next.getValue(); - if (queueItem == null || queueItem.hasReachedBuildTimeout(now)) { + + if (queueItem == null) { this.removeFromQueue(resourceId); } + // Ignore builds that have failed recently + if (ArbitraryDataManager.getInstance().isInFailedBuildsList(queueItem)) { + continue; + } + + try { // Perform the build LOGGER.info("Building {}...", queueItem); @@ -62,8 +71,9 @@ public class ArbitraryDataBuildManager implements Runnable { } catch (IOException | DataException e) { LOGGER.info("Error building {}: {}", queueItem, e.getMessage()); - // Something went wrong - so remove it from the queue - // TODO: we may want to keep track of this in a "cooloff" list to prevent frequent re-attempts + // Something went wrong - so remove it from the queue, and add to failed builds list + queueItem.setFailed(true); + ArbitraryDataManager.getInstance().addToFailedBuildsList(queueItem); this.removeFromQueue(resourceId); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 4a99f55e..be43b20b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -79,6 +79,11 @@ public class ArbitraryDataManager extends Thread { */ public Map arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>()); + /** + * Map to keep track of failed arbitrary transaction builds. + */ + public Map arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>()); + private ArbitraryDataManager() { } @@ -222,12 +227,23 @@ public class ArbitraryDataManager extends Thread { return arbitraryDataFileMessage.getArbitraryDataFile(); } - public void cleanupRequestCache(long now) { + public void cleanupRequestCache(Long now) { + if (now == null) { + return; + } final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); // TODO: fix NPE + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); } + public void cleanupQueues(Long now) { + if (now == null) { + return; + } + arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now)); + arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now)); + } + // Arbitrary data resource cache public boolean isResourceCached(String resourceId) { @@ -272,6 +288,7 @@ public class ArbitraryDataManager extends Thread { } // Build queue + public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) { String resourceId = queueItem.getResourceId(); if (resourceId == null) { @@ -287,6 +304,11 @@ public class ArbitraryDataManager extends Thread { return false; } + // Don't add builds that have failed recently + if (this.isInFailedBuildsList(queueItem)) { + return false; + } + if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) { // Already in queue return true; @@ -318,6 +340,54 @@ public class ArbitraryDataManager extends Thread { } + // Failed builds + + public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + if (this.arbitraryDataFailedBuilds.put(resourceId, queueItem) != null) { + // Already in list + return true; + } + + LOGGER.info("Added {} to failed builds list", resourceId); + + // Added to queue + return true; + } + + public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) { + // Already in list + return true; + } + + // Not in list + return false; + } + + // Network handlers public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { From 029c038a49f5b1146b0bfd79d802335207e02eb1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 16:23:46 +0100 Subject: [PATCH 178/505] Catch runtime exceptions (e.g. IllegalStateException) when using the arbitrary data reader/writer. --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 2 +- src/main/java/org/qortal/api/resource/WebsiteResource.java | 4 ++-- .../controller/arbitrary/ArbitraryDataBuildManager.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 233780fb..1e934b09 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -278,7 +278,7 @@ public class ArbitraryResource { } catch (IOException | DataException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } catch (IllegalStateException e) { + } catch (RuntimeException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 87d439c5..43ba30c7 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -112,7 +112,7 @@ public class WebsiteResource { } catch (IOException | DataException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } catch (IllegalStateException e) { + } catch (RuntimeException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } @@ -214,7 +214,7 @@ public class WebsiteResource { } catch (IOException | DataException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } catch (IllegalStateException e) { + } catch (RuntimeException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index 52eaf5b5..63c33a7b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -69,7 +69,7 @@ public class ArbitraryDataBuildManager implements Runnable { this.removeFromQueue(resourceId); LOGGER.info("Finished building {}", queueItem); - } catch (IOException | DataException e) { + } catch (IOException | DataException | RuntimeException e) { LOGGER.info("Error building {}: {}", queueItem, e.getMessage()); // Something went wrong - so remove it from the queue, and add to failed builds list queueItem.setFailed(true); From bedb87674bafe66751bde4ce128fde6b10e1e778 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 16:24:13 +0100 Subject: [PATCH 179/505] FAILURE_TIMEOUT set to 5 minutes --- .../java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index 75769b27..040ac197 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -21,7 +21,7 @@ public class ArbitraryDataBuildQueueItem { // TODO: interrupt an in-progress build public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds /* The amount of time to remember that a build has failed, to avoid retries */ - public static long FAILURE_TIMEOUT = 1*60*1000L; // 5 minutes + public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { this.resourceId = resourceId; From 851511281174caa7ea2ecaf8bcc07957ab454671 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 16:24:37 +0100 Subject: [PATCH 180/505] Fixed unused variable issue --- src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index d181a53c..8366e7b4 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -127,7 +127,7 @@ public class ArbitraryDataCombiner { private void process() throws IOException { String patchType = metadata.getPatchType(); - ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter, metadata.getPatchType()); + ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter, patchType); merge.compute(); this.finalPath = merge.getMergePath(); } From 42bc12f56d5d7906d09040b6797a089d4dc3fcb7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 16:25:47 +0100 Subject: [PATCH 181/505] Call cleanupQueues() from the Controller --- src/main/java/org/qortal/controller/Controller.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 9d0914b7..5655db0c 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -541,6 +541,8 @@ public class Controller extends Thread { // Clean up arbitrary data request cache ArbitraryDataManager.getInstance().cleanupRequestCache(now); + // Clean up arbitrary data queues and lists + ArbitraryDataManager.getInstance().cleanupQueues(now); // Time to 'checkpoint' uncommitted repository writes? if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) { From 6375b9d14d0e2bee797676f0dd58e72ab1caaeda Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 20:27:18 +0100 Subject: [PATCH 182/505] Fixed bug in getLatestTransaction() filtering. --- src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java index e15bbf37..a8ae0345 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java @@ -130,7 +130,7 @@ public class ArbitraryDataCache { // Find latest transaction for name and service, with any method ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository() - .getLatestTransaction(this.resourceId, this.service, Method.PUT); + .getLatestTransaction(this.resourceId, this.service, null); if (latestTransaction != null) { return latestTransaction.getSignature(); From 11da1f72b1aeb38371b720969bd28a6c8696e6ff Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 20:30:00 +0100 Subject: [PATCH 183/505] Added convenience method to make the code more readable. --- src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java | 4 ++++ src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java index a8ae0345..84947e26 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java @@ -32,6 +32,10 @@ public class ArbitraryDataCache { this.service = service; } + public boolean isCachedDataAvailable() { + return !this.shouldInvalidate(); + } + public boolean shouldInvalidate() { try { // If the user has requested an overwrite, always invalidate the cache diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index d35fa322..d4d2482c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -72,7 +72,7 @@ public class ArbitraryDataReader { // Not in the build queue - so check the cache itself ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false, this.resourceId, this.resourceIdType, this.service); - if (!cache.shouldInvalidate()) { + if (cache.isCachedDataAvailable()) { this.filePath = this.uncompressedPath; return true; } @@ -111,7 +111,8 @@ public class ArbitraryDataReader { try { ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, this.resourceId, this.resourceIdType, this.service); - if (!cache.shouldInvalidate()) { + if (cache.isCachedDataAvailable()) { + // Use cached data this.filePath = this.uncompressedPath; return; } From 77479215a63c3fa02a1c26b3d17de5765acbd4c7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 20:32:47 +0100 Subject: [PATCH 184/505] Throw an exception if a patch file doesn't exist. --- src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index 6a1e6b20..05745ffb 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -126,9 +126,7 @@ public class ArbitraryDataMerge { Path mergePath = Paths.get(this.mergePath.toString(), path.toString()); if (!patchPath.toFile().exists()) { - // Patch file doesn't exist, but its path was included in modifiedPaths - // TODO: We ought to throw an exception here, but skipping for now - return; + throw new IllegalStateException("Patch file doesn't exist, but its path was included in modifiedPaths"); } // Delete an existing file, as we are starting from a duplicate of pathBefore From 59ef66f46de336d813c142767626417730c83e47 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 19 Aug 2021 08:48:39 +0100 Subject: [PATCH 185/505] Handle shutdowns when zipping a large number of files. --- .../java/org/qortal/api/resource/ArbitraryResource.java | 2 +- .../java/org/qortal/api/resource/WebsiteResource.java | 4 ++-- .../java/org/qortal/arbitrary/ArbitraryDataWriter.java | 8 +++++--- src/main/java/org/qortal/utils/ZipUtils.java | 9 +++++++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 1e934b09..46bd9c5b 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -275,7 +275,7 @@ public class ArbitraryResource { ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); try { arbitraryDataWriter.save(); - } catch (IOException | DataException e) { + } catch (IOException | DataException | InterruptedException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (RuntimeException e) { diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 43ba30c7..588a8cd2 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -109,7 +109,7 @@ public class WebsiteResource { ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); try { arbitraryDataWriter.save(); - } catch (IOException | DataException e) { + } catch (IOException | DataException | InterruptedException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (RuntimeException e) { @@ -211,7 +211,7 @@ public class WebsiteResource { ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression); try { arbitraryDataWriter.save(); - } catch (IOException | DataException e) { + } catch (IOException | DataException | InterruptedException e) { LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (RuntimeException e) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index d9f284e7..5165cbf2 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -52,7 +52,7 @@ public class ArbitraryDataWriter { this.compression = compression; } - public void save() throws IllegalStateException, IOException, DataException { + public void save() throws IllegalStateException, IOException, DataException, InterruptedException { try { this.preExecute(); this.process(); @@ -136,7 +136,7 @@ public class ArbitraryDataWriter { this.validatePatch(); } - private void validatePatch() throws IOException { + private void validatePatch() { if (this.filePath == null) { throw new IllegalStateException("Null path after creating patch"); } @@ -158,13 +158,14 @@ public class ArbitraryDataWriter { } } - private void compress() { + private void compress() throws InterruptedException { // Compress the data if requested if (this.compression != Compression.NONE) { this.compressedPath = Paths.get(this.workingPath.toString() + File.separator + "data.zip"); try { if (this.compression == Compression.ZIP) { + LOGGER.info("Compressing..."); ZipUtils.zip(this.filePath.toString(), this.compressedPath.toString(), "data"); } else { @@ -190,6 +191,7 @@ public class ArbitraryDataWriter { this.encryptedPath = Paths.get(this.workingPath.toString() + File.separator + "data.zip.encrypted"); try { // Encrypt the file with AES + LOGGER.info("Encrypting..."); this.aesKey = AES.generateKey(256); AES.encryptFile("AES", this.aesKey, this.filePath.toString(), this.encryptedPath.toString()); diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index 2ccffc63..6e614aef 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -25,6 +25,8 @@ package org.qortal.utils; +import org.qortal.controller.Controller; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -35,7 +37,7 @@ import java.util.zip.ZipOutputStream; public class ZipUtils { - public static void zip(String sourcePath, String destFilePath, String fileName) throws IOException { + public static void zip(String sourcePath, String destFilePath, String fileName) throws IOException, InterruptedException { File sourceFile = new File(sourcePath); if (fileName == null) { fileName = sourceFile.getName(); @@ -47,7 +49,10 @@ public class ZipUtils { fileOutputStream.close(); } - public static void zip(final File fileToZip, final String fileName, final ZipOutputStream zipOut) throws IOException { + public static void zip(final File fileToZip, final String fileName, final ZipOutputStream zipOut) throws IOException, InterruptedException { + if (Controller.isStopping()) { + throw new InterruptedException("Controller is stopping"); + } if (fileToZip.isDirectory()) { if (fileName.endsWith("/")) { zipOut.putNextEntry(new ZipEntry(fileName)); From 1d62ef357d98b5ddb800ae9902dc292fcaf6d0fa Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 19 Aug 2021 09:10:13 +0100 Subject: [PATCH 186/505] Check for null last references at the beginning of the process when creating an arbitrary tx. --- .../api/resource/ArbitraryResource.java | 64 +++++++++-------- .../qortal/api/resource/WebsiteResource.java | 68 ++++++++++--------- 2 files changed, 70 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 46bd9c5b..3ddc217c 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -261,35 +261,42 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } - if (creatorPublicKeyBase58 == null || path == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); - - String name = null; - byte[] secret = null; - Method method = Method.PUT; - Service service = Service.ARBITRARY_DATA; - Compression compression = Compression.NONE; - - ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); - try { - arbitraryDataWriter.save(); - } catch (IOException | DataException | InterruptedException e) { - LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } catch (RuntimeException e) { - LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - - ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); - if (arbitraryDataFile == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - + ArbitraryDataFile arbitraryDataFile = null; try (final Repository repository = RepositoryManager.getRepository()) { + if (creatorPublicKeyBase58 == null || path == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); + final String creatorAddress = Crypto.toAddress(creatorPublicKey); + final byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); + if (lastReference == null) { + LOGGER.info(String.format("Qortal account %s has no last reference", creatorAddress)); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + String name = null; + byte[] secret = null; + Method method = Method.PUT; + Service service = Service.ARBITRARY_DATA; + Compression compression = Compression.NONE; + + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); + try { + arbitraryDataWriter.save(); + } catch (IOException | DataException | InterruptedException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } catch (RuntimeException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); + if (arbitraryDataFile == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid(); if (validationResult != ArbitraryDataFile.ValidationResult.OK) { LOGGER.error("Invalid file: {}", validationResult); @@ -310,9 +317,6 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - final String creatorAddress = Crypto.toAddress(creatorPublicKey); - final byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); - final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); final int size = (int) arbitraryDataFile.size(); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 588a8cd2..9aafe4da 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -96,42 +96,46 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } - if (creatorPublicKeyBase58 == null || path == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); - - String name = "CalDescentTest1"; // TODO: dynamic - ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PATCH; - ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; - ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; - - ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); - try { - arbitraryDataWriter.save(); - } catch (IOException | DataException | InterruptedException e) { - LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } catch (RuntimeException e) { - LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - - ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); - if (arbitraryDataFile == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - - String digest58 = arbitraryDataFile.digest58(); - if (digest58 == null) { - LOGGER.error("Unable to calculate digest"); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - + ArbitraryDataFile arbitraryDataFile = null; try (final Repository repository = RepositoryManager.getRepository()) { + if (creatorPublicKeyBase58 == null || path == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); final String creatorAddress = Crypto.toAddress(creatorPublicKey); final byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); + if (lastReference == null) { + LOGGER.info(String.format("Qortal account %s has no last reference", creatorAddress)); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + + String name = "CalDescentTest1"; // TODO: dynamic + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; // TODO: dynamic + ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; + ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; + + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(path), name, service, method, compression); + try { + arbitraryDataWriter.save(); + } catch (IOException | DataException | InterruptedException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } catch (RuntimeException e) { + LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage()); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); + if (arbitraryDataFile == null) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + String digest58 = arbitraryDataFile.digest58(); + if (digest58 == null) { + LOGGER.error("Unable to calculate digest"); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); From 47ff51ce4eba7a28a4fd8e4ae5680656e9055f31 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 07:49:05 +0100 Subject: [PATCH 187/505] Use a random last reference on the very first transaction for an account This is needed because we want to allow brand new accounts to publish data without a fee. A similar approach to CrossChainResource.buildAtMessage(). We already require PoW on all arbitrary transactions, so no additional logic beyond this should be needed. --- .../api/resource/ArbitraryResource.java | 12 ++++++++--- .../qortal/api/resource/WebsiteResource.java | 12 ++++++++--- .../transaction/ArbitraryTransaction.java | 20 +++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 3ddc217c..b85d6405 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -15,6 +15,7 @@ import java.net.UnknownHostException; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Random; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -53,6 +54,7 @@ import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -269,10 +271,14 @@ public class ArbitraryResource { } byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); final String creatorAddress = Crypto.toAddress(creatorPublicKey); - final byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); + byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); if (lastReference == null) { - LOGGER.info(String.format("Qortal account %s has no last reference", creatorAddress)); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + // Use a random last reference on the very first transaction for an account + // Code copied from CrossChainResource.buildAtMessage() + // We already require PoW on all arbitrary transactions, so no additional logic is needed + Random random = new Random(); + lastReference = new byte[Transformer.SIGNATURE_LENGTH]; + random.nextBytes(lastReference); } String name = null; diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 9aafe4da..559b9bd5 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -15,6 +15,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Random; import com.google.common.io.Resources; import io.swagger.v3.oas.annotations.Operation; @@ -48,6 +49,7 @@ import org.qortal.arbitrary.ArbitraryDataWriter; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; +import org.qortal.transform.Transformer; import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -104,10 +106,14 @@ public class WebsiteResource { } byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); final String creatorAddress = Crypto.toAddress(creatorPublicKey); - final byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); + byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); if (lastReference == null) { - LOGGER.info(String.format("Qortal account %s has no last reference", creatorAddress)); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + // Use a random last reference on the very first transaction for an account + // Code copied from CrossChainResource.buildAtMessage() + // We already require PoW on all arbitrary transactions, so no additional logic is needed + Random random = new Random(); + lastReference = new byte[Transformer.SIGNATURE_LENGTH]; + random.nextBytes(lastReference); } String name = "CalDescentTest1"; // TODO: dynamic diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 0f5bea60..aaa5bd48 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -74,6 +74,26 @@ public class ArbitraryTransaction extends Transaction { this.arbitraryTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, difficulty)); } + @Override + public boolean hasValidReference() throws DataException { + // We shouldn't really get this far, but just in case: + if (this.arbitraryTransactionData.getReference() == null) { + return false; + } + + // If the account current doesn't have a last reference, and the fee is 0, we will allow any value. + // This ensures that the first transaction for an account will be valid whilst still validating + // the last reference from the second transaction onwards. By checking for a zero fee, we ensure + // standard last reference validation when fee > 0. + Account creator = getCreator(); + Long fee = this.arbitraryTransactionData.getFee(); + if (creator.getLastReference() == null && fee == 0) { + return true; + } + + return super.hasValidReference(); + } + @Override public ValidationResult isValid() throws DataException { // Check that some data - or a data hash - has been supplied From ab0aeec434675c2ebb03cfe6b17febc082138e2b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 07:51:27 +0100 Subject: [PATCH 188/505] Default method of serving websites switched from signature to name. Before: GET /site/:signature GET /site/name/:name After: GET /site/signature/:signature GET /site/:name --- .../org/qortal/api/resource/WebsiteResource.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 559b9bd5..2a009340 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -240,15 +240,15 @@ public class WebsiteResource { } @GET - @Path("{signature}") + @Path("/signature/{signature}") public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) { - return this.get(signature, ResourceIdType.SIGNATURE, "/", null, "/site", true); + return this.get(signature, ResourceIdType.SIGNATURE, "/", null, "/site/signature", true); } @GET - @Path("{signature}/{path:.*}") + @Path("/signature/{signature}/{path:.*}") public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) { - return this.get(signature, ResourceIdType.SIGNATURE, inPath,null, "/site", true); + return this.get(signature, ResourceIdType.SIGNATURE, inPath,null, "/site/signature", true); } @GET @@ -258,15 +258,15 @@ public class WebsiteResource { } @GET - @Path("/name/{name}/{path:.*}") + @Path("{name}/{path:.*}") public HttpServletResponse getPathByName(@PathParam("name") String name, @PathParam("path") String inPath) { - return this.get(name, ResourceIdType.NAME, inPath, null, "/site/name", true); + return this.get(name, ResourceIdType.NAME, inPath, null, "/site", true); } @GET - @Path("/name/{name}") + @Path("{name}") public HttpServletResponse getIndexByName(@PathParam("name") String name) { - return this.get(name, ResourceIdType.NAME, "/", null, "/site/name", true); + return this.get(name, ResourceIdType.NAME, "/", null, "/site", true); } @GET From d22a03f1a57e348e30abfd3c65e77e23aa9ded4e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 07:51:58 +0100 Subject: [PATCH 189/505] Fixed misleading exception string. --- src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 87795a18..fcf9df1e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -56,7 +56,8 @@ public class ArbitraryDataBuilder { ArbitraryTransactionData latestPut = repository.getArbitraryRepository() .getLatestTransaction(this.name, this.service, Method.PUT); if (latestPut == null) { - throw new IllegalStateException("Cannot PATCH without existing PUT. Deploy using PUT first."); + throw new IllegalStateException(String.format( + "Couldn't find PUT transaction for name %s and service %s", this.name, this.service)); } this.latestPutTransaction = latestPut; From d01cdeded834ba00e7eee685f812d00697da979b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 07:58:55 +0100 Subject: [PATCH 190/505] Added pagination to ArbitraryDataManager This improves scalability but isn't sufficient for a long term solution. TODO: It probably makes sense to add an additional query for recent transactions only, so that they are fetched quickly. --- .../controller/arbitrary/ArbitraryDataManager.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index be43b20b..6e321ee3 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -104,6 +104,10 @@ public class ArbitraryDataManager extends Thread { ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager()); + // Paginate queries when fetching arbitrary transactions + final int limit = 100; + int offset = 0; + try { while (!isStopping) { Thread.sleep(2000); @@ -120,10 +124,13 @@ public class ArbitraryDataManager extends Thread { // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { - List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true); + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, limit, offset, true); + // LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); if (signatures == null || signatures.isEmpty()) { + offset = 0; continue; } + offset += limit; // Filter out those that already have local data signatures.removeIf(signature -> hasLocalData(repository, signature)); From 51b12567e8f927b6c079c0c59ca619e06544adec Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 09:17:36 +0100 Subject: [PATCH 191/505] Switched ArbitraryDataFile.filePath from String to Path --- .../qortal/arbitrary/ArbitraryDataFile.java | 78 +++++++++---------- .../arbitrary/ArbitraryDataFileChunk.java | 3 +- .../qortal/arbitrary/ArbitraryDataReader.java | 4 +- .../qortal/arbitrary/ArbitraryDataWriter.java | 2 +- 4 files changed, 41 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 98b0c667..aaeaec2d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -48,7 +48,7 @@ public class ArbitraryDataFile { FILE_HASH, TRANSACTION_DATA, NAME - }; + } private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); @@ -56,7 +56,7 @@ public class ArbitraryDataFile { public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static int SHORT_DIGEST_LENGTH = 8; - protected String filePath; + protected Path filePath; protected String hash58; private ArrayList chunks; private byte[] secret; @@ -80,8 +80,8 @@ public class ArbitraryDataFile { this.hash58 = Base58.encode(Crypto.digest(fileContent)); LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); - String outputFilePath = getOutputFilePath(this.hash58, true); - File outputFile = new File(outputFilePath); + Path outputFilePath = getOutputFilePath(this.hash58, true); + File outputFile = outputFilePath.toFile(); try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { outputStream.write(fileContent); this.filePath = outputFilePath; @@ -104,8 +104,11 @@ public class ArbitraryDataFile { return ArbitraryDataFile.fromHash58(Base58.encode(hash)); } - public static ArbitraryDataFile fromPath(String path) { - File file = new File(path); + public static ArbitraryDataFile fromPath(Path path) { + if (path == null) { + return null; + } + File file = path.toFile(); if (file.exists()) { try { byte[] fileContent = Files.readAllBytes(file.toPath()); @@ -113,15 +116,14 @@ public class ArbitraryDataFile { ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); // Copy file to data directory if needed - Path filePath = Paths.get(path); - if (Files.exists(filePath) && !arbitraryDataFile.isInBaseDirectory(path)) { - arbitraryDataFile.copyToDataDirectory(filePath); + if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) { + arbitraryDataFile.copyToDataDirectory(path); } // Or, if it's already in the data directory, we may need to move it - else if (!filePath.equals(arbitraryDataFile.getFilePath())) { + else if (!path.equals(arbitraryDataFile.getFilePath())) { // Wrong path, so relocate - Path dest = Paths.get(arbitraryDataFile.getFilePath()); - FilesystemUtils.moveFile(filePath, dest, true); + Path dest = arbitraryDataFile.getFilePath(); + FilesystemUtils.moveFile(path, dest, true); } return arbitraryDataFile; @@ -133,7 +135,7 @@ public class ArbitraryDataFile { } public static ArbitraryDataFile fromFile(File file) { - return ArbitraryDataFile.fromPath(file.getPath()); + return ArbitraryDataFile.fromPath(Paths.get(file.getPath())); } private boolean createDataDirectory() { @@ -149,21 +151,21 @@ public class ArbitraryDataFile { return true; } - private String copyToDataDirectory(Path sourcePath) { + private Path copyToDataDirectory(Path sourcePath) { if (this.hash58 == null || this.filePath == null) { return null; } - String outputFilePath = getOutputFilePath(this.hash58, true); + Path outputFilePath = getOutputFilePath(this.hash58, true); sourcePath = sourcePath.toAbsolutePath(); - Path destPath = Paths.get(outputFilePath).toAbsolutePath(); + Path destPath = outputFilePath.toAbsolutePath(); try { - return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING).toString(); + return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { throw new IllegalStateException("Unable to copy file to data directory"); } } - public static String getOutputFilePath(String hash58, boolean createDirectories) { + public static Path getOutputFilePath(String hash58, boolean createDirectories) { if (hash58 == null) { return null; } @@ -179,20 +181,19 @@ public class ArbitraryDataFile { throw new IllegalStateException("Unable to create data subdirectory"); } } - return outputDirectory + File.separator + hash58; + return Paths.get(outputDirectory, hash58); } public ValidationResult isValid() { try { // Ensure the file exists on disk - Path path = Paths.get(this.filePath); - if (!Files.exists(path)) { + if (!Files.exists(this.filePath)) { LOGGER.error("File doesn't exist at path {}", this.filePath); return ValidationResult.FILE_NOT_FOUND; } // Validate the file size - long fileSize = Files.size(path); + long fileSize = Files.size(this.filePath); if (fileSize > MAX_FILE_SIZE) { LOGGER.error(String.format("ArbitraryDataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE)); return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE; @@ -276,7 +277,7 @@ public class ArbitraryDataFile { File outputFile = new File(outputPath.toString()); try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) { for (ArbitraryDataFileChunk chunk : this.chunks) { - File sourceFile = new File(chunk.filePath); + File sourceFile = chunk.filePath.toFile(); BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); byte[] buffer = new byte[2048]; int inSize; @@ -306,13 +307,12 @@ public class ArbitraryDataFile { public boolean delete() { // Delete the complete file // ... but only if it's inside the Qortal data or temp directory - Path path = Paths.get(this.filePath); - if (FilesystemUtils.pathInsideDataOrTempPath(path)) { - if (Files.exists(path)) { + if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { + if (Files.exists(this.filePath)) { try { - Files.delete(path); + Files.delete(this.filePath); this.cleanupFilesystem(); - LOGGER.debug("Deleted file {}", path.toString()); + LOGGER.debug("Deleted file {}", this.filePath); return true; } catch (IOException e) { LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath); @@ -340,11 +340,9 @@ public class ArbitraryDataFile { } protected void cleanupFilesystem() { - String path = this.filePath; - // Iterate through two levels of parent directories, and delete if empty for (int i=0; i<2; i++) { - Path directory = Paths.get(path).getParent().toAbsolutePath(); + Path directory = this.filePath.getParent().toAbsolutePath(); try (Stream files = Files.list(directory)) { final long count = files.count(); if (count == 0) { @@ -355,14 +353,13 @@ public class ArbitraryDataFile { } catch (IOException e) { LOGGER.warn("Unable to count files in directory", e); } - path = directory.toString(); + this.filePath = directory; } } public byte[] getBytes() { - Path path = Paths.get(this.filePath); try { - return Files.readAllBytes(path); + return Files.readAllBytes(this.filePath); } catch (IOException e) { LOGGER.error("Unable to read bytes for file"); return null; @@ -372,15 +369,15 @@ public class ArbitraryDataFile { /* Helper methods */ - private boolean isInBaseDirectory(String filePath) { - Path path = Paths.get(filePath).toAbsolutePath(); + private boolean isInBaseDirectory(Path filePath) { + Path path = filePath.toAbsolutePath(); String dataPath = Settings.getInstance().getDataPath(); String basePath = Paths.get(dataPath).toAbsolutePath().toString(); return path.startsWith(basePath); } public boolean exists() { - File file = new File(this.filePath); + File file = this.filePath.toFile(); return file.exists(); } @@ -422,9 +419,8 @@ public class ArbitraryDataFile { } public long size() { - Path path = Paths.get(this.filePath); try { - return Files.size(path); + return Files.size(this.filePath); } catch (IOException e) { return 0; } @@ -464,14 +460,14 @@ public class ArbitraryDataFile { } private File getFile() { - File file = new File(this.filePath); + File file = this.filePath.toFile(); if (file.exists()) { return file; } return null; } - public String getFilePath() { + public Path getFilePath() { return this.filePath; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java index ccd14eae..7b41f100 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java @@ -38,10 +38,9 @@ public class ArbitraryDataFileChunk extends ArbitraryDataFile { return superclassValidationResult; } - Path path = Paths.get(this.filePath); try { // Validate the file size (chunks have stricter limits) - long fileSize = Files.size(path); + long fileSize = Files.size(this.filePath); if (fileSize > CHUNK_SIZE) { LOGGER.error(String.format("DataFileChunk is too large: %d bytes (max chunk size: %d bytes)", fileSize, CHUNK_SIZE)); return ValidationResult.FILE_TOO_LARGE; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index d4d2482c..c2dc5289 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -218,7 +218,7 @@ public class ArbitraryDataReader { // Load data file directly from the hash ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId); // Set filePath to the location of the ArbitraryDataFile - this.filePath = Paths.get(arbitraryDataFile.getFilePath()); + this.filePath = arbitraryDataFile.getFilePath(); } private void fetchFromName() throws IllegalStateException, IOException, DataException { @@ -294,7 +294,7 @@ public class ArbitraryDataReader { throw new IllegalStateException("Unable to validate complete file hash"); } // Set filePath to the location of the ArbitraryDataFile - this.filePath = Paths.get(arbitraryDataFile.getFilePath()); + this.filePath = arbitraryDataFile.getFilePath(); } private void decrypt() { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 5165cbf2..e9c9d454 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -233,7 +233,7 @@ public class ArbitraryDataWriter { } private void split() throws IOException { - this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath.toString()); + this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath); if (this.arbitraryDataFile == null) { throw new IOException("No file available when trying to split"); } From 6730683919a161b503aeb2e778edf2142da9ee45 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 12:55:42 +0100 Subject: [PATCH 192/505] Added arbitrary data cleanup manager This deletes redundant copies of data, and also converts complete files to chunks where needed. The idea being that nodes only hold chunks, since they currently are much more likely to serve a chunk to another peer than they are to serve a complete file. It doesn't yet cleanup files that are unassociated with transactions, nor does it delete anything from the _temp folder. --- .../qortal/arbitrary/ArbitraryDataFile.java | 17 +- .../org/qortal/controller/Controller.java | 11 +- .../ArbitraryDataCleanupManager.java | 281 ++++++++++++++++++ 3 files changed, 301 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index aaeaec2d..4c98fa4c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -315,16 +315,15 @@ public class ArbitraryDataFile { LOGGER.debug("Deleted file {}", this.filePath); return true; } catch (IOException e) { - LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath); + LOGGER.warn("Couldn't delete file at path {}", this.filePath); } } } return false; } - public boolean deleteAll() { - // Delete the complete file - boolean success = this.delete(); + public boolean deleteAllChunks() { + boolean success = false; // Delete the individual chunks if (this.chunks != null && this.chunks.size() > 0) { @@ -339,6 +338,16 @@ public class ArbitraryDataFile { return success; } + public boolean deleteAll() { + // Delete the complete file + boolean fileDeleted = this.delete(); + + // Delete the individual chunks + boolean chunksDeleted = this.deleteAllChunks(); + + return fileDeleted && chunksDeleted; + } + protected void cleanupFilesystem() { // Iterate through two levels of parent directories, and delete if empty for (int i=0; i<2; i++) { diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 5655db0c..6bab9d42 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -14,6 +14,7 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.data.account.MintingAccountData; @@ -446,9 +447,10 @@ public class Controller extends Thread { LOGGER.info("Starting trade-bot"); TradeBot.getInstance(); - // Arbitrary transaction data manager - LOGGER.info("Starting arbitrary-transaction data manager"); + // Arbitrary data controllers + LOGGER.info("Starting arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().start(); + ArbitraryDataCleanupManager.getInstance().start(); // Auto-update service? if (Settings.getInstance().isAutoUpdateEnabled()) { @@ -934,9 +936,10 @@ public class Controller extends Thread { AutoUpdate.getInstance().shutdown(); } - // Arbitrary transaction data manager - LOGGER.info("Shutting down arbitrary-transaction data manager"); + // Arbitrary data controllers + LOGGER.info("Shutting down arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().shutdown(); + ArbitraryDataCleanupManager.getInstance().shutdown(); if (blockMinter != null) { LOGGER.info("Shutting down block minter"); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java new file mode 100644 index 00000000..c85cc964 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -0,0 +1,281 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.controller.Controller; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class ArbitraryDataCleanupManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class); + private static final List ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY); + + private static ArbitraryDataCleanupManager instance; + + private volatile boolean isStopping = false; + + /** + * The amount of time that must pass before a file is treated as stale / not recent. + * We can safely delete files created/accessed longer ago that this, if we have a means of + * rebuilding them. The main purpose of this is to avoid deleting files that are currently + * being used by other parts of the system. + */ + private static long STALE_FILE_TIMEOUT = 60*60*1000; // 1 hour + + + private ArbitraryDataCleanupManager() { + } + + public static ArbitraryDataCleanupManager getInstance() { + if (instance == null) + instance = new ArbitraryDataCleanupManager(); + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Cleanup Manager"); + + // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) + // This can be expanded to have multiple threads processing the build queue when needed + ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); + arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager()); + + // Paginate queries when fetching arbitrary transactions + final int limit = 100; + int offset = 0; + + try { + while (!isStopping) { + Thread.sleep(30000); + + if (NTP.getTime() == null) { + // Don't attempt to make decisions if we haven't synced our time yet + continue; + } + + List peers = Network.getInstance().getHandshakedPeers(); + + // Disregard peers that have "misbehaved" recently + peers.removeIf(Controller.hasMisbehaved); + + // Don't fetch data if we don't have enough up-to-date peers + if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) { + continue; + } + + // Any arbitrary transactions we want to fetch data for? + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, limit, offset, true); + // LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + if (signatures == null || signatures.isEmpty()) { + offset = 0; + continue; + } + offset += limit; + Long now = NTP.getTime(); + + // Loop through the signatures in this batch + for (int i=0; i 0) { + arbitraryDataFile.addChunkHashes(chunkHashes); + } + return arbitraryDataFile.allChunksExist(chunkHashes); + + } + + private boolean isFileHashRecent(byte[] hash, long now) { + try { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); + if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { + // No hash, or file doesn't exist, so it's not recent + return false; + } + Path filePath = arbitraryDataFile.getFilePath(); + + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + long timeSinceCreated = now - attr.creationTime().toMillis(); + long timeSinceModified = now - attr.lastModifiedTime().toMillis(); + + // Check if the file has been created or modified recently + if (timeSinceCreated < STALE_FILE_TIMEOUT) { + return true; + } + if (timeSinceModified < STALE_FILE_TIMEOUT) { + return true; + } + + } catch (IOException e) { + // Can't read file attributes, so assume it's not recent + } + return false; + } + + private void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now) { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + arbitraryDataFile.addChunkHashes(chunkHashes); + + if (!this.isFileHashRecent(completeHash, now)) { + LOGGER.info("Deleting file {} because it can be rebuilt from chunks " + + "if needed", Base58.encode(completeHash)); + + arbitraryDataFile.delete(); + } + } + + private void createChunks(ArbitraryTransactionData arbitraryTransactionData, long now) { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + // Split the file into chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); + if (chunkCount > 1) { + LOGGER.info(String.format("Successfully split %s into %d chunk%s", + Base58.encode(completeHash), chunkCount, (chunkCount == 1 ? "" : "s"))); + + // Verify that the chunk hashes match those in the transaction + if (chunkHashes != null && Arrays.equals(chunkHashes, arbitraryDataFile.chunkHashes())) { + // Ensure they exist on disk + if (arbitraryDataFile.allChunksExist(chunkHashes)) { + + // Now delete the original file if it's not recent + if (!this.isFileHashRecent(completeHash, now)) { + LOGGER.info("Deleting file {} because it can now be rebuilt from " + + "chunks if needed", Base58.encode(completeHash)); + + this.deleteCompleteFile(arbitraryTransactionData, now); + } + else { + // File might be in use. It's best to leave it and it it will be cleaned up later. + } + } + } + } + } + +} From 190f70f332f8b556aad4e9fed77c5c7288b6a188 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 13:01:55 +0100 Subject: [PATCH 193/505] Removed unused, buggy code in HSQLDBArbitraryRepository.save() It's safer to throw an exception and point the user towards ArbitraryDataWriter, rather than maintaining unused code. --- .../hsqldb/HSQLDBArbitraryRepository.java | 42 ++----------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 16841cc1..e7ee0d50 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -16,6 +16,7 @@ import org.qortal.transaction.Transaction.ApprovalStatus; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class HSQLDBArbitraryRepository implements ArbitraryRepository { @@ -23,9 +24,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { private static final int MAX_RAW_DATA_SIZE = 255; // size of VARBINARY protected HSQLDBRepository repository; - - private static final Logger LOGGER = LogManager.getLogger(ArbitraryRepository.class); - + public HSQLDBArbitraryRepository(HSQLDBRepository repository) { this.repository = repository; } @@ -130,42 +129,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return; } - // Store non-trivial payloads in filesystem and convert transaction's data to hash form - byte[] rawData = arbitraryTransactionData.getData(); - - // Calculate hash of data and update our transaction to use that - byte[] dataHash = Crypto.digest(rawData); - arbitraryTransactionData.setData(dataHash); - arbitraryTransactionData.setDataType(DataType.DATA_HASH); - - // Create ArbitraryDataFile - ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(rawData); - - // Verify that the data file is valid, and that it matches the expected hash - ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid(); - if (validationResult != ArbitraryDataFile.ValidationResult.OK) { - arbitraryDataFile.deleteAll(); - throw new DataException("Invalid data file when attempting to store arbitrary transaction data"); - } - if (!dataHash.equals(arbitraryDataFile.digest())) { - arbitraryDataFile.deleteAll(); - throw new DataException("Could not verify hash when attempting to store arbitrary transaction data"); - } - - // Now create chunks if needed - int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); - if (chunkCount > 0) { - LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s"))); - LOGGER.info("{}", arbitraryDataFile.printChunks()); - - // Verify that the chunk hashes match those in the transaction - byte[] chunkHashes = arbitraryDataFile.chunkHashes(); - if (!chunkHashes.equals(arbitraryTransactionData.getChunkHashes())) { - arbitraryDataFile.deleteAll(); - throw new DataException("Could not verify chunk hashes when attempting to store arbitrary transaction data"); - } - - } + throw new IllegalStateException(String.format("Supplied data is larger than maximum size (%i bytes). Please use ArbitraryDataWriter.", MAX_RAW_DATA_SIZE)); } @Override From 8f3620e07b90f1182a6ee93fcc7a5a339bafba7b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 13:44:30 +0100 Subject: [PATCH 194/505] Fixed bug introduced in commit 51b1256 --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 4c98fa4c..b1a8c245 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -349,9 +349,13 @@ public class ArbitraryDataFile { } protected void cleanupFilesystem() { + // It is essential that use a separate path reference in this method + // as we don't want to modify this.filePath + Path path = this.filePath; + // Iterate through two levels of parent directories, and delete if empty for (int i=0; i<2; i++) { - Path directory = this.filePath.getParent().toAbsolutePath(); + Path directory = path.getParent().toAbsolutePath(); try (Stream files = Files.list(directory)) { final long count = files.count(); if (count == 0) { @@ -362,7 +366,7 @@ public class ArbitraryDataFile { } catch (IOException e) { LOGGER.warn("Unable to count files in directory", e); } - this.filePath = directory; + path = directory; } } From 8fa61e628cd86891717a975d7f9b27f3616c126d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 16:56:49 +0100 Subject: [PATCH 195/505] Delete files related to transactions that have a more recent PUT A PUT creates a new base layer meaning anything before that point is no longer needed. These files are now deleted automatically by the cleanup manager. This involved relocating a lot of the cleanup manager methods into a shared utility, so that they could be used by the arbitrary data manager. Without this, they would be fetched from the network again as soon as they were deleted. --- .../qortal/arbitrary/ArbitraryDataFile.java | 16 ++ .../ArbitraryDataCleanupManager.java | 181 ++++--------- .../arbitrary/ArbitraryDataManager.java | 16 ++ .../utils/ArbitraryTransactionUtils.java | 237 ++++++++++++++++++ 4 files changed, 312 insertions(+), 138 deletions(-) create mode 100644 src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index b1a8c245..91dee563 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -422,6 +422,22 @@ public class ArbitraryDataFile { return true; } + public boolean anyChunksExist(byte[] chunks) { + if (chunks == null) { + return false; + } + ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); + while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { + byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; + byteBuffer.get(chunkHash); + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash); + if (chunk.exists()) { + return true; + } + } + return false; + } + public boolean containsChunk(byte[] hash) { for (ArbitraryDataFileChunk chunk : this.chunks) { if (Arrays.equals(hash, chunk.getHash())) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index c85cc964..2e00b448 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -3,10 +3,8 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; -import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.controller.Controller; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.repository.DataException; @@ -14,13 +12,10 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -43,6 +38,14 @@ public class ArbitraryDataCleanupManager extends Thread { private static long STALE_FILE_TIMEOUT = 60*60*1000; // 1 hour + /* + TODO: + - Discard all files relating to transactions for a name/service combination before the most recent PUT + - Delete old files from _temp + - Delete old files not associated with transactions + */ + + private ArbitraryDataCleanupManager() { } @@ -104,7 +107,7 @@ public class ArbitraryDataCleanupManager extends Thread { } // Fetch the transaction data - ArbitraryTransactionData arbitraryTransactionData = this.fetchTransactionData(repository, signature); + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); // Raw data doesn't have any associated files to clean up if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.RAW_DATA) { @@ -112,24 +115,49 @@ public class ArbitraryDataCleanupManager extends Thread { } // Check if we have the complete file - boolean completeFileExists = this.completeFileExists(arbitraryTransactionData); + boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData); - // Check if we have all the chunks - boolean allChunksExist = this.allChunksExist(arbitraryTransactionData); + // Check if we have any of the chunks + boolean anyChunksExist = ArbitraryTransactionUtils.anyChunksExist(arbitraryTransactionData); + boolean transactionHasChunks = (arbitraryTransactionData.getChunkHashes() != null); - if (completeFileExists && arbitraryTransactionData.getChunkHashes() == null) { - // This file doesn't have any chunks because it is too small - // We must not delete anything + if (!completeFileExists && !anyChunksExist) { + // We don't have any files at all for this transaction - nothing to do continue; } + // We have at least 1 chunk or file for this transaction, so we might need to delete them... + + + // Check to see if we have had a more recent PUT + boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); + if (hasMoreRecentPutTransaction) { + // There is a more recent PUT transaction than the one we are currently processing. + // When a PUT is issued, it replaces any layers that would have been there before. + // Therefore any data relating to this older transaction is no longer needed. + LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " + + "Deleting all files.", arbitraryTransactionData.getService(), + arbitraryTransactionData.getName(), Base58.encode(signature))); + + ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData); + } + + if (completeFileExists && !transactionHasChunks) { + // This file doesn't have any chunks because it is too small. + // We must not delete anything. + continue; + } + + // Check if we have all of the chunks + boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData); + if (completeFileExists && allChunksExist) { // We have the complete file and all the chunks, so we can delete // the complete file if it has reached a certain age. LOGGER.info(String.format("Transaction %s has complete file and all chunks", Base58.encode(arbitraryTransactionData.getSignature()))); - this.deleteCompleteFile(arbitraryTransactionData, now); + ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); } if (completeFileExists && !allChunksExist) { @@ -137,7 +165,7 @@ public class ArbitraryDataCleanupManager extends Thread { LOGGER.info(String.format("Transaction %s has complete file but no chunks", Base58.encode(arbitraryTransactionData.getSignature()))); - this.createChunks(arbitraryTransactionData, now); + ArbitraryTransactionUtils.convertFileToChunks(arbitraryTransactionData, now, STALE_FILE_TIMEOUT); } } @@ -155,127 +183,4 @@ public class ArbitraryDataCleanupManager extends Thread { this.interrupt(); } - - private ArbitraryTransactionData fetchTransactionData(final Repository repository, final byte[] signature) { - try { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) - return null; - - return (ArbitraryTransactionData) transactionData; - - } catch (DataException e) { - LOGGER.error("Repository issue when fetching arbitrary transaction data", e); - return null; - } - } - - private boolean completeFileExists(ArbitraryTransactionData transactionData) { - if (transactionData == null) { - return false; - } - - byte[] digest = transactionData.getData(); - - // Load complete file - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); - return arbitraryDataFile.exists(); - - } - - private boolean allChunksExist(ArbitraryTransactionData transactionData) { - if (transactionData == null) { - return false; - } - - byte[] digest = transactionData.getData(); - byte[] chunkHashes = transactionData.getChunkHashes(); - - if (chunkHashes == null) { - // This file doesn't have any chunks - return true; - } - - // Load complete file and chunks - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); - if (chunkHashes != null && chunkHashes.length > 0) { - arbitraryDataFile.addChunkHashes(chunkHashes); - } - return arbitraryDataFile.allChunksExist(chunkHashes); - - } - - private boolean isFileHashRecent(byte[] hash, long now) { - try { - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); - if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { - // No hash, or file doesn't exist, so it's not recent - return false; - } - Path filePath = arbitraryDataFile.getFilePath(); - - BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); - long timeSinceCreated = now - attr.creationTime().toMillis(); - long timeSinceModified = now - attr.lastModifiedTime().toMillis(); - - // Check if the file has been created or modified recently - if (timeSinceCreated < STALE_FILE_TIMEOUT) { - return true; - } - if (timeSinceModified < STALE_FILE_TIMEOUT) { - return true; - } - - } catch (IOException e) { - // Can't read file attributes, so assume it's not recent - } - return false; - } - - private void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now) { - byte[] completeHash = arbitraryTransactionData.getData(); - byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); - - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); - arbitraryDataFile.addChunkHashes(chunkHashes); - - if (!this.isFileHashRecent(completeHash, now)) { - LOGGER.info("Deleting file {} because it can be rebuilt from chunks " + - "if needed", Base58.encode(completeHash)); - - arbitraryDataFile.delete(); - } - } - - private void createChunks(ArbitraryTransactionData arbitraryTransactionData, long now) { - byte[] completeHash = arbitraryTransactionData.getData(); - byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); - - // Split the file into chunks - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); - int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); - if (chunkCount > 1) { - LOGGER.info(String.format("Successfully split %s into %d chunk%s", - Base58.encode(completeHash), chunkCount, (chunkCount == 1 ? "" : "s"))); - - // Verify that the chunk hashes match those in the transaction - if (chunkHashes != null && Arrays.equals(chunkHashes, arbitraryDataFile.chunkHashes())) { - // Ensure they exist on disk - if (arbitraryDataFile.allChunksExist(chunkHashes)) { - - // Now delete the original file if it's not recent - if (!this.isFileHashRecent(completeHash, now)) { - LOGGER.info("Deleting file {} because it can now be rebuilt from " + - "chunks if needed", Base58.encode(completeHash)); - - this.deleteCompleteFile(arbitraryTransactionData, now); - } - else { - // File might be in use. It's best to leave it and it it will be cleaned up later. - } - } - } - } - } - } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 6e321ee3..337bbfa7 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -22,6 +22,7 @@ import org.qortal.arbitrary.ArbitraryDataFileChunk; import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; import org.qortal.utils.Triple; @@ -143,6 +144,21 @@ public class ArbitraryDataManager extends Thread { final int index = new Random().nextInt(signatures.size()); byte[] signature = signatures.get(index); + if (signature == null) { + continue; + } + + // Check to see if we have had a more recent PUT + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData); + if (hasMoreRecentPutTransaction) { + // There is a more recent PUT transaction than the one we are currently processing. + // When a PUT is issued, it replaces any layers that would have been there before. + // Therefore any data relating to this older transaction is no longer needed and we + // shouldn't fetch it from the network. + continue; + } + // Ask our connected peers if they have files for this signature // This process automatically then fetches the files themselves if a peer is found fetchArbitraryDataFileList(signature); diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java new file mode 100644 index 00000000..13098480 --- /dev/null +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -0,0 +1,237 @@ +package org.qortal.utils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; + +public class ArbitraryTransactionUtils { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransactionUtils.class); + + public static ArbitraryTransactionData fetchTransactionData(final Repository repository, final byte[] signature) { + try { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (!(transactionData instanceof ArbitraryTransactionData)) + return null; + + return (ArbitraryTransactionData) transactionData; + + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + return null; + } + } + + public static ArbitraryTransactionData fetchLatestPut(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + if (arbitraryTransactionData == null) { + return null; + } + + String name = arbitraryTransactionData.getName(); + ArbitraryTransactionData.Service service = arbitraryTransactionData.getService(); + + if (name == null || service == null) { + return null; + } + + // Get the most recent PUT for this name and service + ArbitraryTransactionData latestPut; + try { + latestPut = repository.getArbitraryRepository() + .getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT); + } catch (DataException e) { + return null; + } + + return latestPut; + } + + public static boolean hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { + byte[] signature = arbitraryTransactionData.getSignature(); + if (signature == null) { + // We can't make a sensible decision without a signature + // so it's best to assume there is nothing newer + return false; + } + + ArbitraryTransactionData latestPut = ArbitraryTransactionUtils.fetchLatestPut(repository, arbitraryTransactionData); + if (latestPut == null) { + return false; + } + + // If the latest PUT transaction has a newer timestamp, it will override the existing transaction + // Any data relating to the older transaction is no longer needed + boolean hasNewerPut = (latestPut.getTimestamp() > arbitraryTransactionData.getTimestamp()); + return hasNewerPut; + } + + public static boolean completeFileExists(ArbitraryTransactionData transactionData) { + if (transactionData == null) { + return false; + } + + byte[] digest = transactionData.getData(); + + // Load complete file + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + return arbitraryDataFile.exists(); + + } + + public static boolean allChunksExist(ArbitraryTransactionData transactionData) { + if (transactionData == null) { + return false; + } + + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + if (chunkHashes == null) { + // This file doesn't have any chunks, which is the same as us having them all + return true; + } + + // Load complete file and chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + if (chunkHashes != null && chunkHashes.length > 0) { + arbitraryDataFile.addChunkHashes(chunkHashes); + } + return arbitraryDataFile.allChunksExist(chunkHashes); + } + + public static boolean anyChunksExist(ArbitraryTransactionData transactionData) { + if (transactionData == null) { + return false; + } + + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + if (chunkHashes == null) { + // This file doesn't have any chunks, which means none exist + return false; + } + + // Load complete file and chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + if (chunkHashes != null && chunkHashes.length > 0) { + arbitraryDataFile.addChunkHashes(chunkHashes); + } + return arbitraryDataFile.anyChunksExist(chunkHashes); + } + + public static int ourChunkCount(ArbitraryTransactionData transactionData) { + if (transactionData == null) { + return 0; + } + + byte[] digest = transactionData.getData(); + byte[] chunkHashes = transactionData.getChunkHashes(); + + if (chunkHashes == null) { + // This file doesn't have any chunks + return 0; + } + + // Load complete file and chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + if (chunkHashes != null && chunkHashes.length > 0) { + arbitraryDataFile.addChunkHashes(chunkHashes); + } + return arbitraryDataFile.chunkCount(); + } + + public static boolean isFileHashRecent(byte[] hash, long now, long cleanupAfter) { + try { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); + if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { + // No hash, or file doesn't exist, so it's not recent + return false; + } + Path filePath = arbitraryDataFile.getFilePath(); + + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + long timeSinceCreated = now - attr.creationTime().toMillis(); + long timeSinceModified = now - attr.lastModifiedTime().toMillis(); + + // Check if the file has been created or modified recently + if (timeSinceCreated < cleanupAfter) { + return true; + } + if (timeSinceModified < cleanupAfter) { + return true; + } + + } catch (IOException e) { + // Can't read file attributes, so assume it's not recent + } + return false; + } + + public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + arbitraryDataFile.addChunkHashes(chunkHashes); + + if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) { + LOGGER.info("Deleting file {} because it can be rebuilt from chunks " + + "if needed", Base58.encode(completeHash)); + + arbitraryDataFile.delete(); + } + } + + public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + arbitraryDataFile.addChunkHashes(chunkHashes); + arbitraryDataFile.deleteAll(); + } + + public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) { + byte[] completeHash = arbitraryTransactionData.getData(); + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + // Split the file into chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); + if (chunkCount > 1) { + LOGGER.info(String.format("Successfully split %s into %d chunk%s", + Base58.encode(completeHash), chunkCount, (chunkCount == 1 ? "" : "s"))); + + // Verify that the chunk hashes match those in the transaction + if (chunkHashes != null && Arrays.equals(chunkHashes, arbitraryDataFile.chunkHashes())) { + // Ensure they exist on disk + if (arbitraryDataFile.allChunksExist(chunkHashes)) { + + // Now delete the original file if it's not recent + if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) { + LOGGER.info("Deleting file {} because it can now be rebuilt from " + + "chunks if needed", Base58.encode(completeHash)); + + ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter); + } + else { + // File might be in use. It's best to leave it and it it will be cleaned up later. + } + } + } + } + } + +} From 988a83962382ed870c2be6d2d3c633f1600f5cef Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 16:58:43 +0100 Subject: [PATCH 196/505] Improved response value of ArbitraryDataFile.deleteAllChunks() as it was inaccurate. --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 91dee563..d838e56b 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -330,9 +330,8 @@ public class ArbitraryDataFile { Iterator iterator = this.chunks.iterator(); while (iterator.hasNext()) { ArbitraryDataFileChunk chunk = (ArbitraryDataFileChunk) iterator.next(); - chunk.delete(); + success = chunk.delete(); iterator.remove(); - success = true; } } return success; From 00ba16f536422c3fd76d435b7fcef761291f19b4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 16:59:05 +0100 Subject: [PATCH 197/505] Fixed incorrect ArbitraryTransactionTransformer layout. --- .../transform/transaction/ArbitraryTransactionTransformer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java index 7b2b2575..1bcb783b 100644 --- a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java @@ -73,7 +73,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("data", TransformationType.DATA); layout.add("raw data size", TransformationType.INT); // Version 5+ - layout.add("chunk count", TransformationType.INT); // Version 5+ + layout.add("chunk hashes length", TransformationType.INT); // Version 5+ layout.add("chunk hashes", TransformationType.DATA); // Version 5+ layout.add("fee", TransformationType.AMOUNT); From 6cb39795a90888cf45541f5dbb4958d54719eb42 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 17:00:53 +0100 Subject: [PATCH 198/505] Removed requirement to have connected peers in order to cleanup directories. --- .../arbitrary/ArbitraryDataCleanupManager.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 2e00b448..a3500766 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -40,7 +40,6 @@ public class ArbitraryDataCleanupManager extends Thread { /* TODO: - - Discard all files relating to transactions for a name/service combination before the most recent PUT - Delete old files from _temp - Delete old files not associated with transactions */ @@ -77,17 +76,7 @@ public class ArbitraryDataCleanupManager extends Thread { // Don't attempt to make decisions if we haven't synced our time yet continue; } - - List peers = Network.getInstance().getHandshakedPeers(); - - // Disregard peers that have "misbehaved" recently - peers.removeIf(Controller.hasMisbehaved); - - // Don't fetch data if we don't have enough up-to-date peers - if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) { - continue; - } - + // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, limit, offset, true); From 4ba72f7eeb398685063b92510dc9d2f584bb6851 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 19:27:42 +0100 Subject: [PATCH 199/505] Regularly clean up old and unused files/folders in the temp directory Also added code to purge built resource caches, but it is currently disabled. This will become more useful once we implement local storage limits. --- .../ArbitraryDataCleanupManager.java | 88 +++++++++++++++++-- .../utils/ArbitraryTransactionUtils.java | 34 +++---- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index a3500766..a468942d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -3,10 +3,7 @@ package org.qortal.controller.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; -import org.qortal.controller.Controller; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.network.Network; -import org.qortal.network.Peer; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -14,8 +11,13 @@ import org.qortal.settings.Settings; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -35,12 +37,20 @@ public class ArbitraryDataCleanupManager extends Thread { * rebuilding them. The main purpose of this is to avoid deleting files that are currently * being used by other parts of the system. */ - private static long STALE_FILE_TIMEOUT = 60*60*1000; // 1 hour + private static long STALE_FILE_TIMEOUT = 60*60*1000L; // 1 hour + + /** + * The amount of time that must pass before a built resource is cleaned up. This should be + * considerably longer than STALE_FILE_TIMEOUT because building a resource is costly. Longer + * term we could consider tracking when each resource is requested, and only delete those + * that haven't been requested for a large amount of time. We could also consider only purging + * built resources when the disk space is getting low. + */ + private static long PURGE_BUILT_RESOURCES_TIMEOUT = 30*24*60*60*1000L; // 30 days /* TODO: - - Delete old files from _temp - Delete old files not associated with transactions */ @@ -72,11 +82,17 @@ public class ArbitraryDataCleanupManager extends Thread { while (!isStopping) { Thread.sleep(30000); - if (NTP.getTime() == null) { + Long now = NTP.getTime(); + if (now == null) { // Don't attempt to make decisions if we haven't synced our time yet continue; } - + + // Periodically delete any unnecessary files from the temp directory + if (offset == 0 || offset % (limit * 10) == 0) { + this.cleanupTempDirectory(now); + } + // Any arbitrary transactions we want to fetch data for? try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, limit, offset, true); @@ -86,7 +102,7 @@ public class ArbitraryDataCleanupManager extends Thread { continue; } offset += limit; - Long now = NTP.getTime(); + now = NTP.getTime(); // Loop through the signatures in this batch for (int i=0; i cleanupAfter) { + return false; } - if (timeSinceModified < cleanupAfter) { - return true; + if (timeSinceModified > cleanupAfter) { + return false; } } catch (IOException e) { - // Can't read file attributes, so assume it's not recent + // Can't read file attributes, so assume it's recent so that we don't delete something accidentally } - return false; + return true; + } + + public static boolean isFileHashRecent(byte[] hash, long now, long cleanupAfter) { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); + if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { + // No hash, or file doesn't exist, so it's not recent + return false; + } + + Path filePath = arbitraryDataFile.getFilePath(); + return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter); } public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) { From 095083bcfba1a5b6a3981cbd458504966b4b9cbf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 19:54:09 +0100 Subject: [PATCH 200/505] Use lowercase directory names for consistency --- .../java/org/qortal/arbitrary/ArbitraryDataFile.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index d838e56b..58b1bf5b 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -169,19 +169,18 @@ public class ArbitraryDataFile { if (hash58 == null) { return null; } - String hash58First2Chars = hash58.substring(0, 2); - String hash58Next2Chars = hash58.substring(2, 4); - String outputDirectory = Settings.getInstance().getDataPath() + File.separator + hash58First2Chars + File.separator + hash58Next2Chars; - Path outputDirectoryPath = Paths.get(outputDirectory); + String hash58First2Chars = hash58.substring(0, 2).toLowerCase(); + String hash58Next2Chars = hash58.substring(2, 4).toLowerCase(); + Path directory = Paths.get(Settings.getInstance().getDataPath(), hash58First2Chars, hash58Next2Chars); if (createDirectories) { try { - Files.createDirectories(outputDirectoryPath); + Files.createDirectories(directory); } catch (IOException e) { throw new IllegalStateException("Unable to create data subdirectory"); } } - return Paths.get(outputDirectory, hash58); + return Paths.get(directory.toString(), hash58); } public ValidationResult isValid() { From b2c0915a712e4b5e21543b5634ad14e171112c31 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 20:17:02 +0100 Subject: [PATCH 201/505] Removed accidentally duplicated code This was causing two instances of the build manager to run. --- .../controller/arbitrary/ArbitraryDataCleanupManager.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index a468942d..8295d23d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -69,11 +69,6 @@ public class ArbitraryDataCleanupManager extends Thread { public void run() { Thread.currentThread().setName("Arbitrary Data Cleanup Manager"); - // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) - // This can be expanded to have multiple threads processing the build queue when needed - ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); - arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager()); - // Paginate queries when fetching arbitrary transactions final int limit = 100; int offset = 0; From fd795b4361fd49bff88ed0f2e2faf140dd626e65 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Aug 2021 20:19:51 +0100 Subject: [PATCH 202/505] Don't attempt to cleanup the filesystem if a build is in progress. This isn't essential but it helps to reduce unnecessary load and processing which would be better spent on building. --- .../org/qortal/arbitrary/ArbitraryDataReader.java | 3 ++- .../arbitrary/ArbitraryDataCleanupManager.java | 5 +++++ .../controller/arbitrary/ArbitraryDataManager.java | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index c2dc5289..51abe97f 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -129,12 +129,13 @@ public class ArbitraryDataReader { } private void preExecute() { + ArbitraryDataManager.getInstance().setBuildInProgress(true); this.createWorkingDirectory(); this.createUncompressedDirectory(); } private void postExecute() { - + ArbitraryDataManager.getInstance().setBuildInProgress(false); } private void createWorkingDirectory() { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 8295d23d..7e1a3256 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -106,6 +106,11 @@ public class ArbitraryDataCleanupManager extends Thread { continue; } + // Don't interfere with the filesystem whilst a build is in progress + if (ArbitraryDataManager.getInstance().getBuildInProgress()) { + Thread.sleep(5000); + } + // Fetch the transaction data ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 337bbfa7..c54ac43a 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -36,6 +36,8 @@ public class ArbitraryDataManager extends Thread { private static ArbitraryDataManager instance; + private boolean buildInProgress = false; + private volatile boolean isStopping = false; @@ -411,6 +413,15 @@ public class ArbitraryDataManager extends Thread { } + public void setBuildInProgress(boolean buildInProgress) { + this.buildInProgress = buildInProgress; + } + + public boolean getBuildInProgress() { + return this.buildInProgress; + } + + // Network handlers public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { From 5bed5fb8fd92c8b07e4d78b2b470f73914d905ed Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 21 Aug 2021 08:34:46 +0100 Subject: [PATCH 203/505] Removed unnecessary code in ArbitraryResource.uploadFileAtPath() This is now handled by ArbitraryDataWriter instead --- .../api/resource/ArbitraryResource.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index b85d6405..41a8d17f 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -298,25 +298,6 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); } - arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile(); - if (arbitraryDataFile == null) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - - ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid(); - if (validationResult != ArbitraryDataFile.ValidationResult.OK) { - LOGGER.error("Invalid file: {}", validationResult); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - LOGGER.info("Whole file digest: {}", arbitraryDataFile.digest58()); - - int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); - if (chunkCount == 0) { - LOGGER.error("No chunks created"); - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } - LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); - String digest58 = arbitraryDataFile.digest58(); if (digest58 == null) { LOGGER.error("Unable to calculate digest"); From 7397b9fa874b85c38ced77df3bab6a433050cef6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 21 Aug 2021 08:35:12 +0100 Subject: [PATCH 204/505] Added more detail to exception message. --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 58b1bf5b..d997d446 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -161,7 +161,7 @@ public class ArbitraryDataFile { try { return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { - throw new IllegalStateException("Unable to copy file to data directory"); + throw new IllegalStateException(String.format("Unable to copy file %s to data directory %s", sourcePath, destPath)); } } From 52ab19dec67ca9d6bf667a355ba36115ee43895b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 1 Sep 2021 09:11:03 +0100 Subject: [PATCH 205/505] Added method and name to the /site/upload endpoint params. --- .../org/qortal/api/resource/WebsiteResource.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 2a009340..5d85a7c6 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -66,9 +66,10 @@ public class WebsiteResource { @Context ServletContext context; @POST - @Path("/upload/creator/{publickey}") + @Path("/upload/{method}/{publickey}/{name}") @Operation( summary = "Build raw, unsigned, ARBITRARY transaction, based on a user-supplied path to a static website", + description = "Method should be PUT to create a new base layer, or PATCH to add a delta layer. This will soon be automatic. Name is not currently validated against the main chain for ownership by the public key, but it will be before launch. The request body should contain a local path to the static site, and it is essential that this folder contains an index.html or index.htm (this will be validated later).", requestBody = @RequestBody( required = true, content = @Content( @@ -90,7 +91,7 @@ public class WebsiteResource { ) } ) - public String uploadWebsite(@PathParam("publickey") String creatorPublicKeyBase58, String path) { + public String uploadWebsite(@PathParam("method") String methodString, @PathParam("publickey") String publicKey58, @PathParam("name") String name, String path) { Security.checkApiCallAllowed(request); // It's too dangerous to allow user-supplied filenames in weaker security contexts @@ -101,10 +102,10 @@ public class WebsiteResource { ArbitraryDataFile arbitraryDataFile = null; try (final Repository repository = RepositoryManager.getRepository()) { - if (creatorPublicKeyBase58 == null || path == null) { + if (publicKey58 == null || path == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } - byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); + byte[] creatorPublicKey = Base58.decode(publicKey58); final String creatorAddress = Crypto.toAddress(creatorPublicKey); byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress); if (lastReference == null) { @@ -116,8 +117,7 @@ public class WebsiteResource { random.nextBytes(lastReference); } - String name = "CalDescentTest1"; // TODO: dynamic - ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; // TODO: dynamic + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.valueOf(methodString); ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; From 9b4d832d17a8b96f395daa6cc98944a68b54f4fa Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 1 Sep 2021 09:11:50 +0100 Subject: [PATCH 206/505] Default minPeerVersion set to 0.1.0. TODO: revert this if ever merged into the main repo. --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 49a8a42c..230ead8a 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -139,7 +139,7 @@ public class Settings { private int maxRetries = 2; /** Minimum peer version number required in order to sync with them */ - private String minPeerVersion = "1.5.0"; + private String minPeerVersion = "0.1.0"; /** Whether to allow connections with peers below minPeerVersion * If true, we won't sync with them but they can still sync with us, and will show in the peers list * If false, sync will be blocked both ways, and they will not appear in the peers list */ From aa4f77d4deb5dee084254c8f516f13c9db2b3c1b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 15 Oct 2021 09:13:15 +0100 Subject: [PATCH 207/505] Fixed merge issues relating to database updates. Existing data nodes will need to delete their db folder and resync. --- .../hsqldb/HSQLDBDatabaseUpdates.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 535cac6f..29a1fbc1 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -828,29 +828,6 @@ public class HSQLDBDatabaseUpdates { + "signature Signature, nonce INT NOT NULL, presence_type INT NOT NULL, " + "timestamp_signature Signature NOT NULL, " + TRANSACTION_KEYS + ")"); break; - case 34: - // ARBITRARY transaction updates for off-chain data storage - stmt.execute("CREATE TYPE ArbitraryDataHashes AS VARBINARY(8000)"); - // We may want to use a nonce rather than a transaction fee on the data chain - stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0"); - // We need to know the total size of the data file(s) associated with each transaction - stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0"); - // Larger data files need to be split into chunks, for easier transmission and greater decentralization - stmt.execute("ALTER TABLE ArbitraryTransactions ADD chunk_hashes ArbitraryDataHashes"); - // For finding data files by hash - stmt.execute("CREATE INDEX ArbitraryDataIndex ON ArbitraryTransactions (is_data_raw, data)"); - break; - - case 35: - // We need the ability for arbitrary transactions to be associated with a name - stmt.execute("ALTER TABLE ArbitraryTransactions ADD name RegisteredName"); - // A "method" specifies how the data should be applied (e.g. PUT or PATCH) - stmt.execute("ALTER TABLE ArbitraryTransactions ADD update_method INTEGER NOT NULL DEFAULT 0"); - // For public data, the AES shared secret needs to be available. This is more for data obfuscation as apposed to actual encryption. - stmt.execute("ALTER TABLE ArbitraryTransactions ADD secret VARBINARY(32)"); - // We want to support compressed and uncompressed data, as well as different compression algorithms - stmt.execute("ALTER TABLE ArbitraryTransactions ADD compression INTEGER NOT NULL DEFAULT 0"); - break; case 34: { // AT sleep-until-message support @@ -921,6 +898,30 @@ public class HSQLDBDatabaseUpdates { stmt.execute("SET TABLE BlockArchive NEW SPACE"); break; + case 37: + // ARBITRARY transaction updates for off-chain data storage + stmt.execute("CREATE TYPE ArbitraryDataHashes AS VARBINARY(8000)"); + // We may want to use a nonce rather than a transaction fee on the data chain + stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0"); + // We need to know the total size of the data file(s) associated with each transaction + stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0"); + // Larger data files need to be split into chunks, for easier transmission and greater decentralization + stmt.execute("ALTER TABLE ArbitraryTransactions ADD chunk_hashes ArbitraryDataHashes"); + // For finding data files by hash + stmt.execute("CREATE INDEX ArbitraryDataIndex ON ArbitraryTransactions (is_data_raw, data)"); + break; + + case 38: + // We need the ability for arbitrary transactions to be associated with a name + stmt.execute("ALTER TABLE ArbitraryTransactions ADD name RegisteredName"); + // A "method" specifies how the data should be applied (e.g. PUT or PATCH) + stmt.execute("ALTER TABLE ArbitraryTransactions ADD update_method INTEGER NOT NULL DEFAULT 0"); + // For public data, the AES shared secret needs to be available. This is more for data obfuscation as apposed to actual encryption. + stmt.execute("ALTER TABLE ArbitraryTransactions ADD secret VARBINARY(32)"); + // We want to support compressed and uncompressed data, as well as different compression algorithms + stmt.execute("ALTER TABLE ArbitraryTransactions ADD compression INTEGER NOT NULL DEFAULT 0"); + break; + default: // nothing to do return false; From c8d5ac9248dbf7906dbfb7807d2c3d80c36de593 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 15 Oct 2021 11:32:08 +0100 Subject: [PATCH 208/505] Fixed bug in ArbitraryTransactionTransformer.getDataLength() when missing a name. --- .../transform/transaction/ArbitraryTransactionTransformer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java index 1bcb783b..e3827978 100644 --- a/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java @@ -166,7 +166,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { public static int getDataLength(TransactionData transactionData) throws TransformationException { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - int nameLength = Utf8.encodedLength(arbitraryTransactionData.getName()); + int nameLength = (arbitraryTransactionData.getName() != null) ? Utf8.encodedLength(arbitraryTransactionData.getName()) : 0; int secretLength = (arbitraryTransactionData.getSecret() != null) ? arbitraryTransactionData.getSecret().length : 0; int dataLength = (arbitraryTransactionData.getData() != null) ? arbitraryTransactionData.getData().length : 0; int chunkHashesLength = (arbitraryTransactionData.getChunkHashes() != null) ? arbitraryTransactionData.getChunkHashes().length : 0; From f0e13fa492bbbe896329711e9561a393ea3cb727 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 15 Oct 2021 13:58:27 +0100 Subject: [PATCH 209/505] Arbitrary transaction names are now case insensitive --- .../arbitrary/ArbitraryDataBuildQueueItem.java | 2 +- .../arbitrary/ArbitraryDataBuildManager.java | 5 ++++- .../controller/arbitrary/ArbitraryDataManager.java | 13 ++++++++++++- .../hsqldb/HSQLDBArbitraryRepository.java | 4 ++-- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index 040ac197..f89fafda 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -24,7 +24,7 @@ public class ArbitraryDataBuildQueueItem { public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { - this.resourceId = resourceId; + this.resourceId = resourceId.toLowerCase(); this.resourceIdType = resourceIdType; this.service = service; this.creationTimestamp = NTP.getTime(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index 63c33a7b..96f24f08 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -84,6 +84,9 @@ public class ArbitraryDataBuildManager implements Runnable { } private void removeFromQueue(String resourceId) { - ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId); + if (resourceId == null) { + return; + } + ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId.toLowerCase()); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index c54ac43a..d1f625d3 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -272,6 +272,10 @@ public class ArbitraryDataManager extends Thread { // Arbitrary data resource cache public boolean isResourceCached(String resourceId) { + if (resourceId == null) { + return false; + } + resourceId = resourceId.toLowerCase(); // We don't have an entry for this resource ID, it is not cached if (this.arbitraryDataCachedResources == null) { @@ -297,6 +301,11 @@ public class ArbitraryDataManager extends Thread { } public void addResourceToCache(String resourceId) { + if (resourceId == null) { + return; + } + resourceId = resourceId.toLowerCase(); + // Just in case if (this.arbitraryDataCachedResources == null) { this.arbitraryDataCachedResources = new HashMap<>(); @@ -319,6 +328,7 @@ public class ArbitraryDataManager extends Thread { if (resourceId == null) { return false; } + resourceId = resourceId.toLowerCase(); if (this.arbitraryDataBuildQueue == null) { return false; @@ -398,6 +408,7 @@ public class ArbitraryDataManager extends Thread { if (resourceId == null) { return false; } + resourceId = resourceId.toLowerCase(); if (this.arbitraryDataFailedBuilds == null) { return false; @@ -537,7 +548,7 @@ public class ArbitraryDataManager extends Thread { // so that it is rebuilt the next time we serve it if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(arbitraryTransactionData.getChunkHashes())) { if (arbitraryTransactionData.getName() != null) { - String resourceId = arbitraryTransactionData.getName(); + String resourceId = arbitraryTransactionData.getName().toLowerCase(); if (this.arbitraryDataCachedResources.containsKey(resourceId)) { this.arbitraryDataCachedResources.remove(resourceId); } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index e7ee0d50..0e94314c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -229,7 +229,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + "name, update_method, secret, compression FROM ArbitraryTransactions " + "JOIN Transactions USING (signature) " + - "WHERE name = ? AND service = ?"); + "WHERE lower(name) = ? AND service = ?"); if (method != null) { sql.append(" AND update_method = "); @@ -238,7 +238,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { sql.append("ORDER BY created_when DESC LIMIT 1"); - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name, service.value)) { + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value)) { if (resultSet == null) return null; From d6d564c02734c9fa4651076285ef45f77fac61e1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 17 Oct 2021 16:55:32 +0100 Subject: [PATCH 210/505] Fixed refresh interval of loading screen --- src/main/resources/loading/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html index eaf7ee1b..f13c3202 100644 --- a/src/main/resources/loading/index.html +++ b/src/main/resources/loading/index.html @@ -2,7 +2,7 @@ Loading... + + Loading... + + + - -

    Loading... please wait...

    -

    This page will refresh automatically when the content becomes available

    -

    (We can show a Qortal branded loading screen here)

    + + + + + + + + +
    +

    Loading... please wait...

    +

    This page will refresh automatically when the content becomes available

    +

    (We can show a Qortal branded loading screen here)

    +
    + From d0000c61310d2f23aba17fe49edc19c8056fd1d8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 20 Nov 2021 18:52:03 +0000 Subject: [PATCH 355/505] If "build=true" is specified in query string of GET /resource/status/{service}/{name}, build the resource before returning the status --- .../api/resource/ArbitraryResource.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index de367a3c..96abfef8 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -119,6 +119,7 @@ public class ArbitraryResource { @Path("/resource/status/{service}/{name}") @Operation( summary = "Get status of arbitrary resource with supplied service and name", + description = "If build is set to true, the resource will be built synchronously before returning the status.", responses = { @ApiResponse( content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceSummary.class)) @@ -126,16 +127,17 @@ public class ArbitraryResource { } ) public ArbitraryResourceSummary getDefaultResourceStatus(@PathParam("service") Service service, - @PathParam("name") String name) { + @PathParam("name") String name, + @QueryParam("build") Boolean build) { - ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, null); - return resource.getSummary(); + return this.getSummary(service, name, null, build); } @GET @Path("/resource/status/{service}/{name}/{identifier}") @Operation( summary = "Get status of arbitrary resource with supplied service, name and identifier", + description = "If build is set to true, the resource will be built synchronously before returning the status.", responses = { @ApiResponse( content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceSummary.class)) @@ -144,10 +146,10 @@ public class ArbitraryResource { ) public ArbitraryResourceSummary getResourceStatus(@PathParam("service") Service service, @PathParam("name") String name, - @PathParam("identifier") String identifier) { + @PathParam("identifier") String identifier, + @QueryParam("build") Boolean build) { - ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.getSummary(); + return this.getSummary(service, name, identifier, build); } @@ -559,4 +561,20 @@ public class ArbitraryResource { } } + + private ArbitraryResourceSummary getSummary(Service service, String name, String identifier, Boolean build) { + + // If "build=true" has been specified in the query string, build the resource before returning its status + if (build != null && build == true) { + ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null); + try { + reader.loadSynchronously(false); + } catch (DataException | IOException | MissingDataException e) { + // No need to handle exception, as it will be reflected in the status + } + } + + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); + return resource.getSummary(); + } } From b4f31050353c1701d8114a8cc5804e558c9b009a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 21 Nov 2021 14:57:26 +0000 Subject: [PATCH 356/505] Added /render/authorize/{service}/{resourceId}* APIs These allow the UI to pre-authorize a resource and therefore avoid having to pass a sensitive API key to a website or app. --- .../qortal/api/resource/RenderResource.java | 43 +++++++++++++--- .../arbitrary/ArbitraryDataResource.java | 16 ++++++ .../arbitrary/ArbitraryDataRenderManager.java | 50 +++++++++++++++++++ 3 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java diff --git a/src/main/java/org/qortal/api/resource/RenderResource.java b/src/main/java/org/qortal/api/resource/RenderResource.java index ae8bade9..5b2b1333 100644 --- a/src/main/java/org/qortal/api/resource/RenderResource.java +++ b/src/main/java/org/qortal/api/resource/RenderResource.java @@ -25,6 +25,7 @@ import org.qortal.api.Security; import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.*; import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.controller.arbitrary.ArbitraryDataRenderManager; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.repository.DataException; import org.qortal.settings.Settings; @@ -94,11 +95,34 @@ public class RenderResource { return "Unable to generate preview URL"; } + @POST + @Path("authorize/{service}/{resourceId}") + @SecurityRequirement(name = "apiKey") + public boolean authorizeResource(@PathParam("service") Service service, + @PathParam("resourceId") String resourceId) { + Security.checkApiCallAllowed(request); + ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, null); + ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource); + return true; + } + + @POST + @Path("authorize/{service}/{resourceId}/{identifier}") + @SecurityRequirement(name = "apiKey") + public boolean authorizeResource(@PathParam("service") Service service, + @PathParam("resourceId") String resourceId, + @PathParam("identifier") String identifier) { + Security.checkApiCallAllowed(request); + ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier); + ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource); + return true; + } + @GET @Path("/signature/{signature}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) { - Security.checkApiCallAllowed(request); + requirePriorAuthorization(signature, Service.WEBSITE, null); return this.get(signature, ResourceIdType.SIGNATURE, null, "/", null, "/render/signature", true, true); } @@ -106,7 +130,7 @@ public class RenderResource { @Path("/signature/{signature}/{path:.*}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) { - Security.checkApiCallAllowed(request); + requirePriorAuthorization(signature, Service.WEBSITE, null); return this.get(signature, ResourceIdType.SIGNATURE, null, inPath,null, "/render/signature", true, true); } @@ -114,7 +138,7 @@ public class RenderResource { @Path("/hash/{hash}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) { - Security.checkApiCallAllowed(request); + requirePriorAuthorization(hash58, Service.WEBSITE, null); return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, "/", secret58, "/render/hash", true, false); } @@ -123,7 +147,7 @@ public class RenderResource { @SecurityRequirement(name = "apiKey") public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath, @QueryParam("secret") String secret58) { - Security.checkApiCallAllowed(request); + requirePriorAuthorization(hash58, Service.WEBSITE, null); return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, inPath, secret58, "/render/hash", true, false); } @@ -133,7 +157,7 @@ public class RenderResource { public HttpServletResponse getPathByName(@PathParam("service") Service service, @PathParam("name") String name, @PathParam("path") String inPath) { - Security.checkApiCallAllowed(request); + requirePriorAuthorization(name, service, null); String prefix = String.format("/render/%s", service); return this.get(name, ResourceIdType.NAME, service, inPath, null, prefix, true, true); } @@ -143,7 +167,7 @@ public class RenderResource { @SecurityRequirement(name = "apiKey") public HttpServletResponse getIndexByName(@PathParam("service") Service service, @PathParam("name") String name) { - Security.checkApiCallAllowed(request); + requirePriorAuthorization(name, service, null); String prefix = String.format("/render/%s", service); return this.get(name, ResourceIdType.NAME, service, "/", null, prefix, true, true); } @@ -176,4 +200,11 @@ public class RenderResource { return renderer.render(); } + private void requirePriorAuthorization(String resourceId, Service service, String identifier) { + ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier); + if (!ArbitraryDataRenderManager.getInstance().isAuthorized(resource)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Call /render/authorize first"); + } + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 05fc4933..3974939a 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -113,7 +113,23 @@ public class ArbitraryDataResource { } } + private String resourceIdString() { + return resourceId != null ? resourceId : ""; + } + + private String resourceIdTypeString() { + return resourceIdType != null ? resourceIdType.toString() : ""; + } + + private String serviceString() { + return service != null ? service.toString() : ""; + } + private String identifierString() { return identifier != null ? identifier : ""; } + + public String toString() { + return String.format("%s-%s-%s-%s", resourceIdString(), resourceIdTypeString(), serviceString(), identifierString()); + } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java new file mode 100644 index 00000000..693cbf82 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java @@ -0,0 +1,50 @@ +package org.qortal.controller.arbitrary; + +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataResource; +import org.qortal.arbitrary.misc.Service; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class ArbitraryDataRenderManager { + + private static ArbitraryDataRenderManager instance; + + /** + * List to keep track of authorized resources for rendering. + */ + private List authorizedResources = Collections.synchronizedList(new ArrayList<>()); + + + public ArbitraryDataRenderManager() { + + } + + public static ArbitraryDataRenderManager getInstance() { + if (instance == null) + instance = new ArbitraryDataRenderManager(); + + return instance; + } + + public boolean isAuthorized(ArbitraryDataResource resource) { + for (ArbitraryDataResource authorizedResource : this.authorizedResources) { + if (authorizedResource != null && resource != null) { + if (Objects.equals(authorizedResource.toString(), resource.toString())) { + return true; + } + } + } + return false; + } + + public void addToAuthorizedResources(ArbitraryDataResource resource) { + if (!this.isAuthorized(resource)) { + this.authorizedResources.add(resource); + } + } + +} From c588786a06fe767a3d356953a949ad9d25f97d5b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 21 Nov 2021 19:12:01 +0000 Subject: [PATCH 357/505] Added /base64 variation of POST /arbitrary/* APIs This can be used to upload base64 encoded file data directly from the UI. Using base64 because base58 is unusably slow --- .../api/resource/ArbitraryResource.java | 90 +++++++++++++++++-- 1 file changed, 82 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 96abfef8..df2ff09d 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -29,6 +29,7 @@ import javax.ws.rs.core.MediaType; import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.bouncycastle.util.encoders.Base64; import org.qortal.api.*; import org.qortal.api.model.ArbitraryResourceSummary; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; @@ -348,7 +349,39 @@ public class ArbitraryResource { String path) { Security.checkApiCallAllowed(request); - return this.upload(Service.valueOf(serviceString), name, null, path, null); + return this.upload(Service.valueOf(serviceString), name, null, path, null, null); + } + + @POST + @Path("/{service}/{name}/base64") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on user-supplied base64 encoded data", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_OCTET_STREAM, + schema = @Schema(type = "string", format = "byte") + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postBase64EncodedData(@PathParam("service") String serviceString, + @PathParam("name") String name, + String base64) { + Security.checkApiCallAllowed(request); + + return this.upload(Service.valueOf(serviceString), name, null, null, null, base64); } @POST @@ -378,11 +411,11 @@ public class ArbitraryResource { ) @SecurityRequirement(name = "apiKey") public String postString(@PathParam("service") String serviceString, - @PathParam("name") String name, - String string) { + @PathParam("name") String name, + String string) { Security.checkApiCallAllowed(request); - return this.upload(Service.valueOf(serviceString), name, null, null, string); + return this.upload(Service.valueOf(serviceString), name, null, null, string, null); } @@ -418,7 +451,7 @@ public class ArbitraryResource { String path) { Security.checkApiCallAllowed(request); - return this.upload(Service.valueOf(serviceString), name, identifier, path, null); + return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null); } @POST @@ -448,15 +481,48 @@ public class ArbitraryResource { ) @SecurityRequirement(name = "apiKey") public String postString(@PathParam("service") String serviceString, - @PathParam("name") String name, + @PathParam("name") String name, @PathParam("identifier") String identifier, String string) { Security.checkApiCallAllowed(request); - return this.upload(Service.valueOf(serviceString), name, identifier, null, string); + return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null); } - private String upload(Service service, String name, String identifier, String path, String string) { + @POST + @Path("/{service}/{name}/{identifier}/base64") + @Operation( + summary = "Build raw, unsigned, ARBITRARY transaction, based on user supplied base64 encoded data", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_OCTET_STREAM, + schema = @Schema(type = "string", format = "byte") + ) + ), + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + } + ) + @SecurityRequirement(name = "apiKey") + public String postBase64EncodedData(@PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + String base64) { + Security.checkApiCallAllowed(request); + + return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64); + } + + private String upload(Service service, String name, String identifier, String path, String string, String base64) { // Fetch public key from registered name try (final Repository repository = RepositoryManager.getRepository()) { NameData nameData = repository.getNameRepository().fromName(name); @@ -483,6 +549,13 @@ public class ArbitraryResource { writer.close(); path = tempFile.toPath().toString(); } + // ... or base64 encoded raw data + else if (base64 != null) { + File tempFile = File.createTempFile("qortal-", ".tmp"); + tempFile.deleteOnExit(); + Files.write(tempFile.toPath(), Base64.decode(base64)); + path = tempFile.toPath().toString(); + } else { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing path or data string"); } @@ -498,6 +571,7 @@ public class ArbitraryResource { return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData)); } catch (DataException | TransformationException | IllegalStateException e) { + LOGGER.info("Unable to upload data: {}", e.getMessage()); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); } From 02eab89d82ed7a2a531b64ddc2937bc898daf363 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 21 Nov 2021 19:24:20 +0000 Subject: [PATCH 358/505] Fixed bug when trying to delete a file instead of a directory. --- .../controller/arbitrary/ArbitraryDataCleanupManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 095f7eec..1636703f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -222,7 +222,7 @@ public class ArbitraryDataCleanupManager extends Thread { } // If the directory is empty, we still need to delete its parent folder - if (contentsCount == 0 && tempDir.toFile().exists()) { + if (contentsCount == 0 && tempDir.toFile().isDirectory() && tempDir.toFile().exists()) { try { LOGGER.info("Parent directory {} is empty, so deleting it", tempDir); FilesystemUtils.safeDeleteDirectory(tempDir, false); From 908f80a15d094e250338171ee7ec77a3204aa7b7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 22 Nov 2021 08:43:07 +0000 Subject: [PATCH 359/505] Fixed bug when checking if all files exist locally in /arbitrary/status --- src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 3974939a..e4fb9e38 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -80,7 +80,8 @@ public class ArbitraryDataResource { List transactionDataList = new ArrayList<>(this.transactions); for (ArbitraryTransactionData transactionData : transactionDataList) { - if (!ArbitraryTransactionUtils.allChunksExist(transactionData)) { + if (!ArbitraryTransactionUtils.completeFileExists(transactionData) || + !ArbitraryTransactionUtils.allChunksExist(transactionData)) { return false; } } From ec48ebcd797f4958c8885457d390a851eb68d3b4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 23 Nov 2021 09:14:44 +0000 Subject: [PATCH 360/505] Improved resource statuses --- .../api/model/ArbitraryResourceSummary.java | 1 + .../qortal/arbitrary/ArbitraryDataReader.java | 2 +- .../arbitrary/ArbitraryDataResource.java | 65 +++++++++++++++++-- .../arbitrary/ArbitraryDataManager.java | 39 +++++++++-- 4 files changed, 96 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java b/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java index d70eb6fb..f4c10bb2 100644 --- a/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java +++ b/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java @@ -7,6 +7,7 @@ import javax.xml.bind.annotation.XmlAccessorType; public class ArbitraryResourceSummary { public enum ArbitraryResourceStatus { + DOWNLOADING, DOWNLOADED, BUILDING, READY, diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 5c487800..54d014ac 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -327,7 +327,7 @@ public class ArbitraryDataReader { if (requested) { message = String.format("Requested missing data for file %s", arbitraryDataFile); } else { - message = String.format("Unable to reissue request for missing file %s due to rate limit. Please try again later.", arbitraryDataFile); + message = String.format("Unable to reissue request for missing file %s for signature %s due to rate limit. Please try again later.", arbitraryDataFile, Base58.encode(transactionData.getSignature())); } } else { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index e4fb9e38..20677d7a 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -5,12 +5,14 @@ import org.qortal.api.model.ArbitraryResourceSummary.ArbitraryResourceStatus; import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.list.ResourceListManager; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.NTP; import java.util.ArrayList; import java.util.List; @@ -39,6 +41,12 @@ public class ArbitraryDataResource { return new ArbitraryResourceSummary(ArbitraryResourceStatus.UNSUPPORTED); } + // Check if the name is blacklisted + if (ResourceListManager.getInstance() + .listContains("blacklist", "names", this.resourceId, false)) { + return new ArbitraryResourceSummary(ArbitraryResourceStatus.BLACKLISTED); + } + // Firstly check the cache to see if it's already built ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( resourceId, resourceIdType, service, identifier); @@ -58,14 +66,11 @@ public class ArbitraryDataResource { return new ArbitraryResourceSummary(ArbitraryResourceStatus.BUILD_FAILED); } - // Check if the name is blacklisted - if (ResourceListManager.getInstance() - .listContains("blacklist", "names", this.resourceId, false)) { - return new ArbitraryResourceSummary(ArbitraryResourceStatus.BLACKLISTED); - } - // Check if we have all data locally for this resource if (!this.allFilesDownloaded()) { + if (this.madeRecentRequest()) { + return new ArbitraryResourceSummary(ArbitraryResourceStatus.DOWNLOADING); + } return new ArbitraryResourceSummary(ArbitraryResourceStatus.MISSING_DATA); } @@ -92,7 +97,55 @@ public class ArbitraryDataResource { } } + private boolean isRateLimited() { + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + if (ArbitraryDataManager.getInstance().isSignatureRateLimited(transactionData.getSignature())) { + return true; + } + } + return true; + + } catch (DataException e) { + return false; + } + } + + private boolean madeRecentRequest() { + try { + this.fetchTransactions(); + Long now = NTP.getTime(); + if (now == null) { + return false; + } + + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature()); + if (now - lastRequestTime < 30 * 1000L) { + return true; + } + } + return false; + + } catch (DataException e) { + return false; + } + } + + + private void fetchTransactions() throws DataException { + if (this.transactions != null && !this.transactions.isEmpty()) { + // Already fetched + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { // Get the most recent PUT diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 2b0e9936..eecfeb98 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -321,14 +321,22 @@ public class ArbitraryDataManager extends Thread { } long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp; - if (timeSinceLastAttempt > 5 * 60 * 1000L) { - // We haven't tried for at least 5 minutes + if (timeSinceLastAttempt > 10 * 1000L) { + // We haven't tried for at least 10 seconds if (directPeerRequestCount < 5) { // We've made less than 5 total attempts return true; } } + if (timeSinceLastAttempt > 5 * 60 * 1000L) { + // We haven't tried for at least 5 minutes + if (directPeerRequestCount < 10) { + // We've made less than 10 total attempts + return true; + } + } + if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) { // We haven't tried for at least 24 hours return true; @@ -337,6 +345,29 @@ public class ArbitraryDataManager extends Thread { return false; } + public boolean isSignatureRateLimited(byte[] signature) { + String signature58 = Base58.encode(signature); + return !this.shouldMakeFileListRequestForSignature(signature58) + && !this.shouldMakeDirectFileRequestsForSignature(signature58); + } + + public long lastRequestForSignature(byte[] signature) { + String signature58 = Base58.encode(signature); + Triple request = arbitraryDataSignatureRequests.get(signature58); + + if (request == null) { + // Not attempted yet + return 0; + } + + // Extract the components + Long lastAttemptTimestamp = request.getC(); + if (lastAttemptTimestamp != null) { + return lastAttemptTimestamp; + } + return 0; + } + private void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) { Triple request = arbitraryDataSignatureRequests.get(signature58); Long now = NTP.getTime(); @@ -493,8 +524,8 @@ public class ArbitraryDataManager extends Thread { return; } final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); - arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp); + arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp); } public boolean isResourceCached(String resourceId) { From f76a61876888bf2c31d2f1311f5252d3afdab379 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 23 Nov 2021 20:53:09 +0000 Subject: [PATCH 361/505] Display the latest status on the loading screen, updated via API calls on a timer --- src/main/resources/loading/index.html | 60 ++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html index 1d1be4bf..c5d2babe 100644 --- a/src/main/resources/loading/index.html +++ b/src/main/resources/loading/index.html @@ -27,7 +27,63 @@ } - + + @@ -181,7 +237,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI

    Loading... please wait...

    This page will refresh automatically when the content becomes available

    -

    (We can show a Qortal branded loading screen here)

    +

    Loading...

    From 9ef75ebcde745f47992f84b455057811139e640c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 23 Nov 2021 21:15:45 +0000 Subject: [PATCH 362/505] Improved styling of loading panel --- src/main/resources/loading/index.html | 32 ++++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html index c5d2babe..8ac5c39f 100644 --- a/src/main/resources/loading/index.html +++ b/src/main/resources/loading/index.html @@ -16,14 +16,25 @@ text-align: center; color: black; } - h1 { - margin-top: 50px; - } - #panel { + #panel-outer { position: absolute; width: 100%; text-align: center; z-index: 1000; + top: 45%; + -ms-transform: translateY(-50%); + transform: translateY(-50%); + } + #panel { + text-align: center; + background: white; + width: 350px; + margin: auto; + padding: 25px; + border-radius: 30px; + } + #status { + color: #03a9f4; } @@ -50,10 +61,12 @@ if (json.status == "UNSUPPORTED") { textStatus = "Unsupported request"; + document.getElementById("status").style.color = "red"; } else if (json.status == "BLACKLISTED") { textStatus = name + " is blacklisted so content cannot be served"; retryInterval = 5000; + document.getElementById("status").style.color = "red"; } else if (json.status == "READY") { textStatus = "Ready"; @@ -73,6 +86,7 @@ else if (json.status == "MISSING_DATA") { textStatus = "Unable to locate files. Please try again later."; retryInterval = 10000; + document.getElementById("status").style.color = "red"; } else if (json.status == "DOWNLOADED") { textStatus = "Files downloaded"; @@ -234,10 +248,12 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -
    -

    Loading... please wait...

    -

    This page will refresh automatically when the content becomes available

    -

    Loading...

    +
    +
    +

    Loading

    +

    This page will refresh automatically when the content becomes available

    +

    Loading...

    +
    From f6b9ff50c3ebbb5f277b2c0dc62dc9d7bba6598c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 23 Nov 2021 22:21:57 +0000 Subject: [PATCH 363/505] More loading screen improvements --- .../api/resource/ArbitraryResource.java | 2 +- .../arbitrary/ArbitraryDataResource.java | 13 ++++++--- .../arbitrary/ArbitraryDataManager.java | 1 + src/main/resources/loading/index.html | 27 ++++++++++--------- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index df2ff09d..0e3673c0 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -643,7 +643,7 @@ public class ArbitraryResource { ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null); try { reader.loadSynchronously(false); - } catch (DataException | IOException | MissingDataException e) { + } catch (Exception e) { // No need to handle exception, as it will be reflected in the status } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 20677d7a..4df92f34 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -68,7 +68,7 @@ public class ArbitraryDataResource { // Check if we have all data locally for this resource if (!this.allFilesDownloaded()) { - if (this.madeRecentRequest()) { + if (this.isDataPotentiallyAvailable()) { return new ArbitraryResourceSummary(ArbitraryResourceStatus.DOWNLOADING); } return new ArbitraryResourceSummary(ArbitraryResourceStatus.MISSING_DATA); @@ -115,7 +115,12 @@ public class ArbitraryDataResource { } } - private boolean madeRecentRequest() { + /** + * Best guess as to whether data might be available + * This is only used to give an indication to the user of progress + * @return - whether data might be available on the network + */ + private boolean isDataPotentiallyAvailable() { try { this.fetchTransactions(); Long now = NTP.getTime(); @@ -127,7 +132,9 @@ public class ArbitraryDataResource { for (ArbitraryTransactionData transactionData : transactionDataList) { long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature()); - if (now - lastRequestTime < 30 * 1000L) { + // If we haven't requested yet, or requested in the last 30 seconds, there's still a + // chance that data is on its way but hasn't arrived yet + if (lastRequestTime == 0 || now - lastRequestTime < 30 * 1000L) { return true; } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index eecfeb98..7daa5424 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -666,6 +666,7 @@ public class ArbitraryDataManager extends Thread { String peerAddress = peer.getPeerData().getAddress().toString(); LOGGER.info("Adding arbitrary peer: {} for signature {}", peerAddress, Base58.encode(signature)); ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peer); + repository.discardChanges(); repository.getArbitraryRepository().save(arbitraryPeerData); repository.saveChanges(); } diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html index 8ac5c39f..251912e6 100644 --- a/src/main/resources/loading/index.html +++ b/src/main/resources/loading/index.html @@ -5,7 +5,6 @@ Loading...