From 037ec5aef94d264a38bce7bab674aea5f8604b97 Mon Sep 17 00:00:00 2001 From: Mike Hearn Date: Mon, 6 Apr 2015 15:14:31 +0200 Subject: [PATCH] getutxo: Flesh out the JavaDocs, link to the BIP, include brief security discussion, and make Peer support multiple in flight queries at once. --- .../bitcoinj/core/FullPrunedBlockChain.java | 2 +- .../org/bitcoinj/core/GetUTXOsMessage.java | 12 ++++- .../src/main/java/org/bitcoinj/core/Peer.java | 42 ++++++++++------ .../java/org/bitcoinj/core/UTXOsMessage.java | 29 +++++++++-- .../test/java/org/bitcoinj/core/PeerTest.java | 50 ++++++++++++++++--- 5 files changed, 105 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/bitcoinj/core/FullPrunedBlockChain.java b/core/src/main/java/org/bitcoinj/core/FullPrunedBlockChain.java index 23ddb120..fa2dffff 100644 --- a/core/src/main/java/org/bitcoinj/core/FullPrunedBlockChain.java +++ b/core/src/main/java/org/bitcoinj/core/FullPrunedBlockChain.java @@ -81,7 +81,7 @@ public class FullPrunedBlockChain extends AbstractBlockChain { } /** - * Constructs a block chain connected to the given list of wallets and a store. + * Constructs a block chain connected to the given list of wallets and a store. */ public FullPrunedBlockChain(Context context, List listeners, FullPrunedBlockStore blockStore) throws BlockStoreException { super(context, listeners, blockStore); diff --git a/core/src/main/java/org/bitcoinj/core/GetUTXOsMessage.java b/core/src/main/java/org/bitcoinj/core/GetUTXOsMessage.java index a887a492..160d84e1 100644 --- a/core/src/main/java/org/bitcoinj/core/GetUTXOsMessage.java +++ b/core/src/main/java/org/bitcoinj/core/GetUTXOsMessage.java @@ -23,11 +23,19 @@ import java.io.OutputStream; import java.util.List; /** - * This command is supported only by Bitcoin XT nodes, which + *

This command is supported only by Bitcoin XT nodes, which * advertise themselves using the second service bit flag. It requests a query of the UTXO set keyed by a set of * outpoints (i.e. tx hash and output index). The result contains a bitmap of spentness flags, and the contents of * the associated outputs if they were found. The results aren't authenticated by anything, so the peer could lie, - * or a man in the middle could swap out its answer for something else. + * or a man in the middle could swap out its answer for something else. Please consult + * BIP 65 for more information on this + * message.

+ * + *

Note that this message does not let you query the UTXO set by address, script or any other criteria. The + * reason is that Bitcoin nodes don't calculate the necessary database indexes to answer such queries, to save + * space and time. If you want to look up unspent outputs by address, you can either query a block explorer site, + * or you can use the {@link FullPrunedBlockChain} class to build the required indexes yourself. Bear in that it will + * be quite slow and disk intensive to do that!

*/ public class GetUTXOsMessage extends Message { public static final int MIN_PROTOCOL_VERSION = 70002; diff --git a/core/src/main/java/org/bitcoinj/core/Peer.java b/core/src/main/java/org/bitcoinj/core/Peer.java index f3e68495..13209d33 100644 --- a/core/src/main/java/org/bitcoinj/core/Peer.java +++ b/core/src/main/java/org/bitcoinj/core/Peer.java @@ -135,8 +135,10 @@ public class Peer extends PeerSocketHandler { Sha256Hash hash; SettableFuture future; } + // TODO: The types/locking should be rationalised a bit. private final CopyOnWriteArrayList getDataFutures; @GuardedBy("getAddrFutures") private final LinkedList> getAddrFutures; + @Nullable @GuardedBy("lock") private LinkedList> getutxoFutures; // Outstanding pings against this peer and how long the last one took to complete. private final ReentrantLock lastPingTimesLock = new ReentrantLock(); @@ -150,8 +152,6 @@ public class Peer extends PeerSocketHandler { // A settable future which completes (with this) when the connection is open private final SettableFuture connectionOpenFuture = SettableFuture.create(); private final SettableFuture versionHandshakeFuture = SettableFuture.create(); - // A future representing the results of doing a getUTXOs call. - @Nullable private SettableFuture utxosFuture; /** *

Construct a peer that reads/writes from the given block chain.

@@ -402,10 +402,10 @@ public class Peer extends PeerSocketHandler { close(); } } else if (m instanceof UTXOsMessage) { - if (utxosFuture != null) { - SettableFuture future = utxosFuture; - utxosFuture = null; - future.set((UTXOsMessage)m); + if (getutxoFutures != null) { + SettableFuture future = getutxoFutures.pollFirst(); + if (future != null) + future.set((UTXOsMessage) m); } } else if (m instanceof RejectMessage) { log.error("{} {}: Received {}", this, getPeerVersionMessage().subVer, m); @@ -1619,16 +1619,30 @@ public class Peer extends PeerSocketHandler { * Sends a query to the remote peer asking for the unspent transaction outputs (UTXOs) for the given outpoints, * with the memory pool included. The result should be treated only as a hint: it's possible for the returned * outputs to be fictional and not exist in any transaction, and it's possible for them to be spent the moment - * after the query returns. + * after the query returns. Most peers do not support this request. You will need to connect to Bitcoin XT + * peers if you want this to work. + * + * @throws ProtocolException if this peer doesn't support the protocol. */ public ListenableFuture getUTXOs(List outPoints) { - if (utxosFuture != null) - throw new IllegalStateException("Already fetching UTXOs, wait for previous query to complete first."); - if (getPeerVersionMessage().clientVersion < GetUTXOsMessage.MIN_PROTOCOL_VERSION) - throw new IllegalStateException("Peer does not support getutxos protocol version"); - utxosFuture = SettableFuture.create(); - sendMessage(new GetUTXOsMessage(params, outPoints, true)); - return utxosFuture; + lock.lock(); + try { + VersionMessage peerVer = getPeerVersionMessage(); + if (peerVer.clientVersion < GetUTXOsMessage.MIN_PROTOCOL_VERSION) + throw new ProtocolException("Peer does not support getutxos protocol version"); + if ((peerVer.localServices & GetUTXOsMessage.SERVICE_FLAGS_REQUIRED) != GetUTXOsMessage.SERVICE_FLAGS_REQUIRED) + throw new ProtocolException("Peer does not support getutxos protocol flag: find Bitcoin XT nodes."); + SettableFuture future = SettableFuture.create(); + // Add to the list of in flight requests. + if (getutxoFutures == null) + getutxoFutures = new LinkedList>(); + getutxoFutures.add(future); + sendMessage(new GetUTXOsMessage(params, outPoints, true)); + return future; + } finally { + lock.unlock(); + } + } /** diff --git a/core/src/main/java/org/bitcoinj/core/UTXOsMessage.java b/core/src/main/java/org/bitcoinj/core/UTXOsMessage.java index 7e30eabc..d9384882 100644 --- a/core/src/main/java/org/bitcoinj/core/UTXOsMessage.java +++ b/core/src/main/java/org/bitcoinj/core/UTXOsMessage.java @@ -21,7 +21,22 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -/** Message representing a list of unspent transaction outputs, returned in response to sending a GetUTXOsMessage. */ +/** + *

Message representing a list of unspent transaction outputs ("utxos"), returned in response to sending a + * {@link GetUTXOsMessage} ("getutxos"). Note that both this message and the query that generates it are not + * supported by Bitcoin Core. An implementation is available in Bitcoin XT, + * a patch set on top of Core. Thus if you want to use it, you must find some XT peers to connect to. This can be done + * using a {@link org.bitcoinj.net.discovery.HttpDiscovery} class combined with an HTTP/Cartographer seed.

+ * + *

The getutxos/utxos protocol is defined in BIP 65. + * In that document you can find a discussion of the security of this protocol (briefly, there is none). Because the + * data found in this message is not authenticated it should be used carefully. Places where it can be useful are if + * you're querying your own trusted node, if you're comparing answers from multiple nodes simultaneously and don't + * believe there is a MITM on your connection, or if you're only using the returned data as a UI hint and it's OK + * if the data is occasionally wrong. Bear in mind that the answer can be wrong even in the absence of malicious intent + * just through the nature of querying an ever changing data source: the UTXO set may be updated by a new transaction + * immediately after this message is returned.

+ */ public class UTXOsMessage extends Message { private long height; private Sha256Hash chainHead; @@ -65,10 +80,9 @@ public class UTXOsMessage extends Message { stream.write(hits); stream.write(new VarInt(outputs.size()).encode()); for (TransactionOutput output : outputs) { - // TODO: Allow these to be specified, if one day we care about sending this message ourselves - // (currently it's just used for unit testing). - Utils.uint32ToByteStreamLE(0L, stream); // Version - Utils.uint32ToByteStreamLE(0L, stream); // Height + Transaction tx = output.getParentTransaction(); + Utils.uint32ToByteStreamLE(tx != null ? tx.getVersion() : 0L, stream); // Version + Utils.uint32ToByteStreamLE(height, stream); // Height output.bitcoinSerializeToStream(stream); } } @@ -112,14 +126,19 @@ public class UTXOsMessage extends Message { // Not used. } + /** + * Returns a bit map indicating which of the queried outputs were found in the UTXO set. + */ public byte[] getHitMap() { return Arrays.copyOf(hits, hits.length); } + /** Returns the list of outputs that matched the query. */ public List getOutputs() { return new ArrayList(outputs); } + /** Returns the block heights of each output returned in getOutputs(), or MEMPOOL_HEIGHT if not confirmed yet. */ public long[] getHeights() { return Arrays.copyOf(heights, heights.length); } @Override diff --git a/core/src/test/java/org/bitcoinj/core/PeerTest.java b/core/src/test/java/org/bitcoinj/core/PeerTest.java index 49ecc452..5e2f636e 100644 --- a/core/src/test/java/org/bitcoinj/core/PeerTest.java +++ b/core/src/test/java/org/bitcoinj/core/PeerTest.java @@ -16,12 +16,12 @@ package org.bitcoinj.core; +import com.google.common.collect.*; import org.bitcoinj.params.TestNet3Params; import org.bitcoinj.testing.FakeTxBuilder; import org.bitcoinj.testing.InboundMessageQueuer; import org.bitcoinj.testing.TestWithNetworkConnections; import org.bitcoinj.utils.Threading; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.common.util.concurrent.Uninterruptibles; @@ -93,13 +93,13 @@ public class PeerTest extends TestWithNetworkConnections { } private void connect() throws Exception { - connectWithVersion(70001); + connectWithVersion(70001, VersionMessage.NODE_NETWORK); } - private void connectWithVersion(int version) throws Exception { + private void connectWithVersion(int version, int flags) throws Exception { VersionMessage peerVersion = new VersionMessage(unitTestParams, OTHER_PEER_CHAIN_HEIGHT); peerVersion.clientVersion = version; - peerVersion.localServices = VersionMessage.NODE_NETWORK; + peerVersion.localServices = flags; writeTarget = connect(peer, peerVersion); } @@ -542,7 +542,7 @@ public class PeerTest extends TestWithNetworkConnections { @Test public void recursiveDependencyDownload() throws Exception { // Using ping or notfound? - connectWithVersion(70001); + connectWithVersion(70001, VersionMessage.NODE_NETWORK); // Check that we can download all dependencies of an unconfirmed relevant transaction from the mempool. ECKey to = new ECKey(); @@ -638,7 +638,7 @@ public class PeerTest extends TestWithNetworkConnections { @Test public void timeLockedTransactionNew() throws Exception { - connectWithVersion(70001); + connectWithVersion(70001, VersionMessage.NODE_NETWORK); // Test that if we receive a relevant transaction that has a lock time, it doesn't result in a notification // until we explicitly opt in to seeing those. Wallet wallet = new Wallet(unitTestParams); @@ -691,7 +691,7 @@ public class PeerTest extends TestWithNetworkConnections { private void checkTimeLockedDependency(boolean shouldAccept) throws Exception { // Initial setup. - connectWithVersion(70001); + connectWithVersion(70001, VersionMessage.NODE_NETWORK); Wallet wallet = new Wallet(unitTestParams); ECKey key = wallet.freshReceiveKey(); wallet.setAcceptRiskyTransactions(shouldAccept); @@ -760,7 +760,7 @@ public class PeerTest extends TestWithNetworkConnections { disconnectedFuture.set(null); } }); - connectWithVersion(500); + connectWithVersion(500, VersionMessage.NODE_NETWORK); // We must wait uninterruptibly here because connect[WithVersion] generates a peer that interrupts the current // thread when it disconnects. Uninterruptibles.getUninterruptibly(connectedFuture); @@ -815,6 +815,40 @@ public class PeerTest extends TestWithNetworkConnections { Threading.uncaughtExceptionHandler = null; } + @Test + public void getUTXOs() throws Exception { + // Basic test of support for BIP 64: getutxos support. The Lighthouse unit tests exercise this stuff more + // thoroughly. + connectWithVersion(GetUTXOsMessage.MIN_PROTOCOL_VERSION, VersionMessage.NODE_NETWORK | VersionMessage.NODE_GETUTXOS); + Sha256Hash hash1 = Sha256Hash.hash("foo".getBytes()); + TransactionOutPoint op1 = new TransactionOutPoint(unitTestParams, 1, hash1); + Sha256Hash hash2 = Sha256Hash.hash("bar".getBytes()); + TransactionOutPoint op2 = new TransactionOutPoint(unitTestParams, 2, hash1); + + ListenableFuture future1 = peer.getUTXOs(ImmutableList.of(op1)); + ListenableFuture future2 = peer.getUTXOs(ImmutableList.of(op2)); + + GetUTXOsMessage msg1 = (GetUTXOsMessage) outbound(writeTarget); + GetUTXOsMessage msg2 = (GetUTXOsMessage) outbound(writeTarget); + + assertEquals(op1, msg1.getOutPoints().get(0)); + assertEquals(op2, msg2.getOutPoints().get(0)); + assertEquals(1, msg1.getOutPoints().size()); + + assertFalse(future1.isDone()); + + ECKey key = new ECKey(); + TransactionOutput out1 = new TransactionOutput(unitTestParams, null, Coin.CENT, key); + UTXOsMessage response1 = new UTXOsMessage(unitTestParams, ImmutableList.of(out1), new long[]{-1}, Sha256Hash.ZERO_HASH, 1234); + inbound(writeTarget, response1); + assertEquals(future1.get(), response1); + + TransactionOutput out2 = new TransactionOutput(unitTestParams, null, Coin.FIFTY_COINS, key); + UTXOsMessage response2 = new UTXOsMessage(unitTestParams, ImmutableList.of(out2), new long[]{-1}, Sha256Hash.ZERO_HASH, 1234); + inbound(writeTarget, response2); + assertEquals(future1.get(), response2); + } + @Test public void badMessage() throws Exception { // Bring up an actual network connection and feed it bogus data.