getutxo: Flesh out the JavaDocs, link to the BIP, include brief security discussion, and make Peer support multiple in flight queries at once.

core/src/main/java/org/bitcoinj/core/FullPrunedBlockChain.java

@@ -81,7 +81,7 @@ public class FullPrunedBlockChain extends AbstractBlockChain {
     }
 
     /**
-     * Constructs a block chain  connected to the given list of wallets and a store.
+     * Constructs a block chain connected to the given list of wallets and a store.
      */
     public FullPrunedBlockChain(Context context, List<BlockChainListener> listeners, FullPrunedBlockStore blockStore) throws BlockStoreException {
         super(context, listeners, blockStore);

core/src/main/java/org/bitcoinj/core/GetUTXOsMessage.java

@@ -23,11 +23,19 @@ import java.io.OutputStream;
 import java.util.List;
 
 /**
- * This command is supported only by <a href="http://github.com/bitcoinxt/bitcoinxt">Bitcoin XT</a> nodes, which
+ * <p>This command is supported only by <a href="http://github.com/bitcoinxt/bitcoinxt">Bitcoin XT</a> nodes, which
  * advertise themselves using the second service bit flag. It requests a query of the UTXO set keyed by a set of
  * outpoints (i.e. tx hash and output index). The result contains a bitmap of spentness flags, and the contents of
  * the associated outputs if they were found. The results aren't authenticated by anything, so the peer could lie,
- * or a man in the middle could swap out its answer for something else.
+ * or a man in the middle could swap out its answer for something else. Please consult
+ * <a href="https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki">BIP 65</a> for more information on this
+ * message.</p>
+ *
+ * <p>Note that this message does not let you query the UTXO set by address, script or any other criteria. The
+ * reason is that Bitcoin nodes don't calculate the necessary database indexes to answer such queries, to save
+ * space and time. If you want to look up unspent outputs by address, you can either query a block explorer site,
+ * or you can use the {@link FullPrunedBlockChain} class to build the required indexes yourself. Bear in that it will
+ * be quite slow and disk intensive to do that!</p>
  */
 public class GetUTXOsMessage extends Message {
     public static final int MIN_PROTOCOL_VERSION = 70002;

core/src/main/java/org/bitcoinj/core/Peer.java

@@ -135,8 +135,10 @@ public class Peer extends PeerSocketHandler {
         Sha256Hash hash;
         SettableFuture future;
     }
+    // TODO: The types/locking should be rationalised a bit.
     private final CopyOnWriteArrayList<GetDataRequest> getDataFutures;
     @GuardedBy("getAddrFutures") private final LinkedList<SettableFuture<AddressMessage>> getAddrFutures;
+    @Nullable @GuardedBy("lock") private LinkedList<SettableFuture<UTXOsMessage>> getutxoFutures;
 
     // Outstanding pings against this peer and how long the last one took to complete.
     private final ReentrantLock lastPingTimesLock = new ReentrantLock();
@@ -150,8 +152,6 @@ public class Peer extends PeerSocketHandler {
     // A settable future which completes (with this) when the connection is open
     private final SettableFuture<Peer> connectionOpenFuture = SettableFuture.create();
     private final SettableFuture<Peer> versionHandshakeFuture = SettableFuture.create();
-    // A future representing the results of doing a getUTXOs call.
-    @Nullable private SettableFuture<UTXOsMessage> utxosFuture;
 
     /**
      * <p>Construct a peer that reads/writes from the given block chain.</p>
@@ -402,10 +402,10 @@ public class Peer extends PeerSocketHandler {
                 close();
             }
         } else if (m instanceof UTXOsMessage) {
-            if (utxosFuture != null) {
+            if (getutxoFutures != null) {
-                SettableFuture<UTXOsMessage> future = utxosFuture;
+                SettableFuture<UTXOsMessage> future = getutxoFutures.pollFirst();
-                utxosFuture = null;
+                if (future != null)
-                future.set((UTXOsMessage)m);
+                    future.set((UTXOsMessage) m);
             }
         } else if (m instanceof RejectMessage) {
             log.error("{} {}: Received {}", this, getPeerVersionMessage().subVer, m);
@@ -1619,16 +1619,30 @@ public class Peer extends PeerSocketHandler {
      * Sends a query to the remote peer asking for the unspent transaction outputs (UTXOs) for the given outpoints,
      * with the memory pool included. The result should be treated only as a hint: it's possible for the returned
      * outputs to be fictional and not exist in any transaction, and it's possible for them to be spent the moment
-     * after the query returns.
+     * after the query returns. <b>Most peers do not support this request. You will need to connect to Bitcoin XT
+     * peers if you want this to work.</b>
+     *
+     * @throws ProtocolException if this peer doesn't support the protocol.
      */
     public ListenableFuture<UTXOsMessage> getUTXOs(List<TransactionOutPoint> outPoints) {
-        if (utxosFuture != null)
+        lock.lock();
-            throw new IllegalStateException("Already fetching UTXOs, wait for previous query to complete first.");
+        try {
-        if (getPeerVersionMessage().clientVersion < GetUTXOsMessage.MIN_PROTOCOL_VERSION)
+            VersionMessage peerVer = getPeerVersionMessage();
-            throw new IllegalStateException("Peer does not support getutxos protocol version");
+            if (peerVer.clientVersion < GetUTXOsMessage.MIN_PROTOCOL_VERSION)
-        utxosFuture = SettableFuture.create();
+                throw new ProtocolException("Peer does not support getutxos protocol version");
-        sendMessage(new GetUTXOsMessage(params, outPoints, true));
+            if ((peerVer.localServices & GetUTXOsMessage.SERVICE_FLAGS_REQUIRED) != GetUTXOsMessage.SERVICE_FLAGS_REQUIRED)
-        return utxosFuture;
+                throw new ProtocolException("Peer does not support getutxos protocol flag: find Bitcoin XT nodes.");
+            SettableFuture<UTXOsMessage> future = SettableFuture.create();
+            // Add to the list of in flight requests.
+            if (getutxoFutures == null)
+                getutxoFutures = new LinkedList<SettableFuture<UTXOsMessage>>();
+            getutxoFutures.add(future);
+            sendMessage(new GetUTXOsMessage(params, outPoints, true));
+            return future;
+        } finally {
+            lock.unlock();
+        }
+
     }
 
     /**

core/src/main/java/org/bitcoinj/core/UTXOsMessage.java

@@ -21,7 +21,22 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-/** Message representing a list of unspent transaction outputs, returned in response to sending a GetUTXOsMessage. */
+/**
+ * <p>Message representing a list of unspent transaction outputs ("utxos"), returned in response to sending a
+ * {@link GetUTXOsMessage} ("getutxos"). Note that both this message and the query that generates it are not
+ * supported by Bitcoin Core. An implementation is available in <a href="https://github.com/bitcoinxt/bitcoinxt">Bitcoin XT</a>,
+ * a patch set on top of Core. Thus if you want to use it, you must find some XT peers to connect to. This can be done
+ * using a {@link org.bitcoinj.net.discovery.HttpDiscovery} class combined with an HTTP/Cartographer seed.</p>
+ *
+ * <p>The getutxos/utxos protocol is defined in <a href="https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki">BIP 65</a>.
+ * In that document you can find a discussion of the security of this protocol (briefly, there is none). Because the
+ * data found in this message is not authenticated it should be used carefully. Places where it can be useful are if
+ * you're querying your own trusted node, if you're comparing answers from multiple nodes simultaneously and don't
+ * believe there is a MITM on your connection, or if you're only using the returned data as a UI hint and it's OK
+ * if the data is occasionally wrong. Bear in mind that the answer can be wrong even in the absence of malicious intent
+ * just through the nature of querying an ever changing data source: the UTXO set may be updated by a new transaction
+ * immediately after this message is returned.</p>
+ */
 public class UTXOsMessage extends Message {
     private long height;
     private Sha256Hash chainHead;
@@ -65,10 +80,9 @@ public class UTXOsMessage extends Message {
         stream.write(hits);
         stream.write(new VarInt(outputs.size()).encode());
         for (TransactionOutput output : outputs) {
-            // TODO: Allow these to be specified, if one day we care about sending this message ourselves
+            Transaction tx = output.getParentTransaction();
-            // (currently it's just used for unit testing).
+            Utils.uint32ToByteStreamLE(tx != null ? tx.getVersion() : 0L, stream);  // Version
-            Utils.uint32ToByteStreamLE(0L, stream);  // Version
+            Utils.uint32ToByteStreamLE(height, stream);  // Height
-            Utils.uint32ToByteStreamLE(0L, stream);  // Height
             output.bitcoinSerializeToStream(stream);
         }
     }
@@ -112,14 +126,19 @@ public class UTXOsMessage extends Message {
         // Not used.
     }
 
+    /**
+     * Returns a bit map indicating which of the queried outputs were found in the UTXO set.
+     */
     public byte[] getHitMap() {
         return Arrays.copyOf(hits, hits.length);
     }
 
+    /** Returns the list of outputs that matched the query. */
     public List<TransactionOutput> getOutputs() {
         return new ArrayList<TransactionOutput>(outputs);
     }
 
+    /** Returns the block heights of each output returned in getOutputs(), or MEMPOOL_HEIGHT if not confirmed yet. */
     public long[] getHeights() { return Arrays.copyOf(heights, heights.length); }
 
     @Override

core/src/test/java/org/bitcoinj/core/PeerTest.java

@@ -16,12 +16,12 @@
 
 package org.bitcoinj.core;
 
+import com.google.common.collect.*;
 import org.bitcoinj.params.TestNet3Params;
 import org.bitcoinj.testing.FakeTxBuilder;
 import org.bitcoinj.testing.InboundMessageQueuer;
 import org.bitcoinj.testing.TestWithNetworkConnections;
 import org.bitcoinj.utils.Threading;
-import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.SettableFuture;
 import com.google.common.util.concurrent.Uninterruptibles;
@@ -93,13 +93,13 @@ public class PeerTest extends TestWithNetworkConnections {
     }
 
     private void connect() throws Exception {
-        connectWithVersion(70001);
+        connectWithVersion(70001, VersionMessage.NODE_NETWORK);
     }
 
-    private void connectWithVersion(int version) throws Exception {
+    private void connectWithVersion(int version, int flags) throws Exception {
         VersionMessage peerVersion = new VersionMessage(unitTestParams, OTHER_PEER_CHAIN_HEIGHT);
         peerVersion.clientVersion = version;
-        peerVersion.localServices = VersionMessage.NODE_NETWORK;
+        peerVersion.localServices = flags;
         writeTarget = connect(peer, peerVersion);
     }
 
@@ -542,7 +542,7 @@ public class PeerTest extends TestWithNetworkConnections {
     @Test
     public void recursiveDependencyDownload() throws Exception {
         // Using ping or notfound?
-        connectWithVersion(70001);
+        connectWithVersion(70001, VersionMessage.NODE_NETWORK);
         // Check that we can download all dependencies of an unconfirmed relevant transaction from the mempool.
         ECKey to = new ECKey();
 
@@ -638,7 +638,7 @@ public class PeerTest extends TestWithNetworkConnections {
 
     @Test
     public void timeLockedTransactionNew() throws Exception {
-        connectWithVersion(70001);
+        connectWithVersion(70001, VersionMessage.NODE_NETWORK);
         // Test that if we receive a relevant transaction that has a lock time, it doesn't result in a notification
         // until we explicitly opt in to seeing those.
         Wallet wallet = new Wallet(unitTestParams);
@@ -691,7 +691,7 @@ public class PeerTest extends TestWithNetworkConnections {
 
     private void checkTimeLockedDependency(boolean shouldAccept) throws Exception {
         // Initial setup.
-        connectWithVersion(70001);
+        connectWithVersion(70001, VersionMessage.NODE_NETWORK);
         Wallet wallet = new Wallet(unitTestParams);
         ECKey key = wallet.freshReceiveKey();
         wallet.setAcceptRiskyTransactions(shouldAccept);
@@ -760,7 +760,7 @@ public class PeerTest extends TestWithNetworkConnections {
                 disconnectedFuture.set(null);
             }
         });
-        connectWithVersion(500);
+        connectWithVersion(500, VersionMessage.NODE_NETWORK);
         // We must wait uninterruptibly here because connect[WithVersion] generates a peer that interrupts the current
         // thread when it disconnects.
         Uninterruptibles.getUninterruptibly(connectedFuture);
@@ -815,6 +815,40 @@ public class PeerTest extends TestWithNetworkConnections {
         Threading.uncaughtExceptionHandler = null;
     }
 
+    @Test
+    public void getUTXOs() throws Exception {
+        // Basic test of support for BIP 64: getutxos support. The Lighthouse unit tests exercise this stuff more
+        // thoroughly.
+        connectWithVersion(GetUTXOsMessage.MIN_PROTOCOL_VERSION, VersionMessage.NODE_NETWORK | VersionMessage.NODE_GETUTXOS);
+        Sha256Hash hash1 = Sha256Hash.hash("foo".getBytes());
+        TransactionOutPoint op1 = new TransactionOutPoint(unitTestParams, 1, hash1);
+        Sha256Hash hash2 = Sha256Hash.hash("bar".getBytes());
+        TransactionOutPoint op2 = new TransactionOutPoint(unitTestParams, 2, hash1);
+
+        ListenableFuture<UTXOsMessage> future1 = peer.getUTXOs(ImmutableList.of(op1));
+        ListenableFuture<UTXOsMessage> future2 = peer.getUTXOs(ImmutableList.of(op2));
+
+        GetUTXOsMessage msg1 = (GetUTXOsMessage) outbound(writeTarget);
+        GetUTXOsMessage msg2 = (GetUTXOsMessage) outbound(writeTarget);
+
+        assertEquals(op1, msg1.getOutPoints().get(0));
+        assertEquals(op2, msg2.getOutPoints().get(0));
+        assertEquals(1, msg1.getOutPoints().size());
+
+        assertFalse(future1.isDone());
+
+        ECKey key = new ECKey();
+        TransactionOutput out1 = new TransactionOutput(unitTestParams, null, Coin.CENT, key);
+        UTXOsMessage response1 = new UTXOsMessage(unitTestParams, ImmutableList.of(out1), new long[]{-1}, Sha256Hash.ZERO_HASH, 1234);
+        inbound(writeTarget, response1);
+        assertEquals(future1.get(), response1);
+
+        TransactionOutput out2 = new TransactionOutput(unitTestParams, null, Coin.FIFTY_COINS, key);
+        UTXOsMessage response2 = new UTXOsMessage(unitTestParams, ImmutableList.of(out2), new long[]{-1}, Sha256Hash.ZERO_HASH, 1234);
+        inbound(writeTarget, response2);
+        assertEquals(future1.get(), response2);
+    }
+
     @Test
     public void badMessage() throws Exception {
         // Bring up an actual network connection and feed it bogus data.