From 668b176283423a9bb760195e311888eec81a180d Mon Sep 17 00:00:00 2001 From: Mike Hearn Date: Mon, 28 Mar 2011 17:59:10 +0000 Subject: [PATCH] Add a DiskBlockStore and associated unit tests. This removes the need to download the block chain from scratch each time a program is started up. --- src/com/google/bitcoin/core/Block.java | 7 +- src/com/google/bitcoin/core/BlockChain.java | 17 +- .../google/bitcoin/core/DiskBlockStore.java | 164 ++++++++++++++++++ .../google/bitcoin/core/MemoryBlockStore.java | 4 +- src/com/google/bitcoin/core/Peer.java | 8 +- src/com/google/bitcoin/core/Sha256Hash.java | 57 ++++++ .../google/bitcoin/examples/PingService.java | 26 +-- .../google/bitcoin/examples/PrivateKeys.java | 2 +- .../google/bitcoin/core/BlockChainTest.java | 4 +- tests/com/google/bitcoin/core/BlockTest.java | 8 + .../bitcoin/core/DiskBlockStoreTest.java | 47 +++++ 11 files changed, 322 insertions(+), 22 deletions(-) create mode 100644 src/com/google/bitcoin/core/DiskBlockStore.java create mode 100644 src/com/google/bitcoin/core/Sha256Hash.java create mode 100644 tests/com/google/bitcoin/core/DiskBlockStoreTest.java diff --git a/src/com/google/bitcoin/core/Block.java b/src/com/google/bitcoin/core/Block.java index c11bd951..2d6a20a9 100644 --- a/src/com/google/bitcoin/core/Block.java +++ b/src/com/google/bitcoin/core/Block.java @@ -82,7 +82,12 @@ public class Block extends Message { nonce = readUint32(); hash = Utils.reverseBytes(Utils.doubleDigest(bytes, 0, cursor)); - + + if (cursor == bytes.length) { + // This message is just a header, it has no transactions. + return; + } + int numTransactions = (int) readVarInt(); transactions = new ArrayList(numTransactions); for (int i = 0; i < numTransactions; i++) { diff --git a/src/com/google/bitcoin/core/BlockChain.java b/src/com/google/bitcoin/core/BlockChain.java index 6b271147..9a6bafc8 100644 --- a/src/com/google/bitcoin/core/BlockChain.java +++ b/src/com/google/bitcoin/core/BlockChain.java @@ -16,6 +16,7 @@ package com.google.bitcoin.core; +import java.io.File; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; @@ -67,10 +68,17 @@ public class BlockChain { // were downloading the block chain. private final ArrayList unconnectedBlocks = new ArrayList(); - public BlockChain(NetworkParameters params, Wallet wallet) { - // TODO: Let the user pass in a BlockStore object so they can choose how to store the headers. + /** + * Constructs a BlockChain connected to the given wallet and store. To obtain a {@link Wallet} you can construct + * one from scratch, or you can deserialize a saved wallet from disk using {@link Wallet#loadFromFile(java.io.File)} + *

+ * + * For the store you can use a {@link MemoryBlockStore} if you don't care about saving the downloaded data, or a + * {@link DiskBlockStore} if you'd like to ensure fast startup the next time you run the program. + */ + public BlockChain(NetworkParameters params, Wallet wallet, BlockStore blockStore) { try { - blockStore = new MemoryBlockStore(params); + this.blockStore = blockStore; chainHead = blockStore.getChainHead(); LOG("chain head is: " + chainHead.getHeader().toString()); } catch (BlockStoreException e) { @@ -143,7 +151,8 @@ public class BlockChain { if (storedPrev.equals(chainHead)) { // This block connects to the best known block, it is a normal continuation of the system. setChainHead(newStoredBlock); - LOG("Received new block, chain is now " + chainHead.getHeight() + " blocks high"); + LOG("Received block " + block.getHashAsString() + ", chain is now " + chainHead.getHeight() + + " blocks high"); } else { // This block connects to somewhere other than the top of the chain. if (newStoredBlock.moreWorkThan(chainHead)) { diff --git a/src/com/google/bitcoin/core/DiskBlockStore.java b/src/com/google/bitcoin/core/DiskBlockStore.java new file mode 100644 index 00000000..12c6cb88 --- /dev/null +++ b/src/com/google/bitcoin/core/DiskBlockStore.java @@ -0,0 +1,164 @@ +/** + * Copyright 2011 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.bitcoin.core; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +import static com.google.bitcoin.core.Utils.LOG; + +/** + * Stores the block chain to disk but still holds it in memory. This is intended for desktop apps and tests. + * Constrained environments like mobile phones probably won't want to or be able to store all the block headers in RAM. + */ +public class DiskBlockStore implements BlockStore { + private FileOutputStream stream; + private Map blockMap; + private Sha256Hash chainHead; + private NetworkParameters params; + + public DiskBlockStore(NetworkParameters params, File file) throws BlockStoreException { + this.params = params; + blockMap = new HashMap(); + try { + load(file); + stream = new FileOutputStream(file, true); // Do append. + } catch (IOException e) { + LOG(e.toString()); + createNewStore(params, file); + } + } + + private void createNewStore(NetworkParameters params, File file) throws BlockStoreException { + // Create a new block store if the file wasn't found or anything went wrong whilst reading. + blockMap.clear(); + try { + stream = new FileOutputStream(file, false); // Do not append, create fresh. + stream.write(1); // Version. + } catch (IOException e1) { + // We could not load a block store nor could we create a new one! + throw new BlockStoreException(e1); + } + try { + // Set up the genesis block. When we start out fresh, it is by definition the top of the chain. + Block genesis = params.genesisBlock.cloneAsHeader(); + StoredBlock storedGenesis = new StoredBlock(genesis, genesis.getWork(), 0); + this.chainHead = new Sha256Hash(storedGenesis.getHeader().getHash()); + stream.write(this.chainHead.hash); + put(storedGenesis); + } catch (VerificationException e1) { + throw new RuntimeException(e1); // Cannot happen. + } catch (IOException e) { + throw new BlockStoreException(e); + } + } + + private void load(File file) throws IOException, BlockStoreException { + LOG("Reading block store from " + file.getAbsolutePath()); + FileInputStream input = new FileInputStream(file); + // Read a version byte. + int version = input.read(); + if (version == -1) { + // No such file or the file was empty. + throw new FileNotFoundException(file.getName() + " does not exist or is empty"); + } + if (version != 1) { + throw new BlockStoreException("Bad version number: " + version); + } + // Chain head pointer is the first thing in the file. + byte[] chainHeadHash = new byte[32]; + input.read(chainHeadHash); + this.chainHead = new Sha256Hash(chainHeadHash); + LOG("Read chain head from disk: " + this.chainHead); + long now = System.currentTimeMillis(); + // Rest of file is raw block headers. + byte[] headerBytes = new byte[Block.HEADER_SIZE]; + try { + while (true) { + // Read a block from disk. + if (input.read(headerBytes) < 80) { + // End of file. + break; + } + // Parse it. + Block b = new Block(params, headerBytes); + // Look up the previous block it connects to. + StoredBlock prev = get(b.getPrevBlockHash()); + StoredBlock s; + if (prev == null) { + // First block in the stored chain has to be treated specially. + if (b.equals(params.genesisBlock)) { + s = new StoredBlock(params.genesisBlock.cloneAsHeader(), params.genesisBlock.getWork(), 0); + } else { + throw new BlockStoreException("Could not connect " + Utils.bytesToHexString(b.getHash()) + " to " + + Utils.bytesToHexString(b.getPrevBlockHash())); + } + } else { + // Don't try to verify the genesis block to avoid upsetting the unit tests. + b.verify(); + // Calculate its height and total chain work. + s = prev.build(b); + } + // Save in memory. + blockMap.put(new Sha256Hash(b.getHash()), s); + } + } catch (ProtocolException e) { + // Corrupted file. + throw new BlockStoreException(e); + } catch (VerificationException e) { + // Should not be able to happen unless the file contains bad blocks. + throw new BlockStoreException(e); + } + long elapsed = System.currentTimeMillis() - now; + LOG("Block chain read complete in " + elapsed + "ms"); + } + + public synchronized void put(StoredBlock block) throws BlockStoreException { + try { + Sha256Hash hash = new Sha256Hash(block.getHeader().getHash()); + assert blockMap.get(hash) == null : "Attempt to insert duplicate"; + // Append to the end of the file. The other fields in StoredBlock will be recalculated when it's reloaded. + byte[] bytes = block.getHeader().bitcoinSerialize(); + stream.write(bytes); + stream.flush(); + blockMap.put(hash, block); + } catch (IOException e) { + throw new BlockStoreException(e); + } + } + + public synchronized StoredBlock get(byte[] hash) throws BlockStoreException { + return blockMap.get(new Sha256Hash(hash)); + } + + public synchronized StoredBlock getChainHead() throws BlockStoreException { + return blockMap.get(chainHead); + } + + public synchronized void setChainHead(StoredBlock chainHead) throws BlockStoreException { + try { + byte[] hash = chainHead.getHeader().getHash(); + this.chainHead = new Sha256Hash(hash); + // Write out new hash to the first 32 bytes of the file past one (first byte is version number). + stream.getChannel().write(ByteBuffer.wrap(hash), 1); + } catch (IOException e) { + throw new BlockStoreException(e); + } + } +} diff --git a/src/com/google/bitcoin/core/MemoryBlockStore.java b/src/com/google/bitcoin/core/MemoryBlockStore.java index 7c1a2b33..04055006 100644 --- a/src/com/google/bitcoin/core/MemoryBlockStore.java +++ b/src/com/google/bitcoin/core/MemoryBlockStore.java @@ -24,7 +24,7 @@ import java.util.Map; /** * Keeps {@link StoredBlock}s in memory. Used primarily for unit testing. */ -class MemoryBlockStore implements BlockStore { +public class MemoryBlockStore implements BlockStore { // We use a ByteBuffer to hold hashes here because the Java array equals()/hashcode() methods do not operate on // the contents of the array but just inherit the default Object behavior. ByteBuffer provides the functionality // needed to act as a key in a map. @@ -34,7 +34,7 @@ class MemoryBlockStore implements BlockStore { private Map blockMap; private StoredBlock chainHead; - MemoryBlockStore(NetworkParameters params) { + public MemoryBlockStore(NetworkParameters params) { blockMap = new HashMap(); // Insert the genesis block. try { diff --git a/src/com/google/bitcoin/core/Peer.java b/src/com/google/bitcoin/core/Peer.java index 981b288e..39af16a4 100644 --- a/src/com/google/bitcoin/core/Peer.java +++ b/src/com/google/bitcoin/core/Peer.java @@ -325,8 +325,12 @@ public class Peer { // node. If that happens it means the user overrode us somewhere. throw new RuntimeException("Peer does not have block chain"); } - chainCompletionLatch = new CountDownLatch(chainHeight); - blockChainDownload(params.genesisBlock.getHash()); + int blocksToGet = chainHeight - blockChain.getChainHead().getHeight(); + chainCompletionLatch = new CountDownLatch(blocksToGet); + if (blocksToGet > 0) { + // When we just want as many blocks as possible, we can set the target hash to zero. + blockChainDownload(new byte[32]); + } return chainCompletionLatch; } diff --git a/src/com/google/bitcoin/core/Sha256Hash.java b/src/com/google/bitcoin/core/Sha256Hash.java new file mode 100644 index 00000000..70f03ffc --- /dev/null +++ b/src/com/google/bitcoin/core/Sha256Hash.java @@ -0,0 +1,57 @@ +/** + * Copyright 2011 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.bitcoin.core; + +import java.io.Serializable; +import java.util.Arrays; + +// TODO: Switch all code/interfaces to using this class. + +/** + * A Sha256Hash just wraps a byte[] so that equals and hashcode work correctly, allowing it to be used as keys in a + * map. It also checks that the length is correct and provides a bit more type safety. + */ +public class Sha256Hash implements Serializable { + public byte[] hash; + + public Sha256Hash(byte[] hash) { + assert hash.length == 32; + this.hash = hash; + } + + /** Returns true if the hashes are equal. */ + @Override + public boolean equals(Object other) { + if (!(other instanceof Sha256Hash)) return false; + return Arrays.equals(hash, ((Sha256Hash)other).hash); + } + + /** + * Hash code of the byte array as calculated by {@link Arrays.hashCode()}. Note the difference between a SHA256 + * secure hash and the type of quick/dirty hash used by the Java hashCode method which is designed for use in + * hash tables. + */ + @Override + public int hashCode() { + return Arrays.hashCode(hash); + } + + @Override + public String toString() { + return Utils.bytesToHexString(hash); + } +} diff --git a/src/com/google/bitcoin/examples/PingService.java b/src/com/google/bitcoin/examples/PingService.java index 0e642f03..a32bc1b2 100644 --- a/src/com/google/bitcoin/examples/PingService.java +++ b/src/com/google/bitcoin/examples/PingService.java @@ -31,7 +31,7 @@ import java.util.concurrent.TimeUnit; */ public class PingService { public static void main(String[] args) throws Exception { - final NetworkParameters params = NetworkParameters.testNet(); + final NetworkParameters params = NetworkParameters.prodNet(); // Try to read the wallet from storage, create a new one if not possible. Wallet wallet; @@ -46,11 +46,14 @@ public class PingService { // Fetch the first key in the wallet (should be the only key). ECKey key = wallet.keychain.get(0); - // Connect to the localhost node. - System.out.println("Please wait, connecting and downloading block chain. This may take a while."); + // Load the block chain, if there is one stored locally. + System.out.println("Reading block store from disk"); + BlockStore blockStore = new DiskBlockStore(params, new File("pingservice.blockchain")); + // Connect to the localhost node. + System.out.println("Connecting ..."); NetworkConnection conn = new NetworkConnection(InetAddress.getLocalHost(), params); - BlockChain chain = new BlockChain(params, wallet); + BlockChain chain = new BlockChain(params, wallet, blockStore); final Peer peer = new Peer(params, conn, chain); peer.start(); @@ -85,12 +88,15 @@ public class PingService { CountDownLatch progress = peer.startBlockChainDownload(); long max = progress.getCount(); // Racy but no big deal. - long current = max; - while (current > 0) { - double pct = 100.0 - (100.0 * (current / (double)max)); - System.out.println(String.format("Chain download %d%% done", (int)pct)); - progress.await(1, TimeUnit.SECONDS); - current = progress.getCount(); + if (max > 0) { + System.out.println("Downloading block chain. " + (max > 1000 ? "This may take a while." : "")); + long current = max; + while (current > 0) { + double pct = 100.0 - (100.0 * (current / (double)max)); + System.out.println(String.format("Chain download %d%% done", (int)pct)); + progress.await(1, TimeUnit.SECONDS); + current = progress.getCount(); + } } System.out.println("Send coins to: " + key.toAddress(params).toString()); System.out.println("Waiting for coins to arrive. Press Ctrl-C to quit."); diff --git a/src/com/google/bitcoin/examples/PrivateKeys.java b/src/com/google/bitcoin/examples/PrivateKeys.java index e33ebd2b..69652cf9 100644 --- a/src/com/google/bitcoin/examples/PrivateKeys.java +++ b/src/com/google/bitcoin/examples/PrivateKeys.java @@ -47,7 +47,7 @@ public class PrivateKeys { // Find the transactions that involve those coins. NetworkConnection conn = new NetworkConnection(InetAddress.getLocalHost(), params); - BlockChain chain = new BlockChain(params, wallet); + BlockChain chain = new BlockChain(params, wallet, new MemoryBlockStore(params)); Peer peer = new Peer(params, conn, chain); peer.start(); peer.startBlockChainDownload().await(); diff --git a/tests/com/google/bitcoin/core/BlockChainTest.java b/tests/com/google/bitcoin/core/BlockChainTest.java index dac40a4f..7c739f49 100644 --- a/tests/com/google/bitcoin/core/BlockChainTest.java +++ b/tests/com/google/bitcoin/core/BlockChainTest.java @@ -40,13 +40,13 @@ public class BlockChainTest { @Before public void setUp() { - testNetChain = new BlockChain(testNet, new Wallet(testNet)); + testNetChain = new BlockChain(testNet, new Wallet(testNet), new MemoryBlockStore(testNet)); unitTestParams = NetworkParameters.unitTests(); wallet = new Wallet(unitTestParams); wallet.addKey(new ECKey()); coinbaseTo = wallet.keychain.get(0).toAddress(unitTestParams); - chain = new BlockChain(unitTestParams, wallet); + chain = new BlockChain(unitTestParams, wallet, new MemoryBlockStore(unitTestParams)); } @Test diff --git a/tests/com/google/bitcoin/core/BlockTest.java b/tests/com/google/bitcoin/core/BlockTest.java index f9aecb9f..5b8cd7d3 100644 --- a/tests/com/google/bitcoin/core/BlockTest.java +++ b/tests/com/google/bitcoin/core/BlockTest.java @@ -101,6 +101,14 @@ public class BlockTest { } } + @Test + public void testHeaderParse() throws Exception { + Block block = new Block(params, blockBytes); + Block header = block.cloneAsHeader(); + Block reparsed = new Block(params, header.bitcoinSerialize()); + assertEquals(reparsed, header); + } + @Test public void testBitCoinSerialization() throws Exception { // We have to be able to reserialize everything exactly as we found it for hashing to work. This test also diff --git a/tests/com/google/bitcoin/core/DiskBlockStoreTest.java b/tests/com/google/bitcoin/core/DiskBlockStoreTest.java new file mode 100644 index 00000000..58b0d2d6 --- /dev/null +++ b/tests/com/google/bitcoin/core/DiskBlockStoreTest.java @@ -0,0 +1,47 @@ +/** + * Copyright 2011 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.bitcoin.core; + +import org.junit.Test; +import java.io.File; +import static org.junit.Assert.assertEquals; + +public class DiskBlockStoreTest { + @Test + public void testStorage() throws Exception { + File temp = File.createTempFile("bitcoinj-test", null, null); + System.out.println(temp.getAbsolutePath()); + //temp.deleteOnExit(); + + NetworkParameters params = NetworkParameters.unitTests(); + Address to = new ECKey().toAddress(params); + DiskBlockStore store = new DiskBlockStore(params, temp); + // Check the first block in a new store is the genesis block. + StoredBlock genesis = store.getChainHead(); + assertEquals(params.genesisBlock, genesis.getHeader()); + + // Build a new block. + StoredBlock b1 = genesis.build(genesis.getHeader().createNextBlock(to).cloneAsHeader()); + store.put(b1); + store.setChainHead(b1); + // Check we can get it back out again if we rebuild the store object. + store = new DiskBlockStore(params, temp); + StoredBlock b2 = store.get(b1.getHeader().getHash()); + assertEquals(b1, b2); + // Check the chain head was stored correctly also. + assertEquals(b1, store.getChainHead()); + } +}