Use earliest key time minus a week for setting fast catchup time and selecting a checkpoint.

This handles clock drift both in the block headers and possibly wrong times in the users clock (broken timezone, etc).

Resolves issue 460.

core/src/main/java/com/google/bitcoin/core/CheckpointManager.java

@@ -22,7 +22,10 @@ import com.google.bitcoin.store.FullPrunedBlockStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
@@ -34,22 +37,33 @@ import java.util.TreeMap;
 import static com.google.common.base.Preconditions.*;
 
 /**
- * <p>Vends hard-coded {@link StoredBlock}s for blocks throughout the chain. Checkpoints serve several purposes:</p>
+ * <p>Vends hard-coded {@link StoredBlock}s for blocks throughout the chain. Checkpoints serve two purposes:</p>
  * <ol>
  *    <li>They act as a safety mechanism against huge re-orgs that could rewrite large chunks of history, thus
  *    constraining the block chain to be a consensus mechanism only for recent parts of the timeline.</li>
  *    <li>They allow synchronization to the head of the chain for new wallets/users much faster than syncing all
  *    headers from the genesis block.</li>
- *    <li>They mark each BIP30-violating block, which simplifies full verification logic quite significantly. BIP30
- *    handles the case of blocks that contained duplicated coinbase transactions.</li>
  * </ol>
  *
- * <p>Checkpoints are used by a {@link BlockChain} to initialize fresh {@link com.google.bitcoin.store.SPVBlockStore}s,
+ * <p>Checkpoints are used by the SPV {@link BlockChain} to initialize fresh
- * and by {@link FullPrunedBlockChain} to prevent re-orgs beyond them.</p>
+ * {@link com.google.bitcoin.store.SPVBlockStore}s. They are not used by fully validating mode, which instead has a
+ * different concept of checkpoints that are used to hard-code the validity of blocks that violate BIP30 (duplicate
+ * coinbase transactions). Those "checkpoints" can be found in NetworkParameters.</p>
+ *
+ * <p>The file format consists of the string "CHECKPOINTS 1", followed by a uint32 containing the number of signatures
+ * to read. The value may not be larger than 256 (so it could have been a byte but isn't for historical reasons).
+ * If the number of signatures is larger than zero, each 65 byte ECDSA secp256k1 signature then follows. The signatures
+ * sign the hash of all bytes that follow the last signature.</p>
+ *
+ * <p>After the signatures come an int32 containing the number of checkpoints in the file. Then each checkpoint follows
+ * one after the other. A checkpoint is 12 bytes for the total work done field, 4 bytes for the height, 80 bytes
+ * for the block header and then 1 zero byte at the end (i.e. number of transactions in the block: always zero).</p>
  */
 public class CheckpointManager {
     private static final Logger log = LoggerFactory.getLogger(CheckpointManager.class);
 
+    private static final int MAX_SIGNATURES = 256;
+
     // Map of block header time to data.
     protected final TreeMap<Long, StoredBlock> checkpoints = new TreeMap<Long, StoredBlock>();
 
@@ -58,10 +72,11 @@ public class CheckpointManager {
 
     public CheckpointManager(NetworkParameters params, InputStream inputStream) throws IOException {
         this.params = checkNotNull(params);
+        checkNotNull(inputStream);
         DataInputStream dis = null;
         try {
             MessageDigest digest = MessageDigest.getInstance("SHA-256");
-            DigestInputStream digestInputStream = new DigestInputStream(checkNotNull(inputStream), digest);
+            DigestInputStream digestInputStream = new DigestInputStream(inputStream, digest);
             dis = new DataInputStream(digestInputStream);
             digestInputStream.on(false);
             String magic = "CHECKPOINTS 1";
@@ -69,7 +84,7 @@ public class CheckpointManager {
             dis.readFully(header);
             if (!Arrays.equals(header, magic.getBytes("US-ASCII")))
                 throw new IOException("Header bytes did not match expected version");
-            int numSignatures = dis.readInt();
+            int numSignatures = checkPositionIndex(dis.readInt(), MAX_SIGNATURES, "Num signatures out of range");
             for (int i = 0; i < numSignatures; i++) {
                 byte[] sig = new byte[65];
                 dis.readFully(sig);
@@ -104,18 +119,16 @@ public class CheckpointManager {
      * you would want to know the checkpoint before the earliest wallet birthday.
      */
     public StoredBlock getCheckpointBefore(long time) {
-        checkArgument(time > params.getGenesisBlock().getTimeSeconds());
+        try {
-        // This is thread safe because the map never changes after creation.
+            checkArgument(time > params.getGenesisBlock().getTimeSeconds());
-        Map.Entry<Long, StoredBlock> entry = checkpoints.floorEntry(time);
+            // This is thread safe because the map never changes after creation.
-        if (entry == null) {
+            Map.Entry<Long, StoredBlock> entry = checkpoints.floorEntry(time);
-            try {
+            if (entry != null) return entry.getValue();
-                Block genesis = params.getGenesisBlock().cloneAsHeader();
+            Block genesis = params.getGenesisBlock().cloneAsHeader();
-                return new StoredBlock(genesis, genesis.getWork(), 0);
+            return new StoredBlock(genesis, genesis.getWork(), 0);
-            } catch (VerificationException e) {
+        } catch (VerificationException e) {
-                throw new RuntimeException(e);  // Cannot happen.
+            throw new RuntimeException(e);  // Cannot happen.
-            }
         }
-        return entry.getValue();
     }
 
     /** Returns the number of checkpoints that were loaded. */
@@ -129,15 +142,20 @@ public class CheckpointManager {
     }
 
     /**
-     * Convenience method that creates a CheckpointManager, loads the given data, gets the checkpoint for the given
+     * <p>Convenience method that creates a CheckpointManager, loads the given data, gets the checkpoint for the given
      * time, then inserts it into the store and sets that to be the chain head. Useful when you have just created
-     * a new store from scratch and want to use configure it all in one go.
+     * a new store from scratch and want to use configure it all in one go.</p>
+     *
+     * <p>Note that time is adjusted backwards by a week to account for possible clock drift in the block headers.</p>
      */
     public static void checkpoint(NetworkParameters params, InputStream checkpoints, BlockStore store, long time)
             throws IOException, BlockStoreException {
         checkNotNull(params);
         checkNotNull(store);
         checkArgument(!(store instanceof FullPrunedBlockStore), "You cannot use checkpointing with a full store.");
+
+        time -= 86400 * 7;
+
         BufferedInputStream stream = new BufferedInputStream(checkpoints);
         CheckpointManager manager = new CheckpointManager(params, stream);
         StoredBlock checkpoint = manager.getCheckpointBefore(time);

core/src/main/java/com/google/bitcoin/core/PeerGroup.java

@@ -678,10 +678,10 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
             // Fully verifying mode doesn't use this optimization (it can't as it needs to see all transactions).
             if (chain != null && chain.shouldVerifyTransactions())
                 return;
-            long earliestKeyTime = Long.MAX_VALUE;
+            long earliestKeyTimeSecs = Long.MAX_VALUE;
             int elements = 0;
             for (PeerFilterProvider p : peerFilterProviders) {
-                earliestKeyTime = Math.min(earliestKeyTime, p.getEarliestKeyCreationTime());
+                earliestKeyTimeSecs = Math.min(earliestKeyTimeSecs, p.getEarliestKeyCreationTime());
                 elements += p.getBloomFilterElementCount();
             }
 
@@ -704,8 +704,13 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
                         }
                 }
             }
+            // Now adjust the earliest key time backwards by a week to handle the case of clock drift. This can occur
+            // both in block header timestamps and if the users clock was out of sync when the key was first created
+            // (to within a small amount of tolerance).
+            earliestKeyTimeSecs -= 86400 * 7;
+
             // Do this last so that bloomFilter is already set when it gets called.
-            setFastCatchupTimeSecs(earliestKeyTime);
+            setFastCatchupTimeSecs(earliestKeyTimeSecs);
         } finally {
             lock.unlock();
         }

core/src/test/java/com/google/bitcoin/core/PeerGroupTest.java

@@ -28,7 +28,6 @@ import org.junit.Test;
 
 import java.math.BigInteger;
 import java.net.InetSocketAddress;
-import java.util.Date;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.Semaphore;
@@ -336,23 +335,24 @@ public class PeerGroupTest extends TestWithPeerGroup {
 
     @Test
     public void testWalletCatchupTime() throws Exception {
-        // Check the fast catchup time was initialized to something around the current runtime. The wallet was
+        // Check the fast catchup time was initialized to something around the current runtime minus a week.
-        // already added to the peer in setup.
+        // The wallet was already added to the peer in setup.
-        long time = new Date().getTime() / 1000;
+        final int WEEK = 86400 * 7;
-        assertTrue(peerGroup.getFastCatchupTimeSecs() > time - 10000);
+        final long now = Utils.now().getTime() / 1000;
+        assertTrue(peerGroup.getFastCatchupTimeSecs() > now - WEEK - 10000);
         Wallet w2 = new Wallet(params);
         ECKey key1 = new ECKey();
-        key1.setCreationTimeSeconds(time - 86400);  // One day ago.
+        key1.setCreationTimeSeconds(now - 86400);  // One day ago.
         w2.addKey(key1);
         peerGroup.addWallet(w2);
         Threading.waitForUserCode();
-        assertEquals(peerGroup.getFastCatchupTimeSecs(), time - 86400);
+        assertEquals(peerGroup.getFastCatchupTimeSecs(), now - 86400 - WEEK);
         // Adding a key to the wallet should update the fast catchup time.
         ECKey key2 = new ECKey();
-        key2.setCreationTimeSeconds(time - 100000);
+        key2.setCreationTimeSeconds(now - 100000);
         w2.addKey(key2);
         Threading.waitForUserCode();
-        assertEquals(peerGroup.getFastCatchupTimeSecs(), time - 100000);
+        assertEquals(peerGroup.getFastCatchupTimeSecs(), now - WEEK - 100000);
     }
 
     @Test