Qortal/altcoinj
3
0
Fork 0
mirror of https://github.com/Qortal/altcoinj.git synced 2025-02-12 02:05:53 +00:00
Code Issues Projects Releases Wiki Activity

Make ECKey store its creation time. Implement fast catchup using the getheaders command. You can now set a date on a Peer/PeerGroup, before which block bodies will not be fetched. After that they will. Using the date of the earliest key in the wallet means new users can get started faster and with less battery drain as they aren't parsing blocks that are guaranteed to have no relevant transactions.

Browse Source
This commit is contained in:
Mike Hearn 2011-12-29 23:52:08 +00:00
parent da10e0ca69
commit dd2be6eeb4
15 changed files with 281 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/com/google/bitcoin/core/BitcoinSerializer.java
View File

@ -67,9 +67,9 @@ public class BitcoinSerializer {
names.put(Ping.class, "ping");
names.put(VersionAck.class, "verack");
names.put(GetBlocksMessage.class, "getblocks");
names.put(GetHeadersMessage.class, "getheaders");
names.put(GetAddrMessage.class, "getaddr");
names.put(HeadersMessage.class, "headers");
}
/**

11
src/com/google/bitcoin/core/Block.java
View File

@ -95,8 +95,6 @@ public class Block extends Message {
/**
* Contruct a block object from the BitCoin wire format.
* @param params NetworkParameters object.
* @param msg Bitcoin protocol formatted byte array containing message content.
* @param protocolVersion Bitcoin protocol version.
* @param parseLazy Whether to perform a full parse immediately or delay until a read is requested.
* @param parseRetain Whether to retain the backing byte array for quick reserialization.
* If true and the backing byte array is invalidated due to modification of a field then
@ -768,6 +766,13 @@ public class Block extends Message {
return time;
}
/**
* Returns the time at which the block was solved and broadcast, according to the clock of the solving node.
*/
public Date getTime() {
return new Date(getTimeSeconds());
}
void setTime(long time) {
unCacheHeader();
this.time = time;
@ -867,7 +872,7 @@ public class Block extends Message {
// Visible for testing.
public Block createNextBlock(Address to) {
return createNextBlock(to, System.currentTimeMillis() / 1000);
return createNextBlock(to, Utils.now().getTime() / 1000);
}
/**

8
src/com/google/bitcoin/core/DownloadListener.java
View File

@ -58,7 +58,7 @@ public class DownloadListener extends AbstractPeerEventListener {
double pct = 100.0 - (100.0 * (blocksLeft / (double) originalBlocksLeft));
if ((int) pct != lastPercent) {
progress(pct, new Date(block.getTimeSeconds() * 1000));
progress(pct, blocksLeft, new Date(block.getTimeSeconds() * 1000));
lastPercent = (int) pct;
}
}
@ -69,9 +69,9 @@ public class DownloadListener extends AbstractPeerEventListener {
* @param pct the percentage of chain downloaded, estimated
* @param date the date of the last block downloaded
*/
protected void progress(double pct, Date date) {
System.out.println(String.format("Chain download %d%% done, block date %s", (int) pct,
DateFormat.getDateTimeInstance().format(date)));
protected void progress(double pct, int blocksSoFar, Date date) {
System.out.println(String.format("Chain download %d%% done with %d blocks to go, block date %s", (int) pct,
blocksSoFar, DateFormat.getDateTimeInstance().format(date)));
}
/**

15
src/com/google/bitcoin/core/ECKey.java
View File

@ -51,8 +51,14 @@ public class ECKey implements Serializable {
secureRandom = new SecureRandom();
}
// The two parts of the key. If "priv" is set, "pub" can always be calculated. If "pub" is set but not "priv", we
// can only verify signatures not make them.
// TODO: Redesign this class to use consistent internals and more efficient serialization.
private final BigInteger priv;
private final byte[] pub;
// Creation time of the key in seconds since the epoch, or zero if the key was deserialized from a version that did
// not have this field.
private long creationTimeSeconds;
transient private byte[] pubKeyHash;
@ -67,6 +73,15 @@ public class ECKey implements Serializable {
priv = privParams.getD();
// The public key is an encoded point on the elliptic curve. It has no meaning independent of the curve.
pub = pubParams.getQ().getEncoded();
creationTimeSeconds = Utils.now().getTime() / 1000;
}
/**
* Returns the creation time of this key or zero if the key was deserialized from a version that did not store
* that data.
*/
public long getCreationTimeSeconds() {
return creationTimeSeconds;
}
/**

2
src/com/google/bitcoin/core/GetDataMessage.java
View File

@ -27,8 +27,6 @@ public class GetDataMessage extends ListMessage {
* Deserializes a 'getdata' message.
* @param params NetworkParameters object.
* @param msg Bitcoin protocol formatted byte array containing message content.
* @param offset The location of the first msg byte within the array.
* @param protocolVersion Bitcoin protocol version.
* @param parseLazy Whether to perform a full parse immediately or delay until a read is requested.
* @param parseRetain Whether to retain the backing byte array for quick reserialization.
* If true and the backing byte array is invalidated due to modification of a field then

31
src/com/google/bitcoin/core/GetHeadersMessage.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright 2011 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.bitcoin.core;
import java.util.List;
/**
* The "getheaders" command is structurally identical to "getblocks", but has different meaning. On receiving this
* message a Bitcoin node returns matching blocks up to the limit, but without the bodies. It is useful as an
* optimization: when your wallet does not contain any keys created before a particular time, you don't have to download
* the bodies for those blocks because you know there are no relevant transactions.
*/
public class GetHeadersMessage extends GetBlocksMessage {
public GetHeadersMessage(NetworkParameters params, List<Sha256Hash> locator, Sha256Hash stopHash) {
super(params, locator, stopHash);
}
}

6
src/com/google/bitcoin/core/HeadersMessage.java
View File

@ -20,6 +20,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
@ -39,6 +40,11 @@ public class HeadersMessage extends Message {
super(params, payload, 0);
}
public HeadersMessage(NetworkParameters params, Block... headers) throws ProtocolException {
super(params);
blockHeaders = Arrays.asList(headers);
}
@Override
protected void parseLite() throws ProtocolException {
if (length == UNKNOWN_LENGTH) {

116
src/com/google/bitcoin/core/Peer.java
View File

@ -57,6 +57,13 @@ public class Peer {
*/
public static boolean MOBILE_OPTIMIZED = false;
// A time before which we only download block headers, after that point we download block bodies.
private long fastCatchupTimeSecs;
// Whether we are currently downloading headers only or block bodies. Defaults to true, if the fast catchup time
// is set AND our best block is before that date, switch to false until block headers beyond that point have been
// received at which point it gets set to true again. This isn't relevant unless downloadData is true.
private boolean downloadBlockBodies = true;
/**
* Construct a peer that reads/writes from the given block chain. Note that communication won't occur until
* you call connect(), which will set up a new NetworkConnection.
@ -70,6 +77,7 @@ public class Peer {
this.blockChain = blockChain;
this.pendingGetBlockFutures = new ArrayList<GetDataFuture<Block>>();
this.eventListeners = new ArrayList<PeerEventListener>();
this.fastCatchupTimeSecs = params.genesisBlock.getTimeSeconds();
}
/**
@ -151,6 +159,8 @@ public class Peer {
// We don't care about addresses of the network right now. But in future,
// we should save them in the wallet so we don't put too much load on the seed nodes and can
// properly explore the network.
} else if (m instanceof HeadersMessage) {
processHeaders((HeadersMessage) m);
} else {
// TODO: Handle the other messages we can receive.
log.warn("Received unhandled message: {}", m);
@ -176,6 +186,45 @@ public class Peer {
disconnect();
}
private void processHeaders(HeadersMessage m) throws IOException, ProtocolException {
// Runs in network loop thread for this peer.
//
// This can happen if a peer just randomly sends us a "headers" message (should never happen), or more likely
// when we've requested them as part of chain download using fast catchup. We need to add each block to the
// chain if it pre-dates the fast catchup time. If we go past it, we can stop processing the headers and request
// the full blocks from that point on instead.
assert !downloadBlockBodies;
try {
for (int i = 0; i < m.getBlockHeaders().size(); i++) {
Block header = m.getBlockHeaders().get(i);
if (header.getTimeSeconds() < fastCatchupTimeSecs) {
if (blockChain.add(header)) {
// The block was successfully linked into the chain. Notify the user of our progress.
invokeOnBlocksDownloaded(header);
} else {
// This block is unconnected - we don't know how to get from it back to the genesis block yet.
// That must mean that the peer is buggy or malicious because we specifically requested for
// headers that are part of the best chain.
throw new ProtocolException("Got unconnected header from peer: " + header.getHashAsString());
}
} else {
log.info("Passed the fast catchup time, discarding {} headers and requesting full blocks",
m.getBlockHeaders().size() - i);
downloadBlockBodies = true;
blockChainDownload(header.getHash());
return;
}
}
// We added all headers in the message to the chain. Now request some more!
blockChainDownload(Sha256Hash.ZERO_HASH);
} catch (VerificationException e) {
log.warn("Block header verification failed", e);
} catch (ScriptException e) {
// There are no transactions and thus no scripts in these blocks, so this should never happen.
throw new RuntimeException(e);
}
}
private void processBlock(Block m) throws IOException {
// This should called in the network loop thread for this peer
try {
@ -196,11 +245,7 @@ public class Peer {
// This call will synchronize on blockChain.
if (blockChain.add(m)) {
// The block was successfully linked into the chain. Notify the user of our progress.
for (PeerEventListener listener : eventListeners) {
synchronized (listener) {
listener.onBlocksDownloaded(this, m, getPeerBlocksToGet());
}
}
invokeOnBlocksDownloaded(m);
} else {
// This block is unconnected - we don't know how to get from it back to the genesis block yet. That
// must mean that there are blocks we are missing, so do another getblocks with a new block locator
@ -220,6 +265,14 @@ public class Peer {
}
}
private void invokeOnBlocksDownloaded(Block m) {
for (PeerEventListener listener : eventListeners) {
synchronized (listener) {
listener.onBlocksDownloaded(this, m, getPeerBlockHeightDifference());
}
}
}
private void processInv(InventoryMessage inv) throws IOException {
// This should be called in the network loop thread for this peer.
@ -286,6 +339,29 @@ public class Peer {
return future;
}
/**
* When downloading the block chain, the bodies will be skipped for blocks created before the given date. Any
* transactions relevant to the wallet will therefore not be found, but if you know your wallet has no such
* transactions it doesn't matter and can save a lot of bandwidth and processing time. Note that the times of blocks
* isn't known until their headers are available and they are requested in chunks, so some headers may be downloaded
* twice using this scheme, but this optimization can still be a large win for newly created wallets.
*
* @param secondsSinceEpoch Time in seconds since the epoch or 0 to reset to always downloading block bodies.
*/
public void setFastCatchupTime(long secondsSinceEpoch) {
if (secondsSinceEpoch == 0) {
fastCatchupTimeSecs = params.genesisBlock.getTimeSeconds();
downloadBlockBodies = true;
} else {
fastCatchupTimeSecs = secondsSinceEpoch;
// If the given time is before the current chains head block time, then this has no effect (we already
// downloaded everything we need).
if (fastCatchupTimeSecs >= blockChain.getChainHead().getHeader().getTimeSeconds()) {
downloadBlockBodies = false;
}
}
}
// A GetDataFuture wraps the result of a getBlock or (in future) getTransaction so the owner of the object can
// decide whether to wait forever, wait for a short while or check later after doing other work.
private static class GetDataFuture<T extends Message> implements Future<T> {
@ -352,7 +428,7 @@ public class Peer {
private void blockChainDownload(Sha256Hash toHash) throws IOException {
// This may run in ANY thread.
// The block chain download process is a bit complicated. Basically, we start with zero or more blocks in a
// The block chain download process is a bit complicated. Basically, we start with one or more blocks in a
// chain that we have from a previous session. We want to catch up to the head of the chain BUT we don't know
// where that chain is up to or even if the top block we have is even still in the chain - we
// might have got ourselves onto a fork that was later resolved by the network.
@ -376,7 +452,14 @@ public class Peer {
// process.
//
// So this is a complicated process but it has the advantage that we can download a chain of enormous length
// in a relatively stateless manner and with constant/bounded memory usage.
// in a relatively stateless manner and with constant memory usage.
//
// All this is made more complicated by the desire to skip downloading the bodies of blocks that pre-date the
// 'fast catchup time', which is usually set to the creation date of the earliest key in the wallet. Because
// we know there are no transactions using our keys before that date, we need only the headers. To do that we
// use the "getheaders" command. Once we find we've gone past the target date, we throw away the downloaded
// headers and then request the blocks from that point onwards. "getheaders" does not send us an inv, it just
// sends us the data we requested in a "headers" message.
log.info("blockChainDownload({})", toHash.toString());
// TODO: Block locators should be abstracted out rather than special cased here.
@ -400,8 +483,16 @@ public class Peer {
}
blockLocator.add(0, topBlock.getHash());
}
// The stopHash field is set to zero already by the constructor.
if (downloadBlockBodies) {
GetBlocksMessage message = new GetBlocksMessage(params, blockLocator, toHash);
conn.writeMessage(message);
} else {
// Downloading headers for a while instead of full blocks.
GetHeadersMessage message = new GetHeadersMessage(params, blockLocator, toHash);
conn.writeMessage(message);
}
}
/**
@ -412,10 +503,10 @@ public class Peer {
setDownloadData(true);
// TODO: peer might still have blocks that we don't have, and even have a heavier
// chain even if the chain block count is lower.
if (getPeerBlocksToGet() >= 0) {
if (getPeerBlockHeightDifference() >= 0) {
for (PeerEventListener listener : eventListeners) {
synchronized (listener) {
listener.onChainDownloadStarted(this, getPeerBlocksToGet());
listener.onChainDownloadStarted(this, getPeerBlockHeightDifference());
}
}
@ -425,15 +516,16 @@ public class Peer {
}
/**
* @return the number of blocks to get, based on our chain height and the peer reported height
* Returns the difference between our best chain height and the peers, which can either be positive if we are
* behind the peer, or negative if the peer is ahead of us.
*/
private int getPeerBlocksToGet() {
public int getPeerBlockHeightDifference() {
// Chain will overflow signed int blocks in ~41,000 years.
int chainHeight = (int) conn.getVersionMessage().bestHeight;
if (chainHeight <= 0) {
// This should not happen because we shouldn't have given the user a Peer that is to another client-mode
// node. If that happens it means the user overrode us somewhere.
return -1;
throw new RuntimeException("Connected to peer advertising negative chain height.");
}
int blocksToGet = chainHeight - blockChain.getChainHead().getHeight();
return blocksToGet;

14
src/com/google/bitcoin/core/PeerGroup.java
View File

@ -78,6 +78,7 @@ public class PeerGroup {
private BlockStore blockStore;
private BlockChain chain;
private int connectionDelayMillis;
private long fastCatchupTimeSecs;
/**
* Creates a PeerGroup with the given parameters and a default 5 second connection timeout.
@ -95,6 +96,7 @@ public class PeerGroup {
this.params = params;
this.chain = chain;
this.connectionDelayMillis = connectionDelayMillis;
this.fastCatchupTimeSecs = params.genesisBlock.getTimeSeconds();
inactives = new LinkedBlockingQueue<PeerAddress>();
peers = Collections.synchronizedSet(new HashSet<Peer>());
@ -445,6 +447,18 @@ public class PeerGroup {
downloadPeer = peer;
if (downloadPeer != null) {
downloadPeer.setDownloadData(true);
downloadPeer.setFastCatchupTime(fastCatchupTimeSecs);
}
}
/**
* Tells the PeerGroup to download only block headers before a certain time and bodies after that. See
* {@link Peer#setFastCatchupTime(long)} for further explanation.
*/
public synchronized void setFastCatchupTimeSecs(long secondsSinceEpoch) {
fastCatchupTimeSecs = secondsSinceEpoch;
if (downloadPeer != null) {
downloadPeer.setFastCatchupTime(secondsSinceEpoch);
}
}

23
src/com/google/bitcoin/core/Wallet.java
View File

@ -1021,4 +1021,27 @@ public class Wallet implements Serializable {
public Collection<Transaction> getPendingTransactions() {
return Collections.unmodifiableCollection(pending.values());
}
/**
* Returns the earliest creation time of the keys in this wallet, in seconds since the epoch, ie the min of
* {@link com.google.bitcoin.core.ECKey#getCreationTimeSeconds()}. This can return zero if at least one key does
* not have that data (was created before key timestamping was implemented). <p>
*
* This method is most often used in conjunction with {@link PeerGroup#setFastCatchupTimeSecs(long)} in order to
* optimize chain download for new users of wallet apps. Backwards compatibility notice: if you get zero from this
* method, you can instead use the time of the first release of your software, as it's guaranteed no users will
* have wallets pre-dating this time.
*
* @throws IllegalStateException if there are no keys in the wallet.
*/
public long getEarliestKeyCreationTime() {
if (keychain.size() == 0) {
throw new IllegalStateException("No keys in wallet");
}
long earliestTime = Long.MAX_VALUE;
for (ECKey key : keychain) {
earliestTime = Math.min(key.getCreationTimeSeconds(), earliestTime);
}
return earliestTime;
}
}

4
src/com/google/bitcoin/examples/PingService.java
View File

@ -25,7 +25,7 @@ import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.util.logging.LogManager;
import java.util.Date;
/**
* <p>
@ -83,6 +83,8 @@ public class PingService {
final PeerGroup peerGroup = new PeerGroup(blockStore, params, chain);
peerGroup.addAddress(new PeerAddress(InetAddress.getLocalHost()));
// Download headers only until a day ago.
peerGroup.setFastCatchupTimeSecs((new Date().getTime() / 1000) - (60 * 60 * 24));
peerGroup.start();
// We want to know when the balance changes.

6
tests/com/google/bitcoin/core/MockNetworkConnection.java
View File

@ -133,4 +133,10 @@ public class MockNetworkConnection implements NetworkConnection {
else
return null;
}
/** Convenience that does an inbound() followed by returning the value of outbound() */
public Message exchange(Message m) throws InterruptedException {
inbound(m);
return outbound();
}
}

51
tests/com/google/bitcoin/core/PeerTest.java
View File

@ -16,7 +16,8 @@
package com.google.bitcoin.core;
import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
@ -24,9 +25,7 @@ import java.util.List;
import java.util.concurrent.Future;
import static org.easymock.EasyMock.*;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
public class PeerTest extends TestWithNetworkConnections {
private Peer peer;
@ -224,4 +223,48 @@ public class PeerTest extends TestWithNetworkConnections {
assertEquals(b, b3);
conn.disconnect();
}
@Test
public void fastCatchup() throws Exception {
// Check that blocks before the fast catchup point are retrieved using getheaders, and after using getblocks.
// This test is INCOMPLETE because it does not check we handle >2000 blocks correctly.
Block b1 = TestUtils.createFakeBlock(unitTestParams, blockStore).block;
blockChain.add(b1);
Utils.rollMockClock(60 * 10); // 10 minutes later.
Block b2 = TestUtils.makeSolvedTestBlock(unitTestParams, b1);
Utils.rollMockClock(60 * 10); // 10 minutes later.
Block b3 = TestUtils.makeSolvedTestBlock(unitTestParams, b2);
Utils.rollMockClock(60 * 10);
Block b4 = TestUtils.makeSolvedTestBlock(unitTestParams, b3);
conn.setVersionMessageForHeight(unitTestParams, 4);
// Request headers until the last 2 blocks.
peer.setFastCatchupTime((Utils.now().getTime() / 1000) - (600*2) + 1);
runPeerAsync(peer, conn);
peer.startBlockChainDownload();
GetHeadersMessage getheaders = (GetHeadersMessage) conn.outbound();
List<Sha256Hash> expectedLocator = new ArrayList<Sha256Hash>();
expectedLocator.add(b1.getHash());
expectedLocator.add(b1.getPrevBlockHash());
expectedLocator.add(unitTestParams.genesisBlock.getHash());
assertEquals(getheaders.getLocator(), expectedLocator);
assertEquals(getheaders.getStopHash(), Sha256Hash.ZERO_HASH);
// Now send all the headers.
HeadersMessage headers = new HeadersMessage(unitTestParams, b2.cloneAsHeader(),
b3.cloneAsHeader(), b4.cloneAsHeader());
// We expect to be asked for b3 and b4 again, but this time, with a body.
expectedLocator.clear();
expectedLocator.add(b2.getHash());
expectedLocator.add(b1.getHash());
expectedLocator.add(unitTestParams.genesisBlock.getHash());
GetBlocksMessage getblocks = (GetBlocksMessage) conn.exchange(headers);
assertEquals(expectedLocator, getblocks.getLocator());
assertEquals(b3.getHash(), getblocks.getStopHash());
// We're supposed to get an inv here.
InventoryMessage inv = new InventoryMessage(unitTestParams);
inv.addItem(new InventoryItem(InventoryItem.Type.Block, b3.getHash()));
GetDataMessage getdata = (GetDataMessage) conn.exchange(inv);
assertEquals(b3.getHash(), getdata.getItems().get(0).hash);
// All done.
assertEquals(null, conn.exchange(b3));
}
}

16
tests/com/google/bitcoin/core/WalletTest.java
View File

@ -323,4 +323,20 @@ public class WalletTest {
assertEquals(tx1, transactions.get(2));
assertEquals(3, transactions.size());
}
@Test
public void keyCreationTime() throws Exception {
wallet = new Wallet(params);
// No keys throws an exception.
try {
wallet.getEarliestKeyCreationTime();
fail();
} catch (IllegalStateException e) {}
long now = Utils.rollMockClock(0).getTime() / 1000; // Fix the mock clock.
wallet.addKey(new ECKey());
assertEquals(now, wallet.getEarliestKeyCreationTime());
Utils.rollMockClock(60);
wallet.addKey(new ECKey());
assertEquals(now, wallet.getEarliestKeyCreationTime());
}
}