Qortal/altcoinj
3
0
Fork 0
mirror of https://github.com/Qortal/altcoinj.git synced 2025-02-12 18:25:51 +00:00
Code Issues Projects Releases Wiki Activity

Force update Bloom filter when false positive rate gets high

Browse Source
This commit is contained in:
Devrandom 2013-12-08 20:00:04 -08:00 committed by Mike Hearn
parent 751434ba7c
commit ba9415b3ee
4 changed files with 73 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
core/src/main/java/com/google/bitcoin/core/AbstractBlockChain.java
View File

@ -124,6 +124,9 @@ public abstract class AbstractBlockChain {
// were downloading the block chain.
private final LinkedHashMap<Sha256Hash, OrphanBlock> orphanBlocks = new LinkedHashMap<Sha256Hash, OrphanBlock>();
private static final double FP_ESTIMATOR_DECAY = 0.0001;
private double falsePositiveRate;
/**
* Constructs a BlockChain connected to the given list of listeners (eg, wallets) and a store.
*/
@ -265,7 +268,12 @@ public abstract class AbstractBlockChain {
// a false positive, as expected in any Bloom filtering scheme). The filteredTxn list here will usually
// only be full of data when we are catching up to the head of the chain and thus haven't witnessed any
// of the transactions.
return add(block.getBlockHeader(), true, block.getTransactionHashes(), block.getAssociatedTransactions());
boolean success =
add(block.getBlockHeader(), true, block.getTransactionHashes(), block.getAssociatedTransactions());
if (success) {
onFilteredTransactions(block.getTransactionCount());
}
return success;
} catch (BlockStoreException e) {
// TODO: Figure out a better way to propagate this exception to the user.
throw new RuntimeException(e);
@ -518,7 +526,7 @@ public abstract class AbstractBlockChain {
}
}
private static void informListenerForNewTransactions(Block block, NewBlockType newBlockType,
private void informListenerForNewTransactions(Block block, NewBlockType newBlockType,
@Nullable List<Sha256Hash> filteredTxHashList,
@Nullable Map<Sha256Hash, Transaction> filteredTxn,
StoredBlock newStoredBlock, boolean first,
@ -703,7 +711,7 @@ public abstract class AbstractBlockChain {
SIDE_CHAIN
}
private static void sendTransactionsToListener(StoredBlock block, NewBlockType blockType,
private void sendTransactionsToListener(StoredBlock block, NewBlockType blockType,
BlockChainListener listener,
int relativityOffset,
List<Transaction> transactions,
@ -714,6 +722,8 @@ public abstract class AbstractBlockChain {
if (clone)
tx = new Transaction(tx.params, tx.bitcoinSerialize());
listener.receiveFromBlock(tx, block, blockType, relativityOffset++);
} else {
onFalsePositive(tx, block, blockType);
}
} catch (ScriptException e) {
// We don't want scripts we don't understand to break the block chain so just note that this tx was
@ -967,4 +977,27 @@ public abstract class AbstractBlockChain {
}, Threading.SAME_THREAD);
return result;
}
/**
* The upstream server filtered a number of transactions. Update false-positive estimate based
* on this.
*/
public void onFilteredTransactions(int count) {
falsePositiveRate *= Math.pow(1-FP_ESTIMATOR_DECAY, count);
}
/** An irrelevant transaction was received. Update false-positive estimate. */
public void onFalsePositive(Transaction tx, StoredBlock block, AbstractBlockChain.NewBlockType blockType) {
falsePositiveRate += FP_ESTIMATOR_DECAY;
log.warn("false positive, current rate = {}", falsePositiveRate);
}
/** Resets estimates of false positives, used when the filter is sent to the peer. */
public void resetFalsePositiveEstimate() {
falsePositiveRate = 0;
}
public double getFalsePositiveRate() {
return 0;
}
}

5
core/src/main/java/com/google/bitcoin/core/FilteredBlock.java
View File

@ -110,4 +110,9 @@ public class FilteredBlock extends Message {
public Map<Sha256Hash, Transaction> getAssociatedTransactions() {
return Collections.unmodifiableMap(associatedTransactions);
}
/** Number of transactions in this block, before it was filtered */
public int getTransactionCount() {
return merkleTree.transactionCount;
}
}

3
core/src/main/java/com/google/bitcoin/core/PeerFilterProvider.java
View File

@ -18,7 +18,7 @@ package com.google.bitcoin.core;
/**
* An interface which provides the information required to properly filter data downloaded from Peers.
* Note that an implementer is responsible for calling {@link PeerGroup#recalculateFastCatchupAndFilter()} whenever a
* Note that an implementer is responsible for calling {@link PeerGroup#recalculateFastCatchupAndFilter(boolean)} whenever a
* change occurs which effects the data provided via this interface.
*/
public interface PeerFilterProvider {
@ -41,5 +41,6 @@ public interface PeerFilterProvider {
*/
public BloomFilter getBloomFilter(int size, double falsePositiveRate, long nTweak);
/** Whether this filter provider depends on the server updating the filter on all matches */
boolean isRequiringUpdateAllBloomFilter();
}

38
core/src/main/java/com/google/bitcoin/core/PeerGroup.java
View File

@ -113,17 +113,22 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
// This event listener is added to every peer. It's here so when we announce transactions via an "inv", every
// peer can fetch them.
private final AbstractPeerEventListener getDataListener = new AbstractPeerEventListener() {
private final AbstractPeerEventListener peerListener = new AbstractPeerEventListener() {
@Override
public List<Message> getData(Peer peer, GetDataMessage m) {
return handleGetData(m);
}
@Override
public void onBlocksDownloaded(Peer peer, Block block, int blocksLeft) {
handleBlocksDownloaded();
}
};
private int minBroadcastConnections = 0;
private AbstractWalletEventListener walletEventListener = new AbstractWalletEventListener() {
private void onChanged() {
recalculateFastCatchupAndFilter();
recalculateFastCatchupAndFilter(false);
}
@Override public void onScriptsAdded(Wallet wallet, List<Script> scripts) { onChanged(); }
@Override public void onKeysAdded(Wallet wallet, List<ECKey> keys) { onChanged(); }
@ -131,6 +136,14 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
@Override public void onCoinsSent(Wallet wallet, Transaction tx, BigInteger prevBalance, BigInteger newBalance) { onChanged(); }
};
private void handleBlocksDownloaded() {
double rate = chain.getFalsePositiveRate();
if (rate > bloomFilterFPRate * MAX_FP_RATE_INCREASE) {
log.info("Force update Bloom filter due to high false positive rate");
recalculateFastCatchupAndFilter(true);
}
}
private class PeerStartupListener extends AbstractPeerEventListener {
@Override
public void onPeerConnected(Peer peer, int peerCount) {
@ -153,6 +166,8 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
* Users for which low data usage is of utmost concern, 0.0001 may be better, for users
* to whom anonymity is of utmost concern, 0.001 should provide very good privacy */
public static final double DEFAULT_BLOOM_FILTER_FP_RATE = 0.0005;
/** Maximum increase in FP rate before forced refresh of the bloom filter */
public static final double MAX_FP_RATE_INCREASE = 2.0f;
// The false positive rate for bloomFilter
private double bloomFilterFPRate = DEFAULT_BLOOM_FILTER_FP_RATE;
// We use a constant tweak to avoid giving up privacy when we regenerate our filter with new keys
@ -170,6 +185,7 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
*
* @param params Network parameters
*/
public PeerGroup(NetworkParameters params) {
this(params, null);
}
@ -578,7 +594,7 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
// if a key is added. Of course, by then we may have downloaded the chain already. Ideally adding keys would
// automatically rewind the block chain and redownload the blocks to find transactions relevant to those keys,
// all transparently and in the background. But we are a long way from that yet.
recalculateFastCatchupAndFilter();
recalculateFastCatchupAndFilter(false);
updateVersionMessageRelayTxesBeforeFilter(getVersionMessage());
} finally {
lock.unlock();
@ -598,8 +614,11 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
/**
* Recalculates the bloom filter given to peers as well as the timestamp after which full blocks are downloaded
* (instead of only headers).
*
* @param forceFilterUpdate send the bloom filter even if it didn't change. Use
* this if the false positive rate is high due to server auto-update.
*/
public void recalculateFastCatchupAndFilter() {
public void recalculateFastCatchupAndFilter(boolean forceFilterUpdate) {
lock.lock();
try {
// Fully verifying mode doesn't use this optimization (it can't as it needs to see all transactions).
@ -625,10 +644,13 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
BloomFilter filter = new BloomFilter(lastBloomFilterElementCount, bloomFilterFPRate, bloomFilterTweak, bloomFlags);
for (PeerFilterProvider p : peerFilterProviders)
filter.merge(p.getBloomFilter(lastBloomFilterElementCount, bloomFilterFPRate, bloomFilterTweak));
if (!filter.equals(bloomFilter)) {
if (forceFilterUpdate || !filter.equals(bloomFilter)) {
bloomFilter = filter;
for (Peer peer : peers)
peer.setBloomFilter(filter);
// Reset the false positive estimate so that we don't send a flood of filter updates
// if the estimate temporarily overshoots our threshold.
chain.resetFalsePositiveEstimate();
}
}
// Now adjust the earliest key time backwards by a week to handle the case of clock drift. This can occur
@ -655,7 +677,7 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
lock.lock();
try {
this.bloomFilterFPRate = bloomFilterFPRate;
recalculateFastCatchupAndFilter();
recalculateFastCatchupAndFilter(false);
} finally {
lock.unlock();
}
@ -792,7 +814,7 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
}
}
// Make sure the peer knows how to upload transactions that are requested from us.
peer.addEventListener(getDataListener, Threading.SAME_THREAD);
peer.addEventListener(peerListener, Threading.SAME_THREAD);
// And set up event listeners for clients. This will allow them to find out about new transactions and blocks.
for (ListenerRegistration<PeerEventListener> registration : peerEventListeners) {
peer.addEventListenerWithoutOnDisconnect(registration.listener, registration.executor);
@ -976,7 +998,7 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
log.error(e.getMessage());
}
}
peer.removeEventListener(getDataListener);
peer.removeEventListener(peerListener);
for (Wallet wallet : wallets) {
peer.removeWallet(wallet);
}