Qortal/altcoinj
3
0
Fork 0
mirror of https://github.com/Qortal/altcoinj.git synced 2025-02-15 11:45:51 +00:00
Code Issues Projects Releases Wiki Activity

Faster FP tracking using double exponential

Browse Source
This commit is contained in:
Devrandom 2013-12-14 12:08:03 -08:00 committed by Mike Hearn
parent 1293e42aa3
commit 462c75324e
2 changed files with 44 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
core/src/main/java/com/google/bitcoin/core/AbstractBlockChain.java
View File

@ -126,9 +126,13 @@ public abstract class AbstractBlockChain {
private final LinkedHashMap<Sha256Hash, OrphanBlock> orphanBlocks = new LinkedHashMap<Sha256Hash, OrphanBlock>(); private final LinkedHashMap<Sha256Hash, OrphanBlock> orphanBlocks = new LinkedHashMap<Sha256Hash, OrphanBlock>();
// False positive estimation uses an exponential moving average, with alpha = FP_ESTIMATOR_DECAY // False positive estimation uses an exponential moving average, with alpha = FP_ESTIMATOR_DECAY
static final double FP_ESTIMATOR_DECAY = 0.0001; static final double FP_ESTIMATOR_ALPHA = 0.0001;
static final double FP_ESTIMATOR_BETA = 0.01;
protected double falsePositiveRate; protected double falsePositiveRate;
protected double falsePositiveTrend;
protected double previousFalsePositiveRate;
/** /**
* Constructs a BlockChain connected to the given list of listeners (eg, wallets) and a store. * Constructs a BlockChain connected to the given list of listeners (eg, wallets) and a store.
@ -1001,29 +1005,47 @@ public abstract class AbstractBlockChain {
* on the total number of transactions in the original block. * on the total number of transactions in the original block.
* *
* count includes filtered transactions, transactions that were passed in and were relevant * count includes filtered transactions, transactions that were passed in and were relevant
* and transactions that were false positives. * and transactions that were false positives (i.e. includes all transactions in the block).
*/ */
protected void trackFilteredTransactions(int count) { protected void trackFilteredTransactions(int count) {
// Track non-false-positives in batch by multiplying by (1-alpha) count times. Each // Track non-false-positives in batch. Each non-false-positive counts as
// non-false-positive counts as 0.0 towards the estimate. // 0.0 towards the estimate.
// //
// This is slightly off because we are applying false positive tracking before non-FP tracking, // This is slightly off because we are applying false positive tracking before non-FP tracking,
// which counts FP as if they came at the beginning of the block. Assuming uniform FP // which counts FP as if they came at the beginning of the block. Assuming uniform FP
// spread in a block, this will somewhat underestimate the FP rate (5% for 1000 tx block). // spread in a block, this will somewhat underestimate the FP rate (5% for 1000 tx block).
falsePositiveRate *= Math.pow(1-FP_ESTIMATOR_DECAY, count); double alphaDecay = Math.pow(1 - FP_ESTIMATOR_ALPHA, count);
// new_rate = alpha_decay * new_rate
falsePositiveRate = alphaDecay * falsePositiveRate;
double betaDecay = Math.pow(1 - FP_ESTIMATOR_BETA, count);
// trend = beta * (new_rate - old_rate) + beta_decay * trend
falsePositiveTrend =
FP_ESTIMATOR_BETA * count * (falsePositiveRate - previousFalsePositiveRate) +
betaDecay * falsePositiveTrend;
// new_rate += alpha_decay * trend
falsePositiveRate += alphaDecay * falsePositiveTrend;
// Stash new_rate in old_rate
previousFalsePositiveRate = falsePositiveRate;
} }
/* An irrelevant transaction was received. Update false-positive estimate. */ /* Irrelevant transactions were received. Update false-positive estimate. */
void trackFalsePositives(int count) { void trackFalsePositives(int count) {
// Track false positives in batch by adding alpha to the false positive estimate once per count. // Track false positives in batch by adding alpha to the false positive estimate once per count.
// Each false positive counts as 1.0 towards the estimate. // Each false positive counts as 1.0 towards the estimate.
falsePositiveRate += FP_ESTIMATOR_DECAY * count; falsePositiveRate += FP_ESTIMATOR_ALPHA * count;
if (count > 0) if (count > 0)
log.warn("{} false positives, current rate = {}", count, falsePositiveRate); log.debug("{} false positives, current rate = {} trend = {}", count, falsePositiveRate, falsePositiveTrend);
} }
/** Resets estimates of false positives. Used when the filter is sent to the peer. */ /** Resets estimates of false positives. Used when the filter is sent to the peer. */
public void resetFalsePositiveEstimate() { public void resetFalsePositiveEstimate() {
falsePositiveRate = 0; falsePositiveRate = 0;
falsePositiveTrend = 0;
previousFalsePositiveRate = 0;
} }
} }

18
core/src/test/java/com/google/bitcoin/core/BlockChainTest.java
View File

@ -402,18 +402,28 @@ public class BlockChainTest {
@Test @Test
public void falsePositives() throws Exception { public void falsePositives() throws Exception {
double decay = AbstractBlockChain.FP_ESTIMATOR_DECAY; double decay = AbstractBlockChain.FP_ESTIMATOR_ALPHA;
assertTrue(0 == chain.getFalsePositiveRate()); // Exactly assertTrue(0 == chain.getFalsePositiveRate()); // Exactly
chain.trackFalsePositives(55); chain.trackFalsePositives(55);
assertTrue(Math.abs(decay * 55 - chain.getFalsePositiveRate()) < 1e-4); assertEquals(decay * 55, chain.getFalsePositiveRate(), 1e-4);
chain.trackFilteredTransactions(550); chain.trackFilteredTransactions(550);
double rate1 = chain.getFalsePositiveRate();
// Run this scenario a few more time for the filter to converge // Run this scenario a few more time for the filter to converge
for (int i = 1 ; i < 100 ; i++) { for (int i = 1 ; i < 10 ; i++) {
chain.trackFalsePositives(55); chain.trackFalsePositives(55);
chain.trackFilteredTransactions(550); chain.trackFilteredTransactions(550);
} }
assertTrue(Math.abs(0.1 - chain.getFalsePositiveRate()) < 1e-2);
// Ensure we are within 10%
assertEquals(0.1, chain.getFalsePositiveRate(), 0.01);
// Check that we get repeatable results after a reset
chain.resetFalsePositiveEstimate(); chain.resetFalsePositiveEstimate();
assertTrue(0 == chain.getFalsePositiveRate()); // Exactly assertTrue(0 == chain.getFalsePositiveRate()); // Exactly
chain.trackFalsePositives(55);
assertEquals(decay * 55, chain.getFalsePositiveRate(), 1e-4);
chain.trackFilteredTransactions(550);
assertEquals(rate1, chain.getFalsePositiveRate(), 1e-4);
} }
} }