diff --git a/core/java/src/net/i2p/util/DecayingBloomFilter.java b/core/java/src/net/i2p/util/DecayingBloomFilter.java index a1029ae1b..4179edd38 100644 --- a/core/java/src/net/i2p/util/DecayingBloomFilter.java +++ b/core/java/src/net/i2p/util/DecayingBloomFilter.java @@ -37,6 +37,7 @@ public class DecayingBloomFilter { private String _name; private static final int DEFAULT_M = 23; + private static final int DEFAULT_K = 11; private static final boolean ALWAYS_MISS = false; /** noop for DHS */ @@ -56,16 +57,24 @@ public class DecayingBloomFilter { /** @param name just for logging / debugging / stats */ public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name) { + // this is instantiated in four different places, they may have different + // requirements, but for now use this as a gross method of memory reduction. + // m == 23 => 1MB each BloomSHA1 (4 pairs = 8MB total) + this(context, durationMs, entryBytes, name, context.getProperty("router.decayingBloomFilterM", DEFAULT_M)); + } + + /** @param m filter size exponent */ + public DecayingBloomFilter(I2PAppContext context, int durationMs, int entryBytes, String name, int m) { _context = context; _log = context.logManager().getLog(DecayingBloomFilter.class); _entryBytes = entryBytes; _name = name; - // this is instantiated in four different places, they may have different - // requirements, but for now use this as a gross method of memory reduction. - // m == 23 => 1MB each BloomSHA1 (4 pairs = 8MB total) - int m = context.getProperty("router.decayingBloomFilterM", DEFAULT_M); - _current = new BloomSHA1(m, 11); //new BloomSHA1(23, 11); - _previous = new BloomSHA1(m, 11); //new BloomSHA1(23, 11); + int k = DEFAULT_K; + // max is (23,11) or (26,10); see KeySelector for details + if (m > DEFAULT_M) + k--; + _current = new BloomSHA1(m, k); + _previous = new BloomSHA1(m, k); _durationMs = durationMs; int numExtenders = (32+ (entryBytes-1))/entryBytes - 1; if (numExtenders < 0) @@ -83,7 +92,7 @@ public class DecayingBloomFilter { _keepDecaying = true; SimpleTimer.getInstance().addEvent(_decayEvent, _durationMs); if (_log.shouldLog(Log.WARN)) - _log.warn("New DBF " + name + " m = " + m + " entryBytes = " + entryBytes + + _log.warn("New DBF " + name + " m = " + m + " k = " + k + " entryBytes = " + entryBytes + " numExtenders = " + numExtenders + " cycle (s) = " + (durationMs / 1000)); // try to get a handle on memory usage vs. false positives context.statManager().createRateStat("router.decayingBloomFilter." + name + ".size", @@ -260,12 +269,25 @@ public class DecayingBloomFilter { } /** + * This filter is used only for participants and OBEPs, not + * IBGWs, so depending on your assumptions of avg. tunnel length, + * the performance is somewhat better than the gross share BW + * would indicate. + * + * Following stats for m=23, k=11: * Theoretical false positive rate for 16 KBps: 1.17E-21 * Theoretical false positive rate for 24 KBps: 9.81E-20 * Theoretical false positive rate for 32 KBps: 2.24E-18 * Theoretical false positive rate for 256 KBps: 7.45E-9 * Theoretical false positive rate for 512 KBps: 5.32E-6 * Theoretical false positive rate for 1024 KBps: 1.48E-3 + * Then it gets bad: 1280 .67%; 1536 2.0%; 1792 4.4%; 2048 8.2%. + * + * Following stats for m=24, k=10: + * 1280 4.5E-5; 1792 5.6E-4; 2048 0.14% + * + * Following stats for m=25, k=10: + * 1792 2.4E-6; 4096 0.14% */ public static void main(String args[]) { int kbps = 256; diff --git a/core/java/src/org/xlattice/crypto/filters/BloomSHA1.java b/core/java/src/org/xlattice/crypto/filters/BloomSHA1.java index 6da1f0f60..1e42f991f 100644 --- a/core/java/src/org/xlattice/crypto/filters/BloomSHA1.java +++ b/core/java/src/org/xlattice/crypto/filters/BloomSHA1.java @@ -47,6 +47,8 @@ public class BloomSHA1 { protected final int filterBits; protected final int filterWords; +/* (24,11) too big - see KeySelector + public static void main(String args[]) { BloomSHA1 b = new BloomSHA1(24, 11); for (int i = 0; i < 100; i++) { @@ -55,14 +57,17 @@ public class BloomSHA1 { b.insert(v); } } +*/ /** * Creates a filter with 2^m bits and k 'hash functions', where * each hash function is portion of the 160-bit SHA1 hash. - * @param m determines number of bits in filter, defaults to 20 - * @param k number of hash functions, defaults to 8 + * @param m determines number of bits in filter + * @param k number of hash functionsx + * + * See KeySelector for important restriction on max m and k */ public BloomSHA1( int m, int k) { // XXX need to devise more reasonable set of checks diff --git a/core/java/src/org/xlattice/crypto/filters/KeySelector.java b/core/java/src/org/xlattice/crypto/filters/KeySelector.java index 3a1528ed9..64c6a72ba 100644 --- a/core/java/src/org/xlattice/crypto/filters/KeySelector.java +++ b/core/java/src/org/xlattice/crypto/filters/KeySelector.java @@ -51,6 +51,13 @@ public class KeySelector { * @param k number of 'hash functions' * @param bitOffset array of k bit offsets (offset of flag bit in word) * @param wordOffset array of k word offsets (offset of word flag is in) + * + * Note that if k and m are too big, the GenericWordSelector blows up - + * The max for 32-byte keys is m=23 and k=11. + * The precise restriction appears to be: + * ((5k + (k-1)(m-5)) / 8) + 2 < keySizeInBytes + * + * It isn't clear how to fix this. */ public KeySelector (int m, int k, int[] bitOffset, int [] wordOffset) { //if ( (m < 2) || (m > 20)|| (k < 1) @@ -121,7 +128,7 @@ public class KeySelector { } /** * Extracts the k word offsets from a key. Suitable for general - * values of m and k. + * values of m and k. See above for formula for max m and k. */ public class GenericWordSelector implements WordSelector { /** Extract the k offsets into the word offset array */ @@ -176,6 +183,8 @@ public class KeySelector { // bits from third byte bitsToGet -= 8; if (bitsToGet > 0) { + // AIOOBE here if m and k too big (23,11 is the max) + // for a 32-byte key - see above wordOffset[j] |= ((0xff & b[curByte + 2]) >> (8 - bitsToGet)) << (stride - bitsToGet) ; diff --git a/router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java b/router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java index 2b05d0d20..8cc8cee45 100644 --- a/router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java +++ b/router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java @@ -25,14 +25,21 @@ public class BloomFilterIVValidator implements IVValidator { */ private static final int HALFLIFE_MS = 10*60*1000; private static final int MIN_SHARE_KBPS_TO_USE_BLOOM = 64; + private static final int MIN_SHARE_KBPS_FOR_BIG_BLOOM = 512; + private static final int MIN_SHARE_KBPS_FOR_HUGE_BLOOM = 1536; public BloomFilterIVValidator(RouterContext ctx, int KBps) { _context = ctx; // Select the filter based on share bandwidth. - // Note that at rates approaching 1MB, we need to do something else, - // as the Bloom filter false positive rates approach 0.1%. FIXME - if (getShareBandwidth(ctx) < MIN_SHARE_KBPS_TO_USE_BLOOM) + // Note that at rates above 512KB, we increase the filter size + // to keep acceptable false positive rates. + // See DBF, BloomSHA1, and KeySelector for details. + if (KBps < MIN_SHARE_KBPS_TO_USE_BLOOM) _filter = new DecayingHashSet(ctx, HALFLIFE_MS, 16, "TunnelIVV"); // appx. 4MB max + else if (KBps >= MIN_SHARE_KBPS_FOR_HUGE_BLOOM) + _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 25); // 8MB fixed + else if (KBps >= MIN_SHARE_KBPS_FOR_BIG_BLOOM) + _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 24); // 4MB fixed else _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV"); // 2MB fixed ctx.statManager().createRateStat("tunnel.duplicateIV", "Note that a duplicate IV was received", "Tunnels", @@ -48,11 +55,4 @@ public class BloomFilterIVValidator implements IVValidator { return !dup; // return true if it is OK, false if it isn't } public void destroy() { _filter.stopDecaying(); } - - private static int getShareBandwidth(RouterContext ctx) { - int irateKBps = ctx.bandwidthLimiter().getInboundKBytesPerSecond(); - int orateKBps = ctx.bandwidthLimiter().getOutboundKBytesPerSecond(); - double pct = ctx.router().getSharePercentage(); - return (int) (pct * Math.min(irateKBps, orateKBps)); - } } diff --git a/router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java b/router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java index 2ce0b00e6..147dd2f76 100644 --- a/router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java +++ b/router/java/src/net/i2p/router/tunnel/TunnelDispatcher.java @@ -680,9 +680,18 @@ public class TunnelDispatcher implements Service { ******/ public void startup() { - // NB: 256 == assume max rate (size adjusted to handle 256 messages per second) - _validator = new BloomFilterIVValidator(_context, 256); + // Note that we only use the validator for participants and OBEPs, not IBGWs, so + // this BW estimate will be high by about 33% assuming 2-hop tunnels average + _validator = new BloomFilterIVValidator(_context, getShareBandwidth(_context)); } + + private static int getShareBandwidth(RouterContext ctx) { + int irateKBps = ctx.bandwidthLimiter().getInboundKBytesPerSecond(); + int orateKBps = ctx.bandwidthLimiter().getOutboundKBytesPerSecond(); + double pct = ctx.router().getSharePercentage(); + return (int) (pct * Math.min(irateKBps, orateKBps)); + } + public void shutdown() { if (_validator != null) _validator.destroy();