Tunnels: Add new Bloom filter size, increase bandwidth limit (ticket #1505)

router/java/src/net/i2p/router/transport/FIFOBandwidthRefiller.java

@@ -70,8 +70,12 @@ public class FIFOBandwidthRefiller implements Runnable {
     public static final int MIN_INBOUND_BANDWIDTH_PEAK = 3;
     /** For now, until there is some tuning and safe throttling, we set the floor at a 3KBps during burst */
     public static final int MIN_OUTBOUND_BANDWIDTH_PEAK = 3;
-    /** Max for reasonable bloom filter false positive rate. See util/DecayingBloomFilter and tunnel/BloomFilterIVValidator */
+    /**
-    public static final int MAX_OUTBOUND_BANDWIDTH = 4096;
+     *  Max for reasonable Bloom filter false positive rate.
+     *  Do not increase without adding a new Bloom filter size!
+     *  See util/DecayingBloomFilter and tunnel/BloomFilterIVValidator.
+     */
+    public static final int MAX_OUTBOUND_BANDWIDTH = 8192;
     
     /** 
      * how often we replenish the queues.  

router/java/src/net/i2p/router/tunnel/BloomFilterIVValidator.java

@@ -29,12 +29,17 @@ class BloomFilterIVValidator implements IVValidator {
     private static final int MIN_SHARE_KBPS_TO_USE_BLOOM = 64;
     private static final int MIN_SHARE_KBPS_FOR_BIG_BLOOM = 512;
     private static final int MIN_SHARE_KBPS_FOR_HUGE_BLOOM = 1536;
+    private static final int MIN_SHARE_KBPS_FOR_HUGE2_BLOOM = 4096;
     private static final long MIN_MEM_TO_USE_BLOOM = 64*1024*1024l;
     private static final long MIN_MEM_FOR_BIG_BLOOM = 128*1024*1024l;
     private static final long MIN_MEM_FOR_HUGE_BLOOM = 256*1024*1024l;
+    private static final long MIN_MEM_FOR_HUGE2_BLOOM = 384*1024*1024l;
     /** for testing */
     private static final String PROP_FORCE = "router.forceDecayingBloomFilter";
 
+    /**
+     *  @param Kbps share bandwidth
+     */
     public BloomFilterIVValidator(RouterContext ctx, int KBps) {
         _context = ctx;
         // Select the filter based on share bandwidth and memory.
@@ -48,7 +53,11 @@ class BloomFilterIVValidator implements IVValidator {
             if (KBps >= MIN_SHARE_KBPS_TO_USE_BLOOM)
                 warn(maxMemory, KBps, MIN_MEM_TO_USE_BLOOM, MIN_SHARE_KBPS_TO_USE_BLOOM);
             _filter = new DecayingHashSet(ctx, HALFLIFE_MS, 16, "TunnelIVV"); // appx. 4MB max
+        } else if (KBps >= MIN_SHARE_KBPS_FOR_HUGE2_BLOOM && maxMemory >= MIN_MEM_FOR_HUGE2_BLOOM) {
+            _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 26);  // 16MB fixed
         } else if (KBps >= MIN_SHARE_KBPS_FOR_HUGE_BLOOM && maxMemory >= MIN_MEM_FOR_HUGE_BLOOM) {
+            if (KBps >= MIN_SHARE_KBPS_FOR_HUGE2_BLOOM)
+                warn(maxMemory, KBps, MIN_MEM_FOR_HUGE2_BLOOM, MIN_SHARE_KBPS_FOR_HUGE2_BLOOM);
             _filter = new DecayingBloomFilter(ctx, HALFLIFE_MS, 16, "TunnelIVV", 25);  // 8MB fixed
         } else if (KBps >= MIN_SHARE_KBPS_FOR_BIG_BLOOM && maxMemory >= MIN_MEM_FOR_BIG_BLOOM) {
             if (KBps >= MIN_SHARE_KBPS_FOR_HUGE_BLOOM)

router/java/src/net/i2p/router/util/DecayingBloomFilter.java

@@ -372,6 +372,9 @@ public class DecayingBloomFilter {
      *
      *  Following stats for m=25, k=10:
      *  1792 2.4E-6; 4096 0.14%; 5120 0.6%; 6144 1.7%; 8192 6.8%; 10240 15%
+     *
+     *  Following stats for m=26, k=10:
+     *  4096 7.3E-6; 5120 4.5E-5; 6144 1.8E-4; 8192 0.14%; 10240 0.6%, 12288 1.7%
      *</pre>
      */
 /*****
@@ -400,16 +403,23 @@ public class DecayingBloomFilter {
     }
 
     private static void testByLong(int kbps, int m, int numRuns) {
+        System.out.println("Starting 8 byte test");
         int messages = 60 * 10 * kbps;
-        Random r = new Random();
+        java.util.Random r = new java.util.Random();
         DecayingBloomFilter filter = new DecayingBloomFilter(I2PAppContext.getGlobalContext(), 600*1000, 8, "test", m);
         int falsePositives = 0;
         long totalTime = 0;
         double fpr = 0d;
         for (int j = 0; j < numRuns; j++) {
+            // screen out birthday paradoxes (waste of time and space?)
+            java.util.Set<Long> longs = new java.util.HashSet<Long>(messages);
             long start = System.currentTimeMillis();
             for (int i = 0; i < messages; i++) {
-                if (filter.add(r.nextLong())) {
+                long rand;
+                do {
+                    rand = r.nextLong();
+                } while (!longs.add(Long.valueOf(rand)));
+                if (filter.add(rand)) {
                     falsePositives++;
                     //System.out.println("False positive " + falsePositives + " (testByLong j=" + j + " i=" + i + ")");
                 }
@@ -422,13 +432,14 @@ public class DecayingBloomFilter {
         System.out.println("False postive rate should be " + fpr);
         System.out.println("After " + numRuns + " runs pushing " + messages + " entries in "
                            + DataHelper.formatDuration(totalTime/numRuns) + " per run, there were "
-                           + falsePositives + " false positives");
+                           + falsePositives + " false positives (" +
-
+                           (((double) falsePositives) / messages) + ')');
     }
 
     private static void testByBytes(int kbps, int m, int numRuns) {
+        System.out.println("Starting 16 byte test");
         byte iv[][] = new byte[60*10*kbps][16];
-        Random r = new Random();
+        java.util.Random r = new java.util.Random();
         for (int i = 0; i < iv.length; i++)
             r.nextBytes(iv[i]);
 
@@ -452,7 +463,8 @@ public class DecayingBloomFilter {
         System.out.println("False postive rate should be " + fpr);
         System.out.println("After " + numRuns + " runs pushing " + iv.length + " entries in "
                            + DataHelper.formatDuration(totalTime/numRuns) + " per run, there were "
-                           + falsePositives + " false positives");
+                           + falsePositives + " false positives (" +
+                           (((double) falsePositives) / iv.length) + ')');
         //System.out.println("inserted: " + bloom.size() + " with " + bloom.capacity() 
         //                   + " (" + bloom.falsePositives()*100.0d + "% false positive)");
     }