From 4c1050b93a0ad2d08fc022da541b73a436ae0e0a Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 12 Jan 2011 13:19:20 +0000 Subject: [PATCH] * Router: - Add new RandomIterator, use in UDP, peer selector, profile organizer - Add a stat to monitor peer selector run time --- .../kademlia/FloodfillPeerSelector.java | 11 +- .../router/networkdb/kademlia/KBucketSet.java | 3 + .../router/peermanager/ProfileOrganizer.java | 21 ++- .../router/transport/udp/UDPTransport.java | 6 +- .../net/i2p/router/util/RandomIterator.java | 178 ++++++++++++++++++ .../java/src/net/i2p/router/util/package.html | 6 + 6 files changed, 210 insertions(+), 15 deletions(-) create mode 100644 router/java/src/net/i2p/router/util/RandomIterator.java create mode 100644 router/java/src/net/i2p/router/util/package.html diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/FloodfillPeerSelector.java b/router/java/src/net/i2p/router/networkdb/kademlia/FloodfillPeerSelector.java index 4eb4e4825..eaf68fd07 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/FloodfillPeerSelector.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/FloodfillPeerSelector.java @@ -11,6 +11,7 @@ package net.i2p.router.networkdb.kademlia; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -19,6 +20,7 @@ import net.i2p.data.Hash; import net.i2p.data.RouterInfo; import net.i2p.router.RouterContext; import net.i2p.router.peermanager.PeerProfile; +import net.i2p.router.util.RandomIterator; import net.i2p.stat.Rate; import net.i2p.util.Log; @@ -74,6 +76,7 @@ class FloodfillPeerSelector extends PeerSelector { if (peersToIgnore == null) peersToIgnore = new HashSet(1); peersToIgnore.add(_context.routerHash()); + // TODO this is very slow FloodfillSelectionCollector matches = new FloodfillSelectionCollector(key, peersToIgnore, maxNumRouters); if (kbuckets == null) return new ArrayList(); kbuckets.getAll(matches); @@ -104,6 +107,8 @@ class FloodfillPeerSelector extends PeerSelector { */ private List selectFloodfillParticipants(Set toIgnore, KBucketSet kbuckets) { if (kbuckets == null) return Collections.EMPTY_LIST; + // TODO this is very slow - use profile getPeersByCapability('f') instead + _context.statManager().addRateData("netDb.newFSC", 0, 0); FloodfillSelectionCollector matches = new FloodfillSelectionCollector(null, toIgnore, 0); kbuckets.getAll(matches); return matches.getFloodfillParticipants(); @@ -320,7 +325,6 @@ class FloodfillPeerSelector extends PeerSelector { * Group 4: Non-floodfills, sorted by closest-to-the-key */ public List get(int howMany, boolean preferConnected) { - Collections.shuffle(_floodfillMatches, _context.random()); List rv = new ArrayList(howMany); List badff = new ArrayList(howMany); List unconnectedff = new ArrayList(howMany); @@ -329,8 +333,8 @@ class FloodfillPeerSelector extends PeerSelector { // Only add in "good" floodfills here... // Let's say published in last 3h and no failed sends in last 30m // (Forever shitlisted ones are excluded in add() above) - for (int i = 0; found < howMany && i < _floodfillMatches.size(); i++) { - Hash entry = (Hash) _floodfillMatches.get(i); + for (Iterator iter = new RandomIterator(_floodfillMatches); (found < howMany) && iter.hasNext(); ) { + Hash entry = iter.next(); RouterInfo info = _context.netDb().lookupRouterInfoLocally(entry); if (info != null && now - info.getPublished() > 3*60*60*1000) { badff.add(entry); @@ -391,6 +395,7 @@ class FloodfillPeerSelector extends PeerSelector { if (peersToIgnore != null && peersToIgnore.contains(Hash.FAKE_HASH)) { // return non-ff peersToIgnore.addAll(selectFloodfillParticipants(peersToIgnore, kbuckets)); + // TODO this is very slow FloodfillSelectionCollector matches = new FloodfillSelectionCollector(rkey, peersToIgnore, maxNumRouters); kbuckets.getAll(matches); return matches.get(maxNumRouters); diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/KBucketSet.java b/router/java/src/net/i2p/router/networkdb/kademlia/KBucketSet.java index f576d967c..c630e2e9d 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/KBucketSet.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/KBucketSet.java @@ -42,6 +42,7 @@ class KBucketSet { _context = context; _log = context.logManager().getLog(KBucketSet.class); createBuckets(); + context.statManager().createRateStat("netDb.KBSGetAllTime", "Time to add all Hashes to the Collector", "NetworkDatabase", new long[] { 60*60*1000 }); } /** @@ -99,8 +100,10 @@ class KBucketSet { } public void getAll(SelectionCollector collector) { + long start = _context.clock().now(); for (int i = 0; i < _buckets.length; i++) _buckets[i].getEntries(collector); + _context.statManager().addRateData("netDb.KBSGetAllTime", _context.clock().now() - start, 0); } public int pickBucket(Hash key) { diff --git a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java index 03b49269f..fe33c3447 100644 --- a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java +++ b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java @@ -28,6 +28,7 @@ import net.i2p.data.RouterInfo; import net.i2p.router.NetworkDatabaseFacade; import net.i2p.router.RouterContext; import net.i2p.router.tunnel.pool.TunnelPeerSelector; +import net.i2p.router.util.RandomIterator; import net.i2p.stat.Rate; import net.i2p.stat.RateStat; import net.i2p.util.Log; @@ -1043,17 +1044,19 @@ public class ProfileOrganizer { private void locked_selectPeers(Map peers, int howMany, Set toExclude, Set matches) { locked_selectPeers(peers, howMany, toExclude, matches, 0); } + private void locked_selectPeers(Map peers, int howMany, Set toExclude, Set matches, int mask) { - List all = new ArrayList(peers.keySet()); - if (toExclude != null) - all.removeAll(toExclude); - - all.removeAll(matches); - all.remove(_us); - Collections.shuffle(all, _random); + List all = new ArrayList(peers.keySet()); Set IPSet = new HashSet(8); - for (int i = 0; (matches.size() < howMany) && (i < all.size()); i++) { - Hash peer = (Hash)all.get(i); + // use RandomIterator to avoid shuffling the whole thing + for (Iterator iter = new RandomIterator(all); (matches.size() < howMany) && iter.hasNext(); ) { + Hash peer = iter.next(); + if (toExclude != null && toExclude.contains(peer)) + continue; + if (matches.contains(peer)) + continue; + if (_us.equals(peer)) + continue; boolean ok = isSelectable(peer); if (ok) { ok = mask <= 0 || notRestricted(peer, IPSet, mask); diff --git a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java index 8cad7bb0a..e39bf7a6c 100644 --- a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java +++ b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java @@ -34,6 +34,7 @@ import net.i2p.router.RouterContext; import net.i2p.router.transport.Transport; import net.i2p.router.transport.TransportBid; import net.i2p.router.transport.TransportImpl; +import net.i2p.router.util.RandomIterator; import net.i2p.util.ConcurrentHashSet; import net.i2p.util.Log; import net.i2p.util.SimpleScheduler; @@ -2326,9 +2327,8 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority PeerState pickTestPeer(RemoteHostId dontInclude) { List peers = new ArrayList(_peersByIdent.values()); - Collections.shuffle(peers, _context.random()); - for (int i = 0; i < peers.size(); i++) { - PeerState peer = peers.get(i); + for (Iterator iter = new RandomIterator(peers); iter.hasNext(); ) { + PeerState peer = iter.next(); if ( (dontInclude != null) && (dontInclude.equals(peer.getRemoteHostId())) ) continue; RouterInfo peerInfo = _context.netDb().lookupRouterInfoLocally(peer.getRemotePeer()); diff --git a/router/java/src/net/i2p/router/util/RandomIterator.java b/router/java/src/net/i2p/router/util/RandomIterator.java new file mode 100644 index 000000000..db8a560b8 --- /dev/null +++ b/router/java/src/net/i2p/router/util/RandomIterator.java @@ -0,0 +1,178 @@ +package net.i2p.router.util; + +/* + * Modified from: + * http://www.lockergnome.com/awarberg/2007/04/22/random-iterator-in-java/ + * No license, free to use + */ + +//import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Random; + +import net.i2p.util.RandomSource; + +/** + * + * + * This is some Java code I wrote for a school project to save some time when iterating in + * random order over a part of list (until some condition becomes true): + * + * Here is a sample on how to use the code: + * + 
+        for(Iterator iter = new RandomIterator(myObjList); iter.hasNext();){
+            Object o = iter.next();
+            if(someCondition(o) )
+                return o; // iteration stopped early
+        }
+    
+ *
+ * I wrote it to replace a Collection.shuffle call and this code gave us an overall increase in program execution speed of about 25%.
+ * As the javadoc description says, you are better off calling Collection.shuffle if you need to iterate over the entire list. But if you may stop early this class can save you some time, as it did in our case.
+ *
+ * Provides a random iteration over the given list.
+ *
+ * This effect can be achieved by using Collections.shuffle,
+ * which shuffles the entire collection in linear time.
+ *
+ * If the iteration process may end before all items
+ * are processed, this class may give a speed increase
+ * because the shuffling process is performed as items are requested
+ * rather than in the beginning.
+ *
+ * I2P changes:
+ *
+ *   - Use BitSet instead of boolean[]
+ *   - Use I2P RandomSource
+ *   - Done check in next(), throw NSEE
+ *   - Ensure lower and upper bounds are always clear
+ *   - Replace unbounded loop in next(). It is now O(N) time, but now
+ *     the iterator will tend to "clump" results and thus is not truly random.
+ *     *** This class is not recommended for small Lists,
+ *     *** or for iterating through a large portion of a List.
+ *     *** Use Collections.shuffle() instead.
+ *   - Add test code
+ *

+ *
+ */
+public class RandomIterator implements Iterator {
+    /**
+     * Mapping indicating which items were served (by index).
+     * if served[i] then the item with index i in the list
+     * has already been served.
+     *
+     * Note it is possible to save memory here by using
+     * BitSet rather than a boolean array, however it will
+     * increase the running time slightly.
+     */
+    private final BitSet served;
+
+    /** The amount of items served so far */
+    private int servedCount = 0;
+    private final List list;
+    private final int LIST_SIZE;
+
+    /**
+    * The random number generator has a great influence
+    * on the running time of this iterator.
+    *
+    * See, for instance,
+    * http://www.qbrundage.com/michaelb/pubs/e…
+    * for some implementations, which are faster than java.util.Random.
+    */
+    private static final Random rand = RandomSource.getInstance();
+
+    /** Used to narrow the range to take random indexes from */
+    private int lower, upper;
+
+    public RandomIterator(List list){
+        this.list = list;
+        LIST_SIZE = list.size();
+        served = new BitSet(LIST_SIZE);
+        upper = LIST_SIZE - 1;
+    }
+
+    public boolean hasNext() {
+        return servedCount < LIST_SIZE;
+    }
+
+    public E next() {
+        if (!hasNext())
+            throw new NoSuchElementException();
+        int range = upper - lower + 1;
+
+        // This has unbounded behavior, even with lower/upper
+        //int index;
+        //do {
+        //    index = lower + rand.nextInt(range);
+        //} while (served.get(index));
+
+        // This tends to "clump" results, escpecially toward the end of the iteration.
+        // It also tends to leave the first and last few elements until the end.
+        int start = lower + rand.nextInt(range);
+        int index;
+        if ((start % 2) == 0)  // coin flip
+            index = served.nextClearBit(start);
+        else
+            index = previousClearBit(start);
+        if (index < 0)
+            throw new NoSuchElementException("shouldn't happen");
+        servedCount++;
+        served.set(index);
+
+        // check if the range from which random values
+        // are taken can be reduced
+        // I2P - ensure lower and upper are always clear
+        if (hasNext()) {
+            if (index == lower)
+                lower = served.nextClearBit(lower);
+            else if (index == upper)
+                upper = previousClearBit(upper - 1);
+        }
+        return list.get(index);
+    }
+
+    /** just like nextClearBit() */
+    private int previousClearBit(int n) {
+        for (int i = n; i >= lower; i--) {
+            if (!served.get(i)) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    /**
+     *  @throws UnsupportedOperationException always
+     */
+    public void remove() {
+        throw new UnsupportedOperationException();
+    }
+
+/*****
+    public static void main(String[] args) {
+        System.out.println("\n testing with 0");
+        test(0);
+        System.out.println("\n testing with 1");
+        test(1);
+        System.out.println("\n testing with 2");
+        test(2);
+        System.out.println("\n testing with 1000");
+        test(1000);
+    }
+
+    public static void test(int n) {
+        List l = new ArrayList(n);
+        for (int i = 0; i < n; i++) {
+            l.add(Integer.valueOf(i));
+        }
+        for (Iterator iter = new RandomIterator(l); iter.hasNext(); ) {
+            System.out.println(iter.next().toString());
+        }
+    }
+*****/
+}
diff --git a/router/java/src/net/i2p/router/util/package.html b/router/java/src/net/i2p/router/util/package.html
new file mode 100644
index 000000000..bf81eb3fa
--- /dev/null
+++ b/router/java/src/net/i2p/router/util/package.html
@@ -0,0 +1,6 @@
+
+

+These classes define the several useful utilities used
+throughout the router.
+

+