* BundleRouterInfos:

- Move to its own class
   - Run GeoIP, exclude bad countries
   - Exclude class K
   - Exclude dup IPs
   - GeoIP mods for use in I2PAppContext

router/java/src/net/i2p/router/networkdb/kademlia/BundleRouterInfos.java

@@ -0,0 +1,249 @@
+package net.i2p.router.networkdb.kademlia;
+/*
+ * free (adj.): unencumbered; not under the control of others
+ * Written by jrandom in 2003 and released into the public domain
+ * with no warranty of any kind, either expressed or implied.
+ * It probably won't make your computer catch on fire, or eat
+ * your children, but it might.  Use at your own risk.
+ *
+ */
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import gnu.getopt.Getopt;
+
+import net.i2p.I2PAppContext;
+import net.i2p.data.Hash;
+import net.i2p.data.RouterAddress;
+import net.i2p.data.RouterInfo;
+import net.i2p.router.transport.BadCountries;
+import net.i2p.router.transport.GeoIP;
+import net.i2p.util.FileUtil;
+
+/**
+ *  Copy a random selection of 'count' router infos from configDir/netDb
+ *  to 'toDir'. Skip your own router info, and old, hidden, unreachable, and
+ *  introduced routers, and those from bad countries.
+ *
+ *  Used in the build process.
+ *
+ *  @since 0.9.15
+ *
+ */
+public class BundleRouterInfos {
+
+    /**
+     *  Usage: PersistentDataStore -i configDir -o toDir -c count
+     *
+     *  Copy a random selection of 'count' router infos from configDir/netDb
+     *  to 'toDir'. Skip your own router info, and old, hidden, unreachable, and
+     *  introduced routers, and those from bad countries.
+     *
+     *  @since 0.9.15
+     */
+    public static void main(String[] args) {
+        Getopt g = new Getopt("PersistentDataStore", args, "i:o:c:");
+        String in = System.getProperty("user.home") + "/.i2p";
+        String out = "netDb";
+        int count = 200;
+        boolean error = false;
+        int c;
+        while ((c = g.getopt()) != -1) {
+          switch (c) {
+            case 'i':
+                in = g.getOptarg();
+                break;
+
+            case 'o':
+                out = g.getOptarg();
+                break;
+
+            case 'c':
+                String scount = g.getOptarg();
+                try {
+                    count = Integer.parseInt(scount);
+                } catch (NumberFormatException nfe) {
+                    error = true;
+                }
+                break;
+
+            case '?':
+            case ':':
+            default:
+                error = true;
+          }
+        }
+        if (error) {
+            usage();
+            System.exit(1);
+        }
+
+        Properties props = new Properties();
+        props.setProperty(GeoIP.PROP_GEOIP_DIR, System.getProperty("user.dir") + "/installer/resources");
+        GeoIP geoIP = new GeoIP(new I2PAppContext(props));
+
+        File confDir = new File(in);
+        File dbDir = new File(confDir, "netDb");
+        if (!dbDir.exists()) {
+            System.out.println("NetDB directory " + dbDir + " does not exist");
+            System.exit(1);
+        }
+        File myFile = new File(confDir, "router.info");
+        File toDir = new File(out);
+        toDir.mkdirs();
+        InputStream fis = null;
+        Hash me = null;
+        try {
+            fis = new BufferedInputStream(new FileInputStream(myFile));
+            RouterInfo ri = new RouterInfo();
+            ri.readBytes(fis, true);  // true = verify sig on read
+            me = ri.getIdentity().getHash();
+        } catch (Exception e) {
+            //System.out.println("Can't determine our identity");
+        } finally {
+            if (fis != null) try { fis.close(); } catch (IOException ioe) {}
+        }
+
+        int routerCount = 0;
+        List<File> toRead = new ArrayList<File>(2048);
+        for (int j = 0; j < PersistentDataStore.B64.length(); j++) {
+            File subdir = new File(dbDir, PersistentDataStore.DIR_PREFIX + PersistentDataStore.B64.charAt(j));
+            File[] files = subdir.listFiles(PersistentDataStore.RouterInfoFilter.getInstance());
+            if (files == null)
+                continue;
+            routerCount += files.length;
+            for (int i = 0; i < files.length; i++) {
+                toRead.add(files[i]);
+            }
+        }
+        if (toRead.isEmpty()) {
+            System.out.println("No files to copy in " + dbDir);
+            System.exit(1);
+        }
+        Collections.shuffle(toRead);
+        int copied = 0;
+        long tooOld = System.currentTimeMillis() - 7*24*60*60*1000L;
+        Map<String, String> ipMap = new HashMap<String, String>(count);
+        for (File file : toRead) {
+            if (copied >= count)
+                break;
+            Hash key = PersistentDataStore.getRouterInfoHash(file.getName());
+            if (key == null) {
+                System.out.println("Skipping bad " + file);
+                continue;
+            }
+            if (key.equals(me)) {
+                System.out.println("Skipping my RI");
+                continue;
+            }
+            fis = null;
+            try {
+                fis = new BufferedInputStream(new FileInputStream(file));
+                RouterInfo ri = new RouterInfo();
+                ri.readBytes(fis, true);  // true = verify sig on read
+                try { fis.close(); } catch (IOException ioe) {}
+                fis = null;
+                if (ri.getPublished() < tooOld) {
+                    System.out.println("Skipping too old " + key);
+                    continue;
+                }
+                if (ri.getCapabilities().contains("U")) {
+                    System.out.println("Skipping unreachable " + key);
+                    continue;
+                }
+                if (ri.getCapabilities().contains("K")) {
+                    System.out.println("Skipping slow " + key);
+                    continue;
+                }
+                Collection<RouterAddress> addrs = ri.getAddresses();
+                if (addrs.isEmpty()) {
+                    System.out.println("Skipping hidden " + key);
+                    continue;
+                }
+                boolean hasIntro = false;
+                boolean hasIPv4 = false;
+                boolean dupIP = false;
+                for (RouterAddress addr : addrs) {
+                    if ("SSU".equals(addr.getTransportStyle()) && addr.getOption("ihost0") != null) {
+                        hasIntro = true;
+                        break;
+                    }
+                    String host = addr.getHost();
+                    if (host != null && host.contains(".")) {
+                        hasIPv4 = true;
+                        geoIP.add(host);
+                        String old = ipMap.put(host, file.getName());
+                        if (old != null && !old.equals(file.getName())) {
+                            dupIP = true;
+                            break;
+                        }
+                    }
+                }
+                if (dupIP) {
+                    System.out.println("Skipping dup IP " + key);
+                    continue;
+                }
+                if (hasIntro) {
+                    System.out.println("Skipping introduced " + key);
+                    continue;
+                }
+                if (!hasIPv4) {
+                    System.out.println("Skipping IPv6-only " + key);
+                    continue;
+                }
+                File toFile = new File(toDir, file.getName());
+                // We could call ri.write() to avoid simultaneous change by the router
+                boolean ok = FileUtil.copy(file, toFile, true, true);
+                if (ok)
+                    copied++;
+                else
+                    System.out.println("Failed copy of " + file + " to " + toDir);
+            } catch (Exception e) {
+                System.out.println("Skipping bad " + file);
+            } finally {
+                if (fis != null) try { fis.close(); } catch (IOException ioe) {}
+            }
+        }
+        if (copied > 0) {
+            // now do all the geoip lookups, and delete any bad countries
+            geoIP.blockingLookup();
+            for (Map.Entry<String, String> e : ipMap.entrySet()) {
+                String co = geoIP.get(e.getKey());
+                if (co != null) {
+                    if (BadCountries.contains(co)) {
+                        String name = e.getValue();
+                        File toFile = new File(toDir, name);
+                        if (toFile.delete()) {
+                            String full = geoIP.fullName(co);
+                            if (full == null)
+                                full = co;
+                            System.out.println("Skipping " + full + ": " + name);
+                            copied--;
+                        }
+                    }
+                }
+            }
+        }
+        if (copied > 0) {
+            System.out.println("Copied " + copied + " router info files to " + toDir);
+        } else {
+            System.out.println("Failed to copy any files to " + toDir);
+            System.exit(1);
+        }
+    }
+
+    private static void usage() {
+        System.err.println("Usage: PersistentDataStore [-i $HOME/.i2p] [-o netDb/] [-c 200]");
+    }
+}

router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java

@@ -17,7 +17,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -26,13 +25,10 @@ import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.concurrent.ConcurrentHashMap;
 
-import gnu.getopt.Getopt;
-
 import net.i2p.data.Base64;
 import net.i2p.data.DatabaseEntry;
 import net.i2p.data.DataFormatException;
 import net.i2p.data.Hash;
-import net.i2p.data.RouterAddress;
 import net.i2p.data.RouterInfo;
 import net.i2p.router.JobImpl;
 import net.i2p.router.Router;
@@ -58,8 +54,8 @@ class PersistentDataStore extends TransientDataStore {
     
     private final static int READ_DELAY = 2*60*1000;
     private static final String PROP_FLAT = "router.networkDatabase.flat";
-    private static final String DIR_PREFIX = "r";
+    static final String DIR_PREFIX = "r";
-    private static final String B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~";
+    static final String B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~";
     
     /**
      *  @param dbDir relative path
@@ -618,7 +614,7 @@ class PersistentDataStore extends TransientDataStore {
         return DIR_PREFIX + b64.charAt(0) + File.separatorChar + ROUTERINFO_PREFIX + b64 + ROUTERINFO_SUFFIX;
     }
     
-    private static Hash getRouterInfoHash(String filename) {
+    static Hash getRouterInfoHash(String filename) {
         return getHash(filename, ROUTERINFO_PREFIX, ROUTERINFO_SUFFIX);
     }
     
@@ -655,7 +651,7 @@ class PersistentDataStore extends TransientDataStore {
         }
     }
     
-    private final static class RouterInfoFilter implements FilenameFilter {
+    static class RouterInfoFilter implements FilenameFilter {
         private static final FilenameFilter _instance = new RouterInfoFilter();
         public static final FilenameFilter getInstance() { return _instance; }
         public boolean accept(File dir, String name) {
@@ -664,169 +660,4 @@ class PersistentDataStore extends TransientDataStore {
             return (name.startsWith(ROUTERINFO_PREFIX.toUpperCase(Locale.US)) && name.endsWith(ROUTERINFO_SUFFIX.toUpperCase(Locale.US)));
         }
     }
-
-    /**
-     *  Usage: PersistentDataStore -i configDir -o toDir -c count
-     *
-     *  Copy a random selection of 'count' router infos from configDir/netDb
-     *  to 'toDir'. Skip your own router info, and old, hidden, unreachable, and
-     *  introduced routers.
-     *
-     *  Used in the build process, do not comment out.
-     *
-     *  @since 0.9.15
-     */
-    public static void main(String[] args) {
-        Getopt g = new Getopt("PersistentDataStore", args, "i:o:c:");
-        String in = System.getProperty("user.home") + "/.i2p";
-        String out = "netDb";
-        int count = 200;
-        boolean error = false;
-        int c;
-        while ((c = g.getopt()) != -1) {
-          switch (c) {
-            case 'i':
-                in = g.getOptarg();
-                break;
-
-            case 'o':
-                out = g.getOptarg();
-                break;
-
-            case 'c':
-                String scount = g.getOptarg();
-                try {
-                    count = Integer.parseInt(scount);
-                } catch (NumberFormatException nfe) {
-                    error = true;
-                }
-                break;
-
-            case '?':
-            case ':':
-            default:
-                error = true;
-          }
-        }
-        if (error) {
-            usage();
-            System.exit(1);
-        }
-
-        File confDir = new File(in);
-        File dbDir = new File(confDir, "netDb");
-        if (!dbDir.exists()) {
-            System.out.println("NetDB directory " + dbDir + " does not exist");
-            System.exit(1);
-        }
-        File myFile = new File(confDir, "router.info");
-        File toDir = new File(out);
-        toDir.mkdirs();
-        InputStream fis = null;
-        Hash me = null;
-        try {
-            fis = new BufferedInputStream(new FileInputStream(myFile));
-            RouterInfo ri = new RouterInfo();
-            ri.readBytes(fis, true);  // true = verify sig on read
-            me = ri.getIdentity().getHash();
-        } catch (Exception e) {
-            //System.out.println("Can't determine our identity");
-        } finally {
-            if (fis != null) try { fis.close(); } catch (IOException ioe) {}
-        }
-
-        int routerCount = 0;
-        List<File> toRead = new ArrayList<File>(2048);
-        for (int j = 0; j < B64.length(); j++) {
-            File subdir = new File(dbDir, DIR_PREFIX + B64.charAt(j));
-            File[] files = subdir.listFiles(RouterInfoFilter.getInstance());
-            if (files == null)
-                continue;
-            routerCount += files.length;
-            for (int i = 0; i < files.length; i++) {
-                toRead.add(files[i]);
-            }
-        }
-        if (toRead.isEmpty()) {
-            System.out.println("No files to copy in " + dbDir);
-            System.exit(1);
-        }
-        Collections.shuffle(toRead);
-        int copied = 0;
-        long tooOld = System.currentTimeMillis() - 7*24*60*60*1000L;
-        for (File file : toRead) {
-            if (copied >= count)
-                break;
-            Hash key = getRouterInfoHash(file.getName());
-            if (key == null) {
-                System.out.println("Skipping bad " + file);
-                continue;
-            }
-            if (key.equals(me)) {
-                System.out.println("Skipping my RI");
-                continue;
-            }
-            fis = null;
-            try {
-                fis = new BufferedInputStream(new FileInputStream(file));
-                RouterInfo ri = new RouterInfo();
-                ri.readBytes(fis, true);  // true = verify sig on read
-                try { fis.close(); } catch (IOException ioe) {}
-                fis = null;
-                if (ri.getPublished() < tooOld) {
-                    System.out.println("Skipping too old " + key);
-                    continue;
-                }
-                if (ri.getCapabilities().contains("U")) {
-                    System.out.println("Skipping unreachable " + key);
-                    continue;
-                }
-                Collection<RouterAddress> addrs = ri.getAddresses();
-                if (addrs.isEmpty()) {
-                    System.out.println("Skipping hidden " + key);
-                    continue;
-                }
-                boolean hasIntro = false;
-                boolean hasIPv4 = false;
-                for (RouterAddress addr : addrs) {
-                    if ("SSU".equals(addr.getTransportStyle()) && addr.getOption("ihost0") != null) {
-                        hasIntro = true;
-                        break;
-                    }
-                    String host = addr.getHost();
-                    if (host != null && host.contains("."))
-                        hasIPv4 = true;
-                }
-                if (hasIntro) {
-                    System.out.println("Skipping introduced " + key);
-                    continue;
-                }
-                if (!hasIPv4) {
-                    System.out.println("Skipping IPv6-only " + key);
-                    continue;
-                }
-                File toFile = new File(toDir, file.getName());
-                // We could call ri.write() to avoid simultaneous change by the router
-                boolean ok = FileUtil.copy(file, toFile, true, true);
-                if (ok)
-                    copied++;
-                else
-                    System.out.println("Failed copy of " + file + " to " + toDir);
-            } catch (Exception e) {
-                System.out.println("Skipping bad " + file);
-            } finally {
-                if (fis != null) try { fis.close(); } catch (IOException ioe) {}
-            }
-        }
-        if (copied > 0) {
-            System.out.println("Copied " + copied + " router info files to " + toDir);
-        } else {
-            System.out.println("Failed to copy any files to " + toDir);
-            System.exit(1);
-        }
-    }
-
-    private static void usage() {
-        System.err.println("Usage: PersistentDataStore [-i $HOME/.i2p] [-o netDb/] [-c 200]");
-    }
 }

router/java/src/net/i2p/router/transport/BadCountries.java

@@ -9,7 +9,7 @@ import java.util.Set;
  *  Maintain a list of bad places.
  *  @since 0.8.13
  */
-abstract class BadCountries {
+public abstract class BadCountries {
 
     private static final Set<String> _countries;
 

router/java/src/net/i2p/router/transport/GeoIP.java

@@ -16,6 +16,7 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import net.i2p.I2PAppContext;
 import net.i2p.data.Hash;
 import net.i2p.router.Router;
 import net.i2p.router.RouterContext;
@@ -38,11 +39,9 @@ import net.i2p.util.Log;
  *
  * @author zzz
  */
-class GeoIP {
+public class GeoIP {
     private final Log _log;
-    // change to test with main()
+    private final I2PAppContext _context;
-    //private final I2PAppContext _context;
-    private final RouterContext _context;
     private final Map<String, String> _codeToName;
     /** code to itself to prevent String proliferation */
     private final Map<String, String> _codeCache;
@@ -56,8 +55,10 @@ class GeoIP {
     private final AtomicBoolean _lock;
     private int _lookupRunCount;
     
-    //public GeoIP(I2PAppContext context) {
+    /**
-    public GeoIP(RouterContext context) {
+     *  @param context RouterContext in production, I2PAppContext for testing only
+     */
+    public GeoIP(I2PAppContext context) {
         _context = context;
         _log = context.logManager().getLog(GeoIP.class);
         _codeToName = new ConcurrentHashMap<String, String>(512);
@@ -71,6 +72,7 @@ class GeoIP {
     }
     
     static final String PROP_GEOIP_ENABLED = "routerconsole.geoip.enable";
+    public static final String PROP_GEOIP_DIR = "geoip.dir";
     static final String GEOIP_DIR_DEFAULT = "geoip";
     static final String GEOIP_FILE_DEFAULT = "geoip.txt";
     static final String COUNTRY_FILE_DEFAULT = "countries.txt";
@@ -187,7 +189,10 @@ class GeoIP {
     *
     */
     private void readCountryFile() {
-        File geoFile = new File(_context.getBaseDir(), GEOIP_DIR_DEFAULT);
+        String geoDir = _context.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT);
+        File geoFile = new File(geoDir);
+        if (!geoFile.isAbsolute())
+            geoFile = new File(_context.getBaseDir(), geoDir);
         geoFile = new File(geoFile, COUNTRY_FILE_DEFAULT);
         if (!geoFile.exists()) {
             if (_log.shouldLog(Log.WARN))
@@ -246,7 +251,10 @@ class GeoIP {
     *
     */
     private String[] readGeoIPFile(Long[] search) {
-        File geoFile = new File(_context.getBaseDir(), GEOIP_DIR_DEFAULT);
+        String geoDir = _context.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT);
+        File geoFile = new File(geoDir);
+        if (!geoFile.isAbsolute())
+            geoFile = new File(_context.getBaseDir(), geoDir);
         geoFile = new File(geoFile, GEOIP_FILE_DEFAULT);
         if (!geoFile.exists()) {
             if (_log.shouldLog(Log.WARN))
@@ -300,24 +308,28 @@ class GeoIP {
     /**
      *  Put our country code in the config, where others (such as Timestamper) can get it,
      *  and it will be there next time at startup.
+     *
+     *  Does nothing in I2PAppContext
      */
     private void updateOurCountry() {
-        /**** comment out to test with main() */
+        if (! (_context instanceof RouterContext))
-        String oldCountry = _context.router().getConfigSetting(PROP_IP_COUNTRY);
+            return;
-        Hash ourHash = _context.routerHash();
+        RouterContext ctx = (RouterContext) _context;
+        String oldCountry = ctx.router().getConfigSetting(PROP_IP_COUNTRY);
+        Hash ourHash = ctx.routerHash();
         // we should always have a RouterInfo by now, but we had one report of an NPE here
         if (ourHash == null)
             return;
-        String country = _context.commSystem().getCountry(ourHash);
+        String country = ctx.commSystem().getCountry(ourHash);
         if (country != null && !country.equals(oldCountry)) {
-            _context.router().saveConfig(PROP_IP_COUNTRY, country);
+            ctx.router().saveConfig(PROP_IP_COUNTRY, country);
-            if (_context.commSystem().isInBadCountry() && _context.getProperty(Router.PROP_HIDDEN_HIDDEN) == null) {
+            if (ctx.commSystem().isInBadCountry() && ctx.getProperty(Router.PROP_HIDDEN_HIDDEN) == null) {
                 String name = fullName(country);
                 if (name == null)
                     name = country;
                 _log.logAlways(Log.WARN, "Setting hidden mode to protect you in " + name +
                                          ", you may override on the network configuration page");
-                _context.router().rebuildRouterInfo();
+                ctx.router().rebuildRouterInfo();
             }
         }
         /****/