From f84b86a7523d645b305419ba641372d25896c924 Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 3 Sep 2014 15:19:18 +0000 Subject: [PATCH] * BundleRouterInfos: - Move to its own class - Run GeoIP, exclude bad countries - Exclude class K - Exclude dup IPs - GeoIP mods for use in I2PAppContext --- .../networkdb/kademlia/BundleRouterInfos.java | 249 ++++++++++++++++++ .../kademlia/PersistentDataStore.java | 177 +------------ .../i2p/router/transport/BadCountries.java | 2 +- .../src/net/i2p/router/transport/GeoIP.java | 42 +-- 4 files changed, 281 insertions(+), 189 deletions(-) create mode 100644 router/java/src/net/i2p/router/networkdb/kademlia/BundleRouterInfos.java diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/BundleRouterInfos.java b/router/java/src/net/i2p/router/networkdb/kademlia/BundleRouterInfos.java new file mode 100644 index 000000000..ebcde49c6 --- /dev/null +++ b/router/java/src/net/i2p/router/networkdb/kademlia/BundleRouterInfos.java @@ -0,0 +1,249 @@ +package net.i2p.router.networkdb.kademlia; +/* + * free (adj.): unencumbered; not under the control of others + * Written by jrandom in 2003 and released into the public domain + * with no warranty of any kind, either expressed or implied. + * It probably won't make your computer catch on fire, or eat + * your children, but it might. Use at your own risk. + * + */ + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import gnu.getopt.Getopt; + +import net.i2p.I2PAppContext; +import net.i2p.data.Hash; +import net.i2p.data.RouterAddress; +import net.i2p.data.RouterInfo; +import net.i2p.router.transport.BadCountries; +import net.i2p.router.transport.GeoIP; +import net.i2p.util.FileUtil; + +/** + * Copy a random selection of 'count' router infos from configDir/netDb + * to 'toDir'. Skip your own router info, and old, hidden, unreachable, and + * introduced routers, and those from bad countries. + * + * Used in the build process. + * + * @since 0.9.15 + * + */ +public class BundleRouterInfos { + + /** + * Usage: PersistentDataStore -i configDir -o toDir -c count + * + * Copy a random selection of 'count' router infos from configDir/netDb + * to 'toDir'. Skip your own router info, and old, hidden, unreachable, and + * introduced routers, and those from bad countries. + * + * @since 0.9.15 + */ + public static void main(String[] args) { + Getopt g = new Getopt("PersistentDataStore", args, "i:o:c:"); + String in = System.getProperty("user.home") + "/.i2p"; + String out = "netDb"; + int count = 200; + boolean error = false; + int c; + while ((c = g.getopt()) != -1) { + switch (c) { + case 'i': + in = g.getOptarg(); + break; + + case 'o': + out = g.getOptarg(); + break; + + case 'c': + String scount = g.getOptarg(); + try { + count = Integer.parseInt(scount); + } catch (NumberFormatException nfe) { + error = true; + } + break; + + case '?': + case ':': + default: + error = true; + } + } + if (error) { + usage(); + System.exit(1); + } + + Properties props = new Properties(); + props.setProperty(GeoIP.PROP_GEOIP_DIR, System.getProperty("user.dir") + "/installer/resources"); + GeoIP geoIP = new GeoIP(new I2PAppContext(props)); + + File confDir = new File(in); + File dbDir = new File(confDir, "netDb"); + if (!dbDir.exists()) { + System.out.println("NetDB directory " + dbDir + " does not exist"); + System.exit(1); + } + File myFile = new File(confDir, "router.info"); + File toDir = new File(out); + toDir.mkdirs(); + InputStream fis = null; + Hash me = null; + try { + fis = new BufferedInputStream(new FileInputStream(myFile)); + RouterInfo ri = new RouterInfo(); + ri.readBytes(fis, true); // true = verify sig on read + me = ri.getIdentity().getHash(); + } catch (Exception e) { + //System.out.println("Can't determine our identity"); + } finally { + if (fis != null) try { fis.close(); } catch (IOException ioe) {} + } + + int routerCount = 0; + List toRead = new ArrayList(2048); + for (int j = 0; j < PersistentDataStore.B64.length(); j++) { + File subdir = new File(dbDir, PersistentDataStore.DIR_PREFIX + PersistentDataStore.B64.charAt(j)); + File[] files = subdir.listFiles(PersistentDataStore.RouterInfoFilter.getInstance()); + if (files == null) + continue; + routerCount += files.length; + for (int i = 0; i < files.length; i++) { + toRead.add(files[i]); + } + } + if (toRead.isEmpty()) { + System.out.println("No files to copy in " + dbDir); + System.exit(1); + } + Collections.shuffle(toRead); + int copied = 0; + long tooOld = System.currentTimeMillis() - 7*24*60*60*1000L; + Map ipMap = new HashMap(count); + for (File file : toRead) { + if (copied >= count) + break; + Hash key = PersistentDataStore.getRouterInfoHash(file.getName()); + if (key == null) { + System.out.println("Skipping bad " + file); + continue; + } + if (key.equals(me)) { + System.out.println("Skipping my RI"); + continue; + } + fis = null; + try { + fis = new BufferedInputStream(new FileInputStream(file)); + RouterInfo ri = new RouterInfo(); + ri.readBytes(fis, true); // true = verify sig on read + try { fis.close(); } catch (IOException ioe) {} + fis = null; + if (ri.getPublished() < tooOld) { + System.out.println("Skipping too old " + key); + continue; + } + if (ri.getCapabilities().contains("U")) { + System.out.println("Skipping unreachable " + key); + continue; + } + if (ri.getCapabilities().contains("K")) { + System.out.println("Skipping slow " + key); + continue; + } + Collection addrs = ri.getAddresses(); + if (addrs.isEmpty()) { + System.out.println("Skipping hidden " + key); + continue; + } + boolean hasIntro = false; + boolean hasIPv4 = false; + boolean dupIP = false; + for (RouterAddress addr : addrs) { + if ("SSU".equals(addr.getTransportStyle()) && addr.getOption("ihost0") != null) { + hasIntro = true; + break; + } + String host = addr.getHost(); + if (host != null && host.contains(".")) { + hasIPv4 = true; + geoIP.add(host); + String old = ipMap.put(host, file.getName()); + if (old != null && !old.equals(file.getName())) { + dupIP = true; + break; + } + } + } + if (dupIP) { + System.out.println("Skipping dup IP " + key); + continue; + } + if (hasIntro) { + System.out.println("Skipping introduced " + key); + continue; + } + if (!hasIPv4) { + System.out.println("Skipping IPv6-only " + key); + continue; + } + File toFile = new File(toDir, file.getName()); + // We could call ri.write() to avoid simultaneous change by the router + boolean ok = FileUtil.copy(file, toFile, true, true); + if (ok) + copied++; + else + System.out.println("Failed copy of " + file + " to " + toDir); + } catch (Exception e) { + System.out.println("Skipping bad " + file); + } finally { + if (fis != null) try { fis.close(); } catch (IOException ioe) {} + } + } + if (copied > 0) { + // now do all the geoip lookups, and delete any bad countries + geoIP.blockingLookup(); + for (Map.Entry e : ipMap.entrySet()) { + String co = geoIP.get(e.getKey()); + if (co != null) { + if (BadCountries.contains(co)) { + String name = e.getValue(); + File toFile = new File(toDir, name); + if (toFile.delete()) { + String full = geoIP.fullName(co); + if (full == null) + full = co; + System.out.println("Skipping " + full + ": " + name); + copied--; + } + } + } + } + } + if (copied > 0) { + System.out.println("Copied " + copied + " router info files to " + toDir); + } else { + System.out.println("Failed to copy any files to " + toDir); + System.exit(1); + } + } + + private static void usage() { + System.err.println("Usage: PersistentDataStore [-i $HOME/.i2p] [-o netDb/] [-c 200]"); + } +} diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java b/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java index 5e8d863e1..620679b7e 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java @@ -17,7 +17,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -26,13 +25,10 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentHashMap; -import gnu.getopt.Getopt; - import net.i2p.data.Base64; import net.i2p.data.DatabaseEntry; import net.i2p.data.DataFormatException; import net.i2p.data.Hash; -import net.i2p.data.RouterAddress; import net.i2p.data.RouterInfo; import net.i2p.router.JobImpl; import net.i2p.router.Router; @@ -58,8 +54,8 @@ class PersistentDataStore extends TransientDataStore { private final static int READ_DELAY = 2*60*1000; private static final String PROP_FLAT = "router.networkDatabase.flat"; - private static final String DIR_PREFIX = "r"; - private static final String B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~"; + static final String DIR_PREFIX = "r"; + static final String B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~"; /** * @param dbDir relative path @@ -618,7 +614,7 @@ class PersistentDataStore extends TransientDataStore { return DIR_PREFIX + b64.charAt(0) + File.separatorChar + ROUTERINFO_PREFIX + b64 + ROUTERINFO_SUFFIX; } - private static Hash getRouterInfoHash(String filename) { + static Hash getRouterInfoHash(String filename) { return getHash(filename, ROUTERINFO_PREFIX, ROUTERINFO_SUFFIX); } @@ -655,7 +651,7 @@ class PersistentDataStore extends TransientDataStore { } } - private final static class RouterInfoFilter implements FilenameFilter { + static class RouterInfoFilter implements FilenameFilter { private static final FilenameFilter _instance = new RouterInfoFilter(); public static final FilenameFilter getInstance() { return _instance; } public boolean accept(File dir, String name) { @@ -664,169 +660,4 @@ class PersistentDataStore extends TransientDataStore { return (name.startsWith(ROUTERINFO_PREFIX.toUpperCase(Locale.US)) && name.endsWith(ROUTERINFO_SUFFIX.toUpperCase(Locale.US))); } } - - /** - * Usage: PersistentDataStore -i configDir -o toDir -c count - * - * Copy a random selection of 'count' router infos from configDir/netDb - * to 'toDir'. Skip your own router info, and old, hidden, unreachable, and - * introduced routers. - * - * Used in the build process, do not comment out. - * - * @since 0.9.15 - */ - public static void main(String[] args) { - Getopt g = new Getopt("PersistentDataStore", args, "i:o:c:"); - String in = System.getProperty("user.home") + "/.i2p"; - String out = "netDb"; - int count = 200; - boolean error = false; - int c; - while ((c = g.getopt()) != -1) { - switch (c) { - case 'i': - in = g.getOptarg(); - break; - - case 'o': - out = g.getOptarg(); - break; - - case 'c': - String scount = g.getOptarg(); - try { - count = Integer.parseInt(scount); - } catch (NumberFormatException nfe) { - error = true; - } - break; - - case '?': - case ':': - default: - error = true; - } - } - if (error) { - usage(); - System.exit(1); - } - - File confDir = new File(in); - File dbDir = new File(confDir, "netDb"); - if (!dbDir.exists()) { - System.out.println("NetDB directory " + dbDir + " does not exist"); - System.exit(1); - } - File myFile = new File(confDir, "router.info"); - File toDir = new File(out); - toDir.mkdirs(); - InputStream fis = null; - Hash me = null; - try { - fis = new BufferedInputStream(new FileInputStream(myFile)); - RouterInfo ri = new RouterInfo(); - ri.readBytes(fis, true); // true = verify sig on read - me = ri.getIdentity().getHash(); - } catch (Exception e) { - //System.out.println("Can't determine our identity"); - } finally { - if (fis != null) try { fis.close(); } catch (IOException ioe) {} - } - - int routerCount = 0; - List toRead = new ArrayList(2048); - for (int j = 0; j < B64.length(); j++) { - File subdir = new File(dbDir, DIR_PREFIX + B64.charAt(j)); - File[] files = subdir.listFiles(RouterInfoFilter.getInstance()); - if (files == null) - continue; - routerCount += files.length; - for (int i = 0; i < files.length; i++) { - toRead.add(files[i]); - } - } - if (toRead.isEmpty()) { - System.out.println("No files to copy in " + dbDir); - System.exit(1); - } - Collections.shuffle(toRead); - int copied = 0; - long tooOld = System.currentTimeMillis() - 7*24*60*60*1000L; - for (File file : toRead) { - if (copied >= count) - break; - Hash key = getRouterInfoHash(file.getName()); - if (key == null) { - System.out.println("Skipping bad " + file); - continue; - } - if (key.equals(me)) { - System.out.println("Skipping my RI"); - continue; - } - fis = null; - try { - fis = new BufferedInputStream(new FileInputStream(file)); - RouterInfo ri = new RouterInfo(); - ri.readBytes(fis, true); // true = verify sig on read - try { fis.close(); } catch (IOException ioe) {} - fis = null; - if (ri.getPublished() < tooOld) { - System.out.println("Skipping too old " + key); - continue; - } - if (ri.getCapabilities().contains("U")) { - System.out.println("Skipping unreachable " + key); - continue; - } - Collection addrs = ri.getAddresses(); - if (addrs.isEmpty()) { - System.out.println("Skipping hidden " + key); - continue; - } - boolean hasIntro = false; - boolean hasIPv4 = false; - for (RouterAddress addr : addrs) { - if ("SSU".equals(addr.getTransportStyle()) && addr.getOption("ihost0") != null) { - hasIntro = true; - break; - } - String host = addr.getHost(); - if (host != null && host.contains(".")) - hasIPv4 = true; - } - if (hasIntro) { - System.out.println("Skipping introduced " + key); - continue; - } - if (!hasIPv4) { - System.out.println("Skipping IPv6-only " + key); - continue; - } - File toFile = new File(toDir, file.getName()); - // We could call ri.write() to avoid simultaneous change by the router - boolean ok = FileUtil.copy(file, toFile, true, true); - if (ok) - copied++; - else - System.out.println("Failed copy of " + file + " to " + toDir); - } catch (Exception e) { - System.out.println("Skipping bad " + file); - } finally { - if (fis != null) try { fis.close(); } catch (IOException ioe) {} - } - } - if (copied > 0) { - System.out.println("Copied " + copied + " router info files to " + toDir); - } else { - System.out.println("Failed to copy any files to " + toDir); - System.exit(1); - } - } - - private static void usage() { - System.err.println("Usage: PersistentDataStore [-i $HOME/.i2p] [-o netDb/] [-c 200]"); - } } diff --git a/router/java/src/net/i2p/router/transport/BadCountries.java b/router/java/src/net/i2p/router/transport/BadCountries.java index 4a85f2644..dc848d859 100644 --- a/router/java/src/net/i2p/router/transport/BadCountries.java +++ b/router/java/src/net/i2p/router/transport/BadCountries.java @@ -9,7 +9,7 @@ import java.util.Set; * Maintain a list of bad places. * @since 0.8.13 */ -abstract class BadCountries { +public abstract class BadCountries { private static final Set _countries; diff --git a/router/java/src/net/i2p/router/transport/GeoIP.java b/router/java/src/net/i2p/router/transport/GeoIP.java index 3f0990dd3..e379ce05f 100644 --- a/router/java/src/net/i2p/router/transport/GeoIP.java +++ b/router/java/src/net/i2p/router/transport/GeoIP.java @@ -16,6 +16,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import net.i2p.I2PAppContext; import net.i2p.data.Hash; import net.i2p.router.Router; import net.i2p.router.RouterContext; @@ -38,11 +39,9 @@ import net.i2p.util.Log; * * @author zzz */ -class GeoIP { +public class GeoIP { private final Log _log; - // change to test with main() - //private final I2PAppContext _context; - private final RouterContext _context; + private final I2PAppContext _context; private final Map _codeToName; /** code to itself to prevent String proliferation */ private final Map _codeCache; @@ -56,8 +55,10 @@ class GeoIP { private final AtomicBoolean _lock; private int _lookupRunCount; - //public GeoIP(I2PAppContext context) { - public GeoIP(RouterContext context) { + /** + * @param context RouterContext in production, I2PAppContext for testing only + */ + public GeoIP(I2PAppContext context) { _context = context; _log = context.logManager().getLog(GeoIP.class); _codeToName = new ConcurrentHashMap(512); @@ -71,6 +72,7 @@ class GeoIP { } static final String PROP_GEOIP_ENABLED = "routerconsole.geoip.enable"; + public static final String PROP_GEOIP_DIR = "geoip.dir"; static final String GEOIP_DIR_DEFAULT = "geoip"; static final String GEOIP_FILE_DEFAULT = "geoip.txt"; static final String COUNTRY_FILE_DEFAULT = "countries.txt"; @@ -187,7 +189,10 @@ class GeoIP { * */ private void readCountryFile() { - File geoFile = new File(_context.getBaseDir(), GEOIP_DIR_DEFAULT); + String geoDir = _context.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT); + File geoFile = new File(geoDir); + if (!geoFile.isAbsolute()) + geoFile = new File(_context.getBaseDir(), geoDir); geoFile = new File(geoFile, COUNTRY_FILE_DEFAULT); if (!geoFile.exists()) { if (_log.shouldLog(Log.WARN)) @@ -246,7 +251,10 @@ class GeoIP { * */ private String[] readGeoIPFile(Long[] search) { - File geoFile = new File(_context.getBaseDir(), GEOIP_DIR_DEFAULT); + String geoDir = _context.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT); + File geoFile = new File(geoDir); + if (!geoFile.isAbsolute()) + geoFile = new File(_context.getBaseDir(), geoDir); geoFile = new File(geoFile, GEOIP_FILE_DEFAULT); if (!geoFile.exists()) { if (_log.shouldLog(Log.WARN)) @@ -300,24 +308,28 @@ class GeoIP { /** * Put our country code in the config, where others (such as Timestamper) can get it, * and it will be there next time at startup. + * + * Does nothing in I2PAppContext */ private void updateOurCountry() { - /**** comment out to test with main() */ - String oldCountry = _context.router().getConfigSetting(PROP_IP_COUNTRY); - Hash ourHash = _context.routerHash(); + if (! (_context instanceof RouterContext)) + return; + RouterContext ctx = (RouterContext) _context; + String oldCountry = ctx.router().getConfigSetting(PROP_IP_COUNTRY); + Hash ourHash = ctx.routerHash(); // we should always have a RouterInfo by now, but we had one report of an NPE here if (ourHash == null) return; - String country = _context.commSystem().getCountry(ourHash); + String country = ctx.commSystem().getCountry(ourHash); if (country != null && !country.equals(oldCountry)) { - _context.router().saveConfig(PROP_IP_COUNTRY, country); - if (_context.commSystem().isInBadCountry() && _context.getProperty(Router.PROP_HIDDEN_HIDDEN) == null) { + ctx.router().saveConfig(PROP_IP_COUNTRY, country); + if (ctx.commSystem().isInBadCountry() && ctx.getProperty(Router.PROP_HIDDEN_HIDDEN) == null) { String name = fullName(country); if (name == null) name = country; _log.logAlways(Log.WARN, "Setting hidden mode to protect you in " + name + ", you may override on the network configuration page"); - _context.router().rebuildRouterInfo(); + ctx.router().rebuildRouterInfo(); } } /****/