zzz/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* BundleRouterInfos:

Browse Source 
- Move to its own class
   - Run GeoIP, exclude bad countries
   - Exclude class K
   - Exclude dup IPs
   - GeoIP mods for use in I2PAppContext
This commit is contained in:
zzz
2014-09-03 15:19:18 +00:00
parent ca7873eda7
commit f84b86a752
4 changed files with 281 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
router/java/src/net/i2p/router/networkdb/kademlia/BundleRouterInfos.java Normal file
View File

@@ -0,0 +1,249 @@
package net.i2p.router.networkdb.kademlia;
/*
* free (adj.): unencumbered; not under the control of others
* Written by jrandom in 2003 and released into the public domain
* with no warranty of any kind, either expressed or implied.
* It probably won't make your computer catch on fire, or eat
* your children, but it might. Use at your own risk.
*
*/
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import gnu.getopt.Getopt;
import net.i2p.I2PAppContext;
import net.i2p.data.Hash;
import net.i2p.data.RouterAddress;
import net.i2p.data.RouterInfo;
import net.i2p.router.transport.BadCountries;
import net.i2p.router.transport.GeoIP;
import net.i2p.util.FileUtil;
/**
* Copy a random selection of 'count' router infos from configDir/netDb
* to 'toDir'. Skip your own router info, and old, hidden, unreachable, and
* introduced routers, and those from bad countries.
*
* Used in the build process.
*
* @since 0.9.15
*
*/
public class BundleRouterInfos {
/**
* Usage: PersistentDataStore -i configDir -o toDir -c count
*
* Copy a random selection of 'count' router infos from configDir/netDb
* to 'toDir'. Skip your own router info, and old, hidden, unreachable, and
* introduced routers, and those from bad countries.
*
* @since 0.9.15
*/
public static void main(String[] args) {
Getopt g = new Getopt("PersistentDataStore", args, "i:o:c:");
String in = System.getProperty("user.home") + "/.i2p";
String out = "netDb";
int count = 200;
boolean error = false;
int c;
while ((c = g.getopt()) != -1) {
switch (c) {
case 'i':
in = g.getOptarg();
break;
case 'o':
out = g.getOptarg();
break;
case 'c':
String scount = g.getOptarg();
try {
count = Integer.parseInt(scount);
} catch (NumberFormatException nfe) {
error = true;
}
break;
case '?':
case ':':
default:
error = true;
}
}
if (error) {
usage();
System.exit(1);
}
Properties props = new Properties();
props.setProperty(GeoIP.PROP_GEOIP_DIR, System.getProperty("user.dir") + "/installer/resources");
GeoIP geoIP = new GeoIP(new I2PAppContext(props));
File confDir = new File(in);
File dbDir = new File(confDir, "netDb");
if (!dbDir.exists()) {
System.out.println("NetDB directory " + dbDir + " does not exist");
System.exit(1);
}
File myFile = new File(confDir, "router.info");
File toDir = new File(out);
toDir.mkdirs();
InputStream fis = null;
Hash me = null;
try {
fis = new BufferedInputStream(new FileInputStream(myFile));
RouterInfo ri = new RouterInfo();
ri.readBytes(fis, true); // true = verify sig on read
me = ri.getIdentity().getHash();
} catch (Exception e) {
//System.out.println("Can't determine our identity");
} finally {
if (fis != null) try { fis.close(); } catch (IOException ioe) {}
}
int routerCount = 0;
List<File> toRead = new ArrayList<File>(2048);
for (int j = 0; j < PersistentDataStore.B64.length(); j++) {
File subdir = new File(dbDir, PersistentDataStore.DIR_PREFIX + PersistentDataStore.B64.charAt(j));
File[] files = subdir.listFiles(PersistentDataStore.RouterInfoFilter.getInstance());
if (files == null)
continue;
routerCount += files.length;
for (int i = 0; i < files.length; i++) {
toRead.add(files[i]);
}
}
if (toRead.isEmpty()) {
System.out.println("No files to copy in " + dbDir);
System.exit(1);
}
Collections.shuffle(toRead);
int copied = 0;
long tooOld = System.currentTimeMillis() - 7*24*60*60*1000L;
Map<String, String> ipMap = new HashMap<String, String>(count);
for (File file : toRead) {
if (copied >= count)
break;
Hash key = PersistentDataStore.getRouterInfoHash(file.getName());
if (key == null) {
System.out.println("Skipping bad " + file);
continue;
}
if (key.equals(me)) {
System.out.println("Skipping my RI");
continue;
}
fis = null;
try {
fis = new BufferedInputStream(new FileInputStream(file));
RouterInfo ri = new RouterInfo();
ri.readBytes(fis, true); // true = verify sig on read
try { fis.close(); } catch (IOException ioe) {}
fis = null;
if (ri.getPublished() < tooOld) {
System.out.println("Skipping too old " + key);
continue;
}
if (ri.getCapabilities().contains("U")) {
System.out.println("Skipping unreachable " + key);
continue;
}
if (ri.getCapabilities().contains("K")) {
System.out.println("Skipping slow " + key);
continue;
}
Collection<RouterAddress> addrs = ri.getAddresses();
if (addrs.isEmpty()) {
System.out.println("Skipping hidden " + key);
continue;
}
boolean hasIntro = false;
boolean hasIPv4 = false;
boolean dupIP = false;
for (RouterAddress addr : addrs) {
if ("SSU".equals(addr.getTransportStyle()) && addr.getOption("ihost0") != null) {
hasIntro = true;
break;
}
String host = addr.getHost();
if (host != null && host.contains(".")) {
hasIPv4 = true;
geoIP.add(host);
String old = ipMap.put(host, file.getName());
if (old != null && !old.equals(file.getName())) {
dupIP = true;
break;
}
}
}
if (dupIP) {
System.out.println("Skipping dup IP " + key);
continue;
}
if (hasIntro) {
System.out.println("Skipping introduced " + key);
continue;
}
if (!hasIPv4) {
System.out.println("Skipping IPv6-only " + key);
continue;
}
File toFile = new File(toDir, file.getName());
// We could call ri.write() to avoid simultaneous change by the router
boolean ok = FileUtil.copy(file, toFile, true, true);
if (ok)
copied++;
else
System.out.println("Failed copy of " + file + " to " + toDir);
} catch (Exception e) {
System.out.println("Skipping bad " + file);
} finally {
if (fis != null) try { fis.close(); } catch (IOException ioe) {}
}
}
if (copied > 0) {
// now do all the geoip lookups, and delete any bad countries
geoIP.blockingLookup();
for (Map.Entry<String, String> e : ipMap.entrySet()) {
String co = geoIP.get(e.getKey());
if (co != null) {
if (BadCountries.contains(co)) {
String name = e.getValue();
File toFile = new File(toDir, name);
if (toFile.delete()) {
String full = geoIP.fullName(co);
if (full == null)
full = co;
System.out.println("Skipping " + full + ": " + name);
copied--;
}
}
}
}
}
if (copied > 0) {
System.out.println("Copied " + copied + " router info files to " + toDir);
} else {
System.out.println("Failed to copy any files to " + toDir);
System.exit(1);
}
}
private static void usage() {
System.err.println("Usage: PersistentDataStore [-i $HOME/.i2p] [-o netDb/] [-c 200]");
}
}

177
router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java
View File

@@ -17,7 +17,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -26,13 +25,10 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap;
import gnu.getopt.Getopt;
import net.i2p.data.Base64;
import net.i2p.data.DatabaseEntry;
import net.i2p.data.DataFormatException;
import net.i2p.data.Hash;
import net.i2p.data.RouterAddress;
import net.i2p.data.RouterInfo;
import net.i2p.router.JobImpl;
import net.i2p.router.Router;
@@ -58,8 +54,8 @@ class PersistentDataStore extends TransientDataStore {
private final static int READ_DELAY = 2*60*1000;
private static final String PROP_FLAT = "router.networkDatabase.flat";
private static final String DIR_PREFIX = "r";
private static final String B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~";
static final String DIR_PREFIX = "r";
static final String B64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-~";
/**
* @param dbDir relative path
@@ -618,7 +614,7 @@ class PersistentDataStore extends TransientDataStore {
return DIR_PREFIX + b64.charAt(0) + File.separatorChar + ROUTERINFO_PREFIX + b64 + ROUTERINFO_SUFFIX;
}
private static Hash getRouterInfoHash(String filename) {
static Hash getRouterInfoHash(String filename) {
return getHash(filename, ROUTERINFO_PREFIX, ROUTERINFO_SUFFIX);
}
@@ -655,7 +651,7 @@ class PersistentDataStore extends TransientDataStore {
}
}
private final static class RouterInfoFilter implements FilenameFilter {
static class RouterInfoFilter implements FilenameFilter {
private static final FilenameFilter _instance = new RouterInfoFilter();
public static final FilenameFilter getInstance() { return _instance; }
public boolean accept(File dir, String name) {
@@ -664,169 +660,4 @@ class PersistentDataStore extends TransientDataStore {
return (name.startsWith(ROUTERINFO_PREFIX.toUpperCase(Locale.US)) && name.endsWith(ROUTERINFO_SUFFIX.toUpperCase(Locale.US)));
}
}
/**
* Usage: PersistentDataStore -i configDir -o toDir -c count
*
* Copy a random selection of 'count' router infos from configDir/netDb
* to 'toDir'. Skip your own router info, and old, hidden, unreachable, and
* introduced routers.
*
* Used in the build process, do not comment out.
*
* @since 0.9.15
*/
public static void main(String[] args) {
Getopt g = new Getopt("PersistentDataStore", args, "i:o:c:");
String in = System.getProperty("user.home") + "/.i2p";
String out = "netDb";
int count = 200;
boolean error = false;
int c;
while ((c = g.getopt()) != -1) {
switch (c) {
case 'i':
in = g.getOptarg();
break;
case 'o':
out = g.getOptarg();
break;
case 'c':
String scount = g.getOptarg();
try {
count = Integer.parseInt(scount);
} catch (NumberFormatException nfe) {
error = true;
}
break;
case '?':
case ':':
default:
error = true;
}
}
if (error) {
usage();
System.exit(1);
}
File confDir = new File(in);
File dbDir = new File(confDir, "netDb");
if (!dbDir.exists()) {
System.out.println("NetDB directory " + dbDir + " does not exist");
System.exit(1);
}
File myFile = new File(confDir, "router.info");
File toDir = new File(out);
toDir.mkdirs();
InputStream fis = null;
Hash me = null;
try {
fis = new BufferedInputStream(new FileInputStream(myFile));
RouterInfo ri = new RouterInfo();
ri.readBytes(fis, true); // true = verify sig on read
me = ri.getIdentity().getHash();
} catch (Exception e) {
//System.out.println("Can't determine our identity");
} finally {
if (fis != null) try { fis.close(); } catch (IOException ioe) {}
}
int routerCount = 0;
List<File> toRead = new ArrayList<File>(2048);
for (int j = 0; j < B64.length(); j++) {
File subdir = new File(dbDir, DIR_PREFIX + B64.charAt(j));
File[] files = subdir.listFiles(RouterInfoFilter.getInstance());
if (files == null)
continue;
routerCount += files.length;
for (int i = 0; i < files.length; i++) {
toRead.add(files[i]);
}
}
if (toRead.isEmpty()) {
System.out.println("No files to copy in " + dbDir);
System.exit(1);
}
Collections.shuffle(toRead);
int copied = 0;
long tooOld = System.currentTimeMillis() - 7*24*60*60*1000L;
for (File file : toRead) {
if (copied >= count)
break;
Hash key = getRouterInfoHash(file.getName());
if (key == null) {
System.out.println("Skipping bad " + file);
continue;
}
if (key.equals(me)) {
System.out.println("Skipping my RI");
continue;
}
fis = null;
try {
fis = new BufferedInputStream(new FileInputStream(file));
RouterInfo ri = new RouterInfo();
ri.readBytes(fis, true); // true = verify sig on read
try { fis.close(); } catch (IOException ioe) {}
fis = null;
if (ri.getPublished() < tooOld) {
System.out.println("Skipping too old " + key);
continue;
}
if (ri.getCapabilities().contains("U")) {
System.out.println("Skipping unreachable " + key);
continue;
}
Collection<RouterAddress> addrs = ri.getAddresses();
if (addrs.isEmpty()) {
System.out.println("Skipping hidden " + key);
continue;
}
boolean hasIntro = false;
boolean hasIPv4 = false;
for (RouterAddress addr : addrs) {
if ("SSU".equals(addr.getTransportStyle()) && addr.getOption("ihost0") != null) {
hasIntro = true;
break;
}
String host = addr.getHost();
if (host != null && host.contains("."))
hasIPv4 = true;
}
if (hasIntro) {
System.out.println("Skipping introduced " + key);
continue;
}
if (!hasIPv4) {
System.out.println("Skipping IPv6-only " + key);
continue;
}
File toFile = new File(toDir, file.getName());
// We could call ri.write() to avoid simultaneous change by the router
boolean ok = FileUtil.copy(file, toFile, true, true);
if (ok)
copied++;
else
System.out.println("Failed copy of " + file + " to " + toDir);
} catch (Exception e) {
System.out.println("Skipping bad " + file);
} finally {
if (fis != null) try { fis.close(); } catch (IOException ioe) {}
}
}
if (copied > 0) {
System.out.println("Copied " + copied + " router info files to " + toDir);
} else {
System.out.println("Failed to copy any files to " + toDir);
System.exit(1);
}
}
private static void usage() {
System.err.println("Usage: PersistentDataStore [-i $HOME/.i2p] [-o netDb/] [-c 200]");
}
}

2
router/java/src/net/i2p/router/transport/BadCountries.java
View File

@@ -9,7 +9,7 @@ import java.util.Set;
* Maintain a list of bad places.
* @since 0.8.13
*/
abstract class BadCountries {
public abstract class BadCountries {
private static final Set<String> _countries;

42
router/java/src/net/i2p/router/transport/GeoIP.java
View File

@@ -16,6 +16,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import net.i2p.I2PAppContext;
import net.i2p.data.Hash;
import net.i2p.router.Router;
import net.i2p.router.RouterContext;
@@ -38,11 +39,9 @@ import net.i2p.util.Log;
*
* @author zzz
*/
class GeoIP {
public class GeoIP {
private final Log _log;
// change to test with main()
//private final I2PAppContext _context;
private final RouterContext _context;
private final I2PAppContext _context;
private final Map<String, String> _codeToName;
/** code to itself to prevent String proliferation */
private final Map<String, String> _codeCache;
@@ -56,8 +55,10 @@ class GeoIP {
private final AtomicBoolean _lock;
private int _lookupRunCount;
//public GeoIP(I2PAppContext context) {
public GeoIP(RouterContext context) {
/**
* @param context RouterContext in production, I2PAppContext for testing only
*/
public GeoIP(I2PAppContext context) {
_context = context;
_log = context.logManager().getLog(GeoIP.class);
_codeToName = new ConcurrentHashMap<String, String>(512);
@@ -71,6 +72,7 @@ class GeoIP {
}
static final String PROP_GEOIP_ENABLED = "routerconsole.geoip.enable";
public static final String PROP_GEOIP_DIR = "geoip.dir";
static final String GEOIP_DIR_DEFAULT = "geoip";
static final String GEOIP_FILE_DEFAULT = "geoip.txt";
static final String COUNTRY_FILE_DEFAULT = "countries.txt";
@@ -187,7 +189,10 @@ class GeoIP {
*
*/
private void readCountryFile() {
File geoFile = new File(_context.getBaseDir(), GEOIP_DIR_DEFAULT);
String geoDir = _context.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT);
File geoFile = new File(geoDir);
if (!geoFile.isAbsolute())
geoFile = new File(_context.getBaseDir(), geoDir);
geoFile = new File(geoFile, COUNTRY_FILE_DEFAULT);
if (!geoFile.exists()) {
if (_log.shouldLog(Log.WARN))
@@ -246,7 +251,10 @@ class GeoIP {
*
*/
private String[] readGeoIPFile(Long[] search) {
File geoFile = new File(_context.getBaseDir(), GEOIP_DIR_DEFAULT);
String geoDir = _context.getProperty(PROP_GEOIP_DIR, GEOIP_DIR_DEFAULT);
File geoFile = new File(geoDir);
if (!geoFile.isAbsolute())
geoFile = new File(_context.getBaseDir(), geoDir);
geoFile = new File(geoFile, GEOIP_FILE_DEFAULT);
if (!geoFile.exists()) {
if (_log.shouldLog(Log.WARN))
@@ -300,24 +308,28 @@ class GeoIP {
/**
* Put our country code in the config, where others (such as Timestamper) can get it,
* and it will be there next time at startup.
*
* Does nothing in I2PAppContext
*/
private void updateOurCountry() {
/**** comment out to test with main() */
String oldCountry = _context.router().getConfigSetting(PROP_IP_COUNTRY);
Hash ourHash = _context.routerHash();
if (! (_context instanceof RouterContext))
return;
RouterContext ctx = (RouterContext) _context;
String oldCountry = ctx.router().getConfigSetting(PROP_IP_COUNTRY);
Hash ourHash = ctx.routerHash();
// we should always have a RouterInfo by now, but we had one report of an NPE here
if (ourHash == null)
return;
String country = _context.commSystem().getCountry(ourHash);
String country = ctx.commSystem().getCountry(ourHash);
if (country != null && !country.equals(oldCountry)) {
_context.router().saveConfig(PROP_IP_COUNTRY, country);
if (_context.commSystem().isInBadCountry() && _context.getProperty(Router.PROP_HIDDEN_HIDDEN) == null) {
ctx.router().saveConfig(PROP_IP_COUNTRY, country);
if (ctx.commSystem().isInBadCountry() && ctx.getProperty(Router.PROP_HIDDEN_HIDDEN) == null) {
String name = fullName(country);
if (name == null)
name = country;
_log.logAlways(Log.WARN, "Setting hidden mode to protect you in " + name +
", you may override on the network configuration page");
_context.router().rebuildRouterInfo();
ctx.router().rebuildRouterInfo();
}
}
/****/