From ea9a3320a363660790ebcd3ff73818f61ed761e8 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 17 Apr 2018 17:16:51 +0000 Subject: [PATCH] DNSoverHTTPS for SSLEepGet and NTP (ticket #2201) --- core/java/src/net/i2p/util/CommandLine.java | 1 + core/java/src/net/i2p/util/DNSOverHTTPS.java | 454 ++++++++++++++++++ core/java/src/net/i2p/util/ObjectCounter.java | 10 +- core/java/src/net/i2p/util/SSLEepGet.java | 15 +- history.txt | 2 + .../src/net/i2p/router/RouterVersion.java | 2 +- .../src/net/i2p/router/time/NtpClient.java | 44 +- 7 files changed, 515 insertions(+), 13 deletions(-) create mode 100644 core/java/src/net/i2p/util/DNSOverHTTPS.java diff --git a/core/java/src/net/i2p/util/CommandLine.java b/core/java/src/net/i2p/util/CommandLine.java index f5ac357b53..693ee83cca 100644 --- a/core/java/src/net/i2p/util/CommandLine.java +++ b/core/java/src/net/i2p/util/CommandLine.java @@ -32,6 +32,7 @@ public class CommandLine { "net.i2p.time.BuildTime", "net.i2p.util.Addresses", "net.i2p.util.ConvertToHash", + "net.i2p.util.DNSOverHTTPS", "net.i2p.util.EepGet", "net.i2p.util.EepHead", "net.i2p.util.FileUtil", diff --git a/core/java/src/net/i2p/util/DNSOverHTTPS.java b/core/java/src/net/i2p/util/DNSOverHTTPS.java new file mode 100644 index 0000000000..c475496dd4 --- /dev/null +++ b/core/java/src/net/i2p/util/DNSOverHTTPS.java @@ -0,0 +1,454 @@ +package net.i2p.util; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import gnu.getopt.Getopt; + +import net.minidev.json.JSONArray; +import net.minidev.json.JSONObject; +import net.minidev.json.parser.JSONParser; + +import net.i2p.I2PAppContext; +import net.i2p.data.DataHelper; + +/** + * Simple implemetation of DNS over HTTPS. + * Also sets the local clock from the received date header. + * + * https://developers.google.com/speed/public-dns/docs/dns-over-https + * https://developers.cloudflare.com/1.1.1.1/dns-over-https/json-format/ + * + * @since 0.9.35 + */ +public class DNSOverHTTPS implements EepGet.StatusListener { + private final I2PAppContext ctx; + private final Log _log; + private final JSONParser parser; + private final ByteArrayOutputStream baos; + private SSLEepGet.SSLState state; + private long fetchStart; + private int gotDate; + + private static final Map v4Cache = new LHMCache(32); + private static final Map v6Cache = new LHMCache(32); + // v4 URLs to query, ending with '&' + private static final List v4urls = new ArrayList(4); + // v6 URLs to query, ending with '&' + private static final List v6urls = new ArrayList(4); + // consecutive failures + private static final ObjectCounter fails = new ObjectCounter(); + + // Don't look up any of these TLDs + // RFC 2606, 6303, 7393 + // https://www.iana.org/assignments/locally-served-dns-zones/locally-served-dns-zones.xhtml + private static final List locals = Arrays.asList(new String[] { + "localhost", + "in-addr.arpa", "ip6.arpa", "home.arpa", + "i2p", "onion", + "i2p.arpa", "onion.arpa", + "corp", "home", "internal", "intranet", "lan", "local", "private", + "test", "example", "invalid", + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" + } ); + + static { + // Warning: All hostnames MUST be in loop check in lookup() below + // Google + // Certs for 8.8.8.8 and 8.8.4.4 don't work + v4urls.add("https://dns.google.com/resolve?edns_client_subnet=0.0.0.0/0&"); + v6urls.add("https://dns.google.com/resolve?edns_client_subnet=0.0.0.0/0&"); + // Cloudflare cloudflare-dns.com + // https://developers.cloudflare.com/1.1.1.1/nitty-gritty-details/ + // 1.1.1.1 is a privacy centric resolver so it does not send any client IP information + // and does not send the EDNS Client Subnet Header to authoritative servers + v4urls.add("https://1.1.1.1/dns-query?ct=application/dns-json&"); + v4urls.add("https://1.0.0.1/dns-query?ct=application/dns-json&"); + v6urls.add("https://[2606:4700:4700::1111]/dns-query?ct=application/dns-json&"); + v6urls.add("https://[2606:4700:4700::1001]/dns-query?ct=application/dns-json&"); + } + + // keep the timeout very short, as we try multiple addresses, + // and will be falling back to regular DNS. + private static final long TIMEOUT = 3*1000; + private static final int MAX_TTL = 24*60*60; + // don't use a URL after this many consecutive failures + private static final int MAX_FAILS = 3; + private static final int V4_CODE = 1; + private static final int CNAME_CODE = 5; + private static final int V6_CODE = 28; + private static final int MAX_DATE_SETS = 2; + // From RouterClock + private static final int DEFAULT_STRATUM = 8; + + public DNSOverHTTPS(I2PAppContext context) { + this(context, null); + } + + public DNSOverHTTPS(I2PAppContext context, SSLEepGet.SSLState sslState) { + ctx = context; + _log = ctx.logManager().getLog(DNSOverHTTPS.class); + state = sslState; + baos = new ByteArrayOutputStream(512); + parser = new JSONParser(JSONParser.MODE_PERMISSIVE); + } + + public enum Type { V4_ONLY, V6_ONLY, V4_PREFERRED, V6_PREFERRED } + + private static class Result { + public final String ip; + public final long expires; + public Result(String i, long e) { + ip = i; expires = e; + } + } + + /** + * V4_ONLY + * @return null if not found + */ + public String lookup(String host) { + return lookup(host, Type.V4_ONLY); + } + + /** + * Lookup in cache, then query servers + * @return null if not found + */ + public String lookup(String host, Type type) { + if (Addresses.isIPAddress(host)) + return host; + if (host.startsWith("[")) + return host; + host = host.toLowerCase(Locale.US); + if (host.indexOf('.') < 0) + return null; + for (String local : locals) { + if (host.equals(local) || + (host.endsWith(local) && host.charAt(host.length() - local.length() - 1) == '.')) { + return null; + } + } + // don't loop via SSLEepGet + if (host.equals("dns.google.com")) + return "8.8.8.8"; + if (type == Type.V4_ONLY || type == Type.V4_PREFERRED) { + // v4 lookup + String rv = lookup(host, v4Cache); + if (rv != null) + return rv; + } + if (type != Type.V4_ONLY) { + // v6 lookup + String rv = lookup(host, v6Cache); + if (rv != null) + return rv; + } + if (type == Type.V6_PREFERRED) { + // v4 lookup after v6 lookup + String rv = lookup(host, v4Cache); + if (rv != null) + return rv; + } + return query(host, type); + } + + public static void clearCaches() { + synchronized (v4Cache) { + v4Cache.clear(); + } + synchronized (v6Cache) { + v6Cache.clear(); + } + fails.clear(); + } + + /** + * Lookup in cache + * @return null if not found or expired + */ + private static String lookup(String host, Map cache) { + synchronized (cache) { + Result r = cache.get(host); + if (r != null) { + if (r.expires >= System.currentTimeMillis()) + return r.ip; + cache.remove(host); + } + } + return null; + } + + /** + * Query servers + * @return null if not found + */ + private String query(String host, Type type) { + List toQuery = new ArrayList((type == Type.V6_ONLY) ? v6urls : v4urls); + Collections.shuffle(toQuery); + if (type == Type.V4_ONLY || type == Type.V4_PREFERRED) { + // v4 query + String rv = query(host, false, toQuery); + if (rv != null) + return rv; + } + if (type != Type.V4_ONLY) { + // v6 query + String rv = query(host, true, toQuery); + if (rv != null) + return rv; + } + if (type == Type.V6_PREFERRED) { + // v4 query after v6 query + String rv = query(host, false, toQuery); + if (rv != null) + return rv; + } + return null; + } + + /** + * @return null if not found + */ + private String query(String host, boolean isv6, List toQuery) { + for (String url : toQuery) { + if (fails.count(url) > MAX_FAILS) + continue; + int tcode = isv6 ? V6_CODE : V4_CODE; + String furl = url + "name=" + host + "&type=" + tcode; + log("Fetching " + furl); + baos.reset(); + SSLEepGet eepget = new SSLEepGet(ctx, baos, furl, state); + if (ctx.isRouterContext()) + eepget.addStatusListener(this); + else + fetchStart = System.currentTimeMillis(); // debug + String rv = fetch(eepget, host, isv6); + if (rv != null) { + fails.clear(url); + return rv; + } + if (state == null) + state = eepget.getSSLState(); + // we treat all fails the same, whether server responded or not + fails.increment(url); + log("No result from " + furl); + } + return null; + } + + /** + * @return null if not found + */ + private String fetch(SSLEepGet eepget, String host, boolean isv6) { + if (eepget.fetch(TIMEOUT, TIMEOUT, TIMEOUT) && + eepget.getStatusCode() == 200 && baos.size() > 0) { + long end = System.currentTimeMillis(); + log("Got response in " + (end - fetchStart) + "ms"); + byte[] b = baos.toByteArray(); + try { + JSONObject map = (JSONObject) parser.parse(b); + if (map == null) { + log("No map"); + return null; + } + Integer status = (Integer) map.get("Status"); + if (status == null || status.intValue() != 0) { + log("Bad status: " + status); + return null; + } + JSONArray list = (JSONArray) map.get("Answer"); + if (list == null || list.isEmpty()) { + log("No answer"); + return null; + } + log(list.size() + " answers"); + String hostAnswer = host + '.'; + for (Object o : list) { + try { + JSONObject a = (JSONObject) o; + String data = (String) a.get("data"); + if (data == null) { + log("no data"); + continue; + } + Integer typ = (Integer) a.get("type"); + if (typ == null) + continue; + String name = (String) a.get("name"); + if (name == null) + continue; + if (typ.intValue() == CNAME_CODE) { + log("CNAME is: " + data); + hostAnswer = data; + continue; + } + if (isv6) { + if (typ.intValue() != V6_CODE) { + log("type mismatch: " + typ); + continue; + } + if (!Addresses.isIPv6Address(data)) { + log("bad addr: " + data); + continue; + } + } else { + if (typ.intValue() != V4_CODE) { + log("type mismatch: " + typ); + continue; + } + if (!Addresses.isIPv4Address(data)) { + log("bad addr: " + data); + continue; + } + } + if (!hostAnswer.equals(name)) { + log("name mismatch: " + name); + continue; + } + Integer ttl = (Integer) a.get("TTL"); + int ittl = (ttl != null) ? Math.min(ttl.intValue(), MAX_TTL) : 3600; + long expires = end + (ittl * 1000L); + Map cache = isv6 ? v6Cache : v4Cache; + synchronized(cache) { + cache.put(host, new Result(data, expires)); + } + log("Got answer: " + name + ' ' + typ + ' ' + ttl + ' ' + data + " in " + (end - fetchStart) + "ms"); + return data; + } catch (Exception e) { + log("Fail parsing", e); + } + } + } catch (Exception e) { + log("Fail parsing", e); + } + log("Bad response:\n" + new String(b)); + } else { + log("Fail fetching"); + } + return null; + } + + // EepGet status listeners Reseeder + public void attemptFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt, int numRetries, Exception cause) {} + public void bytesTransferred(long alreadyTransferred, int currentWrite, long bytesTransferred, long bytesRemaining, String url) {} + public void transferComplete(long alreadyTransferred, long bytesTransferred, long bytesRemaining, String url, String outputFile, boolean notModified) {} + public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {} + + public void attempting(String url) { + if (gotDate < MAX_DATE_SETS) + fetchStart = System.currentTimeMillis(); + } + + /** + * Use the Date header as a backup time source. + * Code from Reseeder. + * We set the stratum to a lower (better) value than in Reseeder, + * as Cloudflare and Google probably have a better idea than our reseeds. + */ + public void headerReceived(String url, int attemptNum, String key, String val) { + // We do this more than once, because + // the first SSL handshake may take a while, and it may take the server + // a while to render the index page. + if (gotDate < MAX_DATE_SETS && "Date".equals(key)) { + long timeRcvd = System.currentTimeMillis(); + long serverTime = RFC822Date.parse822Date(val); + if (serverTime > 0) { + // add 500ms since it's 1-sec resolution, and add half the RTT + long now = serverTime + 500 + ((timeRcvd - fetchStart) / 2); + long offset = now - ctx.clock().now(); + if (ctx.clock().getUpdatedSuccessfully()) { + // 2nd time better than the first + if (gotDate > 0) + ctx.clock().setNow(now, DEFAULT_STRATUM - 4); + else + ctx.clock().setNow(now, DEFAULT_STRATUM - 3); + log("DNSOverHTTPS adjusting clock by " + + DataHelper.formatDuration(Math.abs(offset))); + } else { + // No peers or NTP yet, this is probably better than the peer average will be for a while + // default stratum - 1, so the peer average is a worse stratum + ctx.clock().setNow(now, DEFAULT_STRATUM - 3); + log("DNSOverHTTPS setting initial clock skew to " + + DataHelper.formatDuration(Math.abs(offset))); + } + gotDate++; + } + } + } + + // End of EepGet status listeners + + private void log(String msg) { + log(msg, null); + } + + private void log(String msg, Throwable t) { + int level = (t != null) ? Log.WARN : Log.INFO; + _log.log(level, msg, t); + } + + public static void main(String[] args) { + Type type = Type.V4_PREFERRED; + boolean error = false; + Getopt g = new Getopt("dnsoverhttps", args, "46fs"); + try { + int c; + while ((c = g.getopt()) != -1) { + switch (c) { + case '4': + type = Type.V4_ONLY; + break; + + case '6': + type = Type.V6_ONLY; + break; + + case 'f': + type = Type.V4_PREFERRED; + break; + + case 's': + type = Type.V6_PREFERRED; + break; + + case '?': + case ':': + default: + error = true; + break; + } // switch + } // while + } catch (RuntimeException e) { + e.printStackTrace(); + error = true; + } + if (error || args.length - g.getOptind() != 1) { + usage(); + System.exit(1); + } + + String url = args[g.getOptind()]; + String result = (new DNSOverHTTPS(I2PAppContext.getGlobalContext())).lookup(url, type); + if (result != null) + System.out.println(type + " lookup for " + url + " is " + result); + else + System.err.println(type + " lookup failed for " + url); + } + + private static void usage() { + System.err.println("DNSOverHTTPS [-fs46] hostname\n" + + " [-f] (IPv4 preferred) (default)\n" + + " [-s] (IPv6 preferred)\n" + + " [-4] (IPv4 only)\n" + + " [-6] (IPv6 only)"); + } +} diff --git a/core/java/src/net/i2p/util/ObjectCounter.java b/core/java/src/net/i2p/util/ObjectCounter.java index ff00a8e398..9e04cd20ea 100644 --- a/core/java/src/net/i2p/util/ObjectCounter.java +++ b/core/java/src/net/i2p/util/ObjectCounter.java @@ -51,11 +51,19 @@ public class ObjectCounter implements Serializable { } /** - * start over + * Start over. Reset the count for all keys to zero. * @since 0.7.11 */ public void clear() { this.map.clear(); } + + /** + * Reset the count for this key to zero + * @since 0.9.36 + */ + public void clear(K h) { + this.map.remove(h); + } } diff --git a/core/java/src/net/i2p/util/SSLEepGet.java b/core/java/src/net/i2p/util/SSLEepGet.java index 58c7e38ac2..e21014aab5 100644 --- a/core/java/src/net/i2p/util/SSLEepGet.java +++ b/core/java/src/net/i2p/util/SSLEepGet.java @@ -101,6 +101,8 @@ public class SSLEepGet extends EepGet { private final ProxyType _proxyType; private static final String CERT_DIR = "certificates/ssl"; + private static final String PROP_USE_DNS_OVER_HTTPS = "eepget.useDNSOverHTTPS"; + private static final boolean DEFAULT_USE_DNS_OVER_HTTPS = true; /** * Not all may be supported. @@ -706,6 +708,17 @@ public class SSLEepGet extends EepGet { if (port == -1) port = 443; + String originalHost = host; + boolean useDNSOverHTTPS = _context.getProperty(PROP_USE_DNS_OVER_HTTPS, DEFAULT_USE_DNS_OVER_HTTPS); + // This duplicates checks in DNSOverHTTPS.lookup() but do it here too so + // we don't even construct it if we don't need it + if (useDNSOverHTTPS && !host.equals("dns.google.com") && !Addresses.isIPAddress(host)) { + DNSOverHTTPS doh = new DNSOverHTTPS(_context, getSSLState()); + String ip = doh.lookup(host); + if (ip != null) + host = ip; + } + if (_shouldProxy) { if (_log.shouldLog(Log.DEBUG)) _log.debug("Connecting to " + _proxyType + " proxy"); @@ -757,7 +770,7 @@ public class SSLEepGet extends EepGet { I2PSSLSocketFactory.setProtocolsAndCiphers(socket); if (!_bypassVerification) { try { - I2PSSLSocketFactory.verifyHostname(_context, socket, host); + I2PSSLSocketFactory.verifyHostname(_context, socket, originalHost); } catch (SSLException ssle) { if (_saveCerts > 0 && _stm != null) saveCerts(host, _stm); diff --git a/history.txt b/history.txt index 964687bef4..6437afe025 100644 --- a/history.txt +++ b/history.txt @@ -1,5 +1,7 @@ 2018-04-17 zzz * Console: Fix sidebar status when updating plugin (ticket #2137) + * Reseed, NTP: Use DNSoverHTTPS (ticket #2201) + * SusiMail: Fix error message on login page 2018-04-16 zzz * Console: Add links to bandwidth graphs on /tunnels diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 0725033fa6..f6e6df3a5c 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 3; + public final static long BUILD = 4; /** for example "-test" */ public final static String EXTRA = ""; diff --git a/router/java/src/net/i2p/router/time/NtpClient.java b/router/java/src/net/i2p/router/time/NtpClient.java index 76498b3b77..e221f6d55f 100644 --- a/router/java/src/net/i2p/router/time/NtpClient.java +++ b/router/java/src/net/i2p/router/time/NtpClient.java @@ -41,7 +41,10 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import net.i2p.I2PAppContext; import net.i2p.data.DataHelper; +import net.i2p.util.Addresses; +import net.i2p.util.DNSOverHTTPS; import net.i2p.util.HexDump; import net.i2p.util.Log; @@ -73,6 +76,8 @@ public class NtpClient { private static final int MIN_PKT_LEN = 48; // IP:reason for servers that sent us a kiss of death private static final Map kisses = new ConcurrentHashMap(2); + private static final String PROP_USE_DNS_OVER_HTTPS = "time.useDNSOverHTTPS"; + private static final boolean DEFAULT_USE_DNS_OVER_HTTPS = true; /** * Query the ntp servers, returning the current time from first one we find @@ -144,23 +149,42 @@ public class NtpClient { */ private static long[] currentTimeAndStratum(String serverName, int timeout, boolean preferIPv6, Log log) { DatagramSocket socket = null; + I2PAppContext ctx = I2PAppContext.getGlobalContext(); + boolean useDNSOverHTTPS = ctx.getProperty(PROP_USE_DNS_OVER_HTTPS, DEFAULT_USE_DNS_OVER_HTTPS); try { // Send request InetAddress address; if (preferIPv6) { - InetAddress[] addrs = InetAddress.getAllByName(serverName); - if (addrs == null || addrs.length == 0) - throw new UnknownHostException(); - address = null; - for (int i = 0; i < addrs.length; i++) { - if (addrs[i] instanceof Inet6Address) { - address = addrs[i]; - break; + String ip = null; + if (useDNSOverHTTPS) { + DNSOverHTTPS doh = new DNSOverHTTPS(ctx); + ip = doh.lookup(serverName, DNSOverHTTPS.Type.V6_PREFERRED); + } + if (ip != null) { + address = InetAddress.getByName(ip); + } else { + // fallback to regular DNS + InetAddress[] addrs = InetAddress.getAllByName(serverName); + if (addrs == null || addrs.length == 0) + throw new UnknownHostException(); + address = null; + for (int i = 0; i < addrs.length; i++) { + if (addrs[i] instanceof Inet6Address) { + address = addrs[i]; + break; + } + if (address == null) + address = addrs[0]; } - if (address == null) - address = addrs[0]; } } else { + if (useDNSOverHTTPS) { + DNSOverHTTPS doh = new DNSOverHTTPS(ctx); + String ip = doh.lookup(serverName, DNSOverHTTPS.Type.V4_ONLY); + if (ip != null) + serverName = ip; + } + // fallback to regular DNS address = InetAddress.getByName(serverName); } String who = address.getHostAddress();