From 239fe518a9ba7f66efb0a137fd53d7aad0b617dd Mon Sep 17 00:00:00 2001
From: zzz <zzz@mail.i2p>
Date: Tue, 21 Oct 2014 18:35:06 +0000
Subject: [PATCH] Update: Partial implementation of su3 news with atom feed. No
 spec yet, just followed str4d's testnews.atom.xml proposal. Atom parsing is
 tested, su3 part is incomplete and untested. Todo: add spec to
 http://i2p-projekt.i2p/en/docs/spec/updates, finish su3 and test.

---
 apps/routerconsole/java/build.xml             |  24 ++
 .../src/net/i2p/router/news/NewsEntry.java    |  27 ++
 .../src/net/i2p/router/news/NewsMetadata.java |  28 ++
 .../net/i2p/router/news/NewsXMLParser.java    | 352 ++++++++++++++++++
 .../src/net/i2p/router/news/RFC3339Date.java  | 122 ++++++
 .../java/src/net/i2p/router/news/package.html |  14 +
 .../router/update/ConsoleUpdateManager.java   |   3 +-
 .../net/i2p/router/update/NewsFetcher.java    | 128 +++++++
 .../net/i2p/router/update/NewsHandler.java    |   7 +-
 .../i2p/router/web/ConfigUpdateHandler.java   |   3 +
 .../i2p/router/web/ConfigUpdateHelper.java    |   6 +-
 build.xml                                     |   7 +-
 core/java/src/net/i2p/crypto/SU3File.java     |   2 +
 core/java/src/net/i2p/update/UpdateType.java  |   4 +-
 14 files changed, 716 insertions(+), 11 deletions(-)
 create mode 100644 apps/routerconsole/java/src/net/i2p/router/news/NewsEntry.java
 create mode 100644 apps/routerconsole/java/src/net/i2p/router/news/NewsMetadata.java
 create mode 100644 apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java
 create mode 100644 apps/routerconsole/java/src/net/i2p/router/news/RFC3339Date.java
 create mode 100644 apps/routerconsole/java/src/net/i2p/router/news/package.html

diff --git a/apps/routerconsole/java/build.xml b/apps/routerconsole/java/build.xml
index 8d9e9ec99..f897cff20 100644
--- a/apps/routerconsole/java/build.xml
+++ b/apps/routerconsole/java/build.xml
@@ -81,6 +81,24 @@
         </javac>
     </target>
 
+    <!-- newsxml.jar only (subset of routerconsole, no war) for Android -->
+    <target name="compileNewsOnly" depends="prepare, depend, dependVersion">
+        <mkdir dir="./build" />
+        <mkdir dir="./build/obj" />
+        <javac 
+            srcdir="./src" 
+            includes="net/i2p/router/news/*.java"
+            debug="true" deprecation="on" source="${javac.version}" target="${javac.version}" 
+            includeAntRuntime="false"
+            destdir="./build/obj">
+            <compilerarg line="${javac.compilerargs}" />
+            <classpath>
+                <pathelement location="../../../core/java/build/i2p.jar" />
+                <pathelement location="../../../router/java/build/router.jar" />
+            </classpath>
+        </javac>
+    </target>
+
     <!-- the jar with the latest message classes from the jsps, and the war too -->
     <target name="jar" depends="jar1, war, bundle" />
 
@@ -119,6 +137,12 @@
         </jar>
     </target>
 
+    <!-- newsxml.jar only (subset of routerconsole, no war) for Android -->
+    <target name="newsxmljar" depends="compileNewsOnly" >
+        <jar destfile="./build/newsxml.jar" basedir="./build/obj" includes="**/*.class">
+        </jar>
+    </target>
+
     <!-- this is tricky because the message classes go in the jar, not in the war -->
     <target name="bundle" depends="jar1, precompilejsp" unless="no.bundle">
         <!-- Update the messages_*.po files.
diff --git a/apps/routerconsole/java/src/net/i2p/router/news/NewsEntry.java b/apps/routerconsole/java/src/net/i2p/router/news/NewsEntry.java
new file mode 100644
index 000000000..de6521b80
--- /dev/null
+++ b/apps/routerconsole/java/src/net/i2p/router/news/NewsEntry.java
@@ -0,0 +1,27 @@
+package net.i2p.router.news;
+
+/**
+ *  One news item.
+ *  Any String fields may be null.
+ *
+ *  @since 0.9.17
+ */
+public class NewsEntry implements Comparable<NewsEntry> {
+    public String title;
+    public String link;
+    public String id;
+    public long updated;
+    public String summary;
+    public String content;
+    public String contentType; // attribute of content
+    public String authorName;  // subnode of author
+
+    /** reverse, newest first */
+    public int compareTo(NewsEntry e) {
+        if (updated > e.updated)
+            return -1;
+        if (updated < e.updated)
+            return 1;
+        return 0;
+    }
+}
diff --git a/apps/routerconsole/java/src/net/i2p/router/news/NewsMetadata.java b/apps/routerconsole/java/src/net/i2p/router/news/NewsMetadata.java
new file mode 100644
index 000000000..732b814a7
--- /dev/null
+++ b/apps/routerconsole/java/src/net/i2p/router/news/NewsMetadata.java
@@ -0,0 +1,28 @@
+package net.i2p.router.news;
+
+import java.util.List;
+
+/**
+ *  The update metadata.
+ *  Any String or List fields may be null.
+ *
+ *  @since 0.9.17
+ */
+public class NewsMetadata {
+    // Standard Atom feed metadata
+    public String feedTitle;
+    public String feedSubtitle;
+    public String feedID;
+    public long feedUpdated;
+
+    // I2P update metadata
+    public long date;
+    public String minVersion;
+    public String minJavaVersion;
+    public String i2pVersion;
+    public String sudTorrent;
+    public String su2Torrent;
+    public String su3Torrent;
+    public List<String> su3Clearnet;
+    public List<String> su3SSL;
+}
diff --git a/apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java b/apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java
new file mode 100644
index 000000000..b59e4b0b2
--- /dev/null
+++ b/apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java
@@ -0,0 +1,352 @@
+package net.i2p.router.news;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import net.i2p.I2PAppContext;
+import net.i2p.util.Log;
+import org.cybergarage.xml.Node;
+import org.cybergarage.xml.ParserException;
+import org.cybergarage.xml.parser.JaxpParser;
+
+/**
+ *  Parse out the news.xml file which is in Atom format (RFC4287).
+ *
+ *  We use the XML parser from the UPnP library.
+ *
+ *  @since 0.9.17
+ */
+public class NewsXMLParser {
+    private final I2PAppContext _context;
+    private final Log _log;
+    private List<NewsEntry> _entries;
+    private NewsMetadata _metadata;
+    private XHTMLMode _mode;
+
+    private static final Set xhtmlWhitelist = new HashSet(Arrays.asList(new String[] {
+        "a", "b", "br", "div", "i", "p", "span", "font", "blockquote", "hr",
+        "del", "ins", "em", "strong", "mark", "sub", "sup", "tt", "code", "strike", "s", "u",
+        "h4", "h5", "h6",
+        "ol", "ul", "li", "dl", "dt", "dd",
+        "table", "tr", "td", "th"
+    }));
+
+    /**
+     *  The action taken when encountering a non-whitelisted
+     *  XHTML element in the feed content.
+     */
+    public enum XHTMLMode {
+        /** abort the parsing on any non-whitelisted element */
+        ABORT,
+        /** remove only the non-whitelisted element */
+        REMOVE_ELEMENT,
+        /** skip the feed entry containing the non-whitelisted element */
+        SKIP_ENTRY,
+        /** disable whitelist checks */
+        ALLOW_ALL
+    }
+
+    public NewsXMLParser(I2PAppContext ctx) { 
+        _context = ctx;
+        _log = ctx.logManager().getLog(NewsXMLParser.class);
+        _mode = XHTMLMode.REMOVE_ELEMENT;
+    }
+
+    /**
+     *  Sets the action taken when encountering a non-whitelisted
+     *  XHTML element in the feed content.
+     *  Must be set before parse().
+     *  Default REMOVE_ELEMENT.
+     */
+    public void setXHTMLMode(XHTMLMode mode) {
+        _mode = mode;
+    }
+
+    /**
+     *  Process the XML file.
+     *
+     *  @param file XML content only. Any su3 or gunzip handling must have
+     *              already happened.
+     *  @throws IOException on any parse error
+     */
+    public void parse(File file) throws IOException {
+        parse(new BufferedInputStream(new FileInputStream(file)));
+    }
+
+    /**
+     *  Process the XML input stream.
+     *
+     *  @param in XML content only. Any su3 or gunzip handling must have
+     *            already happened.
+     *  @throws IOException on any parse error
+     */
+    public void parse(InputStream in) throws IOException {
+        _entries = null;
+        _metadata = null;
+        JaxpParser parser = new JaxpParser();
+        try {
+            Node root = parser.parse(in);
+            extract(root);
+        } catch (ParserException pe) {
+            throw new I2PParserException(pe);
+        }
+    }
+
+    /**
+     *  The news entries.
+     *  Must call parse() first.
+     *
+     *  @return sorted, newest first, null if parse failed
+     */
+    public List<NewsEntry> getEntries() {
+        return _entries;
+    }
+
+    /**
+     *  The news metatdata.
+     *  Must call parse() first.
+     *
+     *  @return null if parse failed
+     */
+    public NewsMetadata getMetadata() {
+        return _metadata;
+    }
+
+    private void extract(Node root) throws I2PParserException {
+        if (!root.getName().equals("feed"))
+            throw new I2PParserException("no feed in XML");
+        _metadata = extractNewsMetadata(root);
+        _entries = extractNewsEntries(root);
+    }
+
+    private static NewsMetadata extractNewsMetadata(Node feed) throws I2PParserException {
+        NewsMetadata rv = new NewsMetadata();
+        Node n = feed.getNode("title");
+        if (n != null)
+            rv.feedTitle = n.getValue();
+        n = feed.getNode("subtitle");
+        if (n != null)
+            rv.feedSubtitle = n.getValue();
+        n = feed.getNode("id");
+        if (n != null)
+            rv.feedID = n.getValue();
+        n = feed.getNode("updated");
+        if (n != null) {
+            String v = n.getValue();
+            if (v != null) {
+                long time = RFC3339Date.parse3339Date(v);
+                if (time > 0)
+                    rv.feedUpdated = time;
+            }
+        }
+
+        Node r = feed.getNode("i2p:release");
+        if (r == null)
+            throw new I2PParserException("no release data in XML");
+        // release attributes
+        String a = r.getAttributeValue("date");
+        if (a.length() > 0) {
+            long time = RFC3339Date.parse3339Date(a);
+            if (time > 0)
+                rv.date = time;
+        }
+        a = r.getAttributeValue("minVersion");
+        if (a.length() > 0)
+            rv.minVersion = a;
+        a = r.getAttributeValue("minJavaVersion");
+        if (a.length() > 0)
+            rv.minJavaVersion = a;
+        // release nodes
+        n = r.getNode("i2p:version");
+        if (n != null)
+            rv.i2pVersion = n.getValue();
+        List<Node> urls = getNodes(r, "i2p:torrent");
+        for (Node t : urls) {
+            // returns "" for none
+            String href = t.getAttributeValue("href");
+            if (href.length() > 0) {
+                String type = t.getAttributeValue("type");
+                if (type.equals("su2"))
+                    rv.su2Torrent = href;
+                else if (type.equals("su3"))
+                    rv.su3Torrent = href;
+            }
+        }
+
+        return rv;
+    }
+
+    private List<NewsEntry> extractNewsEntries(Node feed) throws I2PParserException {
+        List<NewsEntry> rv = new ArrayList<NewsEntry>();
+        List<Node> entries = getNodes(feed, "entry");
+        for (Node entry : entries) {
+            NewsEntry e = new NewsEntry();
+            Node n = entry.getNode("title");
+            if (n != null)
+                e.title = n.getValue();
+            n = entry.getNode("link");
+            if (n != null)
+                e.link = n.getValue();
+            n = entry.getNode("id");
+            if (n != null)
+                e.id = n.getValue();
+            n = entry.getNode("updated");
+            if (n != null) {
+                String v = n.getValue();
+                if (v != null) {
+                    long time = RFC3339Date.parse3339Date(v);
+                    if (time > 0)
+                        e.updated = time;
+                }
+            }
+            n = entry.getNode("summary");
+            if (n != null)
+                e.summary = n.getValue();
+            n = entry.getNode("author");
+            if (n != null) {
+                n = n.getNode("name");
+                if (n != null)
+                    e.authorName = n.getValue();
+            }
+            n = entry.getNode("content");
+            if (n != null) {
+                String a = n.getAttributeValue("type");
+                if (a.length() > 0)
+                    e.contentType = a;
+                // now recursively sanitize
+                // and convert everything in the content to string
+                StringBuilder buf = new StringBuilder(256);
+                for (int i = 0; i < n.getNNodes(); i++) {
+                    Node sn = n.getNode(i);
+                    try {
+                        boolean removed = validate(sn);
+                        if (removed) {
+                            i--;
+                            continue;
+                        }
+                    } catch (I2PParserException ipe) {
+                        switch (_mode) {
+                          case ABORT:
+                            throw ipe;
+                          case SKIP_ENTRY:
+                            if (_log.shouldLog(Log.WARN))
+                                _log.warn("Skipping entry", ipe);
+                            e = null;
+                            break;
+                          case REMOVE_ELEMENT:
+                            if (_log.shouldLog(Log.WARN))
+                                _log.warn("Removing element", ipe);
+                            continue;
+                          case ALLOW_ALL:
+                          default:
+                            break;
+                        }
+                    }
+                    if (e == null)
+                        break;
+                    buf.append(sn.toString());
+                }
+                if (e == null)
+                    continue;
+                e.content = buf.toString();
+            }
+            rv.add(e);
+        }
+        Collections.sort(rv);
+        return rv;
+    }
+
+    /**
+     *  Helper to get all Nodes matching the name
+     */
+    private static List<Node> getNodes(Node node, String name) {
+        List<Node> rv = new ArrayList<Node>();
+        int count = node.getNNodes();
+        for (int i = 0; i < count; i++) {
+            Node n = node.getNode(i);
+            if (n.getName().equals(name))
+                rv.add(n);
+        }
+        return rv;
+    }
+
+    /**
+     *  @throws I2PParserException if any node not in whitelist (depends on mode)
+     *  @return true if node was removed from parent (only for REMOVE_ELEMENT mode)
+     */
+    private boolean validate(Node node) throws I2PParserException {
+        String name = node.getName();
+        //if (_log.shouldLog(Log.DEBUG))
+        //    _log.debug("Validating element: " + name);
+        if (!xhtmlWhitelist.contains(name.toLowerCase(Locale.US))) {
+            switch (_mode) {
+              case ABORT:
+              case SKIP_ENTRY:
+                throw new I2PParserException("Invalid XHTML element \"" + name + '"');
+              case REMOVE_ELEMENT:
+                // fixme this screws up the iteration
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Removing element: " + node);
+                node.getParentNode().removeNode(node);
+                return true;
+              case ALLOW_ALL:
+                if (_log.shouldLog(Log.WARN))
+                    _log.warn("Allowing non-whitelisted element by configuration: " + node);
+                break;
+            }
+        }
+        int count = node.getNNodes();
+        for (int i = 0; i < node.getNNodes(); i++) {
+            boolean removed = validate(node.getNode(i));
+            if (removed)
+                i--;
+        }
+        return false;
+    }
+
+    /**
+     *  Extend IOE since cybergarage ParserException extends Exception
+     */
+    private static class I2PParserException extends IOException {
+        public I2PParserException(String s) {
+            super(s);
+        }
+
+        public I2PParserException(Throwable t) {
+            super("XML Parse Error", t);
+        }
+    }
+
+    public static void main(String[] args) {
+        try {
+            NewsXMLParser parser = new NewsXMLParser(new I2PAppContext());
+            parser.setXHTMLMode(XHTMLMode.ABORT);
+            //parser.setXHTMLMode(XHTMLMode.REMOVE_ELEMENT);
+            //parser.setXHTMLMode(XHTMLMode.SKIP_ENTRY);
+            //parser.setXHTMLMode(XHTMLMode.ALLOW_ALL);
+            parser.parse(new File(args[0]));
+            NewsMetadata ud = parser.getMetadata();
+            List<NewsEntry> entries = parser.getEntries();
+            System.out.println("Latest version is " + ud.i2pVersion);
+            System.out.println("Release timestamp: " + ud.date);
+            System.out.println("Feed timestamp: " + ud.feedUpdated);
+            System.out.println("Found " + entries.size() + " news entries");
+            for (int i = 0; i < entries.size(); i++) {
+                NewsEntry e = entries.get(i);
+                System.out.println("News #" + (i+1) + ": " + e.title + '\n' + e.content);
+            }
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+        }
+    }
+}
diff --git a/apps/routerconsole/java/src/net/i2p/router/news/RFC3339Date.java b/apps/routerconsole/java/src/net/i2p/router/news/RFC3339Date.java
new file mode 100644
index 000000000..10cf242d4
--- /dev/null
+++ b/apps/routerconsole/java/src/net/i2p/router/news/RFC3339Date.java
@@ -0,0 +1,122 @@
+package net.i2p.router.news;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+import java.util.TimeZone;
+
+import net.i2p.util.SystemVersion;
+
+/**
+ *  Adapted from net.i2p.router.util.RFC822Date.
+ *  This only supports parsing of the dates specified by Atom (RFC 4287)
+ *  and a couple of others.
+ *  In particular, 'T' is required, and either 'Z' or a numeric timezone offset is required,
+ *  unless there's no time fields at all.
+ *
+ *  The full variety of RFC 3339 (ISO 8601) dates is not supported by the parser,
+ *  but they could be added in the future.
+ *
+ *  See also: http://stackoverflow.com/questions/6038136/how-do-i-parse-rfc-3339-datetimes-with-java
+ *
+ *  @since 0.9.17
+ */
+public abstract class RFC3339Date {
+
+    // SimpleDateFormat is not thread-safe, methods must be synchronized
+    private static final SimpleDateFormat OUTPUT_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+
+    private static final String TZF1, TZF2;
+    static {
+        if (SystemVersion.isJava7()) {
+            // ISO 8601
+            // These handle timezones like +1000, +10, and +10:00
+            TZF1 = "yyyy-MM-dd'T'HH:mm:ssXXX";
+            TZF2 = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";
+        } else {
+            // These handle timezones like +1000
+            // These do NOT handle timezones like +10:00
+            // This is fixed below
+            TZF1 = "yyyy-MM-dd'T'HH:mm:ssZZZZZ";
+            TZF2 = "yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ";
+        }
+    }
+
+    /**
+     *  This only supports parsing of the dates specified by Atom, RFC 4287,
+     *  together with the date only.
+     */
+    private static final SimpleDateFormat rfc3339DateFormats[] = new SimpleDateFormat[] {
+                 OUTPUT_FORMAT,
+                 // .S or .SS will get the milliseconds wrong,
+                 // e.g. .1 will become 1 ms, .11 will become 11 ms
+                 // This is NOT fixed below
+                 new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US),
+                 new SimpleDateFormat(TZF1, Locale.US),
+                 new SimpleDateFormat(TZF2, Locale.US),
+                 new SimpleDateFormat("yyyy-MM-dd", Locale.US),
+                 // old school for backward compatibility
+                 new SimpleDateFormat("yyyy/MM/dd", Locale.US)
+    };
+
+    //
+    // The router JVM is forced to UTC but do this just in case
+    //
+    static {
+        TimeZone utc = TimeZone.getTimeZone("GMT");
+        for (int i = 0; i < rfc3339DateFormats.length; i++) {
+            rfc3339DateFormats[i].setTimeZone(utc);
+        }
+    }
+
+    /**
+     * Parse the date
+     *
+     * @param s non-null
+     * @return -1 on failure
+     */
+    public synchronized static long parse3339Date(String s) {
+        s = s.trim();
+        // strip the ':' out of the time zone, if present,
+        // for Java 6 where we don't have the 'X' format
+        int len = s.length();
+        if (!SystemVersion.isJava7() &&
+            s.charAt(len - 1) != 'Z' &&
+            s.charAt(len - 3) == ':' &&
+            (s.charAt(len - 6) == '+' || s.charAt(len - 6) == '-')) {
+            s = s.substring(0, len - 3) + s.substring(len - 2);
+        }
+        for (int i = 0; i < rfc3339DateFormats.length; i++) {
+            try {
+                Date date = rfc3339DateFormats[i].parse(s);
+                if (date != null)
+                    return date.getTime();
+            } catch (ParseException pe) {}
+        }
+        return -1;
+    }
+
+    /**
+     * Format is "yyyy-MM-ddTHH:mm:ssZ"
+     *
+     * @since 0.8.2
+     */
+    public synchronized static String to3339Date(long t) {
+        return OUTPUT_FORMAT.format(new Date(t));
+    }
+
+/****
+    public static void main(String[] args) {
+        if (args.length == 1) {
+            try {
+                System.out.println(to3339Date(Long.parseLong(args[0])));
+            } catch (NumberFormatException nfe) {
+                System.out.println(parse3339Date(args[0]));
+            }
+        } else {
+            System.out.println("Usage: RFC3339Date numericDate|stringDate");
+        }
+    }
+****/
+}
diff --git a/apps/routerconsole/java/src/net/i2p/router/news/package.html b/apps/routerconsole/java/src/net/i2p/router/news/package.html
new file mode 100644
index 000000000..c023095cf
--- /dev/null
+++ b/apps/routerconsole/java/src/net/i2p/router/news/package.html
@@ -0,0 +1,14 @@
+<html><body>
+<p>
+Classes to parse the I2P news format, which follows the Atom
+standard with additional metadata for the I2P update notification feature.
+</p>
+<p>
+In the standard router package, this is a router console function, but
+these classes are in their own package so they may also
+be easily bundled in Android.
+</p>
+<p>
+Since 0.9.17.
+</p>
+</body></html>
diff --git a/apps/routerconsole/java/src/net/i2p/router/update/ConsoleUpdateManager.java b/apps/routerconsole/java/src/net/i2p/router/update/ConsoleUpdateManager.java
index 6b7c80bd7..94ac47104 100644
--- a/apps/routerconsole/java/src/net/i2p/router/update/ConsoleUpdateManager.java
+++ b/apps/routerconsole/java/src/net/i2p/router/update/ConsoleUpdateManager.java
@@ -158,8 +158,7 @@ public class ConsoleUpdateManager implements UpdateManager, RouterApp {
         // right at instantiation if the news is already indicating a new version
         Checker c = new NewsHandler(_context, this);
         register(c, NEWS, HTTP, 0);
-        // TODO
-        //register(c, NEWS_SU3, HTTP, 0);
+        register(c, NEWS_SU3, HTTP, 0);
         register(c, ROUTER_SIGNED, HTTP, 0);  // news is an update checker for the router
         Updater u = new UpdateHandler(_context, this);
         register(u, ROUTER_SIGNED, HTTP, 0);
diff --git a/apps/routerconsole/java/src/net/i2p/router/update/NewsFetcher.java b/apps/routerconsole/java/src/net/i2p/router/update/NewsFetcher.java
index ccbaaba34..fc54b3752 100644
--- a/apps/routerconsole/java/src/net/i2p/router/update/NewsFetcher.java
+++ b/apps/routerconsole/java/src/net/i2p/router/update/NewsFetcher.java
@@ -1,22 +1,37 @@
 package net.i2p.router.update;
 
+import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FilterInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.StringTokenizer;
 
+import net.i2p.crypto.SU3File;
 import net.i2p.crypto.TrustedUpdate;
 import net.i2p.data.DataHelper;
 import net.i2p.router.RouterContext;
 import net.i2p.router.RouterVersion;
+import net.i2p.router.news.NewsEntry;
+import net.i2p.router.news.NewsMetadata;
+import net.i2p.router.news.NewsXMLParser;
 import net.i2p.router.util.RFC822Date;
 import net.i2p.router.web.ConfigUpdateHandler;
 import net.i2p.router.web.NewsHelper;
@@ -26,6 +41,8 @@ import static net.i2p.update.UpdateMethod.*;
 import net.i2p.util.EepGet;
 import net.i2p.util.FileUtil;
 import net.i2p.util.Log;
+import net.i2p.util.ReusableGZIPInputStream;
+import net.i2p.util.SecureFileOutputStream;
 import net.i2p.util.SSLEepGet;
 import net.i2p.util.VersionComparator;
 
@@ -328,6 +345,10 @@ class NewsFetcher extends UpdateRunner {
         
         long now = _context.clock().now();
         if (_tempFile.exists()) {
+            if (url.endsWith(".su3")) {
+                processSU3();
+                return;
+            }
             boolean copied = FileUtil.copy(_tempFile, _newsFile, true, false);
             _tempFile.delete();
             if (copied) {
@@ -351,4 +372,111 @@ class NewsFetcher extends UpdateRunner {
     /** override to prevent status update */
     @Override
     public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
+
+    /**
+     *  Process the fetched su3 news file _tempFile
+     *
+     *  @since 0.9.17
+     */
+    private void processSU3() {
+        SU3File su3 = new SU3File(_context, _tempFile);
+        File to1 = new File(_context.getTempDir(), "tmp-" + _context.random().nextInt() + ".xml");
+        File to2 = new File(_context.getTempDir(), "tmp2-" + _context.random().nextInt() + ".xml");
+        String sudVersion;
+        String signingKeyName;
+        try {
+            su3.verifyAndMigrate(to1);
+            int type = su3.getFileType();
+            if (type != SU3File.TYPE_XML && type != SU3File.TYPE_XML_GZ)
+                throw new IOException("bad file type");
+            if (su3.getContentType() != SU3File.CONTENT_NEWS)
+                throw new IOException("bad content type");
+            File xml;
+            if (type == SU3File.TYPE_XML_GZ) {
+                gunzip(to1, to2);
+                xml = to2;
+            } else {
+                xml = to1;
+            }
+            sudVersion = su3.getVersionString();
+            signingKeyName = su3.getSignerString();
+            NewsXMLParser parser = new NewsXMLParser(_context);
+            parser.parse(xml);
+            NewsMetadata data = parser.getMetadata();
+            List<NewsEntry> entries = parser.getEntries();
+            outputOldNewsXML(data, entries);
+        } catch (IOException ioe) {
+            // FIXME
+            //statusDone("<b>" + ioe + ' ' + _("from {0}", _currentURI.toString()) + " </b>");
+            _tempFile.delete();
+            to1.delete();
+            to2.delete();
+            return;
+        }
+    }
+
+    /**
+     *  Gunzip the file
+     *
+     *  @since 0.9.17
+     */
+    private static void gunzip(File from, File to) throws IOException {
+        ReusableGZIPInputStream in = ReusableGZIPInputStream.acquire();
+        OutputStream out = null;
+        try {
+            in.initialize(new FileInputStream(from));
+            out = new SecureFileOutputStream(to);
+            byte buf[] = new byte[4096];
+            int read;
+            while ((read = in.read(buf)) != -1) {
+                out.write(buf, 0, read);
+            }
+        } finally {
+            if (out != null) try { 
+                out.close(); 
+            } catch (IOException ioe) {}
+            ReusableGZIPInputStream.release(in);
+        }
+    }
+
+    /**
+     *  Output in the old format.
+     *  Yes there is a better way.
+     *
+     *  @since 0.9.17
+     */
+    private void outputOldNewsXML(NewsMetadata data, List<NewsEntry> entries) throws IOException {
+        Writer out = null;
+        try {
+            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(_newsFile), "UTF-8"));
+            out.write("<!--\n");
+            out.write("<i2p.news date=\"$Date: 2014-09-20 00:00:00 $\">\n");
+            out.write("<i2p.release ");
+            if (data.i2pVersion != null)
+                out.write(" version=\"" + data.i2pVersion + '"');
+            if (data.minVersion != null)
+                out.write(" minVersion=\"" + data.minVersion + '"');
+            if (data.minJavaVersion != null)
+                out.write(" minJavaVersion=\"" + data.minJavaVersion + '"');
+            if (data.su2Torrent != null)
+                out.write(" su2Torrent=\"" + data.su2Torrent + '"');
+            if (data.su3Torrent != null)
+                out.write(" su3Torrent=\"" + data.su3Torrent + '"');
+            out.write("/>\n");
+            out.write("-->\n");
+            for (NewsEntry e : entries) {
+                if (e.title == null || e.content == null)
+                    continue;
+                out.write("<h3>");
+                out.write(e.title);
+                out.write("</h3>\n");
+                out.write(e.content);
+                out.write("\n\n");
+            }
+        } finally {
+            if (out != null) try { 
+                out.close(); 
+            } catch (IOException ioe) {}
+        }
+    }
 }
diff --git a/apps/routerconsole/java/src/net/i2p/router/update/NewsHandler.java b/apps/routerconsole/java/src/net/i2p/router/update/NewsHandler.java
index 1209cc49d..373d9dac4 100644
--- a/apps/routerconsole/java/src/net/i2p/router/update/NewsHandler.java
+++ b/apps/routerconsole/java/src/net/i2p/router/update/NewsHandler.java
@@ -47,13 +47,12 @@ class NewsHandler extends UpdateHandler implements Checker {
             return null;
         List<URI> updateSources = new ArrayList<URI>(2);
         try {
-            // TODO SU3
+            // This may be su3 or xml
             updateSources.add(new URI(ConfigUpdateHelper.getNewsURL(_context)));
         } catch (URISyntaxException use) {}
         try {
-            // TODO
-            //updateSources.add(new URI(BACKUP_NEWS_URL_SU3));
-            updateSources.add(new URI(BACKUP_NEWS_URL));
+            //updateSources.add(new URI(BACKUP_NEWS_URL));
+            updateSources.add(new URI(BACKUP_NEWS_URL_SU3));
         } catch (URISyntaxException use) {}
         UpdateRunner update = new NewsFetcher(_context, _mgr, updateSources);
         return update;
diff --git a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java
index e33ab2bc7..42f7279b0 100644
--- a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java
+++ b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHandler.java
@@ -30,8 +30,11 @@ public class ConfigUpdateHandler extends FormHandler {
 
     public static final String PROP_NEWS_URL = "router.newsURL";
 //  public static final String DEFAULT_NEWS_URL = "http://dev.i2p.net/cgi-bin/cvsweb.cgi/i2p/news.xml?rev=HEAD";
+    /** very old default */
     public static final String OLD_DEFAULT_NEWS_URL = "http://complication.i2p/news.xml";
+    /** old default */
     public static final String DEFAULT_NEWS_URL = "http://echelon.i2p/i2p/news.xml";
+    /** current default */
     public static final String DEFAULT_NEWS_URL_SU3 = "http://echelon.i2p/i2p/news.su3";
     public static final String PROP_REFRESH_FREQUENCY = "router.newsRefreshFrequency";
     public static final long DEFAULT_REFRESH_FREQ = 36*60*60*1000l;
diff --git a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java
index 4f5daab18..716524fbf 100644
--- a/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java
+++ b/apps/routerconsole/java/src/net/i2p/router/web/ConfigUpdateHelper.java
@@ -32,12 +32,12 @@ public class ConfigUpdateHelper extends HelperBase {
     /** hack to replace the old news location with the new one, even if they have saved
         the update page at some point */
     public static String getNewsURL(I2PAppContext ctx) {
-        // TODO SU3
         String url = ctx.getProperty(ConfigUpdateHandler.PROP_NEWS_URL);
-        if (url != null && !url.equals(ConfigUpdateHandler.OLD_DEFAULT_NEWS_URL))
+        if (url != null && !url.equals(ConfigUpdateHandler.OLD_DEFAULT_NEWS_URL) &&
+            !url.equals(ConfigUpdateHandler.DEFAULT_NEWS_URL))
             return url;
         else
-            return ConfigUpdateHandler.DEFAULT_NEWS_URL;
+            return ConfigUpdateHandler.DEFAULT_NEWS_URL_SU3;
     }
 
     public String getUpdateURL() {
diff --git a/build.xml b/build.xml
index 404c72245..200d86b4a 100644
--- a/build.xml
+++ b/build.xml
@@ -197,6 +197,11 @@
         <ant dir="apps/routerconsole/java/" target="jar" />
     </target>
 
+    <!-- newsxml.jar only (subset of routerconsole, no war) for Android -->
+    <target name="buildNewsXMLJar" depends="buildRouter" >
+        <ant dir="apps/routerconsole/java/" target="newsxmljar" />
+    </target>
+
     <target name="buildJetty" depends="buildCore" >
         <ant dir="apps/jetty" target="build" />
         <copy todir="build/" >
@@ -554,7 +559,7 @@
             <group title="Core SDK (i2p.jar)" packages="net.i2p:net.i2p.*:net.i2p.client:net.i2p.client.*:net.i2p.internal:net.i2p.internal.*:freenet.support.CPUInformation:org.bouncycastle.oldcrypto:org.bouncycastle.oldcrypto.*:gnu.crypto.*:gnu.getopt:gnu.gettext:com.nettgryppa.security:net.metanotion:net.metanotion.*" />
             <group title="Streaming Library" packages="net.i2p.client.streaming:net.i2p.client.streaming.impl" />
             <group title="Router" packages="net.i2p.router:net.i2p.router.*:net.i2p.data.i2np:net.i2p.data.router:org.cybergarage.*:org.freenetproject:org.xlattice.crypto.filters" />
-            <group title="Router Console" packages="net.i2p.router.web:net.i2p.router.update" />
+            <group title="Router Console" packages="net.i2p.router.web:net.i2p.router.update:net.i2p.router.news" />
             <!-- apps and bridges starting here, alphabetical please -->
             <group title="Addressbook Application" packages="net.i2p.addressbook" />
             <group title="BOB Bridge" packages="net.i2p.BOB" />
diff --git a/core/java/src/net/i2p/crypto/SU3File.java b/core/java/src/net/i2p/crypto/SU3File.java
index ef56d8bdd..254433d7f 100644
--- a/core/java/src/net/i2p/crypto/SU3File.java
+++ b/core/java/src/net/i2p/crypto/SU3File.java
@@ -71,6 +71,8 @@ public class SU3File {
     public static final int TYPE_XML = 1;
     /** @since 0.9.15 */
     public static final int TYPE_HTML = 2;
+    /** @since 0.9.17 */
+    public static final int TYPE_XML_GZ = 3;
 
     public static final int CONTENT_UNKNOWN = 0;
     public static final int CONTENT_ROUTER = 1;
diff --git a/core/java/src/net/i2p/update/UpdateType.java b/core/java/src/net/i2p/update/UpdateType.java
index 4c699785b..cf26bec08 100644
--- a/core/java/src/net/i2p/update/UpdateType.java
+++ b/core/java/src/net/i2p/update/UpdateType.java
@@ -25,5 +25,7 @@ public enum UpdateType {
     /** @since 0.9.9 */
     ROUTER_SIGNED_SU3,
     /** @since 0.9.15 */
-    NEWS_SU3
+    NEWS_SU3,
+    /** @since 0.9.17 */
+    ROUTER_DEV_SU3
 }