SU3 News:

- Fix SU3File support (tested)
 - Finish implementation (untested)
 - Output metadata as comments in news.xml
 - Support signed HTML
 - Copy router certs to news

apps/routerconsole/java/src/net/i2p/router/news/NewsXMLParser.java

@@ -15,6 +15,7 @@ import java.util.Set;
 
 import net.i2p.I2PAppContext;
 import net.i2p.util.Log;
+import org.cybergarage.util.Debug;
 import org.cybergarage.xml.Node;
 import org.cybergarage.xml.ParserException;
 import org.cybergarage.xml.parser.JaxpParser;
@@ -329,7 +330,9 @@ public class NewsXMLParser {
 
     public static void main(String[] args) {
         try {
-            NewsXMLParser parser = new NewsXMLParser(new I2PAppContext());
+            I2PAppContext ctx = new I2PAppContext();
+            Debug.initialize(ctx);
+            NewsXMLParser parser = new NewsXMLParser(ctx);
             parser.setXHTMLMode(XHTMLMode.ABORT);
             //parser.setXHTMLMode(XHTMLMode.REMOVE_ELEMENT);
             //parser.setXHTMLMode(XHTMLMode.SKIP_ENTRY);

apps/routerconsole/java/src/net/i2p/router/update/NewsFetcher.java

@@ -16,6 +16,7 @@ import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -345,11 +346,19 @@ class NewsFetcher extends UpdateRunner {
         
         long now = _context.clock().now();
         if (_tempFile.exists()) {
+            File from;
             if (url.endsWith(".su3")) {
-                processSU3();
+                try {
-                return;
+                    from = processSU3();
+                } catch (IOException ioe) {
+                    _log.error("Failed to extract the news file", ioe);
+                    _tempFile.delete();
+                    return;
+                }
+            } else {
+                from = _tempFile;
             }
-            boolean copied = FileUtil.copy(_tempFile, _newsFile, true, false);
+            boolean copied = FileUtil.copy(from, _newsFile, true, false);
             _tempFile.delete();
             if (copied) {
                 String newVer = Long.toString(now);
@@ -374,44 +383,47 @@ class NewsFetcher extends UpdateRunner {
     public void transferFailed(String url, long bytesTransferred, long bytesRemaining, int currentAttempt) {}
 
     /**
-     *  Process the fetched su3 news file _tempFile
+     *  Process the fetched su3 news file _tempFile.
+     *  Handles 3 types of contained files: xml.gz (preferred), xml, and html (old format fake xml)
      *
+     *  @return the temp file contining the HTML-format news.xml
      *  @since 0.9.17
      */
-    private void processSU3() {
+    private File processSU3() throws IOException {
         SU3File su3 = new SU3File(_context, _tempFile);
+        // real xml, maybe gz, maybe not
         File to1 = new File(_context.getTempDir(), "tmp-" + _context.random().nextInt() + ".xml");
+        // real xml
         File to2 = new File(_context.getTempDir(), "tmp2-" + _context.random().nextInt() + ".xml");
-        String sudVersion;
-        String signingKeyName;
         try {
             su3.verifyAndMigrate(to1);
             int type = su3.getFileType();
-            if (type != SU3File.TYPE_XML && type != SU3File.TYPE_XML_GZ)
-                throw new IOException("bad file type");
             if (su3.getContentType() != SU3File.CONTENT_NEWS)
-                throw new IOException("bad content type");
+                throw new IOException("bad content type: " + su3.getContentType());
+            if (type == SU3File.TYPE_HTML)
+                return to1;
+            if (type != SU3File.TYPE_XML && type != SU3File.TYPE_XML_GZ)
+                throw new IOException("bad file type: " + type);
             File xml;
             if (type == SU3File.TYPE_XML_GZ) {
                 gunzip(to1, to2);
                 xml = to2;
+                to1.delete();
             } else {
                 xml = to1;
             }
-            sudVersion = su3.getVersionString();
-            signingKeyName = su3.getSignerString();
             NewsXMLParser parser = new NewsXMLParser(_context);
             parser.parse(xml);
+            xml.delete();
             NewsMetadata data = parser.getMetadata();
             List<NewsEntry> entries = parser.getEntries();
-            outputOldNewsXML(data, entries);
+            String sudVersion = su3.getVersionString();
-        } catch (IOException ioe) {
+            String signingKeyName = su3.getSignerString();
-            // FIXME
+            File to3 = new File(_context.getTempDir(), "tmp3-" + _context.random().nextInt() + ".xml");
-            //statusDone("<b>" + ioe + ' ' + _("from {0}", _currentURI.toString()) + " </b>");
+            outputOldNewsXML(data, entries, sudVersion, signingKeyName, to3);
-            _tempFile.delete();
+            return to3;
-            to1.delete();
+        } finally {
             to2.delete();
-            return;
         }
     }
 
@@ -441,15 +453,22 @@ class NewsFetcher extends UpdateRunner {
 
     /**
      *  Output in the old format.
-     *  Yes there is a better way.
      *
      *  @since 0.9.17
      */
-    private void outputOldNewsXML(NewsMetadata data, List<NewsEntry> entries) throws IOException {
+    private void outputOldNewsXML(NewsMetadata data, List<NewsEntry> entries,
+                                  String sudVersion, String signingKeyName, File to) throws IOException {
         Writer out = null;
         try {
-            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(_newsFile), "UTF-8"));
+            out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(to), "UTF-8"));
             out.write("<!--\n");
+            // su3 and feed metadata
+            out.write("** News version:\t" + DataHelper.stripHTML(sudVersion) + '\n');
+            out.write("** Signed by:\t" + signingKeyName + '\n');
+            out.write("** Feed:\t" + DataHelper.stripHTML(data.feedTitle) + '\n');
+            out.write("** Feed ID:\t" + DataHelper.stripHTML(data.feedID) + '\n');
+            out.write("** Feed Date:\t" + (new Date(data.feedUpdated)) + "UTC\n");
+            // update metadata
             out.write("<i2p.news date=\"$Date: 2014-09-20 00:00:00 $\">\n");
             out.write("<i2p.release ");
             if (data.i2pVersion != null)
@@ -464,9 +483,12 @@ class NewsFetcher extends UpdateRunner {
                 out.write(" su3Torrent=\"" + data.su3Torrent + '"');
             out.write("/>\n");
             out.write("-->\n");
+            if (entries == null)
+                return;
             for (NewsEntry e : entries) {
                 if (e.title == null || e.content == null)
                     continue;
+                out.write("<!-- Entry Date: " + (new Date(e.updated)) + "UTC -->\n");
                 out.write("<h3>");
                 out.write(e.title);
                 out.write("</h3>\n");

core/java/src/net/i2p/crypto/SU3File.java

@@ -66,6 +66,10 @@ public class SU3File {
     private static final int MIN_VERSION_BYTES = 16;
     private static final int VERSION_OFFSET = 40; // Signature.SIGNATURE_BYTES; avoid early ctx init
 
+    /**
+     *  The file type is advisory and is application-dependent.
+     *  The following values are defined but any value 0-255 is allowed.
+     */
     public static final int TYPE_ZIP = 0;
     /** @since 0.9.15 */
     public static final int TYPE_XML = 1;
@@ -81,6 +85,11 @@ public class SU3File {
     /** @since 0.9.15 */
     public static final int CONTENT_NEWS = 4;
 
+    /**
+     *  The ContentType is the trust domain for the content.
+     *  The signer and signature will be checked with the
+     *  trusted certificates for that type.
+     */
     private enum ContentType {
         UNKNOWN(CONTENT_UNKNOWN, "unknown"),
         ROUTER(CONTENT_ROUTER, "router"),
@@ -187,6 +196,10 @@ public class SU3File {
     }
 
     /**
+     *  The ContentType is the trust domain for the content.
+     *  The signer and signature will be checked with the
+     *  trusted certificates for that type.
+     *
      *  This does not check the signature, but it will fail if the signer is unknown,
      *  unless setVerifySignature(false) has been called.
      *
@@ -199,10 +212,13 @@ public class SU3File {
     }
 
     /**
+     *  The file type is advisory and is application-dependent.
+     *  The following values are defined but any value 0-255 is allowed.
+     *
      *  This does not check the signature, but it will fail if the signer is unknown,
      *  unless setVerifySignature(false) has been called.
      *
-     *  @return -1 if unknown
+     *  @return 0-255 or -1 if unknown
      *  @since 0.9.15
      */
     public int getFileType() throws IOException {
@@ -265,8 +281,9 @@ public class SU3File {
             throw new IOException("bad content length");
         skip(in, 1);
         _fileType = in.read();
-        if (_fileType != TYPE_ZIP && _fileType != TYPE_XML)
+        // Allow any file type
-            throw new IOException("bad file type");
+        //if (_fileType != TYPE_ZIP && _fileType != TYPE_XML)
+        //    throw new IOException("bad file type");
         skip(in, 1);
         int cType = in.read();
         _contentType = BY_CODE.get(Integer.valueOf(cType));
@@ -625,6 +642,9 @@ public class SU3File {
         buf.append("Available file types (-f):\n");
         buf.append("      ZIP\t(code: 0) DEFAULT\n");
         buf.append("      XML\t(code: 1)\n");
+        buf.append("      HTML\t(code: 2)\n");
+        buf.append("      XML_GZ\t(code: 3)\n");
+        buf.append("      (user defined)\t(code: 4-255)\n");
         return buf.toString();
     }
 
@@ -743,16 +763,22 @@ public class SU3File {
                 ft = TYPE_ZIP;
             } else if (ftype.equalsIgnoreCase("XML")) {
                 ft = TYPE_XML;
+            } else if (ftype.equalsIgnoreCase("HTML")) {
+                ft = TYPE_HTML;
+            } else if (ftype.equalsIgnoreCase("XML_GZ")) {
+                ft = TYPE_XML_GZ;
             } else {
                 try {
                     ft = Integer.parseInt(ftype);
                 } catch (NumberFormatException nfe) {
                     ft = -1;
                 }
-                if (ft != TYPE_ZIP && ft != TYPE_XML) {
+                if (ft < 0 || ft > 255) {
                     System.out.println("File type " + ftype + " is not supported");
                     return false;
                 }
+                if (ft  > TYPE_XML_GZ)
+                    System.out.println("Warning: File type " + ftype + " is undefined");
             }
         }
         return signCLI(type, ct, ft, inputFile, signedFile, privateKeyFile, version, signerName, keypw);

installer/resources/certificates/news/killyourtv_at_mail.i2p.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFhTCCA22gAwIBAgIELuRWgDANBgkqhkiG9w0BAQ0FADBzMQswCQYDVQQGEwJY
+WDELMAkGA1UECBMCWFgxCzAJBgNVBAcTAlhYMR4wHAYDVQQKExVJMlAgQW5vbnlt
+b3VzIE5ldHdvcmsxDDAKBgNVBAsTA0kyUDEcMBoGA1UEAwwTa2lsbHlvdXJ0dkBt
+YWlsLmkycDAeFw0xMzEwMDYyMTM5MzFaFw0yMzEwMDYyMTM5MzFaMHMxCzAJBgNV
+BAYTAlhYMQswCQYDVQQIEwJYWDELMAkGA1UEBxMCWFgxHjAcBgNVBAoTFUkyUCBB
+bm9ueW1vdXMgTmV0d29yazEMMAoGA1UECxMDSTJQMRwwGgYDVQQDDBNraWxseW91
+cnR2QG1haWwuaTJwMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAig3u
+niLWm0y/TFJtciHgmWUt20FOdQrxkiSZ87G8xjuGfq7TbGIiVDn7pQZcHidpq+Dk
+47sm+Swqhb4psSijj0AXUEVKlV39jF5IZE+VUgmEtMqQbnBkWudaTJPWcEe9T/Kd
+8Oz2jgsnrD/EGVTMKBBjt/gk8VqTWvpCdCF1GhqcCeUTFHzjhN9jtoRCaJ2DClpO
+Px+86+d3s9PqUFo8gcD/dbbyJCMqUCMBLtIy/Ooouxb9cfWtXfyOlphU+enmdvuA
+0BDewb9pOJg2/kVd9/9moDWcBGChLOlfSlxpDwyUtcclcpvwnG7c6o4or6gqLeOf
+AbCpse623utV7fWlFWG7M4AQ/2emhhe4YoMJQnflydzV8bPRJxRTeW1j/9UfpvLT
+nO5LHp0oBXE0GqAPjxuAr+r5IDXFbkKYNjK5oWQB/Ul3LkexulYdCzHWbGd1Ja5b
+sbiOy6t/hH6G8DD75HYb+PQZaNZWBv90EyOq1JDSUPw6nxVbhiBldi3ipc8/1X51
+FbzBqJ+QO1XKrKqxWxBKoTekuy38KRzsmkSCpY+WJ9f0gLOKtxzVO2HNNqqVFGQf
+RGIbrNA0JSRQ1fgelccfrcRIXIZ3B8Tk/wxCIzCY6Yvg2jezz2xJkVdqOUsznS2v
++xJe67PYIAeMVtcfO4kmuCvyIYhsUEpob2n/5lkCAwEAAaMhMB8wHQYDVR0OBBYE
+FCLneov6QMtvra5FSoSLhdymi++rMA0GCSqGSIb3DQEBDQUAA4ICAQAIcqbiwjdQ
+M9VlGBiHe5eVsL6OM9zfRqR1wnRg4Q6ce65XDfEOYleBWaaNJA4BdykcA4fkUN1h
+M2D9FDQScsyPTOuzJ6o75TYh0JOtF51yCi9iuemcosxAwsm90ZXGuMDfDYeyND5c
+PAkWfyCP+jwLYbNo/hkNqyv+XWHXPQmT2adRnPXINVUQuBxVPC//C9wv2uDYWhgS
+f8M425VPp4/R/uks9mlzTx08DwacvouD0YOC+HZE4sWq+2smgeBInMiyr/THYzl+
+baMtYgVs8IKUD2gtjfXZoaQNg3eq5SedSf/5F0S/LCdu9/ccQ8CzSEoVTiQFtO78
+SaU37xai8+QTSVpPuINigxCoXmkubBd+voEmWRcBd/XB5L+u+MFU/jXyyBj2BXVj
+6agqVzY53KVYt23/63QliAUWyxT+ns9gRxVN1jrMhHdiDwsdT4NbzHxg1Su4eiHv
+C/wjD3Dga0BRTEGylpHZGzb1U1rZRHM3ho3f1QkmRPPLcBUMTyUTxJm+GEeuhPvp
++TBf3Kg/YkdpnEMlagqcyHuIrf3m8Z/pTmpOIbekJWbbA7tluvWbMWw2ARB7dUOE
+fHYVISh0DTw2oVXxM82/q8XXHnhEXv2nW3K40x1VabxUN+sF4M/7YA8nJqwsPJei
+749STYJRfZXdIe69M9zpM5unxENAsiPJgQ==
+-----END CERTIFICATE-----

installer/resources/certificates/news/str4d_at_mail.i2p.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFezCCA2OgAwIBAgIEHLJfZzANBgkqhkiG9w0BAQ0FADBuMQswCQYDVQQGEwJY
+WDELMAkGA1UECBMCWFgxCzAJBgNVBAcTAlhYMR4wHAYDVQQKExVJMlAgQW5vbnlt
+b3VzIE5ldHdvcmsxDDAKBgNVBAsTA0kyUDEXMBUGA1UEAwwOc3RyNGRAbWFpbC5p
+MnAwHhcNMTMxMDI2MTExODQxWhcNMjMxMDI2MTExODQxWjBuMQswCQYDVQQGEwJY
+WDELMAkGA1UECBMCWFgxCzAJBgNVBAcTAlhYMR4wHAYDVQQKExVJMlAgQW5vbnlt
+b3VzIE5ldHdvcmsxDDAKBgNVBAsTA0kyUDEXMBUGA1UEAwwOc3RyNGRAbWFpbC5p
+MnAwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCvw0vTay1IPOgxvwe8
+yt5jGakha20kw9qDb6zbEL87EWEkeOzdu7iUC69lkxVP9Ws8EbLtkeMf/CXg6CC1
+e+w8WpOHj5prOsiOlrIO+2I1tKMaMUuJDX2wK4I5ZSw/Kieimh9xqOBZknDmtwjw
+2HPW8rpxMqrScaGAP6sQD8Gh4XKKkLogfxYPzF8NnC6O8vBkFKVU2WSVZ0jPAQfv
+6luPdA+5lES+5UPWr9Yhv/CX4siGKUTxchqJRf2VU4o5BzzXae4asVA/NY7lKgEw
+eDDufbm0mRFWP4mbmXRlODuJ8GMnJbMQkNcAvZUnUcvpSTnGnIvxyxtXP5P6ic8V
+3b9HV2eIsbfO1xrgyr6/9qgGpXcdDJejhvNg6fZgQeO40bOGQYwV8bNvsNQHqnZl
+KsVhsMQkOubMxcHTBadcifi8PmdeJ5hxyyqJmyrwkmg2ijnN521M6YkoBzl+8VAi
+zLmqKZfvN5t+pb9PZ3U3jHfkeIEwDRYRAOsvVqch5+ZfSv8x/Te6o15zDKPJQtWK
+ty42GV1vERw30oSZQdrRRy/+4+HSRs3/Zb368OdAbcr+f/xPvwceYGWPeNNIoZ/x
+xkIQE3xgEK+eJyPM9McjlCAezZZclT7fWfiEYNJAiS3fGALi+a+cGYWWULxCXpz+
+y397OHhZBhnh7D9K8aPePB8tCwIDAQABoyEwHzAdBgNVHQ4EFgQUezvGHq3h1gbC
+Hs2LLVoll5fIUWMwDQYJKoZIhvcNAQENBQADggIBAF7SG1WBcE1r5eyTp/BLFZfG
+iPtvqu+B1L2HutPum/Xf8A5fxR4kcKAKpVdu6vnDzCRAsAC9YvyETgAzI2nfVgLk
+l9YZ31tSi6qxnMsQsV5o9lt/q2Rvsf2Zi/Ir8AlWtvnP8YG0Aj/8AG8MyhMLaIdj
+M2FuakPs8RqEjoJL9dTOC9VTQpNTwBH9guP9UalWYwlkaXDzMoyO4nswT/GpCpg8
+4m4RO6grzdsEIamD/PCBM5f/vq+y08GaqfXpX9+8CbaX3tdzd3x48wPphmdpkptk
+aRELIpLJZiK+Mos7W+0ZS8SHxGDIosjqVsgbZPmk12+VBcVgLOr8W1D7osS4OY59
+2GMUVV/GhoDh8wR/Td5wpZlcPE0NWmljjVg9+1E8ePAyMZy+U1KCiMlRVdRy518O
+dOzzUUQGqGQHosRrH0ypS3MGbMLmbuWFRiz7q/3mUmW2xikH9I1t/6ZMNUvh+IWL
+kGAaEf2JIv/D8+QsC0Un1W09DgvYz7qmKSeHhBixlLe68vgXtz/Fa+rRMsmPrueo
+4wk/u/VyILo0BJP860APJMZbm+DPfGhV9DF9L5Gx9+d/BlduBVGHc+AQSWbU70dS
+eH4/rgUYRikWlgwUxjY8/QQTlfx5xl28tG0xdO9libN22z7UwTGfm48BQIdrTyER
+hqQ7usTy3oaWD85MbJ0q
+-----END CERTIFICATE-----

installer/resources/certificates/news/zzz_at_mail.i2p.crt

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdzCCA1+gAwIBAgIEcwrwsjANBgkqhkiG9w0BAQ0FADBsMQswCQYDVQQGEwJY
+WDELMAkGA1UECBMCWFgxCzAJBgNVBAcTAlhYMR4wHAYDVQQKExVJMlAgQW5vbnlt
+b3VzIE5ldHdvcmsxDDAKBgNVBAsTA0kyUDEVMBMGA1UEAwwMenp6QG1haWwuaTJw
+MB4XDTEzMDkzMDE3NDEyNVoXDTIzMDkzMDE3NDEyNVowbDELMAkGA1UEBhMCWFgx
+CzAJBgNVBAgTAlhYMQswCQYDVQQHEwJYWDEeMBwGA1UEChMVSTJQIEFub255bW91
+cyBOZXR3b3JrMQwwCgYDVQQLEwNJMlAxFTATBgNVBAMMDHp6ekBtYWlsLmkycDCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJV4ptvhlfrcize9/ARz4lUy
+iLtLMSvSST1BdZjLJTwus05EUs0oiqnv9qXYIWGRB97aKlAmqSxsn4ZgBttCgmev
+IkuiZ8kbdqI5YaT98yKW5P2Prt9p9cPbnz5/qjwZ5L9W+k/Itx7bv2pkNEP0NLYo
+NrgHHTb1hsyRxc0lfPYk2BwsIi8hIWBHNrRpR41EWFXfqPcdsxS8cQhxVj4zLG/R
+aMm4H8T+V1R1Khl4R4qqRgXBP305xqqRoawHmZ/S9/RkF0Ji6IYwBq9iWthWol6W
+sMDn1xhZk9765fk+ohAC2XWuGSFCr02JOILRV3x/8OUxT1GYgYjc7FfyWIekg/pZ
+yotlhL2I3SMWOH3PdG58iDY121hq/LsSKM9aP20rwtvssnw+8Aex01YDkI3bM6yO
+HNi+tRojaJcJciBWv6cuiFKvQdxj/mOhOr0u0lHLlJ4jqES8uvVJkS7X/C4BB7ra
+bJYQgumZMYvVQJFIjo8vZxMXue53o65FRidvAUT29ay54UTiL7jRV9w1wHnzLapU
+xT1v7kWpWJcZ1zzC8coJjW+6ijkk38cVLb80u1Q4kEbmP2rDxw6jRvmqg6DcCKjK
+oqDt+XQ6P5grxAxLT+VMfB404WHHwNs6BB841//4ZnXvy3msMONY/5y0fsblURgh
+IS2UG1TAjR+x7+XikGx9AgMBAAGjITAfMB0GA1UdDgQWBBSvx/fCCP8UeHwjN65p
+EoHjgRfiIzANBgkqhkiG9w0BAQ0FAAOCAgEAYgVE1Aa/Ok5k+Jvujbx72bktRWXo
+Y4UfbWH/426VdgqXt3n9XtJUNM2oI4ODwITM4O15SyXQTLJhnvJz5ELcJV8nqviZ
+RjK2HNX1BW7IEta3tacCvVnjzZ265kCT59uW+qmd+5PiaAYI5lYUn8P6pe+6neSa
+HW6ecXCrdxJetSYfUUuKeV6YHpdzfjtZClLmwl91sJUBKcjK+Q9G/cE6HnwcDH1s
+uXr7SgkBt/qc/OlNuu4fnTqUA58TAumdq9cD+eLBilDFrux1HsUZMuBUp64x5oPi
+gme+3VewsczfFEtrxaG6+l6UA40Lerdx9XECZcDCcFsK6MS1uQ2HYjsyZcWnNT3l
+6eDNUbjrllwxDdRAk0cbWiMuc21CFq/1v2QMXk88EiBjEajqzyXUPmKzwFhit6pr
+5kfjfXNq+pxQSCoaqjpzVKjb3CqMhSlC8cLgrPw6HEgGnjCy4cTLFHlVmD64M778
+tj6rE7CntcmUi8GKmZKyaMyUo3QQUcrjO5IQ4+3iGUgMkZuujyjrZiOJbvircPmK
+4IQEXzJ/G00upqtqKstRybaWSbJ/k6iuturtA2n8MJiCBjhLy8dtTgDbFaDaNF7F
+NHeqQjIJDLhYDy6mi4gya3A0ort777Inl/rWYLo067pYM+EWDw66GdpbEIB0Bp71
+pwvcQcjIzbUzEK0=
+-----END CERTIFICATE-----