From 79f934fe1748699b2e8cd3f34a162819037c3783 Mon Sep 17 00:00:00 2001 From: jrandom Date: Thu, 16 Feb 2006 08:24:07 +0000 Subject: [PATCH] 2006-02-16 jrandom * Bugfix to the I2PTunnel web config to properly accept i2cp port settings * Initial sucker refactoring to simplify reuse of the html parsing * Beginnings of hooks to push imported rss/atom out to remote syndie archives automatically (though not enabled currently) * Further SSU peer test cleanup --- .../src/net/i2p/i2ptunnel/web/IndexBean.java | 8 +- apps/i2ptunnel/jsp/editClient.jsp | 2 +- apps/i2ptunnel/jsp/editServer.jsp | 2 +- .../java/src/net/i2p/syndie/Sucker.java | 349 +++++++++--------- .../net/i2p/syndie/web/RemoteArchiveBean.java | 6 + apps/syndie/jsp/smlref.jsp | 5 +- core/java/src/net/i2p/util/EepGet.java | 14 +- history.txt | 9 +- .../src/net/i2p/router/RouterVersion.java | 4 +- .../transport/udp/IntroductionManager.java | 2 + .../router/transport/udp/PeerTestManager.java | 9 +- .../router/transport/udp/UDPTransport.java | 4 +- .../i2p/router/tunnel/TunnelParticipant.java | 4 +- .../i2p/router/tunnel/pool/BuildExecutor.java | 8 +- .../i2p/router/tunnel/pool/BuildHandler.java | 12 +- 15 files changed, 245 insertions(+), 193 deletions(-) diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java index 8b8d49b09..7b3e65e3f 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/web/IndexBean.java @@ -448,9 +448,10 @@ public class IndexBean { /** I2CP host the router is on */ public void setClientHost(String host) { _i2cpHost = (host != null ? host.trim() : null); + System.out.println("set client host [" + host + "]"); } /** I2CP port the router is on */ - public void setClientPort(String port) { + public void setClientport(String port) { _i2cpPort = (port != null ? port.trim() : null); } /** how many hops to use for inbound tunnels */ @@ -636,10 +637,11 @@ public class IndexBean { config.setProperty("description", _description); if (_i2cpHost != null) config.setProperty("i2cpHost", _i2cpHost); - if ( (_i2cpPort != null) && (_i2cpPort.trim().length() > 0) ) + if ( (_i2cpPort != null) && (_i2cpPort.trim().length() > 0) ) { config.setProperty("i2cpPort", _i2cpPort); - else + } else { config.setProperty("i2cpPort", "7654"); + } if (_customOptions != null) { StringTokenizer tok = new StringTokenizer(_customOptions); diff --git a/apps/i2ptunnel/jsp/editClient.jsp b/apps/i2ptunnel/jsp/editClient.jsp index c9378d630..359a7fea0 100644 --- a/apps/i2ptunnel/jsp/editClient.jsp +++ b/apps/i2ptunnel/jsp/editClient.jsp @@ -254,7 +254,7 @@ - +
diff --git a/apps/i2ptunnel/jsp/editServer.jsp b/apps/i2ptunnel/jsp/editServer.jsp index 943357f61..f93731b21 100644 --- a/apps/i2ptunnel/jsp/editServer.jsp +++ b/apps/i2ptunnel/jsp/editServer.jsp @@ -226,7 +226,7 @@ - +
diff --git a/apps/syndie/java/src/net/i2p/syndie/Sucker.java b/apps/syndie/java/src/net/i2p/syndie/Sucker.java index 8478321c5..59a7c16e8 100644 --- a/apps/syndie/java/src/net/i2p/syndie/Sucker.java +++ b/apps/syndie/java/src/net/i2p/syndie/Sucker.java @@ -33,97 +33,78 @@ import net.i2p.syndie.data.BlogURI; import net.i2p.util.EepGet; import net.i2p.util.Log; +/** + * + * todo: + * - factor out the parsing / formatting / posting to let the sucker pull in arbitrary HTML pages + * (importing the images and SMLizing some stuff) + * - push the posts out to a remote syndie instance too + */ public class Sucker { private static final Log _log = I2PAppContext.getGlobalContext().logManager().getLog(Sucker.class); - private String urlToLoad; - private String outputDir="./sucker_out"; - private String historyPath="./sucker.history"; - private String feedTag="feed"; - private File historyFile; - private String proxyPort; - private String proxyHost; - private String pushScript; - private int attachmentCounter=0; - private String messagePath; - private String baseUrl; - private boolean importEnclosures=true; - private boolean importRefs=true; - private boolean pendingEndLink; - private boolean shouldProxy; - private int proxyPortNum; - private String blog; - private boolean pushToSyndie; - private long messageNumber=0; - private BlogManager bm; - private User user; - - // - private List fileNames; - private List fileStreams; - private List fileTypes; - private List tempFiles; // deleted after finished push - private boolean stripNewlines; - - public Sucker() { - } + private SuckerState _state; + + public Sucker() {} /** * Constructor for BlogManager. */ public Sucker(String[] strings) throws IllegalArgumentException { - pushToSyndie=true; - urlToLoad = strings[0]; - blog = strings[1]; - feedTag = strings[2]; - outputDir = "blog-"+blog; + SuckerState state = new SuckerState(); + state.pushToSyndie=true; + state.urlToLoad = strings[0]; + state.blog = strings[1]; + state.feedTag = strings[2]; + state.outputDir = "blog-"+state.blog; try { - historyPath=BlogManager.instance().getRootDir().getCanonicalPath()+"/rss.history"; + state.historyPath=BlogManager.instance().getRootDir().getCanonicalPath()+"/rss.history"; } catch (IOException e) { e.printStackTrace(); } - proxyPort = BlogManager.instance().getDefaultProxyPort(); - proxyHost = BlogManager.instance().getDefaultProxyHost(); + state.proxyPort = BlogManager.instance().getDefaultProxyPort(); + state.proxyHost = BlogManager.instance().getDefaultProxyHost(); - bm = BlogManager.instance(); + state.bm = BlogManager.instance(); Hash blogHash = new Hash(); try { - blogHash.fromBase64(blog); + blogHash.fromBase64(state.blog); } catch (DataFormatException e1) { throw new IllegalArgumentException("ooh, bad $blog"); } - user = bm.getUser(blogHash); - if(user==null) + state.user = state.bm.getUser(blogHash); + if(state.user==null) throw new IllegalArgumentException("wtf, user==null? hash:"+blogHash); + _state = state; } public boolean parseArgs(String args[]) { for (int i = 0; i < args.length; i++) { if ("--load".equals(args[i])) - urlToLoad = args[++i]; + _state.urlToLoad = args[++i]; if ("--outputdir".equals(args[i])) - outputDir = args[++i]; + _state.outputDir = args[++i]; if ("--history".equals(args[i])) - historyPath = args[++i]; + _state.historyPath = args[++i]; if ("--tag".equals(args[i])) - feedTag = args[++i]; + _state.feedTag = args[++i]; if ("--proxyhost".equals(args[i])) - proxyHost = args[++i]; + _state.proxyHost = args[++i]; if ("--proxyport".equals(args[i])) - proxyPort = args[++i]; + _state.proxyPort = args[++i]; if ("--exec".equals(args[i])) - pushScript = args[++i]; + _state.pushScript = args[++i]; if ("--importenclosures".equals(args[i])) - importEnclosures= args[++i].equals("true"); + _state.importEnclosures= args[++i].equals("true"); if ("--importenrefs".equals(args[i])) - importRefs= args[++i].equals("true"); + _state.importRefs= args[++i].equals("true"); } // Cut ending '/' from outputDir - if (outputDir.endsWith("/")) - outputDir = outputDir.substring(0, outputDir.length() - 1); + if (_state.outputDir.endsWith("/")) + _state.outputDir = _state.outputDir.substring(0, _state.outputDir.length() - 1); - if (urlToLoad == null) + if (_state.urlToLoad == null) return false; return true; @@ -131,31 +112,33 @@ public class Sucker { /** * Fetch urlToLoad and call convertToHtml() on any new entries. + * @return list of BlogURI entries posted, if any */ - public void suck() { + public List suck() { + _state.entriesPosted = new ArrayList(); SyndFeed feed; File fetched=null; - tempFiles = new ArrayList(); + _state.tempFiles = new ArrayList(); // Find base url - int idx=urlToLoad.lastIndexOf('/'); + int idx=_state.urlToLoad.lastIndexOf('/'); if(idx>0) - baseUrl=urlToLoad.substring(0,idx); + _state.baseUrl=_state.urlToLoad.substring(0,idx); else - baseUrl=urlToLoad; + _state.baseUrl=_state.urlToLoad; - infoLog("Processing: "+urlToLoad); - debugLog("Base url: "+baseUrl); + infoLog("Processing: "+_state.urlToLoad); + debugLog("Base url: "+_state.baseUrl); // try { File lastIdFile=null; // Get next message number to use (for messageId in history only) - if(!pushToSyndie) { + if(!_state.pushToSyndie) { - lastIdFile = new File(historyPath + ".lastId"); + lastIdFile = new File(_state.historyPath + ".lastId"); if (!lastIdFile.exists()) lastIdFile.createNewFile(); @@ -163,34 +146,34 @@ public class Sucker { try { fis = new FileInputStream(lastIdFile); String number = readLine(fis); - messageNumber = Integer.parseInt(number); + _state.messageNumber = Integer.parseInt(number); } catch (NumberFormatException e) { - messageNumber = 0; + _state.messageNumber = 0; } finally { if (fis != null) try { fis.close(); } catch (IOException ioe) {} } // Create outputDir if missing - File f = new File(outputDir); + File f = new File(_state.outputDir); f.mkdirs(); } else { - messageNumber=bm.getNextBlogEntry(user); + _state.messageNumber=_state.bm.getNextBlogEntry(_state.user); } - _log.debug("message number: " + messageNumber); + _log.debug("message number: " + _state.messageNumber); // Create historyFile if missing - historyFile = new File(historyPath); - if (!historyFile.exists()) - historyFile.createNewFile(); + _state.historyFile = new File(_state.historyPath); + if (!_state.historyFile.exists()) + _state.historyFile.createNewFile(); - shouldProxy = false; - proxyPortNum = -1; - if ( (proxyHost != null) && (proxyPort != null) ) { + _state.shouldProxy = false; + _state.proxyPortNum = -1; + if ( (_state.proxyHost != null) && (_state.proxyPort != null) ) { try { - proxyPortNum = Integer.parseInt(proxyPort); - if (proxyPortNum > 0) - shouldProxy = true; + _state.proxyPortNum = Integer.parseInt(_state.proxyPort); + if (_state.proxyPortNum > 0) + _state.shouldProxy = true; } catch (NumberFormatException nfe) { nfe.printStackTrace(); } @@ -199,12 +182,12 @@ public class Sucker { // fetch int numRetries = 2; fetched = File.createTempFile("sucker", ".fetch"); - EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum, - numRetries, fetched.getAbsolutePath(), urlToLoad); + EepGet get = new EepGet(I2PAppContext.getGlobalContext(), _state.shouldProxy, _state.proxyHost, _state.proxyPortNum, + numRetries, fetched.getAbsolutePath(), _state.urlToLoad); SuckerFetchListener lsnr = new SuckerFetchListener(); get.addStatusListener(lsnr); - _log.debug("fetching [" + urlToLoad + "] / " + shouldProxy + "/" + proxyHost + "/" + proxyHost); + _log.debug("fetching [" + _state.urlToLoad + "] / " + _state.shouldProxy + "/" + _state.proxyHost + "/" + _state.proxyHost); get.fetch(); _log.debug("fetched: " + get.getNotModified() + "/" + get.getETag()); @@ -213,13 +196,13 @@ public class Sucker { _log.debug("success? " + ok); System.err.println("Unable to retrieve the url after " + numRetries + " tries."); fetched.delete(); - return; + return _state.entriesPosted; } _log.debug("fetched successfully? " + ok); if(get.getNotModified()) { debugLog("not modified, saving network bytes from useless fetch"); fetched.delete(); - return; + return _state.entriesPosted; } // Build entry list from fetched rss file @@ -233,19 +216,19 @@ public class Sucker { FileOutputStream hos = null; try { - hos = new FileOutputStream(historyFile, true); + hos = new FileOutputStream(_state.historyFile, true); // Process list backwards to get syndie to display the // entries in the right order. (most recent at top) for (int i = entries.size()-1; i >= 0; i--) { SyndEntry e = (SyndEntry) entries.get(i); - attachmentCounter=0; + _state.attachmentCounter=0; if (_log.shouldLog(Log.DEBUG)) _log.debug("Syndicate entry: " + e.getLink()); - String messageId = convertToSml(e); + String messageId = convertToSml(_state, e); if (messageId!=null) { hos.write(messageId.getBytes()); hos.write("\n".getBytes()); @@ -255,11 +238,11 @@ public class Sucker { if (hos != null) try { hos.close(); } catch (IOException ioe) {} } - if(!pushToSyndie) { + if(!_state.pushToSyndie) { FileOutputStream fos = null; try { fos = new FileOutputStream(lastIdFile); - fos.write(("" + messageNumber).getBytes()); + fos.write(("" + _state.messageNumber).getBytes()); } finally { if (fos != null) try { fos.close(); } catch (IOException ioe) {} } @@ -278,6 +261,7 @@ public class Sucker { if(fetched!=null) fetched.delete(); debugLog("Done."); + return _state.entriesPosted; } public static void main(String[] args) { @@ -302,11 +286,11 @@ public class Sucker { /** * Call the specified script with "$outputDir $id and $time". */ - private boolean execPushScript(String id, String time) { + private static boolean execPushScript(SuckerState state, String id, String time) { try { String ls_str; - String cli = pushScript + " " + outputDir + " " + id + " " + time; + String cli = state.pushScript + " " + state.outputDir + " " + id + " " + time; Process pushScript_proc = Runtime.getRuntime().exec(cli); // get its output (your input) stream @@ -318,7 +302,7 @@ public class Sucker { while (true) { boolean eof = DataHelper.readLine(ls_in, buf); if (buf.length() > 0) - infoLog(pushScript + ": " + buf.toString()); + infoLog(state.pushScript + ": " + buf.toString()); buf.setLength(0); if (eof) break; @@ -343,13 +327,13 @@ public class Sucker { /** * Converts the SyndEntry e to sml and fetches any images as attachments */ - private String convertToSml(SyndEntry e) { + private static String convertToSml(SuckerState state, SyndEntry e) { String subject; - stripNewlines=false; + state.stripNewlines=false; // Calculate messageId, and check if we have got the message already - String feedHash = sha1(urlToLoad); + String feedHash = sha1(state.urlToLoad); String itemHash = sha1(e.getTitle() + e.getDescription()); Date d = e.getPublishedDate(); String time; @@ -357,10 +341,10 @@ public class Sucker { time = "" + d.getTime(); else time = "" + new Date().getTime(); - String outputFileName = outputDir + "/" + messageNumber; + String outputFileName = state.outputDir + "/" + state.messageNumber; String messageId = feedHash + ":" + itemHash + ":" + time + ":" + outputFileName; // Check if we already have this - if (existsInHistory(messageId)) + if (existsInHistory(state, messageId)) return null; infoLog("new: " + messageId); @@ -371,14 +355,14 @@ public class Sucker { subject=e.getTitle(); List cats = e.getCategories(); Iterator iter = cats.iterator(); - String tags = feedTag; + String tags = state.feedTag; while (iter.hasNext()) { SyndCategory c = (SyndCategory) iter.next(); debugLog("Name: "+c.getName()); debugLog("uri:"+c.getTaxonomyUri()); String tag=c.getName(); tag=tag.replaceAll("[^a-zA-z.-_:]","_"); - tags += "\t" + feedTag + "." + tag; + tags += "\t" + state.feedTag + "." + tag; } SyndContent content; @@ -393,7 +377,7 @@ public class Sucker { content = (SyndContent)iter.next(); String c = content.getValue(); debugLog("Content: "+c); - sml += htmlToSml(c); + sml += htmlToSml(state, c); sml += "\n"; } } @@ -408,85 +392,86 @@ public class Sucker { // e.g. postman's rss feed @ http://tracker.postman.i2p/rss.jsp has // baseUrl = http://tracker.postman.i2p // and enclosure URLs are /download.php?id=123&file=blah - if (enclosureURL.startsWith("/") || baseUrl.endsWith("/")) - enclosureURL = baseUrl + enclosureURL; + if (enclosureURL.startsWith("/") || state.baseUrl.endsWith("/")) + enclosureURL = state.baseUrl + enclosureURL; else - enclosureURL = baseUrl + '/' + enclosureURL; + enclosureURL = state.baseUrl + '/' + enclosureURL; } - fetchAttachment(enclosureURL, enc.getType()); // fetches and adds to our streams + fetchAttachment(state, enclosureURL, enc.getType()); // fetches and adds to our streams } } String source=e.getLink(); //Uri(); if(source.indexOf("http")<0) - source=baseUrl+source; + source=state.baseUrl+source; sml += "[link schema=\"web\" location=\""+source+"\"]source[/link]\n"; - if(pushToSyndie) { - debugLog("user.blog: "+user.getBlogStr()); - debugLog("user.id: "+bm.getNextBlogEntry(user)); + if(state.pushToSyndie) { + debugLog("user.blog: "+state.user.getBlogStr()); + debugLog("user.id: "+state.bm.getNextBlogEntry(state.user)); debugLog("subject: "+subject); debugLog("tags: "+tags); debugLog("sml: "+sml); debugLog(""); - BlogURI uri = bm.createBlogEntry( - user, + BlogURI uri = state.bm.createBlogEntry( + state.user, false, subject, tags, null, sml, - fileNames, - fileStreams, - fileTypes); + state.fileNames, + state.fileStreams, + state.fileTypes); if(uri==null) { errorLog("pushToSyndie failure."); return null; - } - else + } else { + state.entriesPosted.add(uri); infoLog("pushToSyndie success, uri: "+uri.toString()); + } } else { FileOutputStream fos; - fos = new FileOutputStream(messagePath); + fos = new FileOutputStream(state.messagePath); sml=subject + "\nTags: " + tags + "\n\n" + sml; fos.write(sml.getBytes()); - if (pushScript != null) { - if (!execPushScript(""+messageNumber, time)) { + if (state.pushScript != null) { + if (!execPushScript(state, ""+state.messageNumber, time)) { errorLog("push script failed"); } else { - infoLog("push script success: nr "+messageNumber); + infoLog("push script success: nr "+state.messageNumber); } } } - messageNumber++; - deleteTempFiles(); + state.messageNumber++; + deleteTempFiles(state); return messageId; } catch (FileNotFoundException e1) { e1.printStackTrace(); } catch (IOException e2) { e2.printStackTrace(); } - deleteTempFiles(); + deleteTempFiles(state); return null; } - private void deleteTempFiles() { - Iterator iter = tempFiles.iterator(); + private static void deleteTempFiles(SuckerState state) { + Iterator iter = state.tempFiles.iterator(); while(iter.hasNext()) { File tempFile = (File)iter.next(); tempFile.delete(); } } - private String htmlToSml(String html) { + private static String htmlToSml(SuckerState state, String html) { String sml=""; int i=0; - pendingEndLink=false; + state.pendingEndLink=false; while(i -> [link][/link][img][/img] + if(state.pendingEndLink) { // -> [link][/link][img][/img] ret="[/link]"; - pendingEndLink=false; + state.pendingEndLink=false; } - ret += "[img attachment=\""+""+ attachmentCounter +"\"]"; + ret += "[img attachment=\""+""+ state.attachmentCounter +"\"]"; a=htmlTagLowerCase.indexOf("alt=\"")+5; if(a>=5) @@ -586,9 +571,9 @@ public class Sucker { ret+="[/img]"; if(imageLink.indexOf("http")<0) - imageLink=baseUrl+"/"+imageLink; + imageLink=state.baseUrl+"/"+imageLink; - fetchAttachment(imageLink); + fetchAttachment(state, imageLink); debugLog("Converted to: "+ret); @@ -608,7 +593,7 @@ public class Sucker { return null; // abort the b0rked tag String link=htmlTag.substring(a,b); if(link.indexOf("http")<0) - link=baseUrl+"/"+link; + link=state.baseUrl+"/"+link; String schema="web"; @@ -616,7 +601,7 @@ public class Sucker { if(htmlTagLowerCase.endsWith("/>")) ret += "[/link]"; else - pendingEndLink=true; + state.pendingEndLink=true; debugLog("Converted to: "+ret); @@ -624,8 +609,8 @@ public class Sucker { } if ("".equals(htmlTagLowerCase)) { - if (pendingEndLink) { - pendingEndLink=false; + if (state.pendingEndLink) { + state.pendingEndLink=false; return "[/link]"; } } @@ -647,7 +632,7 @@ public class Sucker { if("".equals(htmlTagLowerCase)) return "[/b]"; if(htmlTagLowerCase.startsWith("".equals(htmlTagLowerCase)) @@ -675,15 +660,15 @@ public class Sucker { return null; } - private void fetchAttachment(String link) { fetchAttachment(link, null); } - private void fetchAttachment(String link, String suggestedMimeType) { + private static void fetchAttachment(SuckerState state, String link) { fetchAttachment(state, link, null); } + private static void fetchAttachment(SuckerState state, String link, String suggestedMimeType) { link=link.replaceAll("&","&"); infoLog("Fetch attachment from: "+link); File fetched; - if(pushToSyndie) { + if(state.pushToSyndie) { try { // perhaps specify a temp dir? fetched = File.createTempFile("sucker",".attachment"); @@ -692,14 +677,14 @@ public class Sucker { e.printStackTrace(); return; } - tempFiles.add(fetched); + state.tempFiles.add(fetched); } else { - String attachmentPath = messagePath+"."+attachmentCounter; + String attachmentPath = state.messagePath+"."+state.attachmentCounter; fetched = new File(attachmentPath); } int numRetries = 2; // we use eepGet, since it retries and doesn't leak DNS requests like URL does - EepGet get = new EepGet(I2PAppContext.getGlobalContext(), shouldProxy, proxyHost, proxyPortNum, + EepGet get = new EepGet(I2PAppContext.getGlobalContext(), state.shouldProxy, state.proxyHost, state.proxyPortNum, numRetries, fetched.getAbsolutePath(), link); SuckerFetchListener lsnr = new SuckerFetchListener(); get.addStatusListener(lsnr); @@ -710,7 +695,7 @@ public class Sucker { fetched.delete(); return; } - tempFiles.add(fetched); + state.tempFiles.add(fetched); String filename=EepGet.suggestName(link); String contentType = suggestedMimeType; if (contentType == null) @@ -719,38 +704,38 @@ public class Sucker { contentType="text/plain"; debugLog("successful fetch of filename "+filename + " suggested mime type [" + suggestedMimeType + "], fetched mime type [" + get.getContentType() + "], final type [" + contentType + "]"); - if(fileNames==null) fileNames = new ArrayList(); - if(fileTypes==null) fileTypes = new ArrayList(); - if(fileStreams==null) fileStreams = new ArrayList(); - fileNames.add(filename); - fileTypes.add(contentType); + if(state.fileNames==null) state.fileNames = new ArrayList(); + if(state.fileTypes==null) state.fileTypes = new ArrayList(); + if(state.fileStreams==null) state.fileStreams = new ArrayList(); + state.fileNames.add(filename); + state.fileTypes.add(contentType); try { - fileStreams.add(new FileInputStream(fetched)); + state.fileStreams.add(new FileInputStream(fetched)); } catch (FileNotFoundException e) { e.printStackTrace(); } - attachmentCounter++; + state.attachmentCounter++; } - private void errorLog(String string) { + private static void errorLog(String string) { if (_log.shouldLog(Log.ERROR)) _log.error(string); - if(!pushToSyndie) - System.out.println(string); + //if(!pushToSyndie) + // System.out.println(string); } - private void infoLog(String string) { + private static void infoLog(String string) { if (_log.shouldLog(Log.INFO)) _log.info(string); - if(!pushToSyndie) - System.out.println(string); + //if(!pushToSyndie) + // System.out.println(string); } - private void debugLog(String string) { + private static void debugLog(String string) { if (_log.shouldLog(Log.DEBUG)) _log.debug(string); - if(!pushToSyndie) - System.out.println(string); + //if(!pushToSyndie) + // System.out.println(string); } private static int findTagLen(String s) { @@ -769,7 +754,7 @@ public class Sucker { return -1; } - private boolean existsInHistory(String messageId) { + private static boolean existsInHistory(SuckerState state, String messageId) { int idx; idx = messageId.lastIndexOf(":"); String lineToCompare = messageId.substring(0, idx-1); @@ -777,7 +762,7 @@ public class Sucker { lineToCompare = lineToCompare.substring(0, idx-1); FileInputStream his = null; try { - his = new FileInputStream(historyFile); + his = new FileInputStream(state.historyFile); String line; while ((line = readLine(his)) != null) { idx = line.lastIndexOf(":"); @@ -885,5 +870,39 @@ class SuckerFetchListener implements EepGet.StatusListener { public void headerReceived(String url, int currentAttempt, String key, String val) { // ignore } - } + + + +class SuckerState { + String urlToLoad; + String outputDir="./sucker_out"; + String historyPath="./sucker.history"; + String feedTag="feed"; + File historyFile; + String proxyPort; + String proxyHost; + String pushScript; + int attachmentCounter=0; + String messagePath; + String baseUrl; + boolean importEnclosures=true; + boolean importRefs=true; + boolean pendingEndLink; + boolean shouldProxy; + int proxyPortNum; + String blog; + boolean pushToSyndie; + long messageNumber=0; + BlogManager bm; + User user; + List entriesPosted; + + // + List fileNames; + List fileStreams; + List fileTypes; + List tempFiles; // deleted after finished push + + boolean stripNewlines; +} \ No newline at end of file diff --git a/apps/syndie/java/src/net/i2p/syndie/web/RemoteArchiveBean.java b/apps/syndie/java/src/net/i2p/syndie/web/RemoteArchiveBean.java index 1fc7b77b6..92d2a1578 100644 --- a/apps/syndie/java/src/net/i2p/syndie/web/RemoteArchiveBean.java +++ b/apps/syndie/java/src/net/i2p/syndie/web/RemoteArchiveBean.java @@ -583,6 +583,12 @@ public class RemoteArchiveBean { List uris = new ArrayList(entries.length); for (int i = 0; i < entries.length; i++) uris.add(new BlogURI(entries[i])); + postSelectedEntries(user, uris, proxyHost, proxyPort, location); + } + public void postSelectedEntries(User user, List uris, String location) { + postSelectedEntries(user, uris, _proxyHost, _proxyPort, location); + } + public void postSelectedEntries(User user, List uris, String proxyHost, int proxyPort, String location) { if ( (proxyPort > 0) && (proxyHost != null) && (proxyHost.trim().length() > 0) ) { _proxyPort = proxyPort; _proxyHost = proxyHost; diff --git a/apps/syndie/jsp/smlref.jsp b/apps/syndie/jsp/smlref.jsp index b67cfb8a2..ed2bd52ee 100644 --- a/apps/syndie/jsp/smlref.jsp +++ b/apps/syndie/jsp/smlref.jsp @@ -11,12 +11,14 @@ request.setCharacterEncoding("UTF-8");
  • newlines are newlines are newlines.
  • all < and > are replaced with their &symbol;
    • +
  • the [ and ] characters delimit tags, or must be quoted by doubling them up ([[ displays as [, ]] displays as ])
  • [b][/b] = <b>bold</b>
  • [i][/i] = <i>italics</i>
  • [u][/u] = <i>underline</i>
    • +
  • [pre]foo[/pre] = <pre>preformatted section</pre>
  • [cut]more inside[/cut] = <a href="#">more inside...</a>
  • [quote][/quote] = Quoted text
    • -
  • [img attachment="1"]alt[/img] = use attachment 1 as an image with 'alt' as the alt text
    • +
  • [img attachment="1"]alt[/img] = use attachment 1 as an image with 'alt' as the alt text.
  • [attachment id="0"]text[/attachment] = offer attachment 0 as a link in your post
  • [attachment thumbnail="0" id="1"]text[/attachment] = offer attachment 1 as a link around a thumbnail image using attachment 0
  • [link schema="eep" location="http://forum.i2p"]text[/link] = offer a link to an external resource (accessible with the given schema)
    • @@ -25,6 +27,7 @@ request.setCharacterEncoding("UTF-8");
  • [blog name="name" bloghash="base64hash" blogtag="tag"]description[/blog] = link to all posts in the blog with the specified tag
  • [blog name="name" blogtag="tag"]description[/blog] = link to all posts in all blogs with the specified tag
  • [archive name="name" description="they have good stuff" schema="eep" location="http://syndiemedia.i2p/archive/archive.txt"]foo![/archive] = offer an easy way to sync up with a new Syndie archive
    • +
  • [address name="www.i2p" location="Nf3ab-ZFkmI-LyMt7Gjg...vobM57UpqSAAAA" schema="i2p" proto="eep"]official website[/address] = share a pet name reference to the given eepsite (using fields from the addresses page)
SML headers are newline delimited key:value pairs. Example keys are:
    diff --git a/core/java/src/net/i2p/util/EepGet.java b/core/java/src/net/i2p/util/EepGet.java index ff30e0d4a..d73af4b6e 100644 --- a/core/java/src/net/i2p/util/EepGet.java +++ b/core/java/src/net/i2p/util/EepGet.java @@ -148,10 +148,18 @@ public class EepGet { } public static String suggestName(String url) { + int last = url.lastIndexOf('/'); + if ((last < 0) || (url.lastIndexOf('#') > last)) + last = url.lastIndexOf('#'); + if ((last < 0) || (url.lastIndexOf('?') > last)) + last = url.lastIndexOf('?'); + if ((last < 0) || (url.lastIndexOf('=') > last)) + last = url.lastIndexOf('='); + String name = null; - if (url.lastIndexOf('/') >= 0) - name = sanitize(url.substring(url.lastIndexOf('/')+1)); - if (name != null) + if (last >= 0) + name = sanitize(url.substring(last+1)); + if ( (name != null) && (name.length() > 0) ) return name; else return sanitize(url); diff --git a/history.txt b/history.txt index 2f029893a..f656732ca 100644 --- a/history.txt +++ b/history.txt @@ -1,4 +1,11 @@ -$Id: history.txt,v 1.397 2006/02/15 00:33:17 jrandom Exp $ +$Id: history.txt,v 1.398 2006/02/15 08:36:29 jrandom Exp $ + +2006-02-16 jrandom + * Bugfix to the I2PTunnel web config to properly accept i2cp port settings + * Initial sucker refactoring to simplify reuse of the html parsing + * Beginnings of hooks to push imported rss/atom out to remote syndie + archives automatically (though not enabled currently) + * Further SSU peer test cleanup 2006-02-15 jrandom * Add in per-blog RSS feeds to Syndie diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index b69ec7939..dc1d630b9 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -15,9 +15,9 @@ import net.i2p.CoreVersion; * */ public class RouterVersion { - public final static String ID = "$Revision: 1.341 $ $Date: 2006/02/15 00:33:33 $"; + public final static String ID = "$Revision: 1.342 $ $Date: 2006/02/15 08:36:32 $"; public final static String VERSION = "0.6.1.9"; - public final static long BUILD = 26; + public final static long BUILD = 27; public static void main(String args[]) { System.out.println("I2P Router version: " + VERSION + "-" + BUILD); System.out.println("Router ID: " + RouterVersion.ID); diff --git a/router/java/src/net/i2p/router/transport/udp/IntroductionManager.java b/router/java/src/net/i2p/router/transport/udp/IntroductionManager.java index c869832e1..add0d9c3a 100644 --- a/router/java/src/net/i2p/router/transport/udp/IntroductionManager.java +++ b/router/java/src/net/i2p/router/transport/udp/IntroductionManager.java @@ -27,6 +27,7 @@ public class IntroductionManager { _outbound = Collections.synchronizedMap(new HashMap(128)); _inbound = new ArrayList(128); ctx.statManager().createRateStat("udp.receiveRelayIntro", "How often we get a relayed request for us to talk to someone?", "udp", new long[] { 60*1000, 5*60*1000, 10*60*1000 }); + ctx.statManager().createRateStat("udp.receiveRelayRequest", "How often we receive a request to relay to someone else?", "udp", new long[] { 60*1000, 5*60*1000, 10*60*1000 }); } public void reset() { @@ -91,6 +92,7 @@ public class IntroductionManager { } public void receiveRelayRequest(RemoteHostId alice, UDPPacketReader reader) { + _context.statManager().addRateData("udp.receiveRelayRequest", 1, 0); if (_context.router().isHidden()) return; long tag = reader.getRelayRequestReader().readTag(); diff --git a/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java b/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java index 8e53ea433..e893b9cd9 100644 --- a/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java +++ b/router/java/src/net/i2p/router/transport/udp/PeerTestManager.java @@ -47,6 +47,8 @@ class PeerTestManager { _currentTest = null; _currentTestComplete = false; _context.statManager().createRateStat("udp.statusKnownCharlie", "How often the bob we pick passes us to a charlie we already have a session with?", "udp", new long[] { 60*1000, 20*60*1000, 60*60*1000 }); + _context.statManager().createRateStat("udp.receiveTestReply", "How often we get a reply to our peer test?", "udp", new long[] { 60*1000, 20*60*1000, 60*60*1000 }); + _context.statManager().createRateStat("udp.receiveTest", "How often we get a packet requesting us to participate in a peer test?", "udp", new long[] { 60*1000, 20*60*1000, 60*60*1000 }); } private static final int RESEND_TIMEOUT = 5*1000; @@ -146,6 +148,7 @@ class PeerTestManager { * test */ private void receiveTestReply(RemoteHostId from, UDPPacketReader.PeerTestReader testInfo) { + _context.statManager().addRateData("udp.receiveTestReply", 1, 0); PeerTestState test = _currentTest; if (expired()) return; @@ -161,7 +164,7 @@ class PeerTestManager { test.setAlicePort(testInfo.readPort()); if (_log.shouldLog(Log.DEBUG)) - _log.debug("Receive test reply from bob @ " + from.getIP() + " via our " + test.getAlicePort() + "/" + test.getAlicePortFromCharlie()); + _log.debug("Receive test reply from bob @ " + from + " via our " + test.getAlicePort() + "/" + test.getAlicePortFromCharlie()); if (test.getAlicePortFromCharlie() > 0) testComplete(false); } catch (UnknownHostException uhe) { @@ -177,6 +180,7 @@ class PeerTestManager { if (_log.shouldLog(Log.WARN)) _log.warn("Bob chose a charlie we already have a session to, cancelling the test and rerunning (bob: " + _currentTest + ", charlie: " + from + ")"); + _currentTestComplete = true; _context.statManager().addRateData("udp.statusKnownCharlie", 1, 0); honorStatus(CommSystemFacade.STATUS_UNKNOWN); return; @@ -286,6 +290,7 @@ class PeerTestManager { * */ public void receiveTest(RemoteHostId from, UDPPacketReader reader) { + _context.statManager().addRateData("udp.receiveTest", 1, 0); UDPPacketReader.PeerTestReader testInfo = reader.getPeerTestReader(); byte testIP[] = null; int testPort = testInfo.readPort(); @@ -318,7 +323,7 @@ class PeerTestManager { // initiated test } else { if (_log.shouldLog(Log.DEBUG)) - _log.debug("We are charlie, as te testIP/port is " + testIP + ":" + testPort + " and the state is unknown for " + nonce); + _log.debug("We are charlie, as te testIP/port is " + RemoteHostId.toString(testIP) + ":" + testPort + " and the state is unknown for " + nonce); // we are charlie, since alice never sends us her IP and port, only bob does (and, // erm, we're not alice, since it isn't our nonce) receiveFromBobAsCharlie(from, testInfo, nonce, null); diff --git a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java index 66b0c0784..0236121b1 100644 --- a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java +++ b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java @@ -113,7 +113,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority private static final int MAX_CONSECUTIVE_FAILED = 5; - private static final int TEST_FREQUENCY = 3*60*1000; + private static final int TEST_FREQUENCY = 13*60*1000; public UDPTransport(RouterContext ctx) { super(ctx); @@ -973,7 +973,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority buf.append(" pushes: ").append(pushCount); buf.append(" expired? ").append(expired); buf.append(" unacked: ").append(msg.getUnackedSize()); - if (!successful) { + if ( (p != null) && (!successful) ) { buf.append(" consec_failed: ").append(p.getConsecutiveFailedSends()); long timeSinceSend = _context.clock().now() - p.getLastSendFullyTime(); buf.append(" lastFullSend: ").append(timeSinceSend); diff --git a/router/java/src/net/i2p/router/tunnel/TunnelParticipant.java b/router/java/src/net/i2p/router/tunnel/TunnelParticipant.java index 6f27e561f..fe2aa9522 100644 --- a/router/java/src/net/i2p/router/tunnel/TunnelParticipant.java +++ b/router/java/src/net/i2p/router/tunnel/TunnelParticipant.java @@ -68,8 +68,8 @@ public class TunnelParticipant { ok = _inboundEndpointProcessor.retrievePreprocessedData(msg.getData(), 0, msg.getData().length, recvFrom); if (!ok) { - if (_log.shouldLog(Log.ERROR)) - _log.error("Failed to dispatch " + msg + ": processor=" + _processor + if (_log.shouldLog(Log.WARN)) + _log.warn("Failed to dispatch " + msg + ": processor=" + _processor + " inboundEndpoint=" + _inboundEndpointProcessor); return; } diff --git a/router/java/src/net/i2p/router/tunnel/pool/BuildExecutor.java b/router/java/src/net/i2p/router/tunnel/pool/BuildExecutor.java index 79683925e..30c68005e 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/BuildExecutor.java +++ b/router/java/src/net/i2p/router/tunnel/pool/BuildExecutor.java @@ -82,8 +82,8 @@ class BuildExecutor implements Runnable { PooledTunnelCreatorConfig cfg = (PooledTunnelCreatorConfig)expired.get(i); // note the fact that this tunnel request timed out in the peers' profiles. // or... not. - if (_log.shouldLog(Log.ERROR)) - _log.error("Timed out waiting for reply asking for " + cfg); + if (_log.shouldLog(Log.INFO)) + _log.info("Timed out waiting for reply asking for " + cfg); TunnelPool pool = cfg.getTunnelPool(); if (pool != null) pool.buildComplete(cfg); @@ -274,8 +274,8 @@ class BuildExecutor implements Runnable { } long expireBefore = _context.clock().now() + 10*60*1000 - BuildRequestor.REQUEST_TIMEOUT; if (cfg.getExpiration() <= expireBefore) { - if (_log.shouldLog(Log.ERROR)) - _log.error("Build complete for expired tunnel: " + cfg); + if (_log.shouldLog(Log.INFO)) + _log.info("Build complete for expired tunnel: " + cfg); } } diff --git a/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java b/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java index 306ea9968..008598a9a 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java +++ b/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java @@ -221,8 +221,8 @@ class BuildHandler { _context.statManager().addRateData("tunnel.buildClientReject", rtt, rtt); } } else { - if (_log.shouldLog(Log.ERROR)) - _log.error(msg.getUniqueId() + ": Tunnel reply could not be decrypted for tunnel " + cfg); + if (_log.shouldLog(Log.WARN)) + _log.warn(msg.getUniqueId() + ": Tunnel reply could not be decrypted for tunnel " + cfg); } } @@ -247,8 +247,8 @@ class BuildHandler { _context.statManager().addRateData("tunnel.decryptRequestTime", decryptTime, decryptTime); if (req == null) { // no records matched, or the decryption failed. bah - if (_log.shouldLog(Log.ERROR)) - _log.error("The request " + state.msg.getUniqueId() + " could not be decrypted"); + if (_log.shouldLog(Log.WARN)) + _log.warn("The request " + state.msg.getUniqueId() + " could not be decrypted"); return; } @@ -309,8 +309,8 @@ class BuildHandler { public String getName() { return "Timeout looking for next peer for tunnel join"; } public void runJob() { getContext().statManager().addRateData("tunnel.rejectTimeout", 1, 1); - if (_log.shouldLog(Log.ERROR)) - _log.error("Request " + _state.msg.getUniqueId() + if (_log.shouldLog(Log.WARN)) + _log.warn("Request " + _state.msg.getUniqueId() + " could no be satisfied, as the next peer could not be found: " + _nextPeer.toBase64()); getContext().messageHistory().tunnelRejected(_state.fromHash, new TunnelId(_req.readReceiveTunnelId()), _nextPeer, "rejected because we couldn't find " + _nextPeer.toBase64() + ": " +