From b9491b269bfdf51baa9fbdd29182f720ab3ceb80 Mon Sep 17 00:00:00 2001 From: zzz Date: Sun, 20 Apr 2014 22:22:22 +0000 Subject: [PATCH] * SusiMail: - Pipeline initial fetch of messages, huge speedup --- .../src/src/i2p/susi/webmail/MailCache.java | 122 +++++++++++++++++- .../src/src/i2p/susi/webmail/WebMail.java | 42 +++++- .../i2p/susi/webmail/pop3/POP3MailBox.java | 47 +++++++ history.txt | 38 ++++++ .../src/net/i2p/router/RouterVersion.java | 2 +- 5 files changed, 246 insertions(+), 5 deletions(-) diff --git a/apps/susimail/src/src/i2p/susi/webmail/MailCache.java b/apps/susimail/src/src/i2p/susi/webmail/MailCache.java index 6a8482c20..229045119 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/MailCache.java +++ b/apps/susimail/src/src/i2p/susi/webmail/MailCache.java @@ -23,9 +23,14 @@ */ package i2p.susi.webmail; +import i2p.susi.util.ReadBuffer; import i2p.susi.webmail.pop3.POP3MailBox; +import i2p.susi.webmail.pop3.POP3MailBox.FetchRequest; +import java.util.ArrayList; +import java.util.Collection; import java.util.Hashtable; +import java.util.List; /** * @author user @@ -43,6 +48,9 @@ public class MailCache { */ private static final int FETCH_ALL_SIZE = 3072; + /** + * @param mailbox non-null + */ MailCache( POP3MailBox mailbox ) { this.mailbox = mailbox; mails = new Hashtable(); @@ -59,7 +67,6 @@ public class MailCache { Mail mail = null, newMail = null; - if( mailbox != null ) { /* * synchronize update to hashtable */ @@ -95,9 +102,120 @@ public class MailCache { } if( parseHeaders && mail.header != null ) mail.parseHeaders(); - } if( mail != null && mail.deleted ) mail = null; return mail; } + + /** + * Fetch any needed data from pop3 server. + * Mail objects are inserted into the requests. + * + * @since 0.9.13 + */ + public void getMail(Collection requests) { + + List fetches = new ArrayList(); + // Fill in the answers from the cache and make a list of + // requests.to send off + for (MailRequest mr : requests) { + Mail mail = null, newMail = null; + String uidl = mr.getUIDL(); + boolean headerOnly = mr.getHeaderOnly(); + + /* + * synchronize update to hashtable + */ + synchronized(mails) { + mail = mails.get( uidl ); + if( mail == null ) { + newMail = new Mail(); + mails.put( uidl, newMail ); + } + } + if( mail == null ) { + mail = newMail; + mail.uidl = uidl; + mail.size = mailbox.getSize( uidl ); + } + if(!mail.deleted) { + mr.setMail(mail); + if( mail.size <= FETCH_ALL_SIZE) + headerOnly = false; + if( headerOnly ) { + if( mail.header == null ) { + POP3Request pr = new POP3Request(mr, mail, true); + fetches.add(pr); + } + } else { + if( mail.body == null ) { + POP3Request pr = new POP3Request(mr, mail, false); + fetches.add(pr); + } + } + } + } + + if (!fetches.isEmpty()) { + // Send off the fetches + // gaah compiler + List foo = fetches; + List bar = foo; + mailbox.getBodies(bar); + // Process results + for (POP3Request pr : fetches) { + ReadBuffer rb = pr.buf; + if (rb != null) { + Mail mail = pr.mail; + boolean parseHeaders = mail.header == null; + if (pr.getHeaderOnly()) { + mail.header = rb; + } else { + mail.header = rb; + mail.body = rb; + MailPart.parse(mail); + } + if (parseHeaders) + mail.parseHeaders(); + } + } + } + } + + /** + * Incoming to us + */ + public interface MailRequest { + public String getUIDL(); + public boolean getHeaderOnly(); + public void setMail(Mail mail); + } + + /** + * Outgoing to POP3 + */ + private static class POP3Request implements FetchRequest { + public final MailRequest request; + public final Mail mail; + private final boolean headerOnly; + public ReadBuffer buf; + + public POP3Request(MailRequest req, Mail m, boolean hOnly) { + request = req; + mail = m; + headerOnly = hOnly; + } + + public String getUIDL() { + return request.getUIDL(); + } + + public boolean getHeaderOnly() { + return headerOnly; + } + + public void setBuffer(ReadBuffer buffer) { + buf = buffer; + } + } } diff --git a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java index 8a6d3f963..703363be9 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java +++ b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java @@ -50,6 +50,7 @@ import java.util.Comparator; import java.util.Date; import java.util.Enumeration; import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -659,9 +660,21 @@ public class WebMail extends HttpServlet sessionObject.host = host; sessionObject.smtpPort = smtpPortNo; sessionObject.state = STATE_LIST; - sessionObject.mailCache = new MailCache(mailbox); + MailCache mc = new MailCache(mailbox); + sessionObject.mailCache = mc; sessionObject.folder = new Folder(); - sessionObject.folder.setElements(mailbox.getUIDLs() ); + String[] uidls = mailbox.getUIDLs(); + sessionObject.folder.setElements(uidls); + if (uidls.length > 0) { + // prime the cache, request all headers at once + // otherwise they are pulled one at a time by sortBy() below + List reqs = new ArrayList(uidls.length); + for (int i = 0; i < uidls.length; i++) { + reqs.add(new CacheRequest(uidls[i])); + } + mc.getMail(reqs); + } + sessionObject.folder.addSorter( SORT_ID, new IDSorter( sessionObject.mailCache ) ); sessionObject.folder.addSorter( SORT_SENDER, new SenderSorter( sessionObject.mailCache ) ); sessionObject.folder.addSorter( SORT_SUBJECT, new SubjectSorter( sessionObject.mailCache ) ); @@ -682,6 +695,31 @@ public class WebMail extends HttpServlet } } } + + /** + * Outgoing to MailCache + * @since 0.9.13 + */ + private static class CacheRequest implements MailCache.MailRequest { + private final String uidl; + + public CacheRequest(String uidl) { + this.uidl = uidl; + } + + public String getUIDL() { + return uidl; + } + + public boolean getHeaderOnly() { + return true; + } + + public void setMail(Mail mail) { + // do nothing, this just pumps up the cache + } + } + /** * * @param sessionObject diff --git a/apps/susimail/src/src/i2p/susi/webmail/pop3/POP3MailBox.java b/apps/susimail/src/src/i2p/susi/webmail/pop3/POP3MailBox.java index a70c8bbec..8800606d8 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/pop3/POP3MailBox.java +++ b/apps/susimail/src/src/i2p/susi/webmail/pop3/POP3MailBox.java @@ -33,6 +33,7 @@ import java.io.InputStream; import java.net.Socket; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -145,6 +146,44 @@ public class POP3MailBox { return getBody(id); } } + + /** + * Fetch headers and/or bodies. Does not cache. + * ReadBuffer objects are inserted into the requests. + * No total time limit. + * + * @since 0.9.13 + */ + public void getBodies(Collection requests) { + List srs = new ArrayList(requests.size()); + synchronized( synchronizer ) { + for (FetchRequest fr : requests) { + int id = getIDfromUIDL(fr.getUIDL()); + if (id < 0) + continue; + SendRecv sr; + if (fr.getHeaderOnly() && supportsTOP) + sr = new SendRecv("TOP " + id + " 0", Mode.RB); + else + sr = new SendRecv("RETR " + id, Mode.RB); + sr.savedObject = fr; + srs.add(sr); + } + if (srs.isEmpty()) + return; + try { + sendCmds(srs); + } catch (IOException ioe) { + // todo maybe + } + } + for (SendRecv sr : srs) { + if (sr.result) { + FetchRequest fr = (FetchRequest) sr.savedObject; + fr.setBuffer(sr.rb); + } + } + } /** * retrieve message body from pop3 server (via RETR command) @@ -897,6 +936,8 @@ public class POP3MailBox { public boolean result; public ReadBuffer rb; public List ls; + // to remember things + public Object savedObject; /** @param s may be null */ public SendRecv(String s, Mode m) { @@ -905,6 +946,12 @@ public class POP3MailBox { } } + public interface FetchRequest { + public String getUIDL(); + public boolean getHeaderOnly(); + public void setBuffer(ReadBuffer buffer); + } + /** translate */ private static String _(String s) { return Messages.getString(s); diff --git a/history.txt b/history.txt index a8fdeb41a..b337281a2 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,41 @@ +2014-04-20 zzz + * SusiMail: + - Implement extensive pipelining in POP3 for a big speedup + of the initial connection + - Don't require an attachment to be "uploaded" to send it + - Move delete attachment button, hide if no attachments + - Save BCC-to-self preference in the session + - Fix date format in reply + - Close any open POP3 socket when session is unbound + - Don't keep returning user to compose page (ticket #1252) + - Add javascript capture of back button on compose page + +2014-04-19 zzz + * Console: Remove the classpath workarounds for SusiMail, + since it isn't using the jetty classes any more + * SusiMail: + - Increase max size of mails that are fetched in full, + previous limit was so small it never happened. + - Move page nav to top of folder view, hide if only one page + - Refuse to send mail with no "to" + - Reduce default page size as it slows startup + - CSS and layout fixes + - Flush writes in POP3 and SMTP + - Don't wait for SMTP response after QUIT + - Tell the user if there are no messages + - Fix the message view layout + - Message view attachment cleanups + - Pipeline USER and PASS to save a round-trip at startup + - Better synchronization in POP3 + - Properly de-byte-stuff in POP3 + - Remove unnecessary caching in POP3 + - More efficient handling of POP3 responses + - Remove 60s timeout for fetching a message, + so retrieval of large messages doesn't fail + - Use pipelining in SMTP + - Rewrite SMTP response processing + - Translate SMTP error messages + 2014-04-18 zzz * configclients: Don't allow console disable * I2PTunnel IRC Client: Prevent AIOOBE (ticket #1254) diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index c10128fe6..fecba78d6 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 6; + public final static long BUILD = 7; /** for example "-test" */ public final static String EXTRA = "";