* SusiMail:

- Pipeline initial fetch of messages, huge speedup

apps/susimail/src/src/i2p/susi/webmail/MailCache.java

@@ -23,9 +23,14 @@
  */
 package i2p.susi.webmail;
 
+import i2p.susi.util.ReadBuffer;
 import i2p.susi.webmail.pop3.POP3MailBox;
+import i2p.susi.webmail.pop3.POP3MailBox.FetchRequest;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Hashtable;
+import java.util.List;
 
 /**
  * @author user
@@ -43,6 +48,9 @@ public class MailCache {
          */
 	private static final int FETCH_ALL_SIZE = 3072;
 
+	/**
+	 * @param mailbox non-null
+	 */
 	MailCache( POP3MailBox mailbox ) {
 		this.mailbox = mailbox;
 		mails = new Hashtable<String, Mail>();
@@ -59,7 +67,6 @@ public class MailCache {
 		
 		Mail mail = null, newMail = null;
 
-		if( mailbox != null ) {
 			/*
 			 * synchronize update to hashtable
 			 */
@@ -95,9 +102,120 @@ public class MailCache {
 			}
 			if( parseHeaders && mail.header != null )
 				mail.parseHeaders();
-		}
 		if( mail != null && mail.deleted )
 			mail = null;
 		return mail;
 	}
+
+	/**
+	 * Fetch any needed data from pop3 server.
+	 * Mail objects are inserted into the requests.
+	 * 
+	 * @since 0.9.13
+	 */
+	public void getMail(Collection<MailRequest> requests) {
+		
+		List<POP3Request> fetches = new ArrayList<POP3Request>();
+		//  Fill in the answers from the cache and make a list of
+		//  requests.to send off
+		for (MailRequest mr : requests) {
+			Mail mail = null, newMail = null;
+			String uidl = mr.getUIDL();
+			boolean headerOnly = mr.getHeaderOnly();
+
+			/*
+			 * synchronize update to hashtable
+			 */
+			synchronized(mails) {
+				mail = mails.get( uidl );
+				if( mail == null ) {
+					newMail = new Mail();
+					mails.put( uidl, newMail );
+				}
+			}
+			if( mail == null ) {
+				mail = newMail;
+				mail.uidl = uidl;
+				mail.size = mailbox.getSize( uidl );
+			}
+			if(!mail.deleted) {
+				mr.setMail(mail);
+				if( mail.size <= FETCH_ALL_SIZE)
+					headerOnly = false;
+				if( headerOnly ) {
+					if( mail.header == null ) {
+						POP3Request pr = new POP3Request(mr, mail, true);
+						fetches.add(pr);
+					}
+				} else {
+					if( mail.body == null ) {
+						POP3Request pr = new POP3Request(mr, mail, false);
+						fetches.add(pr);
+					}
+				}
+			}
+		}
+
+		if (!fetches.isEmpty()) {
+			//  Send off the fetches
+			// gaah compiler
+			List foo = fetches;
+			List<FetchRequest> bar = foo;
+			mailbox.getBodies(bar);
+			//  Process results
+			for (POP3Request pr : fetches) {
+				ReadBuffer rb = pr.buf;
+				if (rb != null) {
+					Mail mail = pr.mail;
+					boolean parseHeaders = mail.header == null;
+					if (pr.getHeaderOnly()) {
+						mail.header = rb;
+					} else {
+						mail.header = rb;
+						mail.body = rb;
+						MailPart.parse(mail);
+					}
+					if (parseHeaders)
+						mail.parseHeaders();
+				}
+			}
+		}
+	}
+
+	/**
+	 *  Incoming to us
+	 */
+	public interface MailRequest {
+		public String getUIDL();
+		public boolean getHeaderOnly();
+		public void setMail(Mail mail);
+	}
+
+	/**
+	 *  Outgoing to POP3
+	 */
+	private static class POP3Request implements FetchRequest {
+		public final MailRequest request;
+		public final Mail mail;
+		private final boolean headerOnly;
+		public ReadBuffer buf;
+
+		public POP3Request(MailRequest req, Mail m, boolean hOnly) {
+			request = req;
+			mail = m;
+			headerOnly = hOnly;
+		}
+
+		public String getUIDL() {
+			return request.getUIDL();
+		}
+
+		public boolean getHeaderOnly() {
+			return headerOnly;
+		}
+
+		public void setBuffer(ReadBuffer buffer) {
+			buf = buffer;
+		}
+	}
 }

apps/susimail/src/src/i2p/susi/webmail/WebMail.java

@@ -50,6 +50,7 @@ import java.util.Comparator;
 import java.util.Date;
 import java.util.Enumeration;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Locale;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -659,9 +660,21 @@ public class WebMail extends HttpServlet
 						sessionObject.host = host;
 						sessionObject.smtpPort = smtpPortNo;
 						sessionObject.state = STATE_LIST;
-						sessionObject.mailCache = new MailCache(mailbox);
+						MailCache mc = new MailCache(mailbox);
+						sessionObject.mailCache = mc;
 						sessionObject.folder = new Folder<String>();
-						sessionObject.folder.setElements(mailbox.getUIDLs() );
+						String[] uidls = mailbox.getUIDLs();
+						sessionObject.folder.setElements(uidls);
+						if (uidls.length > 0) {
+							// prime the cache, request all headers at once
+							// otherwise they are pulled one at a time by sortBy() below
+							List<MailCache.MailRequest> reqs = new ArrayList<MailCache.MailRequest>(uidls.length);
+							for (int i = 0; i < uidls.length; i++) {
+								reqs.add(new CacheRequest(uidls[i]));
+							}
+							mc.getMail(reqs);
+						}
+						
 						sessionObject.folder.addSorter( SORT_ID, new IDSorter( sessionObject.mailCache ) );
 						sessionObject.folder.addSorter( SORT_SENDER, new SenderSorter( sessionObject.mailCache ) );
 						sessionObject.folder.addSorter( SORT_SUBJECT, new SubjectSorter( sessionObject.mailCache ) );
@@ -682,6 +695,31 @@ public class WebMail extends HttpServlet
 			}
 		}
 	}
+
+	/**
+	 *  Outgoing to MailCache
+	 *  @since 0.9.13
+	 */
+	private static class CacheRequest implements MailCache.MailRequest {
+		private final String uidl;
+
+		public CacheRequest(String uidl) {
+			this.uidl = uidl;
+		}
+
+		public String getUIDL() {
+			return uidl;
+		}
+
+		public boolean getHeaderOnly() {
+			return true;
+		}
+
+		public void setMail(Mail mail) {
+			// do nothing, this just pumps up the cache
+		}
+	}
+
 	/**
 	 * 
 	 * @param sessionObject

apps/susimail/src/src/i2p/susi/webmail/pop3/POP3MailBox.java

@@ -33,6 +33,7 @@ import java.io.InputStream;
 import java.net.Socket;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 
@@ -145,6 +146,44 @@ public class POP3MailBox {
 			return getBody(id);
 		}
 	}
+
+	/**
+	 * Fetch headers and/or bodies. Does not cache.
+	 * ReadBuffer objects are inserted into the requests.
+	 * No total time limit.
+	 * 
+	 * @since 0.9.13
+	 */
+	public void getBodies(Collection<FetchRequest> requests) {
+		List<SendRecv> srs = new ArrayList<SendRecv>(requests.size());
+		synchronized( synchronizer ) {
+			for (FetchRequest fr : requests) {
+				int id = getIDfromUIDL(fr.getUIDL());
+				if (id < 0)
+					continue;
+				SendRecv sr;
+				if (fr.getHeaderOnly() && supportsTOP)
+					sr = new SendRecv("TOP " + id + " 0", Mode.RB);
+				else
+					sr = new SendRecv("RETR " + id, Mode.RB);
+				sr.savedObject = fr;
+				srs.add(sr);
+			}
+			if (srs.isEmpty())
+				return;
+			try {
+				sendCmds(srs);
+			} catch (IOException ioe) {
+				// todo maybe
+			}
+		}
+		for (SendRecv sr : srs) {
+			if (sr.result) {
+				FetchRequest fr = (FetchRequest) sr.savedObject;
+				fr.setBuffer(sr.rb);
+			}
+		}
+	}
 	
 	/**
 	 * retrieve message body from pop3 server (via RETR command)
@@ -897,6 +936,8 @@ public class POP3MailBox {
 		public boolean result;
 		public ReadBuffer rb;
 		public List<String> ls;
+		// to remember things
+		public Object savedObject;
 
 		/** @param s may be null */
 		public SendRecv(String s, Mode m) {
@@ -905,6 +946,12 @@ public class POP3MailBox {
 		}
 	}
 
+	public interface FetchRequest {
+		public String getUIDL();
+		public boolean getHeaderOnly();
+		public void setBuffer(ReadBuffer buffer);
+	}
+
 	/** translate */
 	private static String _(String s) {
 		return Messages.getString(s);

history.txt

@@ -1,3 +1,41 @@
+2014-04-20 zzz
+ * SusiMail:
+   - Implement extensive pipelining in POP3 for a big speedup
+     of the initial connection
+   - Don't require an attachment to be "uploaded" to send it
+   - Move delete attachment button, hide if no attachments
+   - Save BCC-to-self preference in the session
+   - Fix date format in reply
+   - Close any open POP3 socket when session is unbound
+   - Don't keep returning user to compose page (ticket #1252)
+   - Add javascript capture of back button on compose page
+
+2014-04-19 zzz
+ * Console: Remove the classpath workarounds for SusiMail,
+   since it isn't using the jetty classes any more
+ * SusiMail:
+   - Increase max size of mails that are fetched in full,
+     previous limit was so small it never happened.
+   - Move page nav to top of folder view, hide if only one page
+   - Refuse to send mail with no "to"
+   - Reduce default page size as it slows startup
+   - CSS and layout fixes
+   - Flush writes in POP3 and SMTP
+   - Don't wait for SMTP response after QUIT
+   - Tell the user if there are no messages
+   - Fix the message view layout
+   - Message view attachment cleanups
+   - Pipeline USER and PASS to save a round-trip at startup
+   - Better synchronization in POP3
+   - Properly de-byte-stuff in POP3
+   - Remove unnecessary caching in POP3
+   - More efficient handling of POP3 responses
+   - Remove 60s timeout for fetching a message,
+     so retrieval of large messages doesn't fail
+   - Use pipelining in SMTP
+   - Rewrite SMTP response processing
+   - Translate SMTP error messages
+
 2014-04-18 zzz
  * configclients: Don't allow console disable
  * I2PTunnel IRC Client: Prevent AIOOBE (ticket #1254)

router/java/src/net/i2p/router/RouterVersion.java

@@ -18,7 +18,7 @@ public class RouterVersion {
     /** deprecated */
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 6;
+    public final static long BUILD = 7;
 
     /** for example "-test" */
     public final static String EXTRA = "";