zzz/i2p.i2p
1
0
Fork 0
forked from I2P_Developers/i2p.i2p
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

* EepGet, I2PSnark:

Browse Source 
- New I2PSocketEepGet fetches through existing tunnels
        rather than through the proxy
      - Use new eepget for i2psnark
      - Add a fake user agent for non-proxied fetches
      - Cleanups
This commit is contained in:
zzz
2009-08-26 22:15:32 +00:00
parent df4143f036
commit 95aba0c537
9 changed files with 297 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
core/java/src/net/i2p/util/EepGet.java
View File

@@ -27,7 +27,7 @@ import net.i2p.data.DataHelper;
* Bug: a malformed url http://example.i2p (no trailing '/') fails cryptically
*/
public class EepGet {
private I2PAppContext _context;
protected I2PAppContext _context;
protected Log _log;
protected boolean _shouldProxy;
private String _proxyHost;
@@ -35,8 +35,8 @@ public class EepGet {
protected int _numRetries;
private long _minSize; // minimum and maximum acceptable response size, -1 signifies unlimited,
private long _maxSize; // applied both against whole responses and chunks
private String _outputFile;
private OutputStream _outputStream;
protected String _outputFile;
protected OutputStream _outputStream;
/** url we were asked to fetch */
protected String _url;
/** the URL we actually fetch from (may differ from the _url in case of redirect) */
@@ -47,10 +47,10 @@ public class EepGet {
private boolean _keepFetching;
private Socket _proxy;
private OutputStream _proxyOut;
private InputStream _proxyIn;
protected OutputStream _proxyOut;
protected InputStream _proxyIn;
protected OutputStream _out;
private long _alreadyTransferred;
protected long _alreadyTransferred;
private long _bytesTransferred;
protected long _bytesRemaining;
protected int _currentAttempt;
@@ -67,6 +67,10 @@ public class EepGet {
protected long _fetchInactivityTimeout;
protected int _redirects;
protected String _redirectLocation;
/** this will be replaced by the HTTP Proxy if we are using it */
protected static final String USER_AGENT = "Wget/1.11.4";
protected static final long CONNECT_TIMEOUT = 45*1000;
protected static final long INACTIVITY_TIMEOUT = 60*1000;
public EepGet(I2PAppContext ctx, String proxyHost, int proxyPort, int numRetries, String outputFile, String url) {
this(ctx, true, proxyHost, proxyPort, numRetries, outputFile, url);
@@ -118,7 +122,7 @@ public class EepGet {
_transferFailed = false;
_headersRead = false;
_aborted = false;
_fetchHeaderTimeout = 45*1000;
_fetchHeaderTimeout = CONNECT_TIMEOUT;
_listeners = new ArrayList(1);
_etag = etag;
_lastModified = lastModified;
@@ -134,7 +138,7 @@ public class EepGet {
int numRetries = 5;
int markSize = 1024;
int lineLen = 40;
int inactivityTimeout = 60*1000;
long inactivityTimeout = INACTIVITY_TIMEOUT;
String etag = null;
String saveAs = null;
String url = null;
@@ -183,7 +187,7 @@ public class EepGet {
EepGet get = new EepGet(I2PAppContext.getGlobalContext(), true, proxyHost, proxyPort, numRetries, saveAs, url, true, etag);
get.addStatusListener(get.new CLIStatusListener(markSize, lineLen));
get.fetch(45*1000, -1, inactivityTimeout);
get.fetch(CONNECT_TIMEOUT, -1, inactivityTimeout);
}
public static String suggestName(String url) {
@@ -216,7 +220,7 @@ public class EepGet {
return buf.toString();
}
protected static void usage() {
private static void usage() {
System.err.println("EepGet [-p 127.0.0.1:4444] [-n #retries] [-o outputFile] [-m markSize lineLen] [-t timeout] url");
}
@@ -247,7 +251,7 @@ public class EepGet {
public void headerReceived(String url, int currentAttempt, String key, String val);
public void attempting(String url);
}
private class CLIStatusListener implements StatusListener {
protected class CLIStatusListener implements StatusListener {
private int _markSize;
private int _lineSize;
private long _startedOn;
@@ -497,7 +501,7 @@ public class EepGet {
if (_fetchInactivityTimeout > 0)
timeout.setInactivityTimeout(_fetchInactivityTimeout);
else
timeout.setInactivityTimeout(60*1000);
timeout.setInactivityTimeout(INACTIVITY_TIMEOUT);
if (_redirectLocation != null) {
try {
@@ -829,12 +833,12 @@ public class EepGet {
}
}
private void increment(byte[] lookahead, int cur) {
private static void increment(byte[] lookahead, int cur) {
lookahead[0] = lookahead[1];
lookahead[1] = lookahead[2];
lookahead[2] = (byte)cur;
}
private boolean isEndOfHeaders(byte lookahead[]) {
private static boolean isEndOfHeaders(byte lookahead[]) {
byte first = lookahead[0];
byte second = lookahead[1];
byte third = lookahead[2];
@@ -844,7 +848,7 @@ public class EepGet {
/** we ignore any potential \r, since we trim it on write anyway */
private static final byte NL = '\n';
private boolean isNL(byte b) { return (b == NL); }
private static boolean isNL(byte b) { return (b == NL); }
protected void sendRequest(SocketTimeout timeout) throws IOException {
if (_outputStream != null) {
@@ -895,7 +899,7 @@ public class EepGet {
}
protected String getRequest() throws IOException {
StringBuilder buf = new StringBuilder(512);
StringBuilder buf = new StringBuilder(2048);
boolean post = false;
if ( (_postData != null) && (_postData.length() > 0) )
post = true;
@@ -906,7 +910,7 @@ public class EepGet {
String path = url.getPath();
String query = url.getQuery();
if (query != null)
path = path + "?" + query;
path = path + '?' + query;
if (!path.startsWith("/"))
path = "/" + path;
if ( (port == 80) || (port == 443) || (port <= 0) ) path = proto + "://" + host + path;
@@ -923,12 +927,11 @@ public class EepGet {
buf.append(_alreadyTransferred);
buf.append("-\r\n");
}
buf.append("Accept-Encoding: \r\n");
if (_shouldProxy)
buf.append("X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n");
if (!_allowCaching) {
buf.append("Cache-control: no-cache\r\n");
buf.append("Pragma: no-cache\r\n");
buf.append("Cache-control: no-cache\r\n" +
"Pragma: no-cache\r\n");
}
if ((_etag != null) && (_alreadyTransferred <= 0)) {
buf.append("If-None-Match: ");
@@ -942,7 +945,10 @@ public class EepGet {
}
if (post)
buf.append("Content-length: ").append(_postData.length()).append("\r\n");
buf.append("Connection: close\r\n\r\n");
// This will be replaced if we are going through I2PTunnelHTTPClient
buf.append("User-Agent: " + USER_AGENT + "\r\n" +
"Accept-Encoding: \r\n" +
"Connection: close\r\n\r\n");
if (post)
buf.append(_postData);
if (_log.shouldLog(Log.DEBUG))

2
core/java/src/net/i2p/util/EepGetScheduler.java
View File

@@ -7,7 +7,7 @@ import java.util.List;
import net.i2p.I2PAppContext;
/**
*
* @deprecated unused a webapp version would be nice though
*/
public class EepGetScheduler implements EepGet.StatusListener {
private I2PAppContext _context;

4
core/java/src/net/i2p/util/EepHead.java
View File

@@ -93,7 +93,7 @@ public class EepHead extends EepGet {
}
}
protected static void usage() {
private static void usage() {
System.err.println("EepHead [-p 127.0.0.1:4444] [-n #retries] [-t timeout] url");
}
@@ -191,6 +191,8 @@ public class EepHead extends EepGet {
buf.append("Accept-Encoding: \r\n");
if (_shouldProxy)
buf.append("X-Accept-Encoding: x-i2p-gzip;q=1.0, identity;q=0.5, deflate;q=0, gzip;q=0, *;q=0\r\n");
// This will be replaced if we are going through I2PTunnelHTTPClient
buf.append("User-Agent: " + USER_AGENT + "\r\n");
buf.append("Connection: close\r\n\r\n");
if (_log.shouldLog(Log.DEBUG))
_log.debug("Request: [" + buf.toString() + "]");

11
core/java/src/net/i2p/util/SocketTimeout.java
View File

@@ -5,6 +5,15 @@ import java.net.Socket;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* This should be deprecated.
* It is only used by EepGet, and it uses the inefficient SimpleTimer.
* The only advantage seems to be a total timeout period, which is the second
* argument to EepGet.fetch(headerTimeout, totalTimeout, inactivityTimeout),
* which is most likely always set to -1.
*
* Use socket.setsotimeout instead?
*/
public class SocketTimeout implements SimpleTimer.TimedEvent {
private Socket _targetSocket;
private long _startTime;
@@ -69,4 +78,4 @@ public class SocketTimeout implements SimpleTimer.TimedEvent {
buf.append("cancelled? ").append(_cancelled);
return buf.toString();
}
}
}