From 5044f3e58f109cddbd47f14417267cf18b4f05b2 Mon Sep 17 00:00:00 2001 From: zzz Date: Sat, 15 Nov 2014 17:48:11 +0000 Subject: [PATCH 01/13] I2NP: Move some data structures away from ByteArray; offsets were always zero - New BuildRequestRecord constructors - BuildRequestRecord field becomes final byte[222] - IV becomes byte[16] - Build record becomes EncryptedBuildRecord Remove extra copy in BuildRequestRecord.encryptRecord() Remove unused BuildRequestRecord.readOurIdentityMatches() --- .../net/i2p/data/i2np/BuildRequestRecord.java | 109 ++++++++++-------- .../i2p/data/i2np/BuildResponseRecord.java | 13 ++- .../i2p/data/i2np/EncryptedBuildRecord.java | 32 +++++ .../i2p/data/i2np/TunnelBuildMessageBase.java | 13 +-- .../data/i2np/VariableTunnelBuildMessage.java | 5 +- .../i2np/VariableTunnelBuildReplyMessage.java | 5 +- .../router/tunnel/BuildMessageGenerator.java | 42 ++++--- .../router/tunnel/BuildMessageProcessor.java | 35 +++--- .../i2p/router/tunnel/BuildReplyHandler.java | 36 +++--- .../src/net/i2p/router/tunnel/HopConfig.java | 23 +++- .../i2p/router/tunnel/pool/BuildHandler.java | 6 +- .../router/tunnel/pool/BuildRequestor.java | 3 +- 12 files changed, 200 insertions(+), 122 deletions(-) create mode 100644 router/java/src/net/i2p/data/i2np/EncryptedBuildRecord.java diff --git a/router/java/src/net/i2p/data/i2np/BuildRequestRecord.java b/router/java/src/net/i2p/data/i2np/BuildRequestRecord.java index 805c8cc4d..fb719ec2e 100644 --- a/router/java/src/net/i2p/data/i2np/BuildRequestRecord.java +++ b/router/java/src/net/i2p/data/i2np/BuildRequestRecord.java @@ -2,6 +2,7 @@ package net.i2p.data.i2np; import net.i2p.I2PAppContext; import net.i2p.data.ByteArray; +import net.i2p.data.DataFormatException; import net.i2p.data.DataHelper; import net.i2p.data.Hash; import net.i2p.data.PrivateKey; @@ -9,7 +10,8 @@ import net.i2p.data.PublicKey; import net.i2p.data.SessionKey; /** - * Hold the tunnel request record, managing its ElGamal encryption and decryption. + * Holds the unencrypted 222-byte tunnel request record, + * with a constructor for ElGamal decryption and a method for ElGamal encryption. * Iterative AES encryption/decryption is done elsewhere. * * Cleartext: @@ -36,7 +38,7 @@ import net.i2p.data.SessionKey; * */ public class BuildRequestRecord { - private ByteArray _data; + private final byte[] _data; /** * If set in the flag byte, any peer may send a message into this tunnel, but if @@ -55,11 +57,10 @@ public class BuildRequestRecord { /** we show 16 bytes of the peer hash outside the elGamal block */ public static final int PEER_SIZE = 16; - public BuildRequestRecord(ByteArray data) { _data = data; } - public BuildRequestRecord() { } - - public ByteArray getData() { return _data; } - public void setData(ByteArray data) { _data = data; } + /** + * @return 222 bytes, non-null + */ + public byte[] getData() { return _data; } private static final int OFF_RECV_TUNNEL = 0; private static final int OFF_OUR_IDENT = OFF_RECV_TUNNEL + 4; @@ -72,91 +73,101 @@ public class BuildRequestRecord { private static final int OFF_FLAG = OFF_REPLY_IV + IV_SIZE; private static final int OFF_REQ_TIME = OFF_FLAG + 1; private static final int OFF_SEND_MSG_ID = OFF_REQ_TIME + 4; + private static final int PADDING_SIZE = 29; + // 222 + private static final int LENGTH = OFF_SEND_MSG_ID + 4 + PADDING_SIZE; + /** what tunnel ID should this receive messages on */ public long readReceiveTunnelId() { - return DataHelper.fromLong(_data.getData(), _data.getOffset() + OFF_RECV_TUNNEL, 4); - } - /** true if the identity they expect us to be is who we are */ - public boolean readOurIdentityMatches(Hash ourIdentity) { - return DataHelper.eq(ourIdentity.getData(), 0, _data.getData(), _data.getOffset() + OFF_OUR_IDENT, Hash.HASH_LENGTH); + return DataHelper.fromLong(_data, OFF_RECV_TUNNEL, 4); } + /** * What tunnel ID the next hop receives messages on. If this is the outbound tunnel endpoint, * this specifies the tunnel ID to which the reply should be sent. */ public long readNextTunnelId() { - return DataHelper.fromLong(_data.getData(), _data.getOffset() + OFF_SEND_TUNNEL, 4); + return DataHelper.fromLong(_data, OFF_SEND_TUNNEL, 4); } + /** * Read the next hop from the record. If this is the outbound tunnel endpoint, this specifies * the gateway to which the reply should be sent. */ public Hash readNextIdentity() { //byte rv[] = new byte[Hash.HASH_LENGTH]; - //System.arraycopy(_data.getData(), _data.getOffset() + OFF_SEND_IDENT, rv, 0, Hash.HASH_LENGTH); + //System.arraycopy(_data, OFF_SEND_IDENT, rv, 0, Hash.HASH_LENGTH); //return new Hash(rv); - return Hash.create(_data.getData(), _data.getOffset() + OFF_SEND_IDENT); + return Hash.create(_data, OFF_SEND_IDENT); } + /** * Tunnel layer encryption key that the current hop should use */ public SessionKey readLayerKey() { byte key[] = new byte[SessionKey.KEYSIZE_BYTES]; - System.arraycopy(_data.getData(), _data.getOffset() + OFF_LAYER_KEY, key, 0, SessionKey.KEYSIZE_BYTES); + System.arraycopy(_data, OFF_LAYER_KEY, key, 0, SessionKey.KEYSIZE_BYTES); return new SessionKey(key); } + /** * Tunnel IV encryption key that the current hop should use */ public SessionKey readIVKey() { byte key[] = new byte[SessionKey.KEYSIZE_BYTES]; - System.arraycopy(_data.getData(), _data.getOffset() + OFF_IV_KEY, key, 0, SessionKey.KEYSIZE_BYTES); + System.arraycopy(_data, OFF_IV_KEY, key, 0, SessionKey.KEYSIZE_BYTES); return new SessionKey(key); } + /** * Session key that should be used to encrypt the reply */ public SessionKey readReplyKey() { byte key[] = new byte[SessionKey.KEYSIZE_BYTES]; - System.arraycopy(_data.getData(), _data.getOffset() + OFF_REPLY_KEY, key, 0, SessionKey.KEYSIZE_BYTES); + System.arraycopy(_data, OFF_REPLY_KEY, key, 0, SessionKey.KEYSIZE_BYTES); return new SessionKey(key); } + /** * IV that should be used to encrypt the reply */ public byte[] readReplyIV() { byte iv[] = new byte[IV_SIZE]; - System.arraycopy(_data.getData(), _data.getOffset() + OFF_REPLY_IV, iv, 0, IV_SIZE); + System.arraycopy(_data, OFF_REPLY_IV, iv, 0, IV_SIZE); return iv; } + /** * The current hop is the inbound gateway. If this is true, it means anyone can send messages to * this tunnel, but if it is false, only the current predecessor can. * */ public boolean readIsInboundGateway() { - return (_data.getData()[_data.getOffset() + OFF_FLAG] & FLAG_UNRESTRICTED_PREV) != 0; + return (_data[OFF_FLAG] & FLAG_UNRESTRICTED_PREV) != 0; } + /** * The current hop is the outbound endpoint. If this is true, the next identity and next tunnel * fields refer to where the reply should be sent. */ public boolean readIsOutboundEndpoint() { - return (_data.getData()[_data.getOffset() + OFF_FLAG] & FLAG_OUTBOUND_ENDPOINT) != 0; + return (_data[OFF_FLAG] & FLAG_OUTBOUND_ENDPOINT) != 0; } + /** * Time that the request was sent (ms), truncated to the nearest hour */ public long readRequestTime() { - return DataHelper.fromLong(_data.getData(), _data.getOffset() + OFF_REQ_TIME, 4) * (60 * 60 * 1000L); + return DataHelper.fromLong(_data, OFF_REQ_TIME, 4) * (60 * 60 * 1000L); } + /** * What message ID should we send the request to the next hop with. If this is the outbound tunnel endpoint, * this specifies the message ID with which the reply should be sent. */ public long readReplyMessageId() { - return DataHelper.fromLong(_data.getData(), _data.getOffset() + OFF_SEND_MSG_ID, 4); + return DataHelper.fromLong(_data, OFF_SEND_MSG_ID, 4); } /** @@ -164,42 +175,43 @@ public class BuildRequestRecord { * bytes 0-15: truncated SHA-256 of the current hop's identity (the toPeer parameter) * bytes 15-527: ElGamal-2048 encrypted block * + * + * @return non-null */ - public void encryptRecord(I2PAppContext ctx, PublicKey toKey, Hash toPeer, byte out[], int outOffset) { - System.arraycopy(toPeer.getData(), 0, out, outOffset, PEER_SIZE); - byte preEncr[] = new byte[OFF_SEND_MSG_ID + 4 + PADDING_SIZE]; - System.arraycopy(_data.getData(), _data.getOffset(), preEncr, 0, preEncr.length); - byte encrypted[] = ctx.elGamalEngine().encrypt(preEncr, toKey); + public EncryptedBuildRecord encryptRecord(I2PAppContext ctx, PublicKey toKey, Hash toPeer) { + byte[] out = new byte[EncryptedBuildRecord.LENGTH]; + System.arraycopy(toPeer.getData(), 0, out, 0, PEER_SIZE); + byte encrypted[] = ctx.elGamalEngine().encrypt(_data, toKey); // the elg engine formats it kind of weird, giving 257 bytes for each part rather than 256, so // we want to strip out that excess byte and store it in the record - System.arraycopy(encrypted, 1, out, outOffset + PEER_SIZE, 256); - System.arraycopy(encrypted, 258, out, outOffset + 256 + PEER_SIZE, 256); + System.arraycopy(encrypted, 1, out, PEER_SIZE, 256); + System.arraycopy(encrypted, 258, out, 256 + PEER_SIZE, 256); + return new EncryptedBuildRecord(out); } /** * Decrypt the data from the specified record, writing the decrypted record into this instance's - * buffer (but not overwriting the array contained within the old buffer) + * data buffer + * + * Caller MUST check that first 16 bytes of our hash matches first 16 bytes of encryptedRecord + * before calling this. Not checked here. + * + * @throws DataFormatException on decrypt fail + * @since 0.9.18, was decryptRecord() */ - public boolean decryptRecord(I2PAppContext ctx, PrivateKey ourKey, Hash ourIdent, ByteArray encryptedRecord) { - if (DataHelper.eq(ourIdent.getData(), 0, encryptedRecord.getData(), encryptedRecord.getOffset(), PEER_SIZE)) { + public BuildRequestRecord(I2PAppContext ctx, PrivateKey ourKey, + EncryptedBuildRecord encryptedRecord) throws DataFormatException { byte preDecrypt[] = new byte[514]; - System.arraycopy(encryptedRecord.getData(), encryptedRecord.getOffset() + PEER_SIZE, preDecrypt, 1, 256); - System.arraycopy(encryptedRecord.getData(), encryptedRecord.getOffset() + PEER_SIZE + 256, preDecrypt, 258, 256); + System.arraycopy(encryptedRecord.getData(), PEER_SIZE, preDecrypt, 1, 256); + System.arraycopy(encryptedRecord.getData(), PEER_SIZE + 256, preDecrypt, 258, 256); byte decrypted[] = ctx.elGamalEngine().decrypt(preDecrypt, ourKey); if (decrypted != null) { - _data = new ByteArray(decrypted); - _data.setOffset(0); - return true; + _data = decrypted; } else { - return false; + throw new DataFormatException("decrypt fail"); } - } else { - return false; - } } - private static final int PADDING_SIZE = 29; - /** * Populate this instance with data. A new buffer is created to contain the data, with the * necessary randomized padding. @@ -215,14 +227,13 @@ public class BuildRequestRecord { * @param iv iv to be used when encrypting the reply to this build request * @param isInGateway are we the gateway of an inbound tunnel? * @param isOutEndpoint are we the endpoint of an outbound tunnel? + * @since 0.9.18, was createRecord() */ - public void createRecord(I2PAppContext ctx, long receiveTunnelId, Hash peer, long nextTunnelId, Hash nextHop, long nextMsgId, + public BuildRequestRecord(I2PAppContext ctx, long receiveTunnelId, Hash peer, long nextTunnelId, Hash nextHop, long nextMsgId, SessionKey layerKey, SessionKey ivKey, SessionKey replyKey, byte iv[], boolean isInGateway, boolean isOutEndpoint) { - if ( (_data == null) || (_data.getData() != null) ) - _data = new ByteArray(); - byte buf[] = new byte[OFF_SEND_MSG_ID+4+PADDING_SIZE]; - _data.setData(buf); + byte buf[] = new byte[LENGTH]; + _data = buf; /* bytes 0-3: tunnel ID to receive messages as * bytes 4-35: local router identity hash diff --git a/router/java/src/net/i2p/data/i2np/BuildResponseRecord.java b/router/java/src/net/i2p/data/i2np/BuildResponseRecord.java index 5b2ee944a..6087ef343 100644 --- a/router/java/src/net/i2p/data/i2np/BuildResponseRecord.java +++ b/router/java/src/net/i2p/data/i2np/BuildResponseRecord.java @@ -7,12 +7,17 @@ import net.i2p.data.SessionKey; //import net.i2p.util.Log; /** - * Read and write the reply to a tunnel build message record. + * Class that creates an encrypted tunnel build message record. * * The reply record is the same size as the request record (528 bytes). + * + * When decrypted: + * + *
  * Bytes 0-31 contain the hash of bytes 32-527
  * Bytes 32-526 contain random data.
  * Byte 527 contains the reply.
+ *
*/ public class BuildResponseRecord { @@ -20,10 +25,12 @@ public class BuildResponseRecord { * Create a new encrypted response * * @param status the response 0-255 + * @param replyIV 16 bytes * @param responseMessageId unused except for debugging * @return a 528-byte response record */ - public static byte[] create(I2PAppContext ctx, int status, SessionKey replyKey, byte replyIV[], long responseMessageId) { + public static EncryptedBuildRecord create(I2PAppContext ctx, int status, SessionKey replyKey, + byte replyIV[], long responseMessageId) { //Log log = ctx.logManager().getLog(BuildResponseRecord.class); byte rv[] = new byte[TunnelBuildReplyMessage.RECORD_SIZE]; ctx.random().nextBytes(rv, Hash.HASH_LENGTH, TunnelBuildReplyMessage.RECORD_SIZE - Hash.HASH_LENGTH - 1); @@ -35,6 +42,6 @@ public class BuildResponseRecord { ctx.aes().encrypt(rv, 0, rv, 0, replyKey, replyIV, rv.length); //if (log.shouldLog(Log.DEBUG)) // log.debug(responseMessageId + ": after encrypt: " + Base64.encode(rv, 0, 128)); - return rv; + return new EncryptedBuildRecord(rv); } } diff --git a/router/java/src/net/i2p/data/i2np/EncryptedBuildRecord.java b/router/java/src/net/i2p/data/i2np/EncryptedBuildRecord.java new file mode 100644 index 000000000..995f30d90 --- /dev/null +++ b/router/java/src/net/i2p/data/i2np/EncryptedBuildRecord.java @@ -0,0 +1,32 @@ +package net.i2p.data.i2np; + +/* + * free (adj.): unencumbered; not under the control of others + * No warranty of any kind, either expressed or implied. + */ + +import net.i2p.data.SimpleDataStructure; + +/** + * ElGamal-encrypted request or response. + * 528 bytes. Previously stored in a ByteArray. + * May or may not be AES layer-encrypted. + * + * Note that these are layer-encrypted and layer-decrypted in-place. + * Do not cache. + * + * @since 0.9.18 + */ +public class EncryptedBuildRecord extends SimpleDataStructure { + + public final static int LENGTH = TunnelBuildMessageBase.RECORD_SIZE; + + /** @throws IllegalArgumentException if data is not correct length (null is ok) */ + public EncryptedBuildRecord(byte data[]) { + super(data); + } + + public int length() { + return LENGTH; + } +} diff --git a/router/java/src/net/i2p/data/i2np/TunnelBuildMessageBase.java b/router/java/src/net/i2p/data/i2np/TunnelBuildMessageBase.java index bb3f579a3..81e349636 100644 --- a/router/java/src/net/i2p/data/i2np/TunnelBuildMessageBase.java +++ b/router/java/src/net/i2p/data/i2np/TunnelBuildMessageBase.java @@ -1,7 +1,6 @@ package net.i2p.data.i2np; import net.i2p.I2PAppContext; -import net.i2p.data.ByteArray; /** * Base for TBM, TBRM, VTBM, VTBRM @@ -18,7 +17,7 @@ import net.i2p.data.ByteArray; * @since 0.8.8 */ public abstract class TunnelBuildMessageBase extends I2NPMessageImpl { - protected ByteArray _records[]; + protected EncryptedBuildRecord _records[]; protected int RECORD_COUNT; public static final int MAX_RECORD_COUNT = 8; @@ -31,14 +30,14 @@ public abstract class TunnelBuildMessageBase extends I2NPMessageImpl { super(context); if (records > 0) { RECORD_COUNT = records; - _records = new ByteArray[records]; + _records = new EncryptedBuildRecord[records]; } // else will be initialized by readMessage() } - public void setRecord(int index, ByteArray record) { _records[index] = record; } + public void setRecord(int index, EncryptedBuildRecord record) { _records[index] = record; } - public ByteArray getRecord(int index) { return _records[index]; } + public EncryptedBuildRecord getRecord(int index) { return _records[index]; } /** @since 0.7.12 */ public int getRecordCount() { return RECORD_COUNT; } @@ -57,7 +56,7 @@ public abstract class TunnelBuildMessageBase extends I2NPMessageImpl { int off = offset + (i * RECORD_SIZE); byte rec[] = new byte[RECORD_SIZE]; System.arraycopy(data, off, rec, 0, RECORD_SIZE); - setRecord(i, new ByteArray(rec)); + setRecord(i, new EncryptedBuildRecord(rec)); } } @@ -66,7 +65,7 @@ public abstract class TunnelBuildMessageBase extends I2NPMessageImpl { if (remaining < 0) throw new I2NPMessageException("Not large enough (too short by " + remaining + ")"); for (int i = 0; i < RECORD_COUNT; i++) { - System.arraycopy(_records[i].getData(), _records[i].getOffset(), out, curIndex, RECORD_SIZE); + System.arraycopy(_records[i].getData(), 0, out, curIndex, RECORD_SIZE); curIndex += RECORD_SIZE; } return curIndex; diff --git a/router/java/src/net/i2p/data/i2np/VariableTunnelBuildMessage.java b/router/java/src/net/i2p/data/i2np/VariableTunnelBuildMessage.java index 9f7dea7a7..b33cba761 100644 --- a/router/java/src/net/i2p/data/i2np/VariableTunnelBuildMessage.java +++ b/router/java/src/net/i2p/data/i2np/VariableTunnelBuildMessage.java @@ -1,7 +1,6 @@ package net.i2p.data.i2np; import net.i2p.I2PAppContext; -import net.i2p.data.ByteArray; import net.i2p.data.DataHelper; /** @@ -36,7 +35,7 @@ public class VariableTunnelBuildMessage extends TunnelBuildMessage { RECORD_COUNT = r; if (dataSize != calculateWrittenLength()) throw new I2NPMessageException("Wrong length (expects " + calculateWrittenLength() + ", recv " + dataSize + ")"); - _records = new ByteArray[RECORD_COUNT]; + _records = new EncryptedBuildRecord[RECORD_COUNT]; super.readMessage(data, offset + 1, dataSize, type); } @@ -51,7 +50,7 @@ public class VariableTunnelBuildMessage extends TunnelBuildMessage { // can't call super, written length check will fail //return super.writeMessageBody(out, curIndex + 1); for (int i = 0; i < RECORD_COUNT; i++) { - System.arraycopy(_records[i].getData(), _records[i].getOffset(), out, curIndex, RECORD_SIZE); + System.arraycopy(_records[i].getData(), 0, out, curIndex, RECORD_SIZE); curIndex += RECORD_SIZE; } return curIndex; diff --git a/router/java/src/net/i2p/data/i2np/VariableTunnelBuildReplyMessage.java b/router/java/src/net/i2p/data/i2np/VariableTunnelBuildReplyMessage.java index fb7bc5b6a..368104a2a 100644 --- a/router/java/src/net/i2p/data/i2np/VariableTunnelBuildReplyMessage.java +++ b/router/java/src/net/i2p/data/i2np/VariableTunnelBuildReplyMessage.java @@ -1,7 +1,6 @@ package net.i2p.data.i2np; import net.i2p.I2PAppContext; -import net.i2p.data.ByteArray; import net.i2p.data.DataHelper; /** @@ -38,7 +37,7 @@ public class VariableTunnelBuildReplyMessage extends TunnelBuildReplyMessage { RECORD_COUNT = r; if (dataSize != calculateWrittenLength()) throw new I2NPMessageException("Wrong length (expects " + calculateWrittenLength() + ", recv " + dataSize + ")"); - _records = new ByteArray[RECORD_COUNT]; + _records = new EncryptedBuildRecord[RECORD_COUNT]; super.readMessage(data, offset + 1, dataSize, type); } @@ -53,7 +52,7 @@ public class VariableTunnelBuildReplyMessage extends TunnelBuildReplyMessage { // can't call super, written length check will fail //return super.writeMessageBody(out, curIndex + 1); for (int i = 0; i < RECORD_COUNT; i++) { - System.arraycopy(_records[i].getData(), _records[i].getOffset(), out, curIndex, RECORD_SIZE); + System.arraycopy(_records[i].getData(), 0, out, curIndex, RECORD_SIZE); curIndex += RECORD_SIZE; } return curIndex; diff --git a/router/java/src/net/i2p/router/tunnel/BuildMessageGenerator.java b/router/java/src/net/i2p/router/tunnel/BuildMessageGenerator.java index e2044305c..c6ddbb1b0 100644 --- a/router/java/src/net/i2p/router/tunnel/BuildMessageGenerator.java +++ b/router/java/src/net/i2p/router/tunnel/BuildMessageGenerator.java @@ -3,11 +3,11 @@ package net.i2p.router.tunnel; import java.util.List; import net.i2p.I2PAppContext; -import net.i2p.data.ByteArray; import net.i2p.data.Hash; import net.i2p.data.PublicKey; import net.i2p.data.SessionKey; import net.i2p.data.i2np.BuildRequestRecord; +import net.i2p.data.i2np.EncryptedBuildRecord; import net.i2p.data.i2np.I2NPMessage; import net.i2p.data.i2np.TunnelBuildMessage; @@ -54,33 +54,41 @@ public abstract class BuildMessageGenerator { * containing the hop's configuration (as well as the reply info, if it is an outbound endpoint) * * @param msg out parameter + * @throws IllegalArgumentException if hop bigger than config */ public static void createRecord(int recordNum, int hop, TunnelBuildMessage msg, TunnelCreatorConfig cfg, Hash replyRouter, long replyTunnel, I2PAppContext ctx, PublicKey peerKey) { - byte encrypted[] = new byte[TunnelBuildMessage.RECORD_SIZE]; //Log log = ctx.logManager().getLog(BuildMessageGenerator.class); + EncryptedBuildRecord erec; if (peerKey != null) { BuildRequestRecord req = null; if ( (!cfg.isInbound()) && (hop + 1 == cfg.getLength()) ) //outbound endpoint req = createUnencryptedRecord(ctx, cfg, hop, replyRouter, replyTunnel); else req = createUnencryptedRecord(ctx, cfg, hop, null, -1); + if (req == null) + throw new IllegalArgumentException("hop bigger than config"); Hash peer = cfg.getPeer(hop); //if (log.shouldLog(Log.DEBUG)) // log.debug("Record " + recordNum + "/" + hop + "/" + peer.toBase64() // + ": unencrypted = " + Base64.encode(req.getData().getData())); - req.encryptRecord(ctx, peerKey, peer, encrypted, 0); + erec = req.encryptRecord(ctx, peerKey, peer); //if (log.shouldLog(Log.DEBUG)) // log.debug("Record " + recordNum + "/" + hop + ": encrypted = " + Base64.encode(encrypted)); } else { //if (log.shouldLog(Log.DEBUG)) // log.debug("Record " + recordNum + "/" + hop + "/ is blank/random"); + byte encrypted[] = new byte[TunnelBuildMessage.RECORD_SIZE]; ctx.random().nextBytes(encrypted); + erec = new EncryptedBuildRecord(encrypted); } - msg.setRecord(recordNum, new ByteArray(encrypted)); + msg.setRecord(recordNum, erec); } + /** + * Returns null if hop >= cfg.length + */ private static BuildRequestRecord createUnencryptedRecord(I2PAppContext ctx, TunnelCreatorConfig cfg, int hop, Hash replyRouter, long replyTunnel) { //Log log = ctx.logManager().getLog(BuildMessageGenerator.class); @@ -111,11 +119,11 @@ public abstract class BuildMessageGenerator { SessionKey layerKey = hopConfig.getLayerKey(); SessionKey ivKey = hopConfig.getIVKey(); SessionKey replyKey = hopConfig.getReplyKey(); - byte iv[] = hopConfig.getReplyIV().getData(); - if ( (iv == null) || (iv.length != BuildRequestRecord.IV_SIZE) ) { + byte iv[] = hopConfig.getReplyIV(); + if (iv == null) { iv = new byte[BuildRequestRecord.IV_SIZE]; ctx.random().nextBytes(iv); - hopConfig.getReplyIV().setData(iv); + hopConfig.setReplyIV(iv); } boolean isInGW = (cfg.isInbound() && (hop == 0)); boolean isOutEnd = (!cfg.isInbound() && (hop + 1 >= cfg.getLength())); @@ -132,9 +140,9 @@ public abstract class BuildMessageGenerator { // log.debug("Hop " + hop + " has the next message ID of " + nextMsgId + " for " + cfg // + " with replyKey " + replyKey.toBase64() + " and replyIV " + Base64.encode(iv)); - BuildRequestRecord rec= new BuildRequestRecord(); - rec.createRecord(ctx, recvTunnelId, peer, nextTunnelId, nextPeer, nextMsgId, layerKey, ivKey, replyKey, - iv, isInGW, isOutEnd); + BuildRequestRecord rec= new BuildRequestRecord(ctx, recvTunnelId, peer, nextTunnelId, nextPeer, + nextMsgId, layerKey, ivKey, replyKey, + iv, isInGW, isOutEnd); return rec; } else { @@ -143,7 +151,11 @@ public abstract class BuildMessageGenerator { } /** - * Encrypt the records so their hop ident is visible at the appropriate times + * Encrypt the records so their hop ident is visible at the appropriate times. + * + * Note that this layer-encrypts the build records for the message in-place. + * Only call this onece for a given message. + * * @param order list of hop #s as Integers. For instance, if (order.get(1) is 4), it is peer cfg.getPeer(4) */ public static void layeredEncrypt(I2PAppContext ctx, TunnelBuildMessage msg, @@ -151,7 +163,7 @@ public abstract class BuildMessageGenerator { //Log log = ctx.logManager().getLog(BuildMessageGenerator.class); // encrypt the records so that the right elements will be visible at the right time for (int i = 0; i < msg.getRecordCount(); i++) { - ByteArray rec = msg.getRecord(i); + EncryptedBuildRecord rec = msg.getRecord(i); Integer hopNum = order.get(i); int hop = hopNum.intValue(); if ( (isBlank(cfg, hop)) || (!cfg.isInbound() && hop == 1) ) { @@ -166,12 +178,12 @@ public abstract class BuildMessageGenerator { for (int j = hop-1; j >= stop; j--) { HopConfig hopConfig = cfg.getConfig(j); SessionKey key = hopConfig.getReplyKey(); - byte iv[] = hopConfig.getReplyIV().getData(); - int off = rec.getOffset(); + byte iv[] = hopConfig.getReplyIV(); //if (log.shouldLog(Log.DEBUG)) // log.debug(msg.getUniqueId() + ": pre-decrypting record " + i + "/" + hop + " for " + cfg // + " with " + key.toBase64() + "/" + Base64.encode(iv)); - ctx.aes().decrypt(rec.getData(), off, rec.getData(), off, key, iv, TunnelBuildMessage.RECORD_SIZE); + // corrupts the SDS + ctx.aes().decrypt(rec.getData(), 0, rec.getData(), 0, key, iv, TunnelBuildMessage.RECORD_SIZE); } } //if (log.shouldLog(Log.DEBUG)) diff --git a/router/java/src/net/i2p/router/tunnel/BuildMessageProcessor.java b/router/java/src/net/i2p/router/tunnel/BuildMessageProcessor.java index 1a2c6a5b8..7b649e73c 100644 --- a/router/java/src/net/i2p/router/tunnel/BuildMessageProcessor.java +++ b/router/java/src/net/i2p/router/tunnel/BuildMessageProcessor.java @@ -2,12 +2,13 @@ package net.i2p.router.tunnel; import net.i2p.I2PAppContext; import net.i2p.data.Base64; -import net.i2p.data.ByteArray; +import net.i2p.data.DataFormatException; import net.i2p.data.DataHelper; import net.i2p.data.Hash; import net.i2p.data.PrivateKey; import net.i2p.data.SessionKey; import net.i2p.data.i2np.BuildRequestRecord; +import net.i2p.data.i2np.EncryptedBuildRecord; import net.i2p.data.i2np.TunnelBuildMessage; import net.i2p.router.util.DecayingBloomFilter; import net.i2p.router.util.DecayingHashSet; @@ -32,7 +33,10 @@ public class BuildMessageProcessor { * message (so that the reply can be placed in that position after going through the decrypted * request record). * - * @return the current hop's decrypted record + * Note that this layer-decrypts the build records in-place. + * Do not call this more than once for a given message. + * + * @return the current hop's decrypted record or null on failure */ public BuildRequestRecord decrypt(I2PAppContext ctx, TunnelBuildMessage msg, Hash ourHash, PrivateKey privKey) { Log log = ctx.logManager().getLog(getClass()); @@ -44,35 +48,33 @@ public class BuildMessageProcessor { long totalDup = 0; long beforeLoop = System.currentTimeMillis(); for (int i = 0; i < msg.getRecordCount(); i++) { - ByteArray rec = msg.getRecord(i); - int off = rec.getOffset(); + EncryptedBuildRecord rec = msg.getRecord(i); int len = BuildRequestRecord.PEER_SIZE; long beforeEq = System.currentTimeMillis(); - boolean eq = DataHelper.eq(ourHash.getData(), 0, rec.getData(), off, len); + boolean eq = DataHelper.eq(ourHash.getData(), 0, rec.getData(), 0, len); totalEq += System.currentTimeMillis()-beforeEq; if (eq) { long beforeIsDup = System.currentTimeMillis(); - boolean isDup = _filter.add(rec.getData(), off + len, 32); + boolean isDup = _filter.add(rec.getData(), len, 32); totalDup += System.currentTimeMillis()-beforeIsDup; if (isDup) { if (log.shouldLog(Log.WARN)) log.debug(msg.getUniqueId() + ": A record matching our hash was found, but it seems to be a duplicate"); - ctx.statManager().addRateData("tunnel.buildRequestDup", 1, 0); + ctx.statManager().addRateData("tunnel.buildRequestDup", 1); return null; } - BuildRequestRecord req = new BuildRequestRecord(); beforeActualDecrypt = System.currentTimeMillis(); - boolean ok = req.decryptRecord(ctx, privKey, ourHash, rec); - afterActualDecrypt = System.currentTimeMillis(); - if (ok) { + try { + BuildRequestRecord req = new BuildRequestRecord(ctx, privKey, rec); if (log.shouldLog(Log.DEBUG)) log.debug(msg.getUniqueId() + ": A record matching our hash was found and decrypted"); rv = req; - } else { + } catch (DataFormatException dfe) { if (log.shouldLog(Log.DEBUG)) log.debug(msg.getUniqueId() + ": A record matching our hash was found, but could not be decrypted"); return null; // our hop is invalid? b0rkage } + afterActualDecrypt = System.currentTimeMillis(); ourHop = i; } } @@ -89,11 +91,12 @@ public class BuildMessageProcessor { int ivOff = 0; for (int i = 0; i < msg.getRecordCount(); i++) { if (i != ourHop) { - ByteArray data = msg.getRecord(i); + EncryptedBuildRecord data = msg.getRecord(i); if (log.shouldLog(Log.DEBUG)) - log.debug("Encrypting record " + i + "/?/" + data.getOffset() + "/" + data.getValid() + " with replyKey " + replyKey.toBase64() + "/" + Base64.encode(iv, ivOff, 16)); - ctx.aes().encrypt(data.getData(), data.getOffset(), data.getData(), data.getOffset(), replyKey, - iv, ivOff, data.getValid()); + log.debug("Encrypting record " + i + "/? with replyKey " + replyKey.toBase64() + "/" + Base64.encode(iv, ivOff, 16)); + // corrupts SDS + ctx.aes().encrypt(data.getData(), 0, data.getData(), 0, replyKey, + iv, ivOff, data.length()); } } long afterEncrypt = System.currentTimeMillis(); diff --git a/router/java/src/net/i2p/router/tunnel/BuildReplyHandler.java b/router/java/src/net/i2p/router/tunnel/BuildReplyHandler.java index b2c7c84b3..2196a851e 100644 --- a/router/java/src/net/i2p/router/tunnel/BuildReplyHandler.java +++ b/router/java/src/net/i2p/router/tunnel/BuildReplyHandler.java @@ -4,10 +4,10 @@ import java.util.List; import net.i2p.I2PAppContext; import net.i2p.data.Base64; -import net.i2p.data.ByteArray; import net.i2p.data.DataHelper; import net.i2p.data.Hash; import net.i2p.data.SessionKey; +import net.i2p.data.i2np.EncryptedBuildRecord; import net.i2p.data.i2np.TunnelBuildReplyMessage; import net.i2p.util.Log; import net.i2p.util.SimpleByteCache; @@ -34,6 +34,9 @@ public class BuildReplyHandler { * Decrypt the tunnel build reply records. This overwrites the contents of the reply. * Thread safe (no state). * + * Note that this layer-decrypts the build records in-place. + * Do not call this more than once for a given message. + * * @return status for the records (in record order), or null if the replies were not valid. Fake records * always have 0 as their value */ @@ -70,7 +73,10 @@ public class BuildReplyHandler { /** * Decrypt the record (removing the layers of reply encyption) and read out the status * - * @return -1 on decrypt failure + * Note that this layer-decrypts the build records in-place. + * Do not call this more than once for a given message. + * + * @return the status 0-255, or -1 on decrypt failure */ private int decryptRecord(TunnelBuildReplyMessage reply, TunnelCreatorConfig cfg, int recordNum, int hop) { if (BuildMessageGenerator.isBlank(cfg, hop)) { @@ -78,9 +84,8 @@ public class BuildReplyHandler { log.debug(reply.getUniqueId() + ": Record " + recordNum + "/" + hop + " is fake, so consider it valid..."); return 0; } - ByteArray rec = reply.getRecord(recordNum); + EncryptedBuildRecord rec = reply.getRecord(recordNum); byte[] data = rec.getData(); - int off = rec.getOffset(); int start = cfg.getLength() - 1; if (cfg.isInbound()) start--; // the last hop in an inbound tunnel response doesn't actually encrypt @@ -88,35 +93,34 @@ public class BuildReplyHandler { for (int j = start; j >= hop; j--) { HopConfig hopConfig = cfg.getConfig(j); SessionKey replyKey = hopConfig.getReplyKey(); - byte replyIV[] = hopConfig.getReplyIV().getData(); - int replyIVOff = hopConfig.getReplyIV().getOffset(); + byte replyIV[] = hopConfig.getReplyIV(); if (log.shouldLog(Log.DEBUG)) { log.debug(reply.getUniqueId() + ": Decrypting record " + recordNum + "/" + hop + "/" + j + " with replyKey " - + replyKey.toBase64() + "/" + Base64.encode(replyIV, replyIVOff, 16) + ": " + cfg); - log.debug(reply.getUniqueId() + ": before decrypt("+ off + "-"+(off+rec.getValid())+"): " + Base64.encode(data, off, rec.getValid())); - log.debug(reply.getUniqueId() + ": Full reply rec: offset=" + off + ", sz=" + data.length + "/" + rec.getValid() + ", data=" + Base64.encode(data, off, TunnelBuildReplyMessage.RECORD_SIZE)); + + replyKey.toBase64() + "/" + Base64.encode(replyIV) + ": " + cfg); + log.debug(reply.getUniqueId() + ": before decrypt: " + Base64.encode(data)); + log.debug(reply.getUniqueId() + ": Full reply rec: sz=" + data.length + " data=" + Base64.encode(data, 0, TunnelBuildReplyMessage.RECORD_SIZE)); } - ctx.aes().decrypt(data, off, data, off, replyKey, replyIV, replyIVOff, TunnelBuildReplyMessage.RECORD_SIZE); + ctx.aes().decrypt(data, 0, data, 0, replyKey, replyIV, 0, TunnelBuildReplyMessage.RECORD_SIZE); if (log.shouldLog(Log.DEBUG)) - log.debug(reply.getUniqueId() + ": after decrypt: " + Base64.encode(data, off, rec.getValid())); + log.debug(reply.getUniqueId() + ": after decrypt: " + Base64.encode(data)); } // ok, all of the layered encryption is stripped, so lets verify it // (formatted per BuildResponseRecord.create) // don't cache the result //Hash h = ctx.sha().calculateHash(data, off + Hash.HASH_LENGTH, TunnelBuildReplyMessage.RECORD_SIZE-Hash.HASH_LENGTH); byte[] h = SimpleByteCache.acquire(Hash.HASH_LENGTH); - ctx.sha().calculateHash(data, off + Hash.HASH_LENGTH, TunnelBuildReplyMessage.RECORD_SIZE-Hash.HASH_LENGTH, h, 0); - boolean ok = DataHelper.eq(h, 0, data, off, Hash.HASH_LENGTH); + ctx.sha().calculateHash(data, Hash.HASH_LENGTH, TunnelBuildReplyMessage.RECORD_SIZE-Hash.HASH_LENGTH, h, 0); + boolean ok = DataHelper.eq(h, 0, data, 0, Hash.HASH_LENGTH); if (!ok) { if (log.shouldLog(Log.DEBUG)) log.debug(reply.getUniqueId() + ": Failed verification on " + recordNum + "/" + hop + ": " + Base64.encode(h) + " calculated, " + - Base64.encode(data, off, Hash.HASH_LENGTH) + " expected\n" + - "Record: " + Base64.encode(data, off+Hash.HASH_LENGTH, TunnelBuildReplyMessage.RECORD_SIZE-Hash.HASH_LENGTH)); + Base64.encode(data, 0, Hash.HASH_LENGTH) + " expected\n" + + "Record: " + Base64.encode(data, Hash.HASH_LENGTH, TunnelBuildReplyMessage.RECORD_SIZE-Hash.HASH_LENGTH)); SimpleByteCache.release(h); return -1; } else { SimpleByteCache.release(h); - int rv = (int)DataHelper.fromLong(data, off + TunnelBuildReplyMessage.RECORD_SIZE - 1, 1); + int rv = (int)DataHelper.fromLong(data, TunnelBuildReplyMessage.RECORD_SIZE - 1, 1); if (log.shouldLog(Log.DEBUG)) log.debug(reply.getUniqueId() + ": Verified: " + rv + " for record " + recordNum + "/" + hop); return rv; diff --git a/router/java/src/net/i2p/router/tunnel/HopConfig.java b/router/java/src/net/i2p/router/tunnel/HopConfig.java index 4c40a1727..447f6a083 100644 --- a/router/java/src/net/i2p/router/tunnel/HopConfig.java +++ b/router/java/src/net/i2p/router/tunnel/HopConfig.java @@ -1,6 +1,5 @@ package net.i2p.router.tunnel; -import net.i2p.data.ByteArray; import net.i2p.data.DataHelper; import net.i2p.data.Hash; import net.i2p.data.SessionKey; @@ -20,7 +19,7 @@ public class HopConfig { private SessionKey _layerKey; private SessionKey _ivKey; private SessionKey _replyKey; - private ByteArray _replyIV; + private byte[] _replyIV; private long _creation; private long _expiration; //private Map _options; @@ -87,9 +86,23 @@ public class HopConfig { public SessionKey getReplyKey() { return _replyKey; } public void setReplyKey(SessionKey key) { _replyKey = key; } - /** iv used to encrypt the reply sent for the new tunnel creation crypto */ - public ByteArray getReplyIV() { return _replyIV; } - public void setReplyIV(ByteArray iv) { _replyIV = iv; } + /** + * IV used to encrypt the reply sent for the new tunnel creation crypto + * + * @return 16 bytes + */ + public byte[] getReplyIV() { return _replyIV; } + + /** + * IV used to encrypt the reply sent for the new tunnel creation crypto + * + * @throws IllegalArgumentException if not 16 bytes + */ + public void setReplyIV(byte[] iv) { + if (iv.length != REPLY_IV_LENGTH) + throw new IllegalArgumentException(); + _replyIV = iv; + } /** when does this tunnel expire (in ms since the epoch)? */ public long getExpiration() { return _expiration; } diff --git a/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java b/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java index 0de09ff24..cf3cb1053 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java +++ b/router/java/src/net/i2p/router/tunnel/pool/BuildHandler.java @@ -6,7 +6,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import net.i2p.data.Base64; -import net.i2p.data.ByteArray; import net.i2p.data.DataHelper; import net.i2p.data.Hash; import net.i2p.data.router.RouterIdentity; @@ -14,6 +13,7 @@ import net.i2p.data.router.RouterInfo; import net.i2p.data.TunnelId; import net.i2p.data.i2np.BuildRequestRecord; import net.i2p.data.i2np.BuildResponseRecord; +import net.i2p.data.i2np.EncryptedBuildRecord; import net.i2p.data.i2np.I2NPMessage; import net.i2p.data.i2np.TunnelBuildMessage; import net.i2p.data.i2np.TunnelBuildReplyMessage; @@ -782,13 +782,13 @@ class BuildHandler implements Runnable { return; } - byte reply[] = BuildResponseRecord.create(_context, response, req.readReplyKey(), req.readReplyIV(), state.msg.getUniqueId()); + EncryptedBuildRecord reply = BuildResponseRecord.create(_context, response, req.readReplyKey(), req.readReplyIV(), state.msg.getUniqueId()); int records = state.msg.getRecordCount(); int ourSlot = -1; for (int j = 0; j < records; j++) { if (state.msg.getRecord(j) == null) { ourSlot = j; - state.msg.setRecord(j, new ByteArray(reply)); + state.msg.setRecord(j, reply); //if (_log.shouldLog(Log.DEBUG)) // _log.debug("Full reply record for slot " + ourSlot + "/" + ourId + "/" + nextId + "/" + req.readReplyMessageId() // + ": " + Base64.encode(reply)); diff --git a/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java b/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java index 31aaa8665..76ccd0561 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java +++ b/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import net.i2p.data.ByteArray; import net.i2p.data.DataHelper; import net.i2p.data.Hash; import net.i2p.data.PublicKey; @@ -90,7 +89,7 @@ abstract class BuildRequestor { cfg.getConfig(i-1).setSendTunnelId(cfg.getConfig(i).getReceiveTunnelId()); byte iv[] = new byte[16]; ctx.random().nextBytes(iv); - cfg.getConfig(i).setReplyIV(new ByteArray(iv)); + cfg.getConfig(i).setReplyIV(iv); cfg.getConfig(i).setReplyKey(ctx.keyGenerator().generateSessionKey()); } // This is in BuildExecutor.buildTunnel() now From 31cc0764a9a7b98c0a0afc39f7fd7347f5debaf8 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 18 Nov 2014 14:49:23 +0000 Subject: [PATCH 02/13] Logger: Configurable flush interval --- core/java/src/net/i2p/util/LogManager.java | 21 +++++++++++++++++++++ core/java/src/net/i2p/util/LogWriter.java | 16 ++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/core/java/src/net/i2p/util/LogManager.java b/core/java/src/net/i2p/util/LogManager.java index 871037b65..85d948c53 100644 --- a/core/java/src/net/i2p/util/LogManager.java +++ b/core/java/src/net/i2p/util/LogManager.java @@ -61,6 +61,8 @@ public class LogManager { private static final String PROP_DROP = "logger.dropOnOverflow"; /** @since 0.9.3 */ private static final String PROP_DUP = "logger.dropDuplicates"; + /** @since 0.9.18 */ + private static final String PROP_FLUSH = "logger.flushInterval"; public final static String PROP_RECORD_PREFIX = "logger.record."; public final static String DEFAULT_FORMAT = DATE + " " + PRIORITY + " [" + THREAD + "] " + CLASS + ": " + MESSAGE; @@ -125,6 +127,8 @@ public class LogManager { private boolean _dropOnOverflow; private boolean _dropDuplicates; private final AtomicLong _droppedRecords = new AtomicLong(); + // in seconds + private int _flushInterval = (int) (LogWriter.FLUSH_INTERVAL / 1000); private boolean _alreadyNoticedMissingConfig; @@ -160,6 +164,7 @@ public class LogManager { if (_writer != null) return; _writer = new LogWriter(this); + _writer.setFlushInterval(_flushInterval * 1000); // if you enable logging in I2PThread again, you MUST change this back to Thread Thread t = new I2PThread(_writer, "LogWriter"); t.setDaemon(true); @@ -269,6 +274,10 @@ public class LogManager { try { _records.put(record); } catch (InterruptedException ie) {} + } else if (_flushInterval <= 0) { + synchronized (_writer) { + _writer.notifyAll(); + } } } @@ -384,6 +393,17 @@ public class LogManager { _logBufferSize = Integer.parseInt(str); } catch (NumberFormatException nfe) {} + try { + String str = config.getProperty(PROP_FLUSH); + if (str != null) { + _flushInterval = Integer.parseInt(str); + synchronized(this) { + if (_writer != null) + _writer.setFlushInterval(_flushInterval * 1000); + } + } + } catch (NumberFormatException nfe) {} + _dropOnOverflow = Boolean.parseBoolean(config.getProperty(PROP_DROP)); String str = config.getProperty(PROP_DUP); _dropDuplicates = str == null || Boolean.parseBoolean(str); @@ -647,6 +667,7 @@ public class LogManager { rv.setProperty(PROP_DEFAULTLEVEL, Log.toLevelString(_defaultLimit)); rv.setProperty(PROP_DISPLAYONSCREENLEVEL, Log.toLevelString(_onScreenLimit)); rv.setProperty(PROP_CONSOLEBUFFERSIZE, Integer.toString(_consoleBufferSize)); + rv.setProperty(PROP_FLUSH, Integer.toString(_flushInterval)); for (LogLimit lim : _limits) { rv.setProperty(PROP_RECORD_PREFIX + lim.getRootName(), Log.toLevelString(lim.getLimit())); diff --git a/core/java/src/net/i2p/util/LogWriter.java b/core/java/src/net/i2p/util/LogWriter.java index 8f47b0d0e..1d24e34ef 100644 --- a/core/java/src/net/i2p/util/LogWriter.java +++ b/core/java/src/net/i2p/util/LogWriter.java @@ -25,7 +25,9 @@ import java.util.Queue; class LogWriter implements Runnable { /** every 10 seconds? why? Just have the gui force a reread after a change?? */ private final static long CONFIG_READ_INTERVAL = 50 * 1000; - private final static long FLUSH_INTERVAL = 29 * 1000; + final static long FLUSH_INTERVAL = 29 * 1000; + private final static long MIN_FLUSH_INTERVAL = 2*1000; + private final static long MAX_FLUSH_INTERVAL = 5*60*1000; private long _lastReadConfig; private long _numBytesInCurrentFile; // volatile as it changes on log file rotation @@ -38,6 +40,8 @@ class LogWriter implements Runnable { private static final int MAX_DISKFULL_MESSAGES = 8; private int _diskFullMessageCount; private LogRecord _last; + // ms + private volatile long _flushInterval = FLUSH_INTERVAL; public LogWriter(LogManager manager) { _manager = manager; @@ -47,6 +51,14 @@ class LogWriter implements Runnable { public void stopWriting() { _write = false; } + + /** + * @param ms + * @since 0.9.18 + */ + public void setFlushInterval(long interval) { + _flushInterval = Math.min(MAX_FLUSH_INTERVAL, Math.max(MIN_FLUSH_INTERVAL, interval)); + } public void run() { _write = true; @@ -109,7 +121,7 @@ class LogWriter implements Runnable { if (shouldWait) { try { synchronized (this) { - this.wait(FLUSH_INTERVAL); + this.wait(_flushInterval); } } catch (InterruptedException ie) { // nop } From a16d17c422f5c3fe4e33a1ac545487c8468dc800 Mon Sep 17 00:00:00 2001 From: zzz Date: Mon, 24 Nov 2014 18:36:16 +0000 Subject: [PATCH 03/13] SusiMail: Add save-as button Fix encoding for filename in Content-Disposition header New icon from Silk, same license as the others --- apps/susimail/src/icons/drive_edit.png | Bin 0 -> 714 bytes .../src/src/i2p/susi/webmail/Mail.java | 8 ++ .../src/src/i2p/susi/webmail/WebMail.java | 85 +++++++++++++++++- .../themes/susimail/dark/susimail.css | 5 ++ .../themes/susimail/light/susimail.css | 6 ++ 5 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 apps/susimail/src/icons/drive_edit.png diff --git a/apps/susimail/src/icons/drive_edit.png b/apps/susimail/src/icons/drive_edit.png new file mode 100644 index 0000000000000000000000000000000000000000..7923fada4bed2af85dfce000bcc9b9ef6793897d GIT binary patch literal 714 zcmV;*0yX`KP)NDEf2-s&}Y^ic31co00R|9}S-FA@q~ z1q;#;e^rr&XrtIjwGF1}(zLNg!zP<;)}57vSQHN(80NjleBU?o-Wz5Z26kf{c4@Zy zBArgpZf|dgO^2@Qs8*{D*EFr+l}d$TEE0)~5{^*|CitW%it(aPq3QK719D!Rt zgstUQNX^{WYy>KmO33H))&)^=tyTkFCPmsiYYBvUG&l|%N8$5Fh)jdi)eU#&9vcya zNr{LEWVU3L%yGY8*w?y(qr)xm^p2ylHVv_jA@g<-Rh>ihotZ|e)GIj=)SdPn!V<=Y z1$g?-qMW+}CVYe7aDrEJsO+D>X32n!2504-Zjlga{h}HSj&$6nW`5U+~3(a=n wM2C8~*KZr{mthE%IdwgMapK0pmw!#;FKVxMw^?5FRsaA107*qoM6N<$f)5QvuK)l5 literal 0 HcmV?d00001 diff --git a/apps/susimail/src/src/i2p/susi/webmail/Mail.java b/apps/susimail/src/src/i2p/susi/webmail/Mail.java index 95e0829bf..8632ac9c4 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/Mail.java +++ b/apps/susimail/src/src/i2p/susi/webmail/Mail.java @@ -88,6 +88,10 @@ class Mail { error = ""; } + /** + * This may or may not contain the body also. + * @return may be null + */ public synchronized ReadBuffer getHeader() { return header; } @@ -103,6 +107,10 @@ class Mail { return header != null; } + /** + * This contains the header also. + * @return may be null + */ public synchronized ReadBuffer getBody() { return body; } diff --git a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java index c3dbd78c3..ad665e0b6 100644 --- a/apps/susimail/src/src/i2p/susi/webmail/WebMail.java +++ b/apps/susimail/src/src/i2p/susi/webmail/WebMail.java @@ -116,6 +116,7 @@ public class WebMail extends HttpServlet private static final String LOGOUT = "logout"; private static final String RELOAD = "reload"; private static final String SAVE = "save"; + private static final String SAVE_AS = "saveas"; private static final String REFRESH = "refresh"; private static final String CONFIGURE = "configure"; private static final String NEW = "new"; @@ -1298,6 +1299,33 @@ public class WebMail extends HttpServlet return isRaw; } + + /** + * Process save-as link in message view + * + * @param sessionObject + * @param request + * @return If true, we sent the file or 404, do not send any other response + * @since 0.9.18 + */ + private static boolean processSaveAsLink(SessionObject sessionObject, RequestWrapper request, HttpServletResponse response) + { + String str = request.getParameter(SAVE_AS); + if( str == null ) + return false; + Mail mail = sessionObject.mailCache.getMail( sessionObject.showUIDL, MailCache.FetchMode.ALL ); + if( mail != null ) { + if (sendMailSaveAs(sessionObject, mail, response)) + return true; + } + // error if we get here + sessionObject.error += _("Message not found."); + try { + response.sendError(404, _("Message not found.")); + } catch (IOException ioe) {} + return true; + } + /** * @param hashCode * @return the part or null @@ -1631,6 +1659,10 @@ public class WebMail extends HttpServlet // download or raw view sent, or 404 return; } + if (processSaveAsLink(sessionObject, request, response)) { + // download or sent, or 404 + return; + } // If the last message has just been deleted then // sessionObject.state = STATE_LIST and // sessionObject.showUIDL = null @@ -1790,7 +1822,7 @@ public class WebMail extends HttpServlet name = part.name; else name = "part" + part.hashCode(); - String name2 = name.replace( "\\.", "_" ); + String name2 = sanitizeFilename(name); response.setContentType( "application/zip; name=\"" + name2 + ".zip\"" ); response.addHeader( "Content-Disposition:", "attachment; filename=\"" + name2 + ".zip\"" ); ZipEntry entry = new ZipEntry( name ); @@ -1809,6 +1841,54 @@ public class WebMail extends HttpServlet } return shown; } + + /** + * Send the mail to be saved by the browser + * + * @param sessionObject + * @param response + * @return success + * @since 0.9.18 + */ + private static boolean sendMailSaveAs(SessionObject sessionObject, Mail mail, + HttpServletResponse response) + { + ReadBuffer content = mail.getBody(); + + if(content == null) + return false; + String name = mail.subject != null ? sanitizeFilename(mail.subject) : "message"; + try { + response.setContentType("message/rfc822"); + response.setContentLength(content.length); + // cache-control? + response.addHeader( "Content-Disposition:", "attachment; filename=\"" + name + ".eml\"" ); + response.getOutputStream().write(content.content, content.offset, content.length); + return true; + } catch (IOException e) { + e.printStackTrace(); + return false; + } + } + + /** + * Convert the UTF-8 to ISO-8859-1 suitable for inclusion in a header. + * This will result in a bunch of ??? for non-Western languages. + * + * @param sessionObject + * @param response + * @return success + * @since 0.9.18 + */ + private static String sanitizeFilename(String name) { + try { + name = new String(name.getBytes("ISO-8859-1"), "ISO-8859-1"); + } catch( UnsupportedEncodingException uee ) {} + // strip control chars? + name = name.replace('"', '_'); + return name; + } + /** * @param sessionObject * @param request @@ -2255,7 +2335,8 @@ public class WebMail extends HttpServlet out.println( button( NEW, _("New") ) + spacer + button( REPLY, _("Reply") ) + button( REPLYALL, _("Reply All") ) + - button( FORWARD, _("Forward") ) + spacer); + button( FORWARD, _("Forward") ) + spacer + + button( SAVE_AS, _("Save As") ) + spacer); if (sessionObject.reallyDelete) out.println(button2(DELETE, _("Delete"))); else diff --git a/installer/resources/themes/susimail/dark/susimail.css b/installer/resources/themes/susimail/dark/susimail.css index a46a0e586..0d49e6540 100644 --- a/installer/resources/themes/susimail/dark/susimail.css +++ b/installer/resources/themes/susimail/dark/susimail.css @@ -252,6 +252,11 @@ input.configure { min-height: 22px; } +input.saveas { + background: #000 url('/susimail/icons/drive_edit.png') no-repeat 2px center; + min-height: 22px; +} + input[type=file], input.new_upload { background: #000 url('/themes/console/images/add.png') no-repeat 2px center; min-height: 22px; diff --git a/installer/resources/themes/susimail/light/susimail.css b/installer/resources/themes/susimail/light/susimail.css index afb2d082a..df3d749fa 100644 --- a/installer/resources/themes/susimail/light/susimail.css +++ b/installer/resources/themes/susimail/light/susimail.css @@ -274,6 +274,12 @@ input.configure { min-height: 22px; } +input.saveas { + background: #ddf url('/susimail/icons/drive_edit.png') no-repeat 4px center; + padding: 2px 3px 2px 24px; + min-height: 22px; +} + input[type=file], input.new_upload { background: #ddf url('/themes/console/images/add.png') no-repeat 4px center; padding: 2px 3px 2px 24px; From 5383f9f097d221e578d213a098a74eb53b04ea96 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 25 Nov 2014 14:25:42 +0000 Subject: [PATCH 04/13] Profiles: Change slice selection argument from an int to an enum for clarity --- .../router/peermanager/ProfileOrganizer.java | 45 +++++++++++++------ .../tunnel/pool/ClientPeerSelector.java | 7 +-- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java index ca2829b15..7ebc4ce93 100644 --- a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java +++ b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java @@ -367,6 +367,29 @@ public class ProfileOrganizer { return; } + /** + * Replaces integer subTierMode argument, for clarity + * + * @since 0.9.18 + */ + public enum Slice { + + SLICE_ALL(0x00, 0), + SLICE_0_1(0x02, 0), + SLICE_2_3(0x02, 2), + SLICE_0(0x03, 0), + SLICE_1(0x03, 1), + SLICE_2(0x03, 2), + SLICE_3(0x03, 3); + + final int mask, val; + + Slice(int mask, int val) { + this.mask = mask; + this.val = val; + } + } + /** * Return a set of Hashes for peers that are both fast and reliable. If an insufficient * number of peers are both fast and reliable, fall back onto high capacity peers, and if that @@ -388,15 +411,15 @@ public class ProfileOrganizer { * 7: return only from group 3 * */ - public void selectFastPeers(int howMany, Set exclude, Set matches, Hash randomKey, int subTierMode) { + public void selectFastPeers(int howMany, Set exclude, Set matches, Hash randomKey, Slice subTierMode) { getReadLock(); try { - if (subTierMode > 0) { + if (subTierMode != Slice.SLICE_ALL) { int sz = _fastPeers.size(); - if (sz < 6 || (subTierMode >= 4 && sz < 12)) - subTierMode = 0; + if (sz < 6 || (subTierMode.mask >= 3 && sz < 12)) + subTierMode = Slice.SLICE_ALL; } - if (subTierMode > 0) + if (subTierMode != Slice.SLICE_ALL) locked_selectPeers(_fastPeers, howMany, exclude, matches, randomKey, subTierMode); else locked_selectPeers(_fastPeers, howMany, exclude, matches, 2); @@ -1302,7 +1325,8 @@ public class ProfileOrganizer { * 7: return only from group 3 * */ - private void locked_selectPeers(Map peers, int howMany, Set toExclude, Set matches, Hash randomKey, int subTierMode) { + private void locked_selectPeers(Map peers, int howMany, Set toExclude, + Set matches, Hash randomKey, Slice subTierMode) { List all = new ArrayList(peers.keySet()); // use RandomIterator to avoid shuffling the whole thing for (Iterator iter = new RandomIterator(all); (matches.size() < howMany) && iter.hasNext(); ) { @@ -1314,13 +1338,8 @@ public class ProfileOrganizer { if (_us.equals(peer)) continue; int subTier = getSubTier(peer, randomKey); - if (subTierMode >= 4) { - if (subTier != (subTierMode & 0x03)) - continue; - } else { - if ((subTier >> 1) != (subTierMode & 0x01)) - continue; - } + if ((subTier & subTierMode.mask) != subTierMode.val) + continue; boolean ok = isSelectable(peer); if (ok) matches.add(peer); diff --git a/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java b/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java index 2e13c4c57..a96504b7c 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java +++ b/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java @@ -8,6 +8,7 @@ import java.util.Set; import net.i2p.data.Hash; import net.i2p.router.RouterContext; import net.i2p.router.TunnelPoolSettings; +import static net.i2p.router.peermanager.ProfileOrganizer.Slice.*; /** * Pick peers randomly out of the fast pool, and put them into tunnels @@ -49,7 +50,7 @@ class ClientPeerSelector extends TunnelPeerSelector { if (!settings.isInbound()) { // exclude existing OBEPs to get some diversity } - ctx.profileOrganizer().selectFastPeers(1, exclude, matches, settings.getRandomKey(), length == 2 ? 2 : 4); + ctx.profileOrganizer().selectFastPeers(1, exclude, matches, settings.getRandomKey(), length == 2 ? SLICE_0_1 : SLICE_0); matches.remove(ctx.routerHash()); exclude.addAll(matches); rv.addAll(matches); @@ -57,7 +58,7 @@ class ClientPeerSelector extends TunnelPeerSelector { if (length > 2) { // middle hop(s) // group 2 or 3 - ctx.profileOrganizer().selectFastPeers(length - 2, exclude, matches, settings.getRandomKey(), 3); + ctx.profileOrganizer().selectFastPeers(length - 2, exclude, matches, settings.getRandomKey(), SLICE_2_3); matches.remove(ctx.routerHash()); if (matches.size() > 1) { // order the middle peers for tunnels >= 4 hops @@ -75,7 +76,7 @@ class ClientPeerSelector extends TunnelPeerSelector { if (settings.isInbound()) { // exclude existing IBGWs to get some diversity } - ctx.profileOrganizer().selectFastPeers(1, exclude, matches, settings.getRandomKey(), length == 2 ? 3 : 5); + ctx.profileOrganizer().selectFastPeers(1, exclude, matches, settings.getRandomKey(), length == 2 ? SLICE_2_3 : SLICE_1); matches.remove(ctx.routerHash()); rv.addAll(matches); } From 6d6f7fb89b0f82cc7e7bcce62a3d81664db15be1 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 28 Nov 2014 13:45:33 +0000 Subject: [PATCH 05/13] Data: Disallow duplicate keys in a Mapping --- core/java/src/net/i2p/data/DataHelper.java | 23 ++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/core/java/src/net/i2p/data/DataHelper.java b/core/java/src/net/i2p/data/DataHelper.java index d3d49dd2e..b18616716 100644 --- a/core/java/src/net/i2p/data/DataHelper.java +++ b/core/java/src/net/i2p/data/DataHelper.java @@ -108,6 +108,9 @@ public class DataHelper { * for the value. Finally after that comes the literal UTF-8 character ';'. This key=value; * is repeated until there are no more bytes (not characters!) left as defined by the * first two byte integer. + * + * As of 0.9.18, throws DataFormatException on duplicate key + * * @param rawStream stream to read the mapping from * @throws DataFormatException if the format is invalid * @throws IOException if there is a problem reading the data @@ -122,7 +125,14 @@ public class DataHelper { /** * Ditto, load into an existing properties + * + * As of 0.9.18, throws DataFormatException on duplicate key + * * @param props the Properties to load into + * @param rawStream stream to read the mapping from + * @throws DataFormatException if the format is invalid + * @throws IOException if there is a problem reading the data + * @return the parameter props * @since 0.8.13 */ public static Properties readProperties(InputStream rawStream, Properties props) @@ -148,7 +158,9 @@ public class DataHelper { if ((read != semiBuf.length) || (!eq(semiBuf, SEMICOLON_BYTES))) { throw new DataFormatException("Bad value"); } - props.put(key, val); + Object old = props.put(key, val); + if (old != null) + throw new DataFormatException("Duplicate key " + key); } return props; } @@ -299,6 +311,8 @@ public class DataHelper { * Warning - confusing method name, Properties is the target. * Strings must be UTF-8 encoded in the byte array. * + * As of 0.9.18, throws DataFormatException on duplicate key + * * @param source source * @param target returned Properties * @return new offset @@ -333,7 +347,9 @@ public class DataHelper { } catch (IOException ioe) { throw new DataFormatException("Bad value", ioe); } - target.put(key, val); + Object old= target.put(key, val); + if (old != null) + throw new DataFormatException("Duplicate key " + key); } return offset + size; } @@ -398,6 +414,9 @@ public class DataHelper { * - '=' is the only key-termination character (not ':' or whitespace) * * As of 0.9.10, an empty value is allowed. + * + * As in Java Properties, duplicate keys are allowed, last one wins. + * */ public static void loadProps(Properties props, File file) throws IOException { loadProps(props, file, false); From 1773fc0e0d4269ed0dbb9ad1c8815315700fa687 Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 28 Nov 2014 14:23:34 +0000 Subject: [PATCH 06/13] Add more clues to file locations in default config files --- installer/resources/blocklist.txt | 2 ++ installer/resources/clients.config | 2 ++ installer/resources/eepsite/docroot/index.html | 2 ++ installer/resources/eepsite/jetty.xml | 2 ++ installer/resources/i2ptunnel.config | 2 ++ 5 files changed, 10 insertions(+) diff --git a/installer/resources/blocklist.txt b/installer/resources/blocklist.txt index 02440c58e..2262b015b 100644 --- a/installer/resources/blocklist.txt +++ b/installer/resources/blocklist.txt @@ -2,6 +2,8 @@ # If you have a 'split' directory installation, with configuration # files in ~/.i2p (Linux) or %APPDATA%\I2P (Windows), be sure to # edit the file in the configuration directory, NOT the install directory. +# When running as a Linux daemon, the configuration directory is /var/lib/i2p +# and the install directory is /usr/share/i2p . # # Blocking is now enabled by default. # To disable blocking, set router.blocklist.enable=false on configadvanced.jsp, diff --git a/installer/resources/clients.config b/installer/resources/clients.config index cd68fb5c0..bb370c47e 100644 --- a/installer/resources/clients.config +++ b/installer/resources/clients.config @@ -3,6 +3,8 @@ # If you have a 'split' directory installation, with configuration # files in ~/.i2p (Linux) or %APPDATA%\I2P (Windows), be sure to # edit the file in the configuration directory, NOT the install directory. +# When running as a Linux daemon, the configuration directory is /var/lib/i2p +# and the install directory is /usr/share/i2p . # # fire up the web console diff --git a/installer/resources/eepsite/docroot/index.html b/installer/resources/eepsite/docroot/index.html index 8f8f8a27b..1af326b6a 100644 --- a/installer/resources/eepsite/docroot/index.html +++ b/installer/resources/eepsite/docroot/index.html @@ -4,6 +4,8 @@ # If you have a 'split' directory installation, with configuration # files in ~/.i2p (Linux) or %APPDATA%\I2P (Windows), be sure to # edit the file in the configuration directory, NOT the install directory. +# When running as a Linux daemon, the configuration directory is /var/lib/i2p +# and the install directory is /usr/share/i2p . # --> diff --git a/installer/resources/eepsite/jetty.xml b/installer/resources/eepsite/jetty.xml index 4ebecbb57..b52445953 100644 --- a/installer/resources/eepsite/jetty.xml +++ b/installer/resources/eepsite/jetty.xml @@ -32,6 +32,8 @@ + + diff --git a/installer/resources/i2ptunnel.config b/installer/resources/i2ptunnel.config index 7406d7118..0f5092cc0 100644 --- a/installer/resources/i2ptunnel.config +++ b/installer/resources/i2ptunnel.config @@ -3,6 +3,8 @@ # If you have a 'split' directory installation, with configuration # files in ~/.i2p (Linux) or %APPDATA%\I2P (Windows), be sure to # edit the file in the configuration directory, NOT the install directory. +# When running as a Linux daemon, the configuration directory is /var/lib/i2p +# and the install directory is /usr/share/i2p . # # eepproxy From 2d43d349ab67f48896c1c40e736de6cbcb1cb227 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 2 Dec 2014 13:22:26 +0000 Subject: [PATCH 07/13] add more invalid ports --- .../src/net/i2p/router/transport/TransportUtil.java | 13 ++++++++++++- .../net/i2p/router/transport/udp/UDPEndpoint.java | 8 ++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/router/java/src/net/i2p/router/transport/TransportUtil.java b/router/java/src/net/i2p/router/transport/TransportUtil.java index 713ec7612..ef8a7786f 100644 --- a/router/java/src/net/i2p/router/transport/TransportUtil.java +++ b/router/java/src/net/i2p/router/transport/TransportUtil.java @@ -173,11 +173,22 @@ public abstract class TransportUtil { /** * Is this a valid port for us or a remote router? * + * ref: http://i2p-projekt.i2p/en/docs/ports + * * @since 0.9.17 moved from logic in individual transports */ public static boolean isValidPort(int port) { + // update log message in UDPEndpoint if you update this list return port >= 1024 && port <= 65535 && - port != 1900; // UPnP SSDP + port != 1900 && // UPnP SSDP + port != 2827 && // BOB + port != 4444 && // HTTP + port != 4445 && // HTTPS + port != 6668 && // IRC + (!(port >= 7650 && port <= 7664)) && // standard I2P range + port != 8998 && // mtn + port != 31000 && // Wrapper + port != 32000; // Wrapper } } diff --git a/router/java/src/net/i2p/router/transport/udp/UDPEndpoint.java b/router/java/src/net/i2p/router/transport/udp/UDPEndpoint.java index 44ddea143..a1e680182 100644 --- a/router/java/src/net/i2p/router/transport/udp/UDPEndpoint.java +++ b/router/java/src/net/i2p/router/transport/udp/UDPEndpoint.java @@ -113,8 +113,12 @@ class UDPEndpoint implements SocketListener { private DatagramSocket getSocket() { DatagramSocket socket = null; int port = _listenPort; - if (port > 0 && !TransportUtil.isValidPort(port)) - _log.error("Specified UDP port is " + port + ", ports lower than 1024 not recommended"); + if (port > 0 && !TransportUtil.isValidPort(port)) { + _log.error("Specified UDP port " + port + " is not valid, selecting a new port"); + // See isValidPort() for list + _log.error("Invalid ports are: 0-1023, 1900, 2827, 4444, 4445, 6668, 7650-7664, 8998, 31000, 32000, 65536+"); + port = -1; + } for (int i = 0; i < MAX_PORT_RETRIES; i++) { if (port <= 0) { From f1e9f5d4fdf613c67ce3002d5d19600f52182847 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 2 Dec 2014 13:28:48 +0000 Subject: [PATCH 08/13] DatabaseStoreMessage: Mask the unused bits in the type field, in case we ever want to use them for options --- .../java/src/net/i2p/data/i2np/DatabaseStoreMessage.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/router/java/src/net/i2p/data/i2np/DatabaseStoreMessage.java b/router/java/src/net/i2p/data/i2np/DatabaseStoreMessage.java index b75ceabd5..8d50fabd1 100644 --- a/router/java/src/net/i2p/data/i2np/DatabaseStoreMessage.java +++ b/router/java/src/net/i2p/data/i2np/DatabaseStoreMessage.java @@ -105,7 +105,8 @@ public class DatabaseStoreMessage extends FastI2NPMessageImpl { _key = Hash.create(data, curIndex); curIndex += Hash.HASH_LENGTH; - type = (int)DataHelper.fromLong(data, curIndex, 1); + // as of 0.9.18, ignore other 7 bits of the type byte, in preparation for future options + int dbType = data[curIndex] & 0x01; curIndex++; _replyToken = DataHelper.fromLong(data, curIndex, 4); @@ -124,7 +125,7 @@ public class DatabaseStoreMessage extends FastI2NPMessageImpl { _replyGateway = null; } - if (type == DatabaseEntry.KEY_TYPE_LEASESET) { + if (dbType == DatabaseEntry.KEY_TYPE_LEASESET) { _dbEntry = new LeaseSet(); try { _dbEntry.readBytes(new ByteArrayInputStream(data, curIndex, data.length-curIndex)); @@ -133,7 +134,7 @@ public class DatabaseStoreMessage extends FastI2NPMessageImpl { } catch (IOException ioe) { throw new I2NPMessageException("Error reading the leaseSet", ioe); } - } else if (type == DatabaseEntry.KEY_TYPE_ROUTERINFO) { + } else { // dbType == DatabaseEntry.KEY_TYPE_ROUTERINFO _dbEntry = new RouterInfo(); int compressedSize = (int)DataHelper.fromLong(data, curIndex, 2); curIndex += 2; @@ -154,8 +155,6 @@ public class DatabaseStoreMessage extends FastI2NPMessageImpl { } catch (IOException ioe) { throw new I2NPMessageException("Corrupt compressed routerInfo size = " + compressedSize, ioe); } - } else { - throw new I2NPMessageException("Invalid type of key read from the structure - " + type); } //if (!key.equals(_dbEntry.getHash())) // throw new I2NPMessageException("Hash mismatch in DSM"); From 4b2715c36ffa2fb22e94345e98d3b807d97f00e2 Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 2 Dec 2014 13:30:31 +0000 Subject: [PATCH 09/13] RouterInfo: Add convenience method getVersion() --- router/java/src/net/i2p/data/router/RouterInfo.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/router/java/src/net/i2p/data/router/RouterInfo.java b/router/java/src/net/i2p/data/router/RouterInfo.java index fcd807a34..b0ff1ffd5 100644 --- a/router/java/src/net/i2p/data/router/RouterInfo.java +++ b/router/java/src/net/i2p/data/router/RouterInfo.java @@ -261,6 +261,18 @@ public class RouterInfo extends DatabaseEntry { return _options.getProperty(opt); } + /** + * For convenience, the same as getOption("router.version"), + * but returns "0" if unset. + * + * @return non-null, "0" if unknown. + * @since 0.9.18 + */ + public String getVersion() { + String rv = _options.getProperty("router.version"); + return rv != null ? rv : "0"; + } + /** * Configure a set of options or statistics that the router can expose. * Makes a copy. From fb641187b8ddc9047c9399792a1703bf6d46593f Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 2 Dec 2014 13:42:58 +0000 Subject: [PATCH 10/13] use new getVersion() --- .../src/net/i2p/data/i2np/DatabaseLookupMessage.java | 5 ++--- .../net/i2p/router/networkdb/kademlia/StoreJob.java | 12 +++--------- .../net/i2p/router/peermanager/ProfileOrganizer.java | 4 ++-- .../net/i2p/router/transport/ntcp/NTCPTransport.java | 4 ++-- .../net/i2p/router/transport/udp/UDPTransport.java | 6 +++--- .../net/i2p/router/tunnel/pool/BuildRequestor.java | 4 +--- .../i2p/router/tunnel/pool/TunnelPeerSelector.java | 8 +++----- 7 files changed, 16 insertions(+), 27 deletions(-) diff --git a/router/java/src/net/i2p/data/i2np/DatabaseLookupMessage.java b/router/java/src/net/i2p/data/i2np/DatabaseLookupMessage.java index 63c33b063..9aa7530d3 100644 --- a/router/java/src/net/i2p/data/i2np/DatabaseLookupMessage.java +++ b/router/java/src/net/i2p/data/i2np/DatabaseLookupMessage.java @@ -214,9 +214,8 @@ public class DatabaseLookupMessage extends FastI2NPMessageImpl { public static boolean supportsEncryptedReplies(RouterInfo to) { if (to == null) return false; - String v = to.getOption("router.version"); - return v != null && - VersionComparator.comp(v, MIN_ENCRYPTION_VERSION) >= 0; + String v = to.getVersion(); + return VersionComparator.comp(v, MIN_ENCRYPTION_VERSION) >= 0; } /** diff --git a/router/java/src/net/i2p/router/networkdb/kademlia/StoreJob.java b/router/java/src/net/i2p/router/networkdb/kademlia/StoreJob.java index 81bef6950..809b96015 100644 --- a/router/java/src/net/i2p/router/networkdb/kademlia/StoreJob.java +++ b/router/java/src/net/i2p/router/networkdb/kademlia/StoreJob.java @@ -513,9 +513,7 @@ class StoreJob extends JobImpl { * @since 0.7.10 */ private static boolean supportsEncryption(RouterInfo ri) { - String v = ri.getOption("router.version"); - if (v == null) - return false; + String v = ri.getVersion(); return VersionComparator.comp(v, MIN_ENCRYPTION_VERSION) >= 0; } @@ -535,9 +533,7 @@ class StoreJob extends JobImpl { } if (type == null) return false; - String v = ri.getOption("router.version"); - if (v == null) - return false; + String v = ri.getVersion(); String since = type.getSupportedSince(); return VersionComparator.comp(v, since) >= 0; } @@ -549,9 +545,7 @@ class StoreJob extends JobImpl { * @since 0.9.12 */ public static boolean supportsBigLeaseSets(RouterInfo ri) { - String v = ri.getOption("router.version"); - if (v == null) - return false; + String v = ri.getVersion(); return VersionComparator.comp(v, MIN_BIGLEASESET_VERSION) >= 0; } diff --git a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java index 7ebc4ce93..87f751bdc 100644 --- a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java +++ b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java @@ -697,9 +697,9 @@ public class ProfileOrganizer { // they probably don't have a TCP hole punched in their firewall either. RouterInfo info = _context.netDb().lookupRouterInfoLocally(peer); if (info != null) { - String v = info.getOption("router.version"); + String v = info.getVersion(); // this only works if there is no 0.6.1.34! - if (v != null && (!v.equals("0.6.1.33")) && + if ((!v.equals("0.6.1.33")) && v.startsWith("0.6.1.") && info.getTargetAddress("NTCP") == null) l.add(peer); else { diff --git a/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java b/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java index 1a7ec1ed4..b48dab090 100644 --- a/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java +++ b/router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java @@ -375,8 +375,8 @@ public class NTCPTransport extends TransportImpl { if (us != null) { RouterIdentity id = us.getIdentity(); if (id.getSigType() != SigType.DSA_SHA1) { - String v = toAddress.getOption("router.version"); - if (v != null && VersionComparator.comp(v, MIN_SIGTYPE_VERSION) < 0) { + String v = toAddress.getVersion(); + if (VersionComparator.comp(v, MIN_SIGTYPE_VERSION) < 0) { markUnreachable(peer); return null; } diff --git a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java index 80a383c9d..efed5b13b 100644 --- a/router/java/src/net/i2p/router/transport/udp/UDPTransport.java +++ b/router/java/src/net/i2p/router/transport/udp/UDPTransport.java @@ -1232,7 +1232,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority _context.simpleScheduler().addEvent(new RemoveDropList(remote), DROPLIST_PERIOD); } markUnreachable(peerHash); - _context.banlist().banlistRouter(peerHash, "Part of the wrong network, version = " + ((RouterInfo) entry).getOption("router.version")); + _context.banlist().banlistRouter(peerHash, "Part of the wrong network, version = " + ((RouterInfo) entry).getVersion()); //_context.banlist().banlistRouter(peerHash, "Part of the wrong network", STYLE); dropPeer(peerHash, false, "wrong network"); if (_log.shouldLog(Log.WARN)) @@ -1578,8 +1578,8 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority if (us != null) { RouterIdentity id = us.getIdentity(); if (id.getSigType() != SigType.DSA_SHA1) { - String v = toAddress.getOption("router.version"); - if (v != null && VersionComparator.comp(v, MIN_SIGTYPE_VERSION) < 0) { + String v = toAddress.getVersion(); + if (VersionComparator.comp(v, MIN_SIGTYPE_VERSION) < 0) { markUnreachable(to); return null; } diff --git a/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java b/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java index 493db21ab..4e74aedb5 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java +++ b/router/java/src/net/i2p/router/tunnel/pool/BuildRequestor.java @@ -238,9 +238,7 @@ abstract class BuildRequestor { RouterInfo ri = ctx.netDb().lookupRouterInfoLocally(h); if (ri == null) return false; - String v = ri.getOption("router.version"); - if (v == null) - return false; + String v = ri.getVersion(); return VersionComparator.comp(v, MIN_VARIABLE_VERSION) >= 0; } diff --git a/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java b/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java index fa3f5e75d..74fc53e83 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java +++ b/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java @@ -349,9 +349,7 @@ public abstract class TunnelPeerSelector { if (known != null) { for (int i = 0; i < known.size(); i++) { RouterInfo peer = known.get(i); - String v = peer.getOption("router.version"); - if (v == null) - continue; + String v = peer.getVersion(); // RI sigtypes added in 0.9.16 // SSU inbound connection bug fixed in 0.9.17, but it won't bid, so NTCP only, // no need to check @@ -402,8 +400,8 @@ public abstract class TunnelPeerSelector { // so don't exclude it based on published capacity // minimum version check - String v = peer.getOption("router.version"); - if (v == null || VersionComparator.comp(v, MIN_VERSION) < 0) + String v = peer.getVersion(); + if (VersionComparator.comp(v, MIN_VERSION) < 0) return true; // uptime is always spoofed to 90m, so just remove all this From 78d7277298e45375c9727e2f893360b5393a40ee Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 2 Dec 2014 15:11:12 +0000 Subject: [PATCH 11/13] show b32 for local leasesets too --- .../java/src/net/i2p/router/web/NetDbRenderer.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apps/routerconsole/java/src/net/i2p/router/web/NetDbRenderer.java b/apps/routerconsole/java/src/net/i2p/router/web/NetDbRenderer.java index 1bf9881eb..f6c44f9cb 100644 --- a/apps/routerconsole/java/src/net/i2p/router/web/NetDbRenderer.java +++ b/apps/routerconsole/java/src/net/i2p/router/web/NetDbRenderer.java @@ -148,6 +148,13 @@ public class NetDbRenderer { else buf.append(dest.toBase64().substring(0, 6)); buf.append(")
\n"); + String b32 = dest.toBase32(); + buf.append("").append(b32).append("
\n"); + String host = _context.namingService().reverseLookup(dest); + if (host == null) { + buf.append("").append(_("Add to local addressbook")).append("
\n"); + } } else { buf.append(" (").append(_("Destination")).append(' '); String host = _context.namingService().reverseLookup(dest); From e7b50c59403830522ee4228b7a2e0d66780602cd Mon Sep 17 00:00:00 2001 From: zzz Date: Tue, 2 Dec 2014 15:23:50 +0000 Subject: [PATCH 12/13] reduce auto-stop threshold again --- apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java index 4b141e02d..9eab1497b 100644 --- a/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java +++ b/apps/i2psnark/java/src/org/klomp/snark/TrackerClient.java @@ -528,9 +528,9 @@ public class TrackerClient implements Runnable { !snark.isChecking() && info.getSeedCount() > 100 && coordinator.getPeerCount() <= 0 && - _util.getContext().clock().now() > _startedOn + 2*60*60*1000 && + _util.getContext().clock().now() > _startedOn + 30*60*1000 && snark.getTotalLength() > 0 && - uploaded >= snark.getTotalLength() * 5 / 4) { + uploaded >= snark.getTotalLength() / 2) { if (_log.shouldLog(Log.WARN)) _log.warn("Auto stopping " + snark.getBaseName()); snark.setAutoStoppable(false); From 5e67008d264ae3ec7b33aa33c6ea01b96d2aa62a Mon Sep 17 00:00:00 2001 From: zzz Date: Fri, 5 Dec 2014 15:12:51 +0000 Subject: [PATCH 13/13] I2PTunnel: Reduce i2ptunnel threads, more thread pooling. Big savings is on client side (two less threads per connection) - Move client pool from static inI2PTunnelClientBase to TCG. - Use client pool for some server threads - Run some things inline that were formerly threads - Client-side I2PTunnelRunner thread used to do nothing but start 2 more threads; now it runs one inline (like we do for server-side HTTP) - Javadocs and cleanups Was originally intended to reduce load for high-traffic servers but most of the savings for now is on the client side. Ref: http://zzz.i2p/topics/1741 Todo: Figure out how to run the HTTP client-side gunzipper inline too Todo: More server-side improvements --- Client side: before: 4-5 threads, 1-2 pooled I2PTunnel Client Runner (BlockingRunner from client pool) starts I2PTunnelRunner or I2PTunnelHTTPClientRunner and exits starts StreamForwarder toI2P and waits starts StreamForwarder fromI2P and waits starts HTTPResponseOutputStream (HTTP gunzip only) (from client pool) now: 2-3 threads, 1-2 pooled I2PTunnel Client Runner (BlockingRunner from client pool) runs I2PTunnelRunner or I2PTunnelHTTPClientRunner inline starts StreamForwarder toI2P and waits runs StreamForwarder fromI2P inline starts HTTPResponseOutputStream (HTTP gunzip only) (from client pool) --- Server side: before: 1-4 threads, 0-1 pooled Server Handler Pool (Handler from server pool) execpt for standard server, blockingHandle() inline in acceptor starts I2PTunnelRunner or CompressedRequestor and exits starts StreamForwarder toI2P and waits (inline for HTTP) starts StreamForwarder fromI2P and waits (except not for HTTP GET) now: 1-4 threads, 0-2 pooled Server Handler Pool (Handler from server pool) execpt for standard server, blockingHandle() inline in acceptor starts I2PTunnelRunner or CompressedRequestor and exits (using client pool) starts StreamForwarder toI2P and waits (inline for HTTP) starts StreamForwarder fromI2P and waits (except not for HTTP GET) --- .../i2ptunnel/HTTPResponseOutputStream.java | 30 ++++-- .../net/i2p/i2ptunnel/I2PTunnelClient.java | 6 +- .../i2p/i2ptunnel/I2PTunnelClientBase.java | 96 +++++-------------- .../i2p/i2ptunnel/I2PTunnelConnectClient.java | 4 +- .../i2p/i2ptunnel/I2PTunnelHTTPClient.java | 14 ++- .../i2ptunnel/I2PTunnelHTTPClientRunner.java | 3 +- .../i2p/i2ptunnel/I2PTunnelHTTPServer.java | 12 +-- .../net/i2p/i2ptunnel/I2PTunnelIRCClient.java | 7 +- .../net/i2p/i2ptunnel/I2PTunnelIRCServer.java | 4 +- .../net/i2p/i2ptunnel/I2PTunnelRunner.java | 12 +-- .../net/i2p/i2ptunnel/I2PTunnelServer.java | 26 ++++- .../net/i2p/i2ptunnel/TunnelController.java | 2 +- .../i2p/i2ptunnel/TunnelControllerGroup.java | 80 +++++++++++++++- .../i2p/i2ptunnel/irc/I2PTunnelDCCClient.java | 4 +- .../i2p/i2ptunnel/irc/I2PTunnelDCCServer.java | 4 +- .../i2ptunnel/socks/I2PSOCKSIRCTunnel.java | 9 +- .../i2p/i2ptunnel/socks/I2PSOCKSTunnel.java | 4 +- 17 files changed, 203 insertions(+), 114 deletions(-) diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java index 244e76c6e..bc1104d83 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/HTTPResponseOutputStream.java @@ -21,6 +21,7 @@ import net.i2p.I2PAppContext; import net.i2p.data.ByteArray; import net.i2p.util.BigPipedInputStream; import net.i2p.util.ByteCache; +import net.i2p.util.I2PAppThread; import net.i2p.util.Log; import net.i2p.util.ReusableGZIPInputStream; @@ -251,16 +252,27 @@ class HTTPResponseOutputStream extends FilterOutputStream { //out.flush(); PipedInputStream pi = BigPipedInputStream.getInstance(); PipedOutputStream po = new PipedOutputStream(pi); - // Run in the client thread pool, as there should be an unused thread - // there after the accept(). - // Overridden in I2PTunnelHTTPServer, where it does not use the client pool. - try { - I2PTunnelClientBase.getClientExecutor().execute(new Pusher(pi, out)); - } catch (RejectedExecutionException ree) { - // shouldn't happen - throw ree; - } + Runnable r = new Pusher(pi, out); out = po; + // TODO we should be able to do this inline somehow + TunnelControllerGroup tcg = TunnelControllerGroup.getInstance(); + if (tcg != null) { + // Run in the client thread pool, as there should be an unused thread + // there after the accept(). + // Overridden in I2PTunnelHTTPServer, where it does not use the client pool. + try { + tcg.getClientExecutor().execute(r); + } catch (RejectedExecutionException ree) { + // shouldn't happen + throw ree; + } + } else { + // Fallback in case TCG.getInstance() is null, never instantiated + // and we were not started by TCG. + // Maybe a plugin loaded before TCG? Should be rare. + Thread t = new I2PAppThread(r, "Pusher"); + t.start(); + } } private class Pusher implements Runnable { diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClient.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClient.java index 2c448d946..6a552719e 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClient.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClient.java @@ -122,9 +122,11 @@ public class I2PTunnelClient extends I2PTunnelClientBase { int port = addr.getPort(); i2ps = createI2PSocket(clientDest, port); i2ps.setReadTimeout(readTimeout); - Thread t = new I2PTunnelRunner(s, i2ps, sockLock, null, null, mySockets, + I2PTunnelRunner t = new I2PTunnelRunner(s, i2ps, sockLock, null, null, mySockets, (I2PTunnelRunner.FailCallback) null); - t.start(); + // we are called from an unlimited thread pool, so run inline + //t.start(); + t.run(); } catch (Exception ex) { if (_log.shouldLog(Log.INFO)) _log.info("Error connecting", ex); diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClientBase.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClientBase.java index 673928552..030e4a0b6 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClientBase.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelClientBase.java @@ -16,12 +16,8 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Properties; -import java.util.concurrent.Executors; -import java.util.concurrent.SynchronousQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicLong; import javax.net.ssl.SSLServerSocket; @@ -77,18 +73,7 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna // true if we are chained from a server. private boolean chained; - /** how long to wait before dropping an idle thread */ - private static final long HANDLER_KEEPALIVE_MS = 2*60*1000; - - /** - * We keep a static pool of socket handlers for all clients, - * as there is no need for isolation on the client side. - * Extending classes may use it for other purposes. - * Not for use by servers, as there is no limit on threads. - */ - private static volatile ThreadPoolExecutor _executor; - private static int _executorThreadCount; - private static final Object _executorLock = new Object(); + private volatile ThreadPoolExecutor _executor; public static final String PROP_USE_SSL = I2PTunnelServer.PROP_USE_SSL; @@ -116,11 +101,6 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna _context.statManager().createRateStat("i2ptunnel.client.buildRunTime", "How long it takes to run a queued socket into an i2ptunnel runner?", "I2PTunnel", new long[] { 60*1000, 10*60*1000, 60*60*1000 }); _log = _context.logManager().getLog(getClass()); - synchronized (_executorLock) { - if (_executor == null) - _executor = new CustomThreadPoolExecutor(); - } - Thread t = new I2PAppThread(this, "Client " + tunnel.listenHost + ':' + localPort); t.start(); open = true; @@ -184,11 +164,6 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna _context.statManager().createRateStat("i2ptunnel.client.buildRunTime", "How long it takes to run a queued socket into an i2ptunnel runner?", "I2PTunnel", new long[] { 60*1000, 10*60*1000, 60*60*1000 }); _log = _context.logManager().getLog(getClass()); - synchronized (_executorLock) { - if (_executor == null) - _executor = new CustomThreadPoolExecutor(); - } - // normalize path so we can find it if (pkf != null) { File keyFile = new File(pkf); @@ -361,6 +336,16 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna return socketManager; } + /** + * Kill the shared client, so that on restart in android + * we won't latch onto the old one + * + * @since 0.9.18 + */ + protected static synchronized void killSharedClient() { + socketManager = null; + } + /** * This may take a LONG time. * @@ -653,6 +638,16 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna } } + TunnelControllerGroup tcg = TunnelControllerGroup.getInstance(); + if (tcg != null) { + _executor = tcg.getClientExecutor(); + } else { + // Fallback in case TCG.getInstance() is null, never instantiated + // and we were not started by TCG. + // Maybe a plugin loaded before TCG? Should be rare. + // Never shut down. + _executor = new TunnelControllerGroup.CustomThreadPoolExecutor(); + } while (open) { Socket s = ss.accept(); manageConnection(s); @@ -672,30 +667,6 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna } } - /** - * @return may be null if no class has been instantiated - * @since 0.8.8 - */ - static ThreadPoolExecutor getClientExecutor() { - return _executor; - } - - /** - * @since 0.8.8 - */ - static void killClientExecutor() { - synchronized (_executorLock) { - if (_executor != null) { - _executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); - _executor.shutdownNow(); - _executor = null; - } - // kill the shared client, so that on restart in android - // we won't latch onto the old one - socketManager = null; - } - } - /** * Manage the connection just opened on the specified socket * @@ -721,26 +692,6 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna } } - /** - * Not really needed for now but in case we want to add some hooks like afterExecute(). - */ - private static class CustomThreadPoolExecutor extends ThreadPoolExecutor { - public CustomThreadPoolExecutor() { - super(0, Integer.MAX_VALUE, HANDLER_KEEPALIVE_MS, TimeUnit.MILLISECONDS, - new SynchronousQueue(), new CustomThreadFactory()); - } - } - - /** just to set the name and set Daemon */ - private static class CustomThreadFactory implements ThreadFactory { - public Thread newThread(Runnable r) { - Thread rv = Executors.defaultThreadFactory().newThread(r); - rv.setName("I2PTunnel Client Runner " + (++_executorThreadCount)); - rv.setDaemon(true); - return rv; - } - } - /** * Blocking runner, used during the connection establishment */ @@ -822,7 +773,10 @@ public abstract class I2PTunnelClientBase extends I2PTunnelTask implements Runna /** * Manage a connection in a separate thread. This only works if - * you do not override manageConnection() + * you do not override manageConnection(). + * + * This is run in a thread from an unlimited-size thread pool, + * so it may block or run indefinitely. */ protected abstract void clientConnectionRun(Socket s); } diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelConnectClient.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelConnectClient.java index 7690e2c65..5e8207a26 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelConnectClient.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelConnectClient.java @@ -292,7 +292,9 @@ public class I2PTunnelConnectClient extends I2PTunnelHTTPClientBase implements R response = SUCCESS_RESPONSE; OnTimeout onTimeout = new OnTimeout(s, s.getOutputStream(), targetRequest, usingWWWProxy, currentProxy, requestId); Thread t = new I2PTunnelRunner(s, i2ps, sockLock, data, response, mySockets, onTimeout); - t.start(); + // we are called from an unlimited thread pool, so run inline + //t.start(); + t.run(); } catch (IOException ex) { _log.info(getPrefix(requestId) + "Error trying to connect", ex); handleClientException(ex, out, targetRequest, usingWWWProxy, currentProxy, requestId); diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java index 625ae863a..4b869243d 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClient.java @@ -972,7 +972,9 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn response = null; } Thread t = new I2PTunnelOutproxyRunner(s, outSocket, sockLock, data, response, onTimeout); - t.start(); + // we are called from an unlimited thread pool, so run inline + //t.start(); + t.run(); return; } @@ -1091,6 +1093,7 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn sktOpts.setPort(remotePort); I2PSocket i2ps = createI2PSocket(clientDest, sktOpts); OnTimeout onTimeout = new OnTimeout(s, s.getOutputStream(), targetRequest, usingWWWProxy, currentProxy, requestId); + Thread t; if (method.toUpperCase(Locale.US).equals("CONNECT")) { byte[] data; byte[] response; @@ -1101,13 +1104,14 @@ public class I2PTunnelHTTPClient extends I2PTunnelHTTPClientBase implements Runn data = null; response = SUCCESS_RESPONSE; } - Thread t = new I2PTunnelRunner(s, i2ps, sockLock, data, response, mySockets, onTimeout); - t.start(); + t = new I2PTunnelRunner(s, i2ps, sockLock, data, response, mySockets, onTimeout); } else { byte[] data = newRequest.toString().getBytes("ISO-8859-1"); - Thread t = new I2PTunnelHTTPClientRunner(s, i2ps, sockLock, data, mySockets, onTimeout); - t.start(); + t = new I2PTunnelHTTPClientRunner(s, i2ps, sockLock, data, mySockets, onTimeout); } + // we are called from an unlimited thread pool, so run inline + //t.start(); + t.run(); } catch(IOException ex) { if(_log.shouldLog(Log.INFO)) { _log.info(getPrefix(requestId) + "Error trying to connect", ex); diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClientRunner.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClientRunner.java index e12b839d3..51425cbd0 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClientRunner.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPClientRunner.java @@ -86,7 +86,8 @@ public class I2PTunnelHTTPClientRunner extends I2PTunnelRunner { // ignore } t1.join(30*1000); - t2.join(30*1000); + // t2 = fromI2P now run inline + //t2.join(30*1000); } } diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java index ac1d2d39a..d4d06c424 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelHTTPServer.java @@ -302,16 +302,16 @@ public class I2PTunnelHTTPServer extends I2PTunnelServer { if (_log.shouldLog(Log.DEBUG)) _log.debug("Modified header: [" + modifiedHeader + "]"); + Runnable t; if (allowGZIP && useGZIP) { - I2PAppThread req = new I2PAppThread( - new CompressedRequestor(s, socket, modifiedHeader, getTunnel().getContext(), _log), - Thread.currentThread().getName()+".hc"); - req.start(); + t = new CompressedRequestor(s, socket, modifiedHeader, getTunnel().getContext(), _log); } else { - Thread t = new I2PTunnelRunner(s, socket, slock, null, modifiedHeader.getBytes(), + t = new I2PTunnelRunner(s, socket, slock, null, modifiedHeader.getBytes(), null, (I2PTunnelRunner.FailCallback) null); - t.start(); } + // run in the unlimited client pool + //t.start(); + _clientExecutor.execute(t); long afterHandle = getTunnel().getContext().clock().now(); long timeToHandle = afterHandle - afterAccept; diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java index 3915ae688..344c72197 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCClient.java @@ -136,8 +136,11 @@ public class I2PTunnelIRCClient extends I2PTunnelClientBase { DCCHelper dcc = _dccEnabled ? new DCC(s.getLocalAddress().getAddress()) : null; Thread in = new I2PAppThread(new IrcInboundFilter(s,i2ps, expectedPong, _log, dcc), "IRC Client " + _clientId + " in", true); in.start(); - Thread out = new I2PAppThread(new IrcOutboundFilter(s,i2ps, expectedPong, _log, dcc), "IRC Client " + _clientId + " out", true); - out.start(); + //Thread out = new I2PAppThread(new IrcOutboundFilter(s,i2ps, expectedPong, _log, dcc), "IRC Client " + _clientId + " out", true); + Runnable out = new IrcOutboundFilter(s,i2ps, expectedPong, _log, dcc); + // we are called from an unlimited thread pool, so run inline + //out.start(); + out.run(); } catch (Exception ex) { // generally NoRouteToHostException if (_log.shouldLog(Log.WARN)) diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCServer.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCServer.java index f39df586f..f82c15417 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCServer.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelIRCServer.java @@ -140,7 +140,9 @@ public class I2PTunnelIRCServer extends I2PTunnelServer implements Runnable { Socket s = getSocket(socket.getPeerDestination().calculateHash(), socket.getLocalPort()); Thread t = new I2PTunnelRunner(s, socket, slock, null, modifiedRegistration.getBytes(), null, (I2PTunnelRunner.FailCallback) null); - t.start(); + // run in the unlimited client pool + //t.start(); + _clientExecutor.execute(t); } catch (SocketException ex) { try { // Send a response so the user doesn't just see a disconnect diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelRunner.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelRunner.java index d4bd1d455..d0dfc74ec 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelRunner.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelRunner.java @@ -62,8 +62,6 @@ public class I2PTunnelRunner extends I2PAppThread implements I2PSocket.SocketErr private long totalSent; private long totalReceived; - private static final AtomicLong __forwarderId = new AtomicLong(); - /** * For use in new constructor * @since 0.9.14 @@ -268,9 +266,10 @@ public class I2PTunnelRunner extends I2PAppThread implements I2PSocket.SocketErr in = new BufferedInputStream(in, 2*NETWORK_BUFFER_SIZE); StreamForwarder toI2P = new StreamForwarder(in, i2pout, true); StreamForwarder fromI2P = new StreamForwarder(i2pin, out, false); - // TODO can we run one of these inline and save a thread? toI2P.start(); - fromI2P.start(); + // We are already a thread, so run the second one inline + //fromI2P.start(); + fromI2P.run(); synchronized (finishLock) { while (!finished) { finishLock.wait(); @@ -384,7 +383,8 @@ public class I2PTunnelRunner extends I2PAppThread implements I2PSocket.SocketErr // ignore } t1.join(30*1000); - t2.join(30*1000); + // t2 = fromI2P now run inline + //t2.join(30*1000); } /** @@ -426,7 +426,7 @@ public class I2PTunnelRunner extends I2PAppThread implements I2PSocket.SocketErr _toI2P = toI2P; direction = (toI2P ? "toI2P" : "fromI2P"); _cache = ByteCache.getInstance(32, NETWORK_BUFFER_SIZE); - setName("StreamForwarder " + _runnerId + '.' + __forwarderId.incrementAndGet()); + setName("StreamForwarder " + _runnerId + '.' + direction); } @Override diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelServer.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelServer.java index 71d2643b2..db1f0d07e 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelServer.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/I2PTunnelServer.java @@ -80,6 +80,7 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable { protected I2PTunnelTask task; protected boolean bidir; private ThreadPoolExecutor _executor; + protected volatile ThreadPoolExecutor _clientExecutor; private final Map _socketMap = new ConcurrentHashMap(4); /** unused? port should always be specified */ @@ -470,6 +471,16 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable { if (_usePool) { _executor = new CustomThreadPoolExecutor(getHandlerCount(), "ServerHandler pool " + remoteHost + ':' + remotePort); } + TunnelControllerGroup tcg = TunnelControllerGroup.getInstance(); + if (tcg != null) { + _clientExecutor = tcg.getClientExecutor(); + } else { + // Fallback in case TCG.getInstance() is null, never instantiated + // and we were not started by TCG. + // Maybe a plugin loaded before TCG? Should be rare. + // Never shut down. + _clientExecutor = new TunnelControllerGroup.CustomThreadPoolExecutor(); + } while (open) { try { I2PServerSocket ci2pss = i2pss; @@ -563,6 +574,17 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable { } } + /** + * This is run in a thread from a limited-size thread pool via Handler.run(), + * except for a standard server (this class, no extension, as determined in getUsePool()), + * it is run directly in the acceptor thread (see run()). + * + * In either case, this method and any overrides must spawn a thread and return quickly. + * If blocking while reading the headers (as in HTTP and IRC), the thread pool + * may be exhausted. + * + * See PROP_USE_POOL, DEFAULT_USE_POOL, PROP_HANDLER_COUNT, DEFAULT_HANDLER_COUNT + */ protected void blockingHandle(I2PSocket socket) { if (_log.shouldLog(Log.INFO)) _log.info("Incoming connection to '" + toString() + "' port " + socket.getLocalPort() + @@ -577,7 +599,9 @@ public class I2PTunnelServer extends I2PTunnelTask implements Runnable { afterSocket = getTunnel().getContext().clock().now(); Thread t = new I2PTunnelRunner(s, socket, slock, null, null, null, (I2PTunnelRunner.FailCallback) null); - t.start(); + // run in the unlimited client pool + //t.start(); + _clientExecutor.execute(t); long afterHandle = getTunnel().getContext().clock().now(); long timeToHandle = afterHandle - afterAccept; diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelController.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelController.java index cb6831949..ad5256fa6 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelController.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelController.java @@ -425,7 +425,7 @@ public class TunnelController implements Logging { // We use _sessions AND the tunnel sessions as // _sessions will be null for delay-open tunnels - see acquire(). // We want the current sessions. - Set sessions = new HashSet(_tunnel.getSessions()); + Set sessions = new HashSet(_tunnel.getSessions()); if (_sessions != null) sessions.addAll(_sessions); return sessions; diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelControllerGroup.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelControllerGroup.java index af8818abd..4c28e1b03 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelControllerGroup.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/TunnelControllerGroup.java @@ -9,6 +9,13 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicLong; import net.i2p.I2PAppContext; import net.i2p.app.*; @@ -48,6 +55,21 @@ public class TunnelControllerGroup implements ClientApp { */ private final Map> _sessions; + /** + * We keep a pool of socket handlers for all clients, + * as there is no need for isolation on the client side. + * Extending classes may use it for other purposes. + * + * May also be used by servers, carefully, + * as there is no limit on threads. + */ + private ThreadPoolExecutor _executor; + private static final AtomicLong _executorThreadCount = new AtomicLong(); + private final Object _executorLock = new Object(); + /** how long to wait before dropping an idle thread */ + private static final long HANDLER_KEEPALIVE_MS = 2*60*1000; + + /** * In I2PAppContext will instantiate if necessary and always return non-null. * As of 0.9.4, when in RouterContext, will return null (except in Android) @@ -206,8 +228,7 @@ public class TunnelControllerGroup implements ClientApp { if (_instance == this) _instance = null; } -/// fixme static - I2PTunnelClientBase.killClientExecutor(); + killClientExecutor(); changeState(STOPPED); } @@ -500,4 +521,59 @@ public class TunnelControllerGroup implements ClientApp { } } } + + /** + * @return non-null + * @since 0.8.8 Moved from I2PTunnelClientBase in 0.9.18 + */ + ThreadPoolExecutor getClientExecutor() { + synchronized (_executorLock) { + if (_executor == null) + _executor = new CustomThreadPoolExecutor(); + } + return _executor; + } + + /** + * @since 0.8.8 Moved from I2PTunnelClientBase in 0.9.18 + */ + private void killClientExecutor() { + synchronized (_executorLock) { + if (_executor != null) { + _executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + _executor.shutdownNow(); + _executor = null; + } + } + // kill the shared client, so that on restart in android + // we won't latch onto the old one + I2PTunnelClientBase.killSharedClient(); + } + + /** + * Not really needed for now but in case we want to add some hooks like afterExecute(). + * Package private for fallback in case TCG.getInstance() is null, never instantiated + * but a plugin still needs it... should be rare. + * + * @since 0.9.18 Moved from I2PTunnelClientBase + */ + static class CustomThreadPoolExecutor extends ThreadPoolExecutor { + public CustomThreadPoolExecutor() { + super(0, Integer.MAX_VALUE, HANDLER_KEEPALIVE_MS, TimeUnit.MILLISECONDS, + new SynchronousQueue(), new CustomThreadFactory()); + } + } + + /** + * Just to set the name and set Daemon + * @since 0.9.18 Moved from I2PTunnelClientBase + */ + private static class CustomThreadFactory implements ThreadFactory { + public Thread newThread(Runnable r) { + Thread rv = Executors.defaultThreadFactory().newThread(r); + rv.setName("I2PTunnel Client Runner " + _executorThreadCount.incrementAndGet()); + rv.setDaemon(true); + return rv; + } + } } diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCClient.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCClient.java index e22662103..fcca71cb9 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCClient.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCClient.java @@ -76,7 +76,9 @@ public class I2PTunnelDCCClient extends I2PTunnelClientBase { try { i2ps = createI2PSocket(dest, opts); Thread t = new Runner(s, i2ps); - t.start(); + // we are called from an unlimited thread pool, so run inline + //t.start(); + t.run(); } catch (Exception ex) { _log.error("Could not make DCC connection to " + _dest + ':' + _remotePort, ex); closeSocket(s); diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCServer.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCServer.java index 4b1b6b8f5..cab6513ad 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCServer.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/irc/I2PTunnelDCCServer.java @@ -111,7 +111,9 @@ public class I2PTunnelDCCServer extends I2PTunnelServer { _sockList.add(socket); Thread t = new I2PTunnelRunner(s, socket, slock, null, null, _sockList, (I2PTunnelRunner.FailCallback) null); - t.start(); + // run in the unlimited client pool + //t.start(); + _clientExecutor.execute(t); local.socket = socket; local.expire = getTunnel().getContext().clock().now() + OUTBOUND_EXPIRE; _active.put(Integer.valueOf(myPort), local); diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSIRCTunnel.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSIRCTunnel.java index b4363afe0..8484ef0c2 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSIRCTunnel.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSIRCTunnel.java @@ -55,9 +55,12 @@ public class I2PSOCKSIRCTunnel extends I2PSOCKSTunnel { Thread in = new I2PAppThread(new IrcInboundFilter(clientSock, destSock, expectedPong, _log), "SOCKS IRC Client " + id + " in", true); in.start(); - Thread out = new I2PAppThread(new IrcOutboundFilter(clientSock, destSock, expectedPong, _log), - "SOCKS IRC Client " + id + " out", true); - out.start(); + //Thread out = new I2PAppThread(new IrcOutboundFilter(clientSock, destSock, expectedPong, _log), + // "SOCKS IRC Client " + id + " out", true); + Runnable out = new IrcOutboundFilter(clientSock, destSock, expectedPong, _log); + // we are called from an unlimited thread pool, so run inline + //out.start(); + out.run(); } catch (SOCKSException e) { if (_log.shouldLog(Log.WARN)) _log.warn("Error from SOCKS connection", e); diff --git a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSTunnel.java b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSTunnel.java index 89674c592..498d75a18 100644 --- a/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSTunnel.java +++ b/apps/i2ptunnel/java/src/net/i2p/i2ptunnel/socks/I2PSOCKSTunnel.java @@ -56,7 +56,9 @@ public class I2PSOCKSTunnel extends I2PTunnelClientBase { I2PSocket destSock = serv.getDestinationI2PSocket(this); Thread t = new I2PTunnelRunner(clientSock, destSock, sockLock, null, null, mySockets, (I2PTunnelRunner.FailCallback) null); - t.start(); + // we are called from an unlimited thread pool, so run inline + //t.start(); + t.run(); } catch (SOCKSException e) { if (_log.shouldLog(Log.WARN)) _log.warn("Error from SOCKS connection", e);