Rewrite utils into Java, cache the persistable data of shared files to reduce object churn

This commit is contained in:
Zlatin Balevsky
2019-10-05 22:50:32 +01:00
parent 730d2202fd
commit 8b0668a134
8 changed files with 133 additions and 89 deletions

View File

@@ -1,13 +0,0 @@
package com.muwire.core
import net.i2p.crypto.SigType
class Constants {
public static final byte PERSONA_VERSION = (byte)1
public static final SigType SIG_TYPE = SigType.EdDSA_SHA512_Ed25519
public static final int MAX_HEADER_SIZE = 0x1 << 14
public static final int MAX_HEADERS = 16
public static final String SPLIT_PATTERN = "[\\*\\+\\-,\\.:;\\(\\)=_/\\\\\\!\\\"\\\'\\\$%\\|\\[\\]\\{\\}\\?]"
}

View File

@@ -0,0 +1,7 @@
package com.muwire.core
class SplitPattern {
public static final String SPLIT_PATTERN = "[\\*\\+\\-,\\.:;\\(\\)=_/\\\\\\!\\\"\\\'\\\$%\\|\\[\\]\\{\\}\\?]";
}

View File

@@ -136,17 +136,12 @@ class PersisterService extends Service {
private def toJson(File f, SharedFile sf) { private def toJson(File f, SharedFile sf) {
def json = [:] def json = [:]
json.file = Base64.encode DataUtil.encodei18nString(f.toString()) json.file = sf.getB64EncodedFileName()
json.length = sf.getCachedLength() json.length = sf.getCachedLength()
InfoHash ih = sf.getInfoHash() InfoHash ih = sf.getInfoHash()
json.infoHash = Base64.encode ih.getRoot() json.infoHash = sf.getB64EncodedHashRoot()
json.pieceSize = sf.getPieceSize() json.pieceSize = sf.getPieceSize()
byte [] tmp = new byte [32] json.hashList = sf.getB64EncodedHashList()
json.hashList = []
for (int i = 0;i < ih.getHashList().length / 32; i++) {
System.arraycopy(ih.getHashList(), i * 32, tmp, 0, 32)
json.hashList.add Base64.encode(tmp)
}
if (sf instanceof DownloadedFile) { if (sf instanceof DownloadedFile) {
json.sources = sf.sources.stream().map( {d -> d.toBase64()}).collect(Collectors.toList()) json.sources = sf.sources.stream().map( {d -> d.toBase64()}).collect(Collectors.toList())

View File

@@ -1,6 +1,6 @@
package com.muwire.core.search package com.muwire.core.search
import com.muwire.core.Constants import com.muwire.core.SplitPattern
class SearchIndex { class SearchIndex {
@@ -32,7 +32,7 @@ class SearchIndex {
} }
private static String[] split(String source) { private static String[] split(String source) {
source = source.replaceAll(Constants.SPLIT_PATTERN, " ").toLowerCase() source = source.replaceAll(SplitPattern.SPLIT_PATTERN, " ").toLowerCase()
String [] split = source.split(" ") String [] split = source.split(" ")
def rv = [] def rv = []
split.each { if (it.length() > 0) rv << it } split.each { if (it.length() > 0) rv << it }

View File

@@ -0,0 +1,11 @@
package com.muwire.core;
import net.i2p.crypto.SigType;
public class Constants {
public static final byte PERSONA_VERSION = (byte)1;
public static final SigType SIG_TYPE = SigType.EdDSA_SHA512_Ed25519;
public static final int MAX_HEADER_SIZE = 0x1 << 14;
public static final int MAX_HEADERS = 16;
}

View File

@@ -2,6 +2,12 @@ package com.muwire.core;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import com.muwire.core.util.DataUtil;
import net.i2p.data.Base64;
public class SharedFile { public class SharedFile {
@@ -11,6 +17,10 @@ public class SharedFile {
private final String cachedPath; private final String cachedPath;
private final long cachedLength; private final long cachedLength;
private final String b64EncodedFileName;
private final String b64EncodedHashRoot;
private final List<String> b64EncodedHashList;
public SharedFile(File file, InfoHash infoHash, int pieceSize) throws IOException { public SharedFile(File file, InfoHash infoHash, int pieceSize) throws IOException {
this.file = file; this.file = file;
@@ -18,6 +28,16 @@ public class SharedFile {
this.pieceSize = pieceSize; this.pieceSize = pieceSize;
this.cachedPath = file.getAbsolutePath(); this.cachedPath = file.getAbsolutePath();
this.cachedLength = file.length(); this.cachedLength = file.length();
this.b64EncodedFileName = Base64.encode(DataUtil.encodei18nString(file.toString()));
this.b64EncodedHashRoot = Base64.encode(infoHash.getRoot());
List<String> b64List = new ArrayList<String>();
byte[] tmp = new byte[32];
for (int i = 0; i < infoHash.getHashList().length / 32; i++) {
System.arraycopy(infoHash.getHashList(), i * 32, tmp, 0, 32);
b64List.add(Base64.encode(tmp));
}
this.b64EncodedHashList = b64List;
} }
public File getFile() { public File getFile() {
@@ -40,6 +60,18 @@ public class SharedFile {
rv++; rv++;
return rv; return rv;
} }
public String getB64EncodedFileName() {
return b64EncodedFileName;
}
public String getB64EncodedHashRoot() {
return b64EncodedHashRoot;
}
public List<String> getB64EncodedHashList() {
return b64EncodedHashList;
}
public String getCachedPath() { public String getCachedPath() {
return cachedPath; return cachedPath;

View File

@@ -1,122 +1,134 @@
package com.muwire.core.util package com.muwire.core.util;
import java.lang.reflect.Field import java.io.ByteArrayOutputStream;
import java.lang.reflect.Method import java.io.DataOutputStream;
import java.nio.ByteBuffer import java.io.IOException;
import java.nio.charset.StandardCharsets import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import com.muwire.core.Constants import com.muwire.core.Constants;
import net.i2p.data.Base64 import net.i2p.data.Base64;
class DataUtil { public class DataUtil {
private final static int MAX_SHORT = (0x1 << 16) - 1 private final static int MAX_SHORT = (0x1 << 16) - 1;
static void writeUnsignedShort(int value, OutputStream os) { static void writeUnsignedShort(int value, OutputStream os) throws IOException {
if (value > MAX_SHORT || value < 0) if (value > MAX_SHORT || value < 0)
throw new IllegalArgumentException("$value invalid") throw new IllegalArgumentException("$value invalid");
byte lsb = (byte) (value & 0xFF) byte lsb = (byte) (value & 0xFF);
byte msb = (byte) (value >> 8) byte msb = (byte) (value >> 8);
os.write(msb) os.write(msb);
os.write(lsb) os.write(lsb);
} }
private final static int MAX_HEADER = 0x7FFFFF private final static int MAX_HEADER = 0x7FFFFF;
static void packHeader(int length, byte [] header) { static void packHeader(int length, byte [] header) {
if (header.length != 3) if (header.length != 3)
throw new IllegalArgumentException("header length $header.length") throw new IllegalArgumentException("header length $header.length");
if (length < 0 || length > MAX_HEADER) if (length < 0 || length > MAX_HEADER)
throw new IllegalArgumentException("length $length") throw new IllegalArgumentException("length $length");
header[2] = (byte) (length & 0xFF) header[2] = (byte) (length & 0xFF);
header[1] = (byte) ((length >> 8) & 0xFF) header[1] = (byte) ((length >> 8) & 0xFF);
header[0] = (byte) ((length >> 16) & 0x7F) header[0] = (byte) ((length >> 16) & 0x7F);
} }
static int readLength(byte [] header) { static int readLength(byte [] header) {
if (header.length != 3) if (header.length != 3)
throw new IllegalArgumentException("header length $header.length") throw new IllegalArgumentException("header length $header.length");
return (((int)(header[0] & 0x7F)) << 16) | return (((int)(header[0] & 0x7F)) << 16) |
(((int)(header[1] & 0xFF) << 8)) | (((int)(header[1] & 0xFF) << 8)) |
((int)header[2] & 0xFF) ((int)header[2] & 0xFF);
} }
static String readi18nString(byte [] encoded) { static String readi18nString(byte [] encoded) {
if (encoded.length < 2) if (encoded.length < 2)
throw new IllegalArgumentException("encoding too short $encoded.length") throw new IllegalArgumentException("encoding too short $encoded.length");
int length = ((encoded[0] & 0xFF) << 8) | (encoded[1] & 0xFF) int length = ((encoded[0] & 0xFF) << 8) | (encoded[1] & 0xFF);
if (encoded.length != length + 2) if (encoded.length != length + 2)
throw new IllegalArgumentException("encoding doesn't match length, expected $length found $encoded.length") throw new IllegalArgumentException("encoding doesn't match length, expected $length found $encoded.length");
byte [] string = new byte[length] byte [] string = new byte[length];
System.arraycopy(encoded, 2, string, 0, length) System.arraycopy(encoded, 2, string, 0, length);
new String(string, StandardCharsets.UTF_8) return new String(string, StandardCharsets.UTF_8);
} }
static byte[] encodei18nString(String string) { public static byte[] encodei18nString(String string) {
byte [] utf8 = string.getBytes(StandardCharsets.UTF_8) byte [] utf8 = string.getBytes(StandardCharsets.UTF_8);
if (utf8.length > Short.MAX_VALUE) if (utf8.length > Short.MAX_VALUE)
throw new IllegalArgumentException("String in utf8 too long $utf8.length") throw new IllegalArgumentException("String in utf8 too long $utf8.length");
def baos = new ByteArrayOutputStream() ByteArrayOutputStream baos = new ByteArrayOutputStream();
def daos = new DataOutputStream(baos) DataOutputStream daos = new DataOutputStream(baos);
daos.writeShort((short) utf8.length) try {
daos.write(utf8) daos.writeShort((short) utf8.length);
daos.close() daos.write(utf8);
baos.toByteArray() daos.close();
} catch (IOException impossible) {
throw new IllegalStateException(impossible);
}
return baos.toByteArray();
} }
public static String readTillRN(InputStream is) { public static String readTillRN(InputStream is) throws IOException {
def baos = new ByteArrayOutputStream() ByteArrayOutputStream baos = new ByteArrayOutputStream();
while(baos.size() < (Constants.MAX_HEADER_SIZE)) { while(baos.size() < (Constants.MAX_HEADER_SIZE)) {
byte read = is.read() int read = is.read();
if (read == -1) if (read == -1)
throw new IOException() throw new IOException();
if (read == '\r') { if (read == '\r') {
if (is.read() != '\n') if (is.read() != '\n')
throw new IOException("invalid header") throw new IOException("invalid header");
break break;
} }
baos.write(read) baos.write(read);
} }
new String(baos.toByteArray(), StandardCharsets.US_ASCII) return new String(baos.toByteArray(), StandardCharsets.US_ASCII);
} }
public static String encodeXHave(List<Integer> pieces, int totalPieces) { public static String encodeXHave(List<Integer> pieces, int totalPieces) {
int bytes = totalPieces / 8 int bytes = totalPieces / 8;
if (totalPieces % 8 != 0) if (totalPieces % 8 != 0)
bytes++ bytes++;
byte[] raw = new byte[bytes] byte[] raw = new byte[bytes];
pieces.each { for (int it : pieces) {
int byteIdx = it / 8 int byteIdx = it / 8;
int offset = it % 8 int offset = it % 8;
int mask = 0x80 >>> offset int mask = 0x80 >>> offset;
raw[byteIdx] |= mask raw[byteIdx] |= mask;
} }
Base64.encode(raw) return Base64.encode(raw);
} }
public static List<Integer> decodeXHave(String xHave) { public static List<Integer> decodeXHave(String xHave) {
byte [] availablePieces = Base64.decode(xHave) byte [] availablePieces = Base64.decode(xHave);
List<Integer> available = new ArrayList<>() List<Integer> available = new ArrayList<>();
availablePieces.eachWithIndex {b, i -> for (int i = 0; i < availablePieces.length; i ++) {
byte b = availablePieces[i];
for (int j = 0; j < 8 ; j++) { for (int j = 0; j < 8 ; j++) {
byte mask = 0x80 >>> j byte mask = (byte) (0x80 >>> j);
if ((b & mask) == mask) { if ((b & mask) == mask) {
available.add(i * 8 + j) available.add(i * 8 + j);
} }
} }
} }
available return available;
} }
public static Exception findRoot(Exception e) { public static Throwable findRoot(Throwable e) {
while(e.getCause() != null) while(e.getCause() != null)
e = e.getCause() e = e.getCause();
e return e;
} }
public static void tryUnmap(ByteBuffer cb) { public static void tryUnmap(ByteBuffer cb) {

View File

@@ -13,10 +13,10 @@ import javax.annotation.Nonnull
import javax.inject.Inject import javax.inject.Inject
import javax.swing.JTable import javax.swing.JTable
import com.muwire.core.Constants
import com.muwire.core.Core import com.muwire.core.Core
import com.muwire.core.Persona import com.muwire.core.Persona
import com.muwire.core.SharedFile import com.muwire.core.SharedFile
import com.muwire.core.SplitPattern
import com.muwire.core.download.Downloader import com.muwire.core.download.Downloader
import com.muwire.core.download.DownloadStartedEvent import com.muwire.core.download.DownloadStartedEvent
import com.muwire.core.download.UIDownloadCancelledEvent import com.muwire.core.download.UIDownloadCancelledEvent
@@ -80,7 +80,7 @@ class MainFrameController {
searchEvent = new SearchEvent(searchHash : root, uuid : uuid, oobInfohash: true) searchEvent = new SearchEvent(searchHash : root, uuid : uuid, oobInfohash: true)
} else { } else {
// this can be improved a lot // this can be improved a lot
def replaced = search.toLowerCase().trim().replaceAll(Constants.SPLIT_PATTERN, " ") def replaced = search.toLowerCase().trim().replaceAll(SplitPattern.SPLIT_PATTERN, " ")
def terms = replaced.split(" ") def terms = replaced.split(" ")
def nonEmpty = [] def nonEmpty = []
terms.each { if (it.length() > 0) nonEmpty << it } terms.each { if (it.length() > 0) nonEmpty << it }