idk/muwire
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite utils into Java, cache the persistable data of shared files to reduce object churn

Browse Source
This commit is contained in:
Zlatin Balevsky
2019-10-05 22:50:32 +01:00
parent 730d2202fd
commit 8b0668a134
8 changed files with 133 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
core/src/main/groovy/com/muwire/core/Constants.groovy
View File

@@ -1,13 +0,0 @@
package com.muwire.core
import net.i2p.crypto.SigType
class Constants {
public static final byte PERSONA_VERSION = (byte)1
public static final SigType SIG_TYPE = SigType.EdDSA_SHA512_Ed25519
public static final int MAX_HEADER_SIZE = 0x1 << 14
public static final int MAX_HEADERS = 16
public static final String SPLIT_PATTERN = "[\\*\\+\\-,\\.:;\\(\\)=_/\\\\\\!\\\"\\\'\\\$%\\|\\[\\]\\{\\}\\?]"
}

7
core/src/main/groovy/com/muwire/core/SplitPattern.groovy Normal file
View File

@@ -0,0 +1,7 @@
package com.muwire.core
class SplitPattern {
public static final String SPLIT_PATTERN = "[\\*\\+\\-,\\.:;\\(\\)=_/\\\\\\!\\\"\\\'\\\$%\\|\\[\\]\\{\\}\\?]";
}

11
core/src/main/groovy/com/muwire/core/files/PersisterService.groovy
View File

@@ -136,17 +136,12 @@ class PersisterService extends Service {
private def toJson(File f, SharedFile sf) {
def json = [:]
json.file = Base64.encode DataUtil.encodei18nString(f.toString())
json.file = sf.getB64EncodedFileName()
json.length = sf.getCachedLength()
InfoHash ih = sf.getInfoHash()
json.infoHash = Base64.encode ih.getRoot()
json.infoHash = sf.getB64EncodedHashRoot()
json.pieceSize = sf.getPieceSize()
byte [] tmp = new byte [32]
json.hashList = []
for (int i = 0;i < ih.getHashList().length / 32; i++) {
System.arraycopy(ih.getHashList(), i * 32, tmp, 0, 32)
json.hashList.add Base64.encode(tmp)
}
json.hashList = sf.getB64EncodedHashList()
if (sf instanceof DownloadedFile) {
json.sources = sf.sources.stream().map( {d -> d.toBase64()}).collect(Collectors.toList())

4
core/src/main/groovy/com/muwire/core/search/SearchIndex.groovy
View File

@@ -1,6 +1,6 @@
package com.muwire.core.search
import com.muwire.core.Constants
import com.muwire.core.SplitPattern
class SearchIndex {
@@ -32,7 +32,7 @@ class SearchIndex {
}
private static String[] split(String source) {
source = source.replaceAll(Constants.SPLIT_PATTERN, " ").toLowerCase()
source = source.replaceAll(SplitPattern.SPLIT_PATTERN, " ").toLowerCase()
String [] split = source.split(" ")
def rv = []
split.each { if (it.length() > 0) rv << it }

11
core/src/main/java/com/muwire/core/Constants.java Normal file
View File

@@ -0,0 +1,11 @@
package com.muwire.core;
import net.i2p.crypto.SigType;
public class Constants {
public static final byte PERSONA_VERSION = (byte)1;
public static final SigType SIG_TYPE = SigType.EdDSA_SHA512_Ed25519;
public static final int MAX_HEADER_SIZE = 0x1 << 14;
public static final int MAX_HEADERS = 16;
}

32
core/src/main/java/com/muwire/core/SharedFile.java
View File

@@ -2,6 +2,12 @@ package com.muwire.core;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import com.muwire.core.util.DataUtil;
import net.i2p.data.Base64;
public class SharedFile {
@@ -12,12 +18,26 @@ public class SharedFile {
private final String cachedPath;
private final long cachedLength;
private final String b64EncodedFileName;
private final String b64EncodedHashRoot;
private final List<String> b64EncodedHashList;
public SharedFile(File file, InfoHash infoHash, int pieceSize) throws IOException {
this.file = file;
this.infoHash = infoHash;
this.pieceSize = pieceSize;
this.cachedPath = file.getAbsolutePath();
this.cachedLength = file.length();
this.b64EncodedFileName = Base64.encode(DataUtil.encodei18nString(file.toString()));
this.b64EncodedHashRoot = Base64.encode(infoHash.getRoot());
List<String> b64List = new ArrayList<String>();
byte[] tmp = new byte[32];
for (int i = 0; i < infoHash.getHashList().length / 32; i++) {
System.arraycopy(infoHash.getHashList(), i * 32, tmp, 0, 32);
b64List.add(Base64.encode(tmp));
}
this.b64EncodedHashList = b64List;
}
public File getFile() {
@@ -41,6 +61,18 @@ public class SharedFile {
return rv;
}
public String getB64EncodedFileName() {
return b64EncodedFileName;
}
public String getB64EncodedHashRoot() {
return b64EncodedHashRoot;
}
public List<String> getB64EncodedHashList() {
return b64EncodedHashList;
}
public String getCachedPath() {
return cachedPath;
}

140
core/src/main/groovy/com/muwire/core/util/DataUtil.groovy → core/src/main/java/com/muwire/core/util/DataUtil.java
View File

@@ -1,122 +1,134 @@
package com.muwire.core.util
package com.muwire.core.util;
import java.lang.reflect.Field
import java.lang.reflect.Method
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import com.muwire.core.Constants
import com.muwire.core.Constants;
import net.i2p.data.Base64
import net.i2p.data.Base64;
class DataUtil {
public class DataUtil {
private final static int MAX_SHORT = (0x1 << 16) - 1
private final static int MAX_SHORT = (0x1 << 16) - 1;
static void writeUnsignedShort(int value, OutputStream os) {
static void writeUnsignedShort(int value, OutputStream os) throws IOException {
if (value > MAX_SHORT || value < 0)
throw new IllegalArgumentException("$value invalid")
throw new IllegalArgumentException("$value invalid");
byte lsb = (byte) (value & 0xFF)
byte msb = (byte) (value >> 8)
byte lsb = (byte) (value & 0xFF);
byte msb = (byte) (value >> 8);
os.write(msb)
os.write(lsb)
os.write(msb);
os.write(lsb);
}
private final static int MAX_HEADER = 0x7FFFFF
private final static int MAX_HEADER = 0x7FFFFF;
static void packHeader(int length, byte [] header) {
if (header.length != 3)
throw new IllegalArgumentException("header length $header.length")
throw new IllegalArgumentException("header length $header.length");
if (length < 0 || length > MAX_HEADER)
throw new IllegalArgumentException("length $length")
throw new IllegalArgumentException("length $length");
header[2] = (byte) (length & 0xFF)
header[1] = (byte) ((length >> 8) & 0xFF)
header[0] = (byte) ((length >> 16) & 0x7F)
header[2] = (byte) (length & 0xFF);
header[1] = (byte) ((length >> 8) & 0xFF);
header[0] = (byte) ((length >> 16) & 0x7F);
}
static int readLength(byte [] header) {
if (header.length != 3)
throw new IllegalArgumentException("header length $header.length")
throw new IllegalArgumentException("header length $header.length");
return (((int)(header[0] & 0x7F)) << 16) |
(((int)(header[1] & 0xFF) << 8)) |
((int)header[2] & 0xFF)
((int)header[2] & 0xFF);
}
static String readi18nString(byte [] encoded) {
if (encoded.length < 2)
throw new IllegalArgumentException("encoding too short $encoded.length")
int length = ((encoded[0] & 0xFF) << 8) | (encoded[1] & 0xFF)
throw new IllegalArgumentException("encoding too short $encoded.length");
int length = ((encoded[0] & 0xFF) << 8) | (encoded[1] & 0xFF);
if (encoded.length != length + 2)
throw new IllegalArgumentException("encoding doesn't match length, expected $length found $encoded.length")
byte [] string = new byte[length]
System.arraycopy(encoded, 2, string, 0, length)
new String(string, StandardCharsets.UTF_8)
throw new IllegalArgumentException("encoding doesn't match length, expected $length found $encoded.length");
byte [] string = new byte[length];
System.arraycopy(encoded, 2, string, 0, length);
return new String(string, StandardCharsets.UTF_8);
}
static byte[] encodei18nString(String string) {
byte [] utf8 = string.getBytes(StandardCharsets.UTF_8)
public static byte[] encodei18nString(String string) {
byte [] utf8 = string.getBytes(StandardCharsets.UTF_8);
if (utf8.length > Short.MAX_VALUE)
throw new IllegalArgumentException("String in utf8 too long $utf8.length")
def baos = new ByteArrayOutputStream()
def daos = new DataOutputStream(baos)
daos.writeShort((short) utf8.length)
daos.write(utf8)
daos.close()
baos.toByteArray()
throw new IllegalArgumentException("String in utf8 too long $utf8.length");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream daos = new DataOutputStream(baos);
try {
daos.writeShort((short) utf8.length);
daos.write(utf8);
daos.close();
} catch (IOException impossible) {
throw new IllegalStateException(impossible);
}
return baos.toByteArray();
}
public static String readTillRN(InputStream is) {
def baos = new ByteArrayOutputStream()
public static String readTillRN(InputStream is) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
while(baos.size() < (Constants.MAX_HEADER_SIZE)) {
byte read = is.read()
int read = is.read();
if (read == -1)
throw new IOException()
throw new IOException();
if (read == '\r') {
if (is.read() != '\n')
throw new IOException("invalid header")
break
throw new IOException("invalid header");
break;
}
baos.write(read)
baos.write(read);
}
new String(baos.toByteArray(), StandardCharsets.US_ASCII)
return new String(baos.toByteArray(), StandardCharsets.US_ASCII);
}
public static String encodeXHave(List<Integer> pieces, int totalPieces) {
int bytes = totalPieces / 8
int bytes = totalPieces / 8;
if (totalPieces % 8 != 0)
bytes++
byte[] raw = new byte[bytes]
pieces.each {
int byteIdx = it / 8
int offset = it % 8
int mask = 0x80 >>> offset
raw[byteIdx] |= mask
bytes++;
byte[] raw = new byte[bytes];
for (int it : pieces) {
int byteIdx = it / 8;
int offset = it % 8;
int mask = 0x80 >>> offset;
raw[byteIdx] |= mask;
}
Base64.encode(raw)
return Base64.encode(raw);
}
public static List<Integer> decodeXHave(String xHave) {
byte [] availablePieces = Base64.decode(xHave)
List<Integer> available = new ArrayList<>()
availablePieces.eachWithIndex {b, i ->
byte [] availablePieces = Base64.decode(xHave);
List<Integer> available = new ArrayList<>();
for (int i = 0; i < availablePieces.length; i ++) {
byte b = availablePieces[i];
for (int j = 0; j < 8 ; j++) {
byte mask = 0x80 >>> j
byte mask = (byte) (0x80 >>> j);
if ((b & mask) == mask) {
available.add(i * 8 + j)
available.add(i * 8 + j);
}
}
}
available
return available;
}
public static Exception findRoot(Exception e) {
public static Throwable findRoot(Throwable e) {
while(e.getCause() != null)
e = e.getCause()
e
e = e.getCause();
return e;
}
public static void tryUnmap(ByteBuffer cb) {

4
gui/griffon-app/controllers/com/muwire/gui/MainFrameController.groovy
View File

@@ -13,10 +13,10 @@ import javax.annotation.Nonnull
import javax.inject.Inject
import javax.swing.JTable
import com.muwire.core.Constants
import com.muwire.core.Core
import com.muwire.core.Persona
import com.muwire.core.SharedFile
import com.muwire.core.SplitPattern
import com.muwire.core.download.Downloader
import com.muwire.core.download.DownloadStartedEvent
import com.muwire.core.download.UIDownloadCancelledEvent
@@ -80,7 +80,7 @@ class MainFrameController {
searchEvent = new SearchEvent(searchHash : root, uuid : uuid, oobInfohash: true)
} else {
// this can be improved a lot
def replaced = search.toLowerCase().trim().replaceAll(Constants.SPLIT_PATTERN, " ")
def replaced = search.toLowerCase().trim().replaceAll(SplitPattern.SPLIT_PATTERN, " ")
def terms = replaced.split(" ")
def nonEmpty = []
terms.each { if (it.length() > 0) nonEmpty << it }