forked from I2P_Developers/i2p.i2p
This is MaxMind-DB-Reader-java release 1.2.2 2017-02-17
retrieved from <a href="https://github.com/maxmind/MaxMind-DB-Reader-java">github</a>. For reading MaxMind GeoLite2 database files. Unmodified as a baseline for future merges. Does not compile. To be heavily modified to remove the dependency on the large com.fasterxml.jackson.databind JSON package, and use POJOs instead, see following revs. Apache 2.0 license.
This commit is contained in:
81
router/java/src/com/maxmind/db/BufferHolder.java
Normal file
81
router/java/src/com/maxmind/db/BufferHolder.java
Normal file
@@ -0,0 +1,81 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.FileChannel.MapMode;
|
||||
|
||||
import com.maxmind.db.Reader.FileMode;
|
||||
|
||||
final class BufferHolder {
|
||||
// DO NOT PASS OUTSIDE THIS CLASS. Doing so will remove thread safety.
|
||||
private final ByteBuffer buffer;
|
||||
|
||||
BufferHolder(File database, FileMode mode) throws IOException {
|
||||
final RandomAccessFile file = new RandomAccessFile(database, "r");
|
||||
boolean threw = true;
|
||||
try {
|
||||
final FileChannel channel = file.getChannel();
|
||||
if (mode == FileMode.MEMORY) {
|
||||
this.buffer = ByteBuffer.wrap(new byte[(int) channel.size()]);
|
||||
if (channel.read(this.buffer) != this.buffer.capacity()) {
|
||||
throw new IOException("Unable to read "
|
||||
+ database.getName()
|
||||
+ " into memory. Unexpected end of stream.");
|
||||
}
|
||||
} else {
|
||||
this.buffer = channel.map(MapMode.READ_ONLY, 0, channel.size());
|
||||
}
|
||||
threw = false;
|
||||
} finally {
|
||||
try {
|
||||
// Also closes the underlying channel.
|
||||
file.close();
|
||||
} catch (final IOException e) {
|
||||
// If an exception was underway when we entered the finally
|
||||
// block, don't stomp over it due to an error closing the file
|
||||
// and channel.
|
||||
if (!threw) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a ThreadBuffer from the provided URL.
|
||||
*
|
||||
* @param stream the source of my bytes.
|
||||
* @throws IOException if unable to read from your source.
|
||||
* @throws NullPointerException if you provide a NULL InputStream
|
||||
*/
|
||||
BufferHolder(InputStream stream) throws IOException {
|
||||
if (null == stream) {
|
||||
throw new NullPointerException("Unable to use a NULL InputStream");
|
||||
}
|
||||
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
final byte[] bytes = new byte[16 * 1024];
|
||||
int br;
|
||||
while (-1 != (br = stream.read(bytes))) {
|
||||
baos.write(bytes, 0, br);
|
||||
}
|
||||
this.buffer = ByteBuffer.wrap(baos.toByteArray());
|
||||
}
|
||||
|
||||
// This is just to ease unit testing
|
||||
BufferHolder(ByteBuffer buffer) {
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns a duplicate of the underlying ByteBuffer. The returned ByteBuffer
|
||||
* should not be shared between threads.
|
||||
*/
|
||||
synchronized ByteBuffer get() {
|
||||
return this.buffer.duplicate();
|
||||
}
|
||||
}
|
||||
47
router/java/src/com/maxmind/db/CHMCache.java
Normal file
47
router/java/src/com/maxmind/db/CHMCache.java
Normal file
@@ -0,0 +1,47 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
/**
|
||||
* A simplistic cache using a {@link ConcurrentHashMap}. There's no eviction
|
||||
* policy, it just fills up until reaching the specified capacity <small>(or
|
||||
* close enough at least, bounds check is not atomic :)</small>
|
||||
*/
|
||||
public class CHMCache implements NodeCache {
|
||||
|
||||
private static final int DEFAULT_CAPACITY = 4096;
|
||||
|
||||
private final int capacity;
|
||||
private final ConcurrentHashMap<Integer, JsonNode> cache;
|
||||
private boolean cacheFull = false;
|
||||
|
||||
public CHMCache() {
|
||||
this(DEFAULT_CAPACITY);
|
||||
}
|
||||
|
||||
public CHMCache(int capacity) {
|
||||
this.capacity = capacity;
|
||||
this.cache = new ConcurrentHashMap<Integer, JsonNode>(capacity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public JsonNode get(int key, Loader loader) throws IOException {
|
||||
Integer k = key;
|
||||
JsonNode value = cache.get(k);
|
||||
if (value == null) {
|
||||
value = loader.load(key);
|
||||
if (!cacheFull) {
|
||||
if (cache.size() < capacity) {
|
||||
cache.put(k, value);
|
||||
} else {
|
||||
cacheFull = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
}
|
||||
15
router/java/src/com/maxmind/db/ClosedDatabaseException.java
Normal file
15
router/java/src/com/maxmind/db/ClosedDatabaseException.java
Normal file
@@ -0,0 +1,15 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Signals that the underlying database has been closed.
|
||||
*/
|
||||
public class ClosedDatabaseException extends IOException {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
ClosedDatabaseException() {
|
||||
super("The MaxMind DB has been closed.");
|
||||
}
|
||||
}
|
||||
297
router/java/src/com/maxmind/db/Decoder.java
Normal file
297
router/java/src/com/maxmind/db/Decoder.java
Normal file
@@ -0,0 +1,297 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.CharacterCodingException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.CharsetDecoder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.*;
|
||||
|
||||
/*
|
||||
* Decoder for MaxMind DB data.
|
||||
*
|
||||
* This class CANNOT be shared between threads
|
||||
*/
|
||||
final class Decoder {
|
||||
|
||||
private static final Charset UTF_8 = Charset.forName("UTF-8");
|
||||
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private static final int[] POINTER_VALUE_OFFSETS = { 0, 0, 1 << 11, (1 << 19) + ((1) << 11), 0 };
|
||||
|
||||
// XXX - This is only for unit testings. We should possibly make a
|
||||
// constructor to set this
|
||||
boolean POINTER_TEST_HACK = false;
|
||||
|
||||
private final NodeCache cache;
|
||||
|
||||
private final long pointerBase;
|
||||
|
||||
private final CharsetDecoder utfDecoder = UTF_8.newDecoder();
|
||||
|
||||
private final ByteBuffer buffer;
|
||||
|
||||
static enum Type {
|
||||
EXTENDED, POINTER, UTF8_STRING, DOUBLE, BYTES, UINT16, UINT32, MAP, INT32, UINT64, UINT128, ARRAY, CONTAINER, END_MARKER, BOOLEAN, FLOAT;
|
||||
|
||||
// Java clones the array when you call values(). Caching it increased
|
||||
// the speed by about 5000 requests per second on my machine.
|
||||
final static Type[] values = Type.values();
|
||||
|
||||
public static Type get(int i) {
|
||||
return Type.values[i];
|
||||
}
|
||||
|
||||
private static Type get(byte b) {
|
||||
// bytes are signed, but we want to treat them as unsigned here
|
||||
return Type.get(b & 0xFF);
|
||||
}
|
||||
|
||||
public static Type fromControlByte(int b) {
|
||||
// The type is encoded in the first 3 bits of the byte.
|
||||
return Type.get((byte) ((0xFF & b) >>> 5));
|
||||
}
|
||||
}
|
||||
|
||||
Decoder(NodeCache cache, ByteBuffer buffer, long pointerBase) {
|
||||
this.cache = cache;
|
||||
this.pointerBase = pointerBase;
|
||||
this.buffer = buffer;
|
||||
}
|
||||
|
||||
private final NodeCache.Loader cacheLoader = new NodeCache.Loader() {
|
||||
@Override
|
||||
public JsonNode load(int key) throws IOException {
|
||||
return decode(key);
|
||||
}
|
||||
};
|
||||
|
||||
JsonNode decode(int offset) throws IOException {
|
||||
if (offset >= this.buffer.capacity()) {
|
||||
throw new InvalidDatabaseException(
|
||||
"The MaxMind DB file's data section contains bad data: "
|
||||
+ "pointer larger than the database.");
|
||||
}
|
||||
|
||||
this.buffer.position(offset);
|
||||
return decode();
|
||||
}
|
||||
|
||||
JsonNode decode() throws IOException {
|
||||
int ctrlByte = 0xFF & this.buffer.get();
|
||||
|
||||
Type type = Type.fromControlByte(ctrlByte);
|
||||
|
||||
// Pointers are a special case, we don't read the next 'size' bytes, we
|
||||
// use the size to determine the length of the pointer and then follow
|
||||
// it.
|
||||
if (type.equals(Type.POINTER)) {
|
||||
int pointerSize = ((ctrlByte >>> 3) & 0x3) + 1;
|
||||
int base = pointerSize == 4 ? (byte) 0 : (byte) (ctrlByte & 0x7);
|
||||
int packed = this.decodeInteger(base, pointerSize);
|
||||
long pointer = packed + this.pointerBase + POINTER_VALUE_OFFSETS[pointerSize];
|
||||
|
||||
// for unit testing
|
||||
if (this.POINTER_TEST_HACK) {
|
||||
return new LongNode(pointer);
|
||||
}
|
||||
|
||||
int targetOffset = (int) pointer;
|
||||
int position = buffer.position();
|
||||
JsonNode node = cache.get(targetOffset, cacheLoader);
|
||||
buffer.position(position);
|
||||
return node;
|
||||
}
|
||||
|
||||
if (type.equals(Type.EXTENDED)) {
|
||||
int nextByte = this.buffer.get();
|
||||
|
||||
int typeNum = nextByte + 7;
|
||||
|
||||
if (typeNum < 8) {
|
||||
throw new InvalidDatabaseException(
|
||||
"Something went horribly wrong in the decoder. An extended type "
|
||||
+ "resolved to a type number < 8 (" + typeNum
|
||||
+ ")");
|
||||
}
|
||||
|
||||
type = Type.get(typeNum);
|
||||
}
|
||||
|
||||
int size = ctrlByte & 0x1f;
|
||||
if (size >= 29) {
|
||||
int bytesToRead = size - 28;
|
||||
int i = this.decodeInteger(bytesToRead);
|
||||
switch (size) {
|
||||
case 29:
|
||||
size = 29 + i;
|
||||
break;
|
||||
case 30:
|
||||
size = 285 + i;
|
||||
break;
|
||||
default:
|
||||
size = 65821 + (i & (0x0FFFFFFF >>> 32 - 8 * bytesToRead));
|
||||
}
|
||||
}
|
||||
|
||||
return this.decodeByType(type, size);
|
||||
}
|
||||
|
||||
private JsonNode decodeByType(Type type, int size)
|
||||
throws IOException {
|
||||
switch (type) {
|
||||
case MAP:
|
||||
return this.decodeMap(size);
|
||||
case ARRAY:
|
||||
return this.decodeArray(size);
|
||||
case BOOLEAN:
|
||||
return Decoder.decodeBoolean(size);
|
||||
case UTF8_STRING:
|
||||
return new TextNode(this.decodeString(size));
|
||||
case DOUBLE:
|
||||
return this.decodeDouble(size);
|
||||
case FLOAT:
|
||||
return this.decodeFloat(size);
|
||||
case BYTES:
|
||||
return new BinaryNode(this.getByteArray(size));
|
||||
case UINT16:
|
||||
return this.decodeUint16(size);
|
||||
case UINT32:
|
||||
return this.decodeUint32(size);
|
||||
case INT32:
|
||||
return this.decodeInt32(size);
|
||||
case UINT64:
|
||||
return this.decodeBigInteger(size);
|
||||
case UINT128:
|
||||
return this.decodeBigInteger(size);
|
||||
default:
|
||||
throw new InvalidDatabaseException(
|
||||
"Unknown or unexpected type: " + type.name());
|
||||
}
|
||||
}
|
||||
|
||||
private String decodeString(int size) throws CharacterCodingException {
|
||||
int oldLimit = buffer.limit();
|
||||
buffer.limit(buffer.position() + size);
|
||||
String s = utfDecoder.decode(buffer).toString();
|
||||
buffer.limit(oldLimit);
|
||||
return s;
|
||||
}
|
||||
|
||||
private IntNode decodeUint16(int size) {
|
||||
return new IntNode(this.decodeInteger(size));
|
||||
}
|
||||
|
||||
private IntNode decodeInt32(int size) {
|
||||
return new IntNode(this.decodeInteger(size));
|
||||
}
|
||||
|
||||
private long decodeLong(int size) {
|
||||
long integer = 0;
|
||||
for (int i = 0; i < size; i++) {
|
||||
integer = (integer << 8) | (this.buffer.get() & 0xFF);
|
||||
}
|
||||
return integer;
|
||||
}
|
||||
|
||||
private LongNode decodeUint32(int size) {
|
||||
return new LongNode(this.decodeLong(size));
|
||||
}
|
||||
|
||||
private int decodeInteger(int size) {
|
||||
return this.decodeInteger(0, size);
|
||||
}
|
||||
|
||||
private int decodeInteger(int base, int size) {
|
||||
return Decoder.decodeInteger(this.buffer, base, size);
|
||||
}
|
||||
|
||||
static int decodeInteger(ByteBuffer buffer, int base, int size) {
|
||||
int integer = base;
|
||||
for (int i = 0; i < size; i++) {
|
||||
integer = (integer << 8) | (buffer.get() & 0xFF);
|
||||
}
|
||||
return integer;
|
||||
}
|
||||
|
||||
private BigIntegerNode decodeBigInteger(int size) {
|
||||
byte[] bytes = this.getByteArray(size);
|
||||
return new BigIntegerNode(new BigInteger(1, bytes));
|
||||
}
|
||||
|
||||
private DoubleNode decodeDouble(int size) throws InvalidDatabaseException {
|
||||
if (size != 8) {
|
||||
throw new InvalidDatabaseException(
|
||||
"The MaxMind DB file's data section contains bad data: "
|
||||
+ "invalid size of double.");
|
||||
}
|
||||
return new DoubleNode(this.buffer.getDouble());
|
||||
}
|
||||
|
||||
private FloatNode decodeFloat(int size) throws InvalidDatabaseException {
|
||||
if (size != 4) {
|
||||
throw new InvalidDatabaseException(
|
||||
"The MaxMind DB file's data section contains bad data: "
|
||||
+ "invalid size of float.");
|
||||
}
|
||||
return new FloatNode(this.buffer.getFloat());
|
||||
}
|
||||
|
||||
private static BooleanNode decodeBoolean(int size)
|
||||
throws InvalidDatabaseException {
|
||||
switch (size) {
|
||||
case 0:
|
||||
return BooleanNode.FALSE;
|
||||
case 1:
|
||||
return BooleanNode.TRUE;
|
||||
default:
|
||||
throw new InvalidDatabaseException(
|
||||
"The MaxMind DB file's data section contains bad data: "
|
||||
+ "invalid size of boolean.");
|
||||
}
|
||||
}
|
||||
|
||||
private JsonNode decodeArray(int size) throws IOException {
|
||||
|
||||
List<JsonNode> array = new ArrayList<JsonNode>(size);
|
||||
for (int i = 0; i < size; i++) {
|
||||
JsonNode r = this.decode();
|
||||
array.add(r);
|
||||
}
|
||||
|
||||
return new ArrayNode(OBJECT_MAPPER.getNodeFactory(), Collections.unmodifiableList(array));
|
||||
}
|
||||
|
||||
private JsonNode decodeMap(int size) throws IOException {
|
||||
int capacity = (int) (size / 0.75F + 1.0F);
|
||||
Map<String, JsonNode> map = new HashMap<String, JsonNode>(capacity);
|
||||
|
||||
for (int i = 0; i < size; i++) {
|
||||
String key = this.decode().asText();
|
||||
JsonNode value = this.decode();
|
||||
map.put(key, value);
|
||||
}
|
||||
|
||||
return new ObjectNode(OBJECT_MAPPER.getNodeFactory(), Collections.unmodifiableMap(map));
|
||||
}
|
||||
|
||||
private byte[] getByteArray(int length) {
|
||||
return Decoder.getByteArray(this.buffer, length);
|
||||
}
|
||||
|
||||
private static byte[] getByteArray(ByteBuffer buffer, int length) {
|
||||
byte[] bytes = new byte[length];
|
||||
buffer.get(bytes);
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
28
router/java/src/com/maxmind/db/InvalidDatabaseException.java
Normal file
28
router/java/src/com/maxmind/db/InvalidDatabaseException.java
Normal file
@@ -0,0 +1,28 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Signals that there was an issue reading from the MaxMind DB file due to
|
||||
* unexpected data formatting. This generally suggests that the database is
|
||||
* corrupt or otherwise not in a format supported by the reader.
|
||||
*/
|
||||
public class InvalidDatabaseException extends IOException {
|
||||
|
||||
private static final long serialVersionUID = 6161763462364823003L;
|
||||
|
||||
/**
|
||||
* @param message A message describing the reason why the exception was thrown.
|
||||
*/
|
||||
public InvalidDatabaseException(String message) {
|
||||
super(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param message A message describing the reason why the exception was thrown.
|
||||
* @param cause The cause of the exception.
|
||||
*/
|
||||
public InvalidDatabaseException(String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
}
|
||||
}
|
||||
151
router/java/src/com/maxmind/db/Metadata.java
Normal file
151
router/java/src/com/maxmind/db/Metadata.java
Normal file
@@ -0,0 +1,151 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public final class Metadata {
|
||||
private final int binaryFormatMajorVersion;
|
||||
private final int binaryFormatMinorVersion;
|
||||
|
||||
private final long buildEpoch;
|
||||
|
||||
private final String databaseType;
|
||||
|
||||
private final JsonNode description;
|
||||
|
||||
private final int ipVersion;
|
||||
|
||||
private final JsonNode languages;
|
||||
|
||||
private final int nodeByteSize;
|
||||
|
||||
private final int nodeCount;
|
||||
|
||||
private final int recordSize;
|
||||
|
||||
private final int searchTreeSize;
|
||||
|
||||
Metadata(JsonNode metadata) {
|
||||
this.binaryFormatMajorVersion = metadata.get(
|
||||
"binary_format_major_version").asInt();
|
||||
this.binaryFormatMinorVersion = metadata.get(
|
||||
"binary_format_minor_version").asInt();
|
||||
this.buildEpoch = metadata.get("build_epoch").asLong();
|
||||
this.databaseType = metadata.get("database_type").asText();
|
||||
this.languages = metadata.get("languages");
|
||||
this.description = metadata.get("description");
|
||||
this.ipVersion = metadata.get("ip_version").asInt();
|
||||
this.nodeCount = metadata.get("node_count").asInt();
|
||||
this.recordSize = metadata.get("record_size").asInt();
|
||||
this.nodeByteSize = this.recordSize / 4;
|
||||
this.searchTreeSize = this.nodeCount * this.nodeByteSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the major version number for the database's binary format.
|
||||
*/
|
||||
public int getBinaryFormatMajorVersion() {
|
||||
return this.binaryFormatMajorVersion;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the minor version number for the database's binary format.
|
||||
*/
|
||||
public int getBinaryFormatMinorVersion() {
|
||||
return this.binaryFormatMinorVersion;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the date of the database build.
|
||||
*/
|
||||
public Date getBuildDate() {
|
||||
return new Date(this.buildEpoch * 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return a string that indicates the structure of each data record
|
||||
* associated with an IP address. The actual definition of these
|
||||
* structures is left up to the database creator.
|
||||
*/
|
||||
public String getDatabaseType() {
|
||||
return this.databaseType;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return map from language code to description in that language.
|
||||
*/
|
||||
public Map<String, String> getDescription() {
|
||||
return new ObjectMapper().convertValue(this.description,
|
||||
new TypeReference<HashMap<String, String>>() {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @return whether the database contains IPv4 or IPv6 address data. The only
|
||||
* possible values are 4 and 6.
|
||||
*/
|
||||
public int getIpVersion() {
|
||||
return this.ipVersion;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list of languages supported by the database.
|
||||
*/
|
||||
public List<String> getLanguages() {
|
||||
return new ObjectMapper().convertValue(this.languages,
|
||||
new TypeReference<ArrayList<String>>() {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the nodeByteSize
|
||||
*/
|
||||
int getNodeByteSize() {
|
||||
return this.nodeByteSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of nodes in the search tree.
|
||||
*/
|
||||
int getNodeCount() {
|
||||
return this.nodeCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the number of bits in a record in the search tree. Note that each
|
||||
* node consists of two records.
|
||||
*/
|
||||
int getRecordSize() {
|
||||
return this.recordSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the searchTreeSize
|
||||
*/
|
||||
int getSearchTreeSize() {
|
||||
return this.searchTreeSize;
|
||||
}
|
||||
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
*
|
||||
* @see java.lang.Object#toString()
|
||||
*/
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Metadata [binaryFormatMajorVersion="
|
||||
+ this.binaryFormatMajorVersion + ", binaryFormatMinorVersion="
|
||||
+ this.binaryFormatMinorVersion + ", buildEpoch="
|
||||
+ this.buildEpoch + ", databaseType=" + this.databaseType
|
||||
+ ", description=" + this.description + ", ipVersion="
|
||||
+ this.ipVersion + ", nodeCount=" + this.nodeCount
|
||||
+ ", recordSize=" + this.recordSize + "]";
|
||||
}
|
||||
}
|
||||
26
router/java/src/com/maxmind/db/NoCache.java
Normal file
26
router/java/src/com/maxmind/db/NoCache.java
Normal file
@@ -0,0 +1,26 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
/**
|
||||
* A no-op cache singleton.
|
||||
*/
|
||||
public class NoCache implements NodeCache {
|
||||
|
||||
private static final NoCache INSTANCE = new NoCache();
|
||||
|
||||
private NoCache() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public JsonNode get(int key, Loader loader) throws IOException {
|
||||
return loader.load(key);
|
||||
}
|
||||
|
||||
public static NoCache getInstance() {
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
}
|
||||
15
router/java/src/com/maxmind/db/NodeCache.java
Normal file
15
router/java/src/com/maxmind/db/NodeCache.java
Normal file
@@ -0,0 +1,15 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
public interface NodeCache {
|
||||
|
||||
public interface Loader {
|
||||
JsonNode load(int key) throws IOException;
|
||||
}
|
||||
|
||||
public JsonNode get(int key, Loader loader) throws IOException;
|
||||
|
||||
}
|
||||
309
router/java/src/com/maxmind/db/Reader.java
Normal file
309
router/java/src/com/maxmind/db/Reader.java
Normal file
@@ -0,0 +1,309 @@
|
||||
package com.maxmind.db;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
|
||||
/**
|
||||
* Instances of this class provide a reader for the MaxMind DB format. IP
|
||||
* addresses can be looked up using the <code>get</code> method.
|
||||
*/
|
||||
public final class Reader implements Closeable {
|
||||
private static final int DATA_SECTION_SEPARATOR_SIZE = 16;
|
||||
private static final byte[] METADATA_START_MARKER = {(byte) 0xAB,
|
||||
(byte) 0xCD, (byte) 0xEF, 'M', 'a', 'x', 'M', 'i', 'n', 'd', '.',
|
||||
'c', 'o', 'm'};
|
||||
|
||||
private final int ipV4Start;
|
||||
private final Metadata metadata;
|
||||
private final AtomicReference<BufferHolder> bufferHolderReference;
|
||||
private final NodeCache cache;
|
||||
|
||||
/**
|
||||
* The file mode to use when opening a MaxMind DB.
|
||||
*/
|
||||
public enum FileMode {
|
||||
/**
|
||||
* The default file mode. This maps the database to virtual memory. This
|
||||
* often provides similar performance to loading the database into real
|
||||
* memory without the overhead.
|
||||
*/
|
||||
MEMORY_MAPPED,
|
||||
/**
|
||||
* Loads the database into memory when the reader is constructed.
|
||||
*/
|
||||
MEMORY
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a Reader for the MaxMind DB format, with no caching. The file
|
||||
* passed to it must be a valid MaxMind DB file such as a GeoIP2 database
|
||||
* file.
|
||||
*
|
||||
* @param database the MaxMind DB file to use.
|
||||
* @throws IOException if there is an error opening or reading from the file.
|
||||
*/
|
||||
public Reader(File database) throws IOException {
|
||||
this(database, NoCache.getInstance());
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a Reader for the MaxMind DB format, with the specified backing
|
||||
* cache. The file passed to it must be a valid MaxMind DB file such as a
|
||||
* GeoIP2 database file.
|
||||
*
|
||||
* @param database the MaxMind DB file to use.
|
||||
* @param cache backing cache instance
|
||||
* @throws IOException if there is an error opening or reading from the file.
|
||||
*/
|
||||
public Reader(File database, NodeCache cache) throws IOException {
|
||||
this(database, FileMode.MEMORY_MAPPED, cache);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a Reader with no caching, as if in mode
|
||||
* {@link FileMode#MEMORY}, without using a <code>File</code> instance.
|
||||
*
|
||||
* @param source the InputStream that contains the MaxMind DB file.
|
||||
* @throws IOException if there is an error reading from the Stream.
|
||||
*/
|
||||
public Reader(InputStream source) throws IOException {
|
||||
this(source, NoCache.getInstance());
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a Reader with the specified backing cache, as if in mode
|
||||
* {@link FileMode#MEMORY}, without using a <code>File</code> instance.
|
||||
*
|
||||
* @param source the InputStream that contains the MaxMind DB file.
|
||||
* @param cache backing cache instance
|
||||
* @throws IOException if there is an error reading from the Stream.
|
||||
*/
|
||||
public Reader(InputStream source, NodeCache cache) throws IOException {
|
||||
this(new BufferHolder(source), "<InputStream>", cache);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a Reader for the MaxMind DB format, with no caching. The file
|
||||
* passed to it must be a valid MaxMind DB file such as a GeoIP2 database
|
||||
* file.
|
||||
*
|
||||
* @param database the MaxMind DB file to use.
|
||||
* @param fileMode the mode to open the file with.
|
||||
* @throws IOException if there is an error opening or reading from the file.
|
||||
*/
|
||||
public Reader(File database, FileMode fileMode) throws IOException {
|
||||
this(database, fileMode, NoCache.getInstance());
|
||||
}
|
||||
|
||||
/**
|
||||
* Constructs a Reader for the MaxMind DB format, with the specified backing
|
||||
* cache. The file passed to it must be a valid MaxMind DB file such as a
|
||||
* GeoIP2 database file.
|
||||
*
|
||||
* @param database the MaxMind DB file to use.
|
||||
* @param fileMode the mode to open the file with.
|
||||
* @param cache backing cache instance
|
||||
* @throws IOException if there is an error opening or reading from the file.
|
||||
*/
|
||||
public Reader(File database, FileMode fileMode, NodeCache cache) throws IOException {
|
||||
this(new BufferHolder(database, fileMode), database.getName(), cache);
|
||||
}
|
||||
|
||||
private Reader(BufferHolder bufferHolder, String name, NodeCache cache) throws IOException {
|
||||
this.bufferHolderReference = new AtomicReference<BufferHolder>(
|
||||
bufferHolder);
|
||||
|
||||
if (cache == null) {
|
||||
throw new NullPointerException("Cache cannot be null");
|
||||
}
|
||||
this.cache = cache;
|
||||
|
||||
ByteBuffer buffer = bufferHolder.get();
|
||||
int start = this.findMetadataStart(buffer, name);
|
||||
|
||||
Decoder metadataDecoder = new Decoder(this.cache, buffer, start);
|
||||
this.metadata = new Metadata(metadataDecoder.decode(start));
|
||||
|
||||
this.ipV4Start = this.findIpV4StartNode(buffer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Looks up the <code>address</code> in the MaxMind DB.
|
||||
*
|
||||
* @param ipAddress the IP address to look up.
|
||||
* @return the record for the IP address.
|
||||
* @throws IOException if a file I/O error occurs.
|
||||
*/
|
||||
public JsonNode get(InetAddress ipAddress) throws IOException {
|
||||
ByteBuffer buffer = this.getBufferHolder().get();
|
||||
int pointer = this.findAddressInTree(buffer, ipAddress);
|
||||
if (pointer == 0) {
|
||||
return null;
|
||||
}
|
||||
return this.resolveDataPointer(buffer, pointer);
|
||||
}
|
||||
|
||||
private BufferHolder getBufferHolder() throws ClosedDatabaseException {
|
||||
BufferHolder bufferHolder = this.bufferHolderReference.get();
|
||||
if (bufferHolder == null) {
|
||||
throw new ClosedDatabaseException();
|
||||
}
|
||||
return bufferHolder;
|
||||
}
|
||||
|
||||
private int findAddressInTree(ByteBuffer buffer, InetAddress address)
|
||||
throws InvalidDatabaseException {
|
||||
byte[] rawAddress = address.getAddress();
|
||||
|
||||
int bitLength = rawAddress.length * 8;
|
||||
int record = this.startNode(bitLength);
|
||||
|
||||
for (int i = 0; i < bitLength; i++) {
|
||||
if (record >= this.metadata.getNodeCount()) {
|
||||
break;
|
||||
}
|
||||
int b = 0xFF & rawAddress[i / 8];
|
||||
int bit = 1 & (b >> 7 - (i % 8));
|
||||
record = this.readNode(buffer, record, bit);
|
||||
}
|
||||
if (record == this.metadata.getNodeCount()) {
|
||||
// record is empty
|
||||
return 0;
|
||||
} else if (record > this.metadata.getNodeCount()) {
|
||||
// record is a data pointer
|
||||
return record;
|
||||
}
|
||||
throw new InvalidDatabaseException("Something bad happened");
|
||||
}
|
||||
|
||||
private int startNode(int bitLength) {
|
||||
// Check if we are looking up an IPv4 address in an IPv6 tree. If this
|
||||
// is the case, we can skip over the first 96 nodes.
|
||||
if (this.metadata.getIpVersion() == 6 && bitLength == 32) {
|
||||
return this.ipV4Start;
|
||||
}
|
||||
// The first node of the tree is always node 0, at the beginning of the
|
||||
// value
|
||||
return 0;
|
||||
}
|
||||
|
||||
private int findIpV4StartNode(ByteBuffer buffer)
|
||||
throws InvalidDatabaseException {
|
||||
if (this.metadata.getIpVersion() == 4) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int node = 0;
|
||||
for (int i = 0; i < 96 && node < this.metadata.getNodeCount(); i++) {
|
||||
node = this.readNode(buffer, node, 0);
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
private int readNode(ByteBuffer buffer, int nodeNumber, int index)
|
||||
throws InvalidDatabaseException {
|
||||
int baseOffset = nodeNumber * this.metadata.getNodeByteSize();
|
||||
|
||||
switch (this.metadata.getRecordSize()) {
|
||||
case 24:
|
||||
buffer.position(baseOffset + index * 3);
|
||||
return Decoder.decodeInteger(buffer, 0, 3);
|
||||
case 28:
|
||||
int middle = buffer.get(baseOffset + 3);
|
||||
|
||||
if (index == 0) {
|
||||
middle = (0xF0 & middle) >>> 4;
|
||||
} else {
|
||||
middle = 0x0F & middle;
|
||||
}
|
||||
buffer.position(baseOffset + index * 4);
|
||||
return Decoder.decodeInteger(buffer, middle, 3);
|
||||
case 32:
|
||||
buffer.position(baseOffset + index * 4);
|
||||
return Decoder.decodeInteger(buffer, 0, 4);
|
||||
default:
|
||||
throw new InvalidDatabaseException("Unknown record size: "
|
||||
+ this.metadata.getRecordSize());
|
||||
}
|
||||
}
|
||||
|
||||
private JsonNode resolveDataPointer(ByteBuffer buffer, int pointer)
|
||||
throws IOException {
|
||||
int resolved = (pointer - this.metadata.getNodeCount())
|
||||
+ this.metadata.getSearchTreeSize();
|
||||
|
||||
if (resolved >= buffer.capacity()) {
|
||||
throw new InvalidDatabaseException(
|
||||
"The MaxMind DB file's search tree is corrupt: "
|
||||
+ "contains pointer larger than the database.");
|
||||
}
|
||||
|
||||
// We only want the data from the decoder, not the offset where it was
|
||||
// found.
|
||||
Decoder decoder = new Decoder(this.cache, buffer,
|
||||
this.metadata.getSearchTreeSize() + DATA_SECTION_SEPARATOR_SIZE);
|
||||
return decoder.decode(resolved);
|
||||
}
|
||||
|
||||
/*
|
||||
* Apparently searching a file for a sequence is not a solved problem in
|
||||
* Java. This searches from the end of the file for metadata start.
|
||||
*
|
||||
* This is an extremely naive but reasonably readable implementation. There
|
||||
* are much faster algorithms (e.g., Boyer-Moore) for this if speed is ever
|
||||
* an issue, but I suspect it won't be.
|
||||
*/
|
||||
private int findMetadataStart(ByteBuffer buffer, String databaseName)
|
||||
throws InvalidDatabaseException {
|
||||
int fileSize = buffer.capacity();
|
||||
|
||||
FILE:
|
||||
for (int i = 0; i < fileSize - METADATA_START_MARKER.length + 1; i++) {
|
||||
for (int j = 0; j < METADATA_START_MARKER.length; j++) {
|
||||
byte b = buffer.get(fileSize - i - j - 1);
|
||||
if (b != METADATA_START_MARKER[METADATA_START_MARKER.length - j
|
||||
- 1]) {
|
||||
continue FILE;
|
||||
}
|
||||
}
|
||||
return fileSize - i;
|
||||
}
|
||||
throw new InvalidDatabaseException(
|
||||
"Could not find a MaxMind DB metadata marker in this file ("
|
||||
+ databaseName + "). Is this a valid MaxMind DB file?");
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the metadata for the MaxMind DB file.
|
||||
*/
|
||||
public Metadata getMetadata() {
|
||||
return this.metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* <p>
|
||||
* Closes the database.
|
||||
* </p>
|
||||
* <p>
|
||||
* If you are using <code>FileMode.MEMORY_MAPPED</code>, this will
|
||||
* <em>not</em> unmap the underlying file due to a limitation in Java's
|
||||
* <code>MappedByteBuffer</code>. It will however set the reference to
|
||||
* the buffer to <code>null</code>, allowing the garbage collector to
|
||||
* collect it.
|
||||
* </p>
|
||||
*
|
||||
* @throws IOException if an I/O error occurs.
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
this.bufferHolderReference.set(null);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user