diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..02a748a --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,2 @@ +2023-12-17 0.0.1 + - Initial checkin diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..1edfe34 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,17 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +======================================================================== + +Includes code from syndie 1.107 (Public Domain) + +Includes sqlite-jdbc-3.44.0.0.jar from https://github.com/xerial/sqlite-jdbc (Apache 2.0) diff --git a/build.xml b/build.xml new file mode 100644 index 0000000..4d4543b --- /dev/null +++ b/build.xml @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/resources/ddl_update0.txt b/resources/ddl_update0.txt new file mode 100644 index 0000000..f77711b --- /dev/null +++ b/resources/ddl_update0.txt @@ -0,0 +1,47 @@ +-- +-- Version 1 sqlite3 database schema +-- + +-- this is the magic that makes it not dog slow +-- https://www.sqlite.org/wal.html +PRAGMA journal_mode = WAL; + +BEGIN TRANSACTION; + +-- info table +CREATE TABLE info ("key" TEXT PRIMARY KEY NOT NULL, "value" TEXT NOT NULL); +INSERT INTO info VALUES('version', '1'); +INSERT INTO info VALUES('created', unixepoch("now")); +INSERT INTO info VALUES('upgraded', unixepoch("now")); + +-- lists table +-- dots in a table name is really bad +-- there must be a way to escape it but let's just map to an internal name +CREATE TABLE lists ("list" TEXT PRIMARY KEY NOT NULL, "internal" TEXT NOT NULL); +INSERT INTO lists VALUES('privatehosts.txt', 'privatehoststxt'); +INSERT INTO lists VALUES('userhosts.txt', 'userhoststxt'); +INSERT INTO lists VALUES('hosts.txt', 'hoststxt'); + +-- list tables +-- hostname to primary entry (preferred, usually non-DSA) and optional alt (DSA) entry numbers +CREATE TABLE "privatehoststxt" ("hostname" TEXT PRIMARY KEY NOT NULL, "entry" INTEGER NOT NULL, "alt" INTEGER); +CREATE TABLE "userhoststxt" ("hostname" TEXT PRIMARY KEY NOT NULL, "entry" INTEGER NOT NULL, "alt" INTEGER); +CREATE TABLE "hoststxt" ("hostname" TEXT PRIMARY KEY NOT NULL, "entry" INTEGER NOT NULL, "alt" INTEGER); + +-- entry table +-- id to dest, added time, modified time, notes, source, verified +-- Don't use AUTOINCREMENT +-- https://www.sqlite.org/autoinc.html +-- CREATE TABLE entries ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "dest" BLOB NOT NULL, "a" DATETIME NOT NULL, "m" DATETIME, "notes" TEXT, "s" TEXT, "v" BOOLEAN NOT NULL); +CREATE TABLE entries ("id" INTEGER PRIMARY KEY NOT NULL, "dest" BLOB NOT NULL, "a" DATETIME NOT NULL, "m" DATETIME, "notes" TEXT, "s" TEXT, "v" BOOLEAN NOT NULL); + +-- reverse lookup table +-- first 8 bytes of hash to comma separated list of hostnames +CREATE TABLE "reverse" ("hash" INTEGER PRIMARY KEY NOT NULL, "hostnames" TEXT NOT NULL); + +COMMIT TRANSACTION; + +-- https://www.sqlite.org/pragma.html#synchronous +PRAGMA main.synchronous = NORMAL; +-- https://www.sqlite.org/pragma.html#pragma_wal_checkpoint +PRAGMA main.wal_checkpoint(FULL); diff --git a/scripts/makeplugin.sh b/scripts/makeplugin.sh new file mode 100755 index 0000000..57bdcb2 --- /dev/null +++ b/scripts/makeplugin.sh @@ -0,0 +1,147 @@ +#!/bin/sh +# +# basic packaging up of a plugin +# +# usage: makeplugin.sh plugindir +# +# zzz 2010-02 +# zzz 2014-08 added support for su3 files +# + +if [ -z "$I2P" -a -d "$PWD/../i2p.i2p/pkg-temp" ]; then + export I2P=../i2p.i2p/pkg-temp +fi + +if [ ! -d "$I2P" ]; then + echo "Can't locate your I2P installation. Please add a environment variable named I2P with the path to the folder as value" + echo "On OSX this solved with running: export I2P=/Applications/i2p if default install directory is used." + exit 1 +fi + +PUBKEYDIR=$HOME/.i2p-plugin-keys +PUBKEYFILE=$PUBKEYDIR/plugin-public-signing.key +PRIVKEYFILE=$PUBKEYDIR/plugin-private-signing.key +B64KEYFILE=$PUBKEYDIR/plugin-public-signing.txt +PUBKEYSTORE=$PUBKEYDIR/plugin-su3-public-signing.crt +PRIVKEYSTORE=$PUBKEYDIR/plugin-su3-keystore.ks +KEYTYPE=RSA_SHA512_4096 + +PLUGINDIR=${1:-plugin} + +PC=plugin.config +PCT=${PC}.tmp + +if [ ! -d $PLUGINDIR ] +then + echo "You must have a $PLUGINDIR directory" + exit 1 +fi + +if [ ! -f $PLUGINDIR/$PC ] +then + echo "You must have a $PLUGINDIR/$PC file" + exit 1 +fi + +SIGNER=`grep '^signer=' $PLUGINDIR/$PC` +if [ "$?" -ne "0" ] +then + echo "You must have a plugin name in $PC" + echo 'For example name=foo' + exit 1 +fi +SIGNER=`echo $SIGNER | cut -f 2 -d '='` + +if [ ! -f $PRIVKEYFILE ] +then + echo "Creating new XPI2P DSA keys" + mkdir -p $PUBKEYDIR || exit 1 + java -cp $I2P/lib/i2p.jar net.i2p.crypto.TrustedUpdate keygen $PUBKEYFILE $PRIVKEYFILE || exit 1 + java -cp $I2P/lib/i2p.jar net.i2p.data.Base64 encode $PUBKEYFILE $B64KEYFILE || exit 1 + rm -rf logs/ + chmod 444 $PUBKEYFILE $B64KEYFILE + chmod 400 $PRIVKEYFILE + echo "Created new XPI2P keys: $PUBKEYFILE $PRIVKEYFILE" +fi + +if [ ! -f $PRIVKEYSTORE ] +then + echo "Creating new SU3 $KEYTYPE keys for $SIGNER" + java -cp $I2P/lib/i2p.jar net.i2p.crypto.SU3File keygen -t $KEYTYPE $PUBKEYSTORE $PRIVKEYSTORE $SIGNER || exit 1 + echo '*** Save your password in a safe place!!! ***' + rm -rf logs/ + # copy to the router dir so verify will work + CDIR=$I2P/certificates/plugin + mkdir -p $CDIR || exit 1 + CFILE=$CDIR/`echo $SIGNER | sed s/@/_at_/`.crt + cp $PUBKEYSTORE $CFILE + chmod 444 $PUBKEYSTORE + chmod 400 $PRIVKEYSTORE + chmod 644 $CFILE + echo "Created new SU3 keys: $PUBKEYSTORE $PRIVKEYSTORE" + echo "Copied public key to $CFILE for testing" +fi + +rm -f plugin.zip + +OPWD=$PWD +cd $PLUGINDIR + +grep -q '^name=' $PC +if [ "$?" -ne "0" ] +then + echo "You must have a plugin name in $PC" + echo 'For example name=foo' + exit 1 +fi + +grep -q '^version=' $PC +if [ "$?" -ne "0" ] +then + echo "You must have a version in $PC" + echo 'For example version=0.1.2' + exit 1 +fi + +# update the date +grep -v '^date=' $PC > $PCT +DATE=`date '+%s000'` +echo "date=$DATE" >> $PCT +mv $PCT $PC || exit 1 + +# add our Base64 key +grep -v '^key=' $PC > $PCT +B64KEY=`cat $B64KEYFILE` +echo "key=$B64KEY" >> $PCT || exit 1 +mv $PCT $PC || exit 1 + +# zip it +zip -r $OPWD/plugin.zip * || exit 1 + +# get the version and use it for the sud header +VERSION=`grep '^version=' $PC | cut -f 2 -d '='` +# get the name and use it for the file name +NAME=`grep '^name=' $PC | cut -f 2 -d '='` +XPI2P=${NAME}.xpi2p +SU3=${NAME}.su3 +cd $OPWD + +# sign it +echo "Signing version $VERSION by $SIGNER" +java -cp $I2P/lib/i2p.jar net.i2p.crypto.TrustedUpdate sign plugin.zip $XPI2P $PRIVKEYFILE $VERSION || exit 1 +java -cp $I2P/lib/i2p.jar net.i2p.crypto.SU3File sign -c PLUGIN -t $KEYTYPE plugin.zip $SU3 $PRIVKEYSTORE $VERSION $SIGNER || exit 1 +rm -f plugin.zip + +# verify +echo 'Verifying. ...' +java -cp $I2P/lib/i2p.jar net.i2p.crypto.TrustedUpdate showversion $XPI2P || exit 1 +java -cp $I2P/lib/i2p.jar -Drouter.trustedUpdateKeys=$B64KEY net.i2p.crypto.TrustedUpdate verifysig $XPI2P || exit 1 +java -cp $I2P/lib/i2p.jar net.i2p.crypto.SU3File showversion $SU3 || exit 1 +java -cp $I2P/lib/i2p.jar net.i2p.crypto.SU3File verifysig -k $PUBKEYSTORE $SU3 || exit 1 +rm -rf logs/ + +echo 'Plugin files created: ' +wc -c $XPI2P +wc -c $SU3 + +exit 0 diff --git a/scripts/plugin.config b/scripts/plugin.config new file mode 100644 index 0000000..fc2b8a6 --- /dev/null +++ b/scripts/plugin.config @@ -0,0 +1,15 @@ +name=sqldb +signer=zzz-plugin@mail.i2p +consoleLinkName=SQLDB +consoleLinkURL=/dns +consoleLinkTooltip=sqlite3 addressbook database +icon-code=iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAMAAACdt4HsAAABm1BMVEUAAAAZGRkZGRkaGhoZGRkZGRkZGRkZGRkYGBgZGRkYGBgZGRkYGBgZGRkZGRkXFxcYGBgZGRkZGRkZGRkZGRkAAAAZGRnziRYTExPzfhbzbhbzdxbzgxbzlRYLCwvzjhbzcxbMzMzzkBb~~~~~~Pl0dHRdXV3GxsYnJyfe3t7OycbBwcGcnJyHh4cjEgP39~f96dSurq5~f397e3tra2tlZWU7OzseHh7haRSGSAsOBwGpqKeWlpbgpnRUVFRNTU00NDTxmynPbRJwNgpXLwdEJQY4IAUVCwLy8vL83sK7u7uysrL7yqf6v5Tdr4H4t3H4rGz2pEvnjUnqnkQvLy~zii30gyjxdCLqdhXLexK9bhGQTQ1-QAtlLgn-9u~Pxr7Uvqr71KbYuJz6yZn5v4T3pVv3s1rkmVr1nzr1kzLyjiPzex~pbBXJYxLJXRK~XxG4aBCYRQ5iNQlMKgctGgT7-~vas4zhnGj2qlNERETlhRTefxTedRTVfxOsZhC0YBC4UhCpWw9wQgr4sH~kqF7sizVpSyfWXxOZVw56Yx8pAAAAFXRSTlMAjPQOr8vV7eVwaVtPNyUa27WCwoZvol0xAAAEx0lEQVRYw72XZ1PbQBCGXQEnVMNJAjtyjpPcbWxsMB2SQOg1tCT0EiA9kN578rOzd4eQZUuO8iXPjGfkmdv3dm-v7DqsaKht8XhrBKDG6~HXNjj-iTpnlVBClbPOrrXb18htJElmSNK5hs9tx9zp4saCXOgOq6qaH-qWkCQzEZf~rxKXqvnU0c6pCUKIohDKREKNckeqL1U0r~cyc6kzRpTJg8OTILBysji~nyEkpnKJ5npr-8suZp7sUHbvXTFybX6SvEhKVMJ12creSaeXUx1bB6utJqzsKx0pthZOc3sPnX4kpsw-uNozmr5xtZzVGRIboU54zOyb6PQ5kjkWxbdphNBd0YTFDFGpE03m88v9ZD8Ew04R8Fw0I7RLsrKZD05qH1EWqKePEUNz~GGPIY4FMkUVnCXrT-2nlMNWyhhirLM~66MIpZ-0FnGoJKjCZUP-XWCfJTx3txHnNvt3hoBxQ0rvKf2g4CreD83ggErmg4xRxPkUBE7ZZ1~QwALJgQveov0L9iNkpo3Rg855Bn82-Odmm5FZMgIKF7vaXQ0BxDIil7-rCWzo39eDRsRMDIKo1k6WHxxIkcV2TloTGG3~iDjP20s5VlJ6JtywglLHTIDzCHHo2mtfvYEyZjskWEfugg8cSCqrAc4NdEGf9tFTLvBgKwku-JhAFazAi93QOeOojLGQCQfUhUZ2~7EUnoicp6icL6IJ95VOcKGOb2I5NqklaBOZkN589ritlJmYDMvII4iSBW2f9iErxt63GlgkURZDAzigkmtXOHeQNaNXjLAYGhy1sASJjCZ7iiqwbnRhcgoup1pHiyCgiVktMMsI-N428G0C0UVogl1E5kWOdQQmV8xrIkhwsXgFqZu8DnG-oop8CBVzTLolOJI1gjxEVq5yxlFFjBdtgIRlocYBSciTVf0cVOTs7qOAjqLCTnCwLLZzepENbn79-JYP39IEwttBThrZ5IwNx2FNIM7z8gvZ5jMMX8NDIEAXcRgHGbYi0O~LFfwGFpGmcRnznTyGbMJv7Ft4mabRAxsJL7VSxpFtrsPwIww3QhM9zSjxir1AyD70sRpMwFZuYYdpYI~vY~uIwE5SpoepAQS64iLw-98E7rMsQvnXKEgFvAQubSLb9MHwI1yQhKrzKy37EhblJrJNLwzfmz6~0upYDJDIPmSTsQ-0aGIRsOKzkVYGg8H3yB7jX4KUOSzzCPjD0rm99hnZ4sbDNspaPAcO-PSnLTLYa-ckbmjP7FxEpk-bXt7k8c-~GffeCV1wC-fBAX~x8z7dX3HmJ-uGl3Eny5734gIjilXruAMlzOEoLzA0vDSVeBhZ8DRk5DtLYXNpkZWKdFsJiAZu4RQrssrKvGTCQsH4si7FB1iZZ1JoDuBhq9OvcxRPskLTtNRN4Zz5~aMzx~wXPBbFdhj3Ryt58GMHd-nFtpkP0SzOSWX3TxtnbRBPR1m5X6nhyEciaonEO17UDG5H8rzhqNjyyHIO44FhuViAbt6X8Uhn5ZYHqG8WmER4GuN-dWhEYq4Ull7txXG2izdd3nobbZ8sF8LJBGZE4JcY6CrYavsAt98lcA0kLb8ZCofDw8sSor4DLqfbVutbZd76Nvrc~6P51tt~v97-t1i3~38A57d5M52iLPcAAAAASUVORK5CYII= +description=BitTorrent OpenTracker +author=zzz +updateURL=http://stats.i2p/i2p/plugins/sqldb-update.xpi2p +updateURL.su3=http://stats.i2p/i2p/plugins/sqldb-update.su3 +websiteURL=http://zzz.i2p/forums/16 +license=Apache 2.0 +min-jetty-version=9 +min-i2p-version=0.9.31 +update-only=true diff --git a/src/build.xml b/src/build.xml new file mode 100644 index 0000000..bba06f4 --- /dev/null +++ b/src/build.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/java/net/i2p/client/naming/sqldb/DBClient.java b/src/java/net/i2p/client/naming/sqldb/DBClient.java new file mode 100644 index 0000000..1a7ae63 --- /dev/null +++ b/src/java/net/i2p/client/naming/sqldb/DBClient.java @@ -0,0 +1,1138 @@ +package net.i2p.client.naming.sqldb; + +import java.io.IOException; +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import net.i2p.I2PAppContext; +import net.i2p.data.DataFormatException; +import net.i2p.data.DataHelper; +import net.i2p.data.Destination; +import net.i2p.data.Hash; +import net.i2p.util.Log; + +import net.i2p.client.naming.sqldb.SQLNamingService.DestEntry; + + +/** + * The interface to the database itself + * + * Not thread safe! Caller must synch. + * + * Adapted from syndie (public domain) + */ +class DBClient { + + static { + try { + Class.forName("org.sqlite.JDBC"); + } catch(Exception exc) {} + } + + private final I2PAppContext _context; + private final Log _log; + private final File _db; + private final List _lists; + private final Map _internal; + + private Connection _con; + private String _url; + private Thread _shutdownHook; + private boolean _shutdownInProgress; + + private static final String REVERSE_SEPARATOR = ","; + + + /** + * @param rootDir should be a SecureFile + */ + public DBClient(I2PAppContext ctx, File db) { + _context = ctx; + _log = ctx.logManager().getLog(getClass()); + _db = db; + _lists = new ArrayList(3); + _internal = new HashMap(3); + } + + /** + * Initialize the DB, update to latest version if necessary, and connect. + * Does nothing if already connected. + */ + public void connect() throws IOException { + try { + if (_con != null && !_con.isClosed()) { + if (_log.shouldWarn()) + _log.warn("connect() called, already connected"); + } + long start = System.currentTimeMillis(); + //String url = "jdbc:sqlite::memory:"; + String url = "jdbc:sqlite:file:" + _db; + _url = url; + _con = DriverManager.getConnection(url); + if (_con == null) + throw new SQLException("Unable to connect to [" + url + "]"); + String version = _con.getMetaData().getDatabaseProductVersion(); + if (_log.shouldDebug()) + _log.debug("connection successful, version: " + version); + + // process all updates + if (!_db.exists() || _db.canWrite()) { + DBInit dbi = new DBInit(_context, _con); + dbi.initDB(); + } else { + _log.logAlways(Log.WARN, "Not upgrading read-only database " + _db); + } + // init the list cache + getLists(); + // requires list cache configured + configure(); + } catch (SQLException se) { + if (_con != null) try { _con.close(); } catch (SQLException se2) {} + _con = null; + handleSQLException(se); + } + + if (_shutdownHook == null) { + _shutdownHook = new Thread(new Runnable() { + public void run() { + _shutdownInProgress = true; + close(); + } + }, "DB shutdown"); + Runtime.getRuntime().addShutdownHook(_shutdownHook); + } + } + + public void disconnect() { + try { + if ( (_con != null) && (!_con.isClosed()) ) { + if (_log.shouldInfo()) + _log.info("Disconnecting from DB"); + _con.close(); + } + } catch (SQLException se) { + if (_log.shouldWarn()) + _log.warn("Error disconnecting", se); + } + _con = null; + } + + public void close() { + try { + if (_con == null) return; + if (_con.isClosed()) return; + _con.close(); + _log.logAlways(Log.WARN, "Database shutdown complete"); + } catch (SQLException se) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Error closing the connection and shutting down the database", se); + } + Thread hook = _shutdownHook; + _shutdownHook = null; + if (!_shutdownInProgress && (hook != null)) + Runtime.getRuntime().removeShutdownHook(hook); + } + + // pragma + // this is the magic that makes it not dog slow + private static final String PRAGMA_JOURNAL = "PRAGMA journal_mode = WAL"; + private static final String PRAGMA_SYNCH = "PRAGMA main.synchronous = NORMAL"; + private static final String PRAGMA_CHECKPOINT = "PRAGMA main.wal_checkpoint(FULL)"; + // in 4K pages; default 1000 which is way too big + private static final String PRAGMA_AUTOCHECKPOINT = "PRAGMA main.wal_autocheckpoint = 25"; + + // info + private static final String SQL_GET_INFO = "SELECT key, value FROM info"; + private static final String SQL_GET_VERSION = "SELECT value FROM info WHERE key = version"; + private static final String SQL_GET_CREATED = "SELECT value FROM info WHERE key = created"; + private static final String SQL_GET_UPDATED = "SELECT value FROM info WHERE key = updated"; + + // lists + private static final String SQL_GET_LISTS = "SELECT * FROM lists"; + + // list + // ?? will be replaced by table name by setTable() + private static final String SQL_COUNT_ENTRIES = "SELECT COUNT(*) FROM ??"; + private static final String SQL_GET_PRIMARY = "SELECT entry FROM ?? WHERE hostname = ?"; + private static final String SQL_GET_ENTRIES = "SELECT entry, alt FROM ?? WHERE hostname = ?"; + private static final String SQL_ADD_ENTRIES = "INSERT INTO ?? VALUES(?, ?, ?)"; + private static final String SQL_UPDATE_ENTRIES = "UPDATE ?? SET entry = ?, alt = ? WHERE hostname = ?"; + private static final String SQL_GET_ALL_HOSTS = "SELECT hostname FROM ?? ORDER BY hostname LIMIT ? OFFSET ?"; + private static final String SQL_SEARCH_HOSTS = "SELECT hostname FROM ?? WHERE hostname GLOB ? ORDER BY hostname LIMIT ? OFFSET ?"; + private static final String SQL_SEARCH2_HOSTS = "SELECT hostname FROM ?? WHERE hostname GLOB ? AND hostname GLOB ? ORDER BY hostname LIMIT ? OFFSET ?"; + private static final String SQL_GET_ALL_DESTS = "SELECT hostname, entry FROM ?? ORDER BY hostname LIMIT ? OFFSET ?"; + private static final String SQL_SEARCH_DESTS = "SELECT hostname, entry FROM ?? WHERE hostname GLOB ? ORDER BY hostname LIMIT ? OFFSET ?"; + private static final String SQL_SEARCH2_DESTS = "SELECT hostname, entry FROM ?? WHERE hostname GLOB ? AND hostname GLOB ? ORDER BY hostname LIMIT ? OFFSET ?"; + private static final String SQL_GET_ALL_ENTRIES = "SELECT hostname, entry FROM ?? ORDER BY hostname"; + private static final String SQL_DELETE_ENTRIES = "DELETE FROM ?? WHERE hostname = ? RETURNING entry, alt"; + + // entries + private static final String SQL_GET_DEST = "SELECT dest FROM entries WHERE id = ?"; + private static final String SQL_GET_ENTRY = "SELECT dest, a, m, notes, s, v FROM entries WHERE id = ?"; + private static final String SQL_SET_ENTRY = "INSERT INTO entries (dest, a, m, notes, s, v) VALUES(?, ?, ?, ?, ?, ?) RETURNING id"; + private static final String SQL_UPDATE_ENTRY = "UPDATE entries SET a = ?, m = ?, notes = ?, s = ?, v = ? WHERE id = ?"; + private static final String SQL_DELETE_ENTRY = "DELETE FROM entries WHERE id = ? RETURNING dest, a, m, notes, s, v"; + + // reverse + private static final String SQL_GET_REVERSE = "SELECT hostnames FROM reverse WHERE hash = ?"; + private static final String SQL_SET_REVERSE = "INSERT INTO reverse VALUES(?, ?)"; + private static final String SQL_UPDATE_REVERSE = "UPDATE reverse SET hostnames = ? WHERE hash = ?"; + private static final String SQL_DELETE_REVERSE = "DELETE FROM reverse WHERE hash = ?"; + + // misc + private static final String SQL_GET_CHANGED_ROWS = "SELECT changes()"; + + + //// getters //// + + public Properties getInfo() throws IOException { + Properties rv = new Properties(); + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_GET_INFO); + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + rv.setProperty(rs.getString(1), rs.getString(2)); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + public List getLists() throws IOException { + if (!_lists.isEmpty()) + return _lists; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_GET_LISTS); + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + String list = rs.getString(1); + String internal = rs.getString(2); + _lists.add(list); + _internal.put(list, internal); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return _lists; + } + + public int getSize(String list) throws IOException { + list = internal(list); + PreparedStatement stmt = null; + int rv = 0; + try { + stmt = _con.prepareStatement(setTable(SQL_COUNT_ENTRIES, list)); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + rv = rs.getInt(1); + } + } catch (SQLException se) { + // don't throw + if (_log.shouldWarn()) + _log.warn("no list " + list, se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + /* + * @param search may be null + * @param beginWith may be null + * @param limit 0 for all + */ + public Set getNames(String list, String search, String beginWith, int limit, int skip) throws IOException { + long start = System.currentTimeMillis(); + list = internal(list); + Set rv = new TreeSet(); + if (limit <= 0) + limit = Integer.MAX_VALUE; + if (skip < 0) + skip = 0; + PreparedStatement stmt = null; + try { + if (search == null && beginWith == null) { + stmt = _con.prepareStatement(setTable(SQL_GET_ALL_HOSTS, list)); + stmt.setInt(1, limit); + stmt.setInt(2, skip); + } else if (search != null && beginWith == null) { + stmt = _con.prepareStatement(setTable(SQL_SEARCH_HOSTS, list)); + stmt.setString(1, '*' + search + '*'); + stmt.setInt(2, limit); + stmt.setInt(3, skip); + } else if (search == null && beginWith != null) { + stmt = _con.prepareStatement(setTable(SQL_SEARCH_HOSTS, list)); + stmt.setString(1, beginWith + '*'); + stmt.setInt(2, limit); + stmt.setInt(3, skip); + } else { + stmt = _con.prepareStatement(setTable(SQL_SEARCH2_HOSTS, list)); + stmt.setString(1, '*' + search + '*'); + stmt.setString(2, beginWith + '*'); + stmt.setInt(3, limit); + stmt.setInt(4, skip); + } + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + String name = rs.getString(1); + rv.add(name); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + if (_log.shouldDebug()) _log.debug("getNames took " + (System.currentTimeMillis() - start)); + return rv; + } + + /* + * Primary dests only, no alts + * + * @param search may be null + * @param beginWith may be null + * @param limit 0 for all + * @return sorted map + */ + public TreeMap getDests(String list, String search, String beginWith, int limit, int skip) throws IOException { + list = internal(list); + TreeMap rv = new TreeMap(); + if (limit <= 0) + limit = Integer.MAX_VALUE; + if (skip < 0) + skip = 0; + PreparedStatement stmt = null; + try { + if (search == null && beginWith == null) { + stmt = _con.prepareStatement(setTable(SQL_GET_ALL_DESTS, list)); + stmt.setInt(1, limit); + stmt.setInt(2, skip); + } else if (search != null && beginWith == null) { + stmt = _con.prepareStatement(setTable(SQL_SEARCH_DESTS, list)); + stmt.setString(1, '*' + search + '*'); + stmt.setInt(2, limit); + stmt.setInt(3, skip); + } else if (search == null && beginWith != null) { + stmt = _con.prepareStatement(setTable(SQL_SEARCH_DESTS, list)); + stmt.setString(1, beginWith + '*'); + stmt.setInt(2, limit); + stmt.setInt(3, skip); + } else { + stmt = _con.prepareStatement(setTable(SQL_SEARCH2_DESTS, list)); + stmt.setString(1, '*' + search + '*'); + stmt.setString(2, beginWith + '*'); + stmt.setInt(3, limit); + stmt.setInt(4, skip); + } + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + String name = rs.getString(1); + int id = rs.getInt(2); + // TODO join + Destination d = getDest(id); + if (d == null) + throw new SQLException("No entry?? " + id); + rv.put(name, d); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + /* + * Primary dests only, no alts + * + * @param search may be null + * @param beginWith may be null + * @param limit 0 for all + * @return sorted map + */ + public TreeMap getEntries(String list, String search, String beginWith) throws IOException { + list = internal(list); + TreeMap rv = new TreeMap(); + int limit = Integer.MAX_VALUE; + int skip = 0; + PreparedStatement stmt = null; + try { + if (search == null && beginWith == null) { + stmt = _con.prepareStatement(setTable(SQL_GET_ALL_DESTS, list)); + stmt.setInt(2, limit); + stmt.setInt(3, skip); + } else if (search != null && beginWith == null) { + stmt = _con.prepareStatement(setTable(SQL_SEARCH_DESTS, list)); + stmt.setString(2, '*' + search + '*'); + stmt.setInt(3, limit); + stmt.setInt(4, skip); + } else if (search == null && beginWith != null) { + stmt = _con.prepareStatement(setTable(SQL_SEARCH_DESTS, list)); + stmt.setString(2, beginWith + '*'); + stmt.setInt(3, limit); + stmt.setInt(4, skip); + } else { + stmt = _con.prepareStatement(setTable(SQL_SEARCH2_DESTS, list)); + stmt.setString(1, '*' + search + '*'); + stmt.setString(2, beginWith + '*'); + stmt.setInt(3, limit); + stmt.setInt(4, skip); + } + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + String name = rs.getString(1); + int id = rs.getInt(2); + DestEntry d = getEntry(id); + if (d == null) + throw new SQLException("No entry?? " + id); + long altid = rs.getLong(2); + if (altid != 0) { + // add the alt to the rv + DestEntry alt = getEntry(altid); + if (alt == null) + throw new SQLException("No alt entry?? " + altid); + d.destList = new ArrayList(2); + d.destList.add(d.dest); + d.destList.add(alt.dest); + d.propsList = new ArrayList(2); + d.propsList.add(d.props); + d.propsList.add(alt.props); + } + rv.put(name, d); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + public DestEntry getEntry(String hostname) throws IOException { + DestEntry rv = null; + for (String list : getLists()) { + rv = getEntry(list, hostname); + if (rv != null) + break; + } + return rv; + } + + public Destination getDest(String hostname) throws IOException { + Destination rv = null; + for (String list : getLists()) { + rv = getDest(list, hostname); + if (rv != null) + break; + } + return rv; + } + + public List getDests(String hostname) throws IOException { + List rv = null; + for (String list : getLists()) { + rv = getDests(list, hostname); + if (rv != null) + break; + } + return rv; + } + + public DestEntry getEntry(String list, String hostname) throws IOException { + list = internal(list); + DestEntry rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(setTable(SQL_GET_ENTRIES, list)); + stmt.setString(1, hostname); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + long id = rs.getLong(1); + if (id == 0) + throw new SQLException("No entry??"); + rv = getEntry(id); + if (rv == null) + throw new SQLException("No entry?? " + id); + long altid = rs.getLong(2); + if (altid != 0) { + // add the alt to the rv + DestEntry alt = getEntry(altid); + if (alt == null) + throw new SQLException("No alt entry?? " + altid); + rv.destList = new ArrayList(2); + rv.destList.add(rv.dest); + rv.destList.add(alt.dest); + rv.propsList = new ArrayList(2); + rv.propsList.add(rv.props); + rv.propsList.add(alt.props); + } + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + public Destination getDest(String list, String hostname) throws IOException { + list = internal(list); + Destination rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(setTable(SQL_GET_PRIMARY, list)); + stmt.setString(1, hostname); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + long id = rs.getLong(1); + if (id == 0) + throw new SQLException("No entry??"); + rv = getDest(id); + if (rv == null) + throw new SQLException("No entry?? " + id); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + public List getDests(String list, String hostname) throws IOException { + list = internal(list); + List rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(setTable(SQL_GET_ENTRIES, list)); + stmt.setString(1, hostname); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + long id = rs.getLong(1); + if (id == 0) + throw new SQLException("No entry??"); + Destination d = getDest(id); + if (d == null) + throw new SQLException("No entry?? " + id); + rv = new ArrayList(2); + rv.add(d); + long altid = rs.getLong(2); + if (altid != 0) { + // add the alt to the rv + Destination alt = getDest(altid); + if (alt == null) + throw new SQLException("No alt entry?? " + altid); + rv.add(alt); + } + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + //// setters / updaters //// + + /** + * May exist or not. One dest. + */ + public void addEntry(String list, String hostname, Destination dest, Properties props) throws IOException { + //long start = System.currentTimeMillis(); + list = internal(list); + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(setTable(SQL_GET_ENTRIES, list)); + stmt.setString(1, hostname); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + // existing + long id = rs.getLong(1); + if (id == 0) + throw new SQLException("No entry??"); + Destination d = getDest(id); + if (!dest.equals(d)) + throw new SQLException("mismatch" + id); + updateEntry(id, props); + } else { + // new + long id = setEntry(dest, props); + try { stmt.close(); } catch (SQLException se) {} + stmt = _con.prepareStatement(setTable(SQL_ADD_ENTRIES, list)); + stmt.setString(1, hostname); + stmt.setLong(2, id); + int rows = stmt.executeUpdate(); + addReverse(dest.getHash(), hostname); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + //if (_log.shouldDebug()) _log.debug("addEntry(s) took " + (System.currentTimeMillis() - start)); + } + + /** + * May exist or not. Multi-dest. + */ + public void addEntry(String list, String hostname, List dests, List plist) throws IOException { + //long start = System.currentTimeMillis(); + list = internal(list); + Destination dest = dests.get(0); + Properties props = plist.get(0); + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(setTable(SQL_GET_ENTRIES, list)); + stmt.setString(1, hostname); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + // existing + long id = rs.getLong(1); + if (id == 0) + throw new SQLException("No entry??"); + long id2 = rs.getLong(2); + if (id2 == 0 && dests.size() == 1) { + // one entry before, one after + updateEntry(id, props); + } else if (id2 != 0 && dests.size() == 1) { + // two entries before, one after + Destination d1 = getDest(id); + Destination d2 = getDest(id2); + if (d1.equals(dest)) { + updateEntry(id, props); + removeEntry(id2); + deleteReverse(d2.getHash(), hostname); + updateEntry(list, hostname, id, 0); + } else if (d2.equals(dest)) { + updateEntry(id2, plist.get(1)); + removeEntry(id); + deleteReverse(d1.getHash(), hostname); + updateEntry(list, hostname, id2, 0); + } else { + throw new SQLException("mismatch"); + } + } else if (id2 == 0 && dests.size() > 1) { + // one entry before, two after + Destination d1 = getDest(id); + Destination d2 = dests.get(1); + if (d1.equals(dest)) { + // add second + id2 = setEntry(d2, plist.get(1)); + addReverse(d2.getHash(), hostname); + } else if (d2.equals(dest)) { + // add first + id2 = id; + id = setEntry(d1, props); + addReverse(d1.getHash(), hostname); + } else { + throw new SQLException("mismatch"); + } + updateEntry(list, hostname, id, id2); + } else { + // two entries before, two after + Destination d1 = getDest(id); + Destination d2 = getDest(id2); + if (d1.equals(dest) && d2.equals(dest)) { + // same order + updateEntry(id, props); + updateEntry(id2, plist.get(1)); + } else { + // swap order? + throw new SQLException("mismatch"); + } + } + } else { + // new + long id = setEntry(dest, props); + addReverse(dest.getHash(), hostname); + long id2 = 0; + if (dests.size() > 1) { + Destination dest2 = dests.get(1); + Properties props2 = plist.get(1); + id2 = setEntry(dest2, props2); + addReverse(dest2.getHash(), hostname); + } + try { stmt.close(); } catch (SQLException se) {} + stmt = _con.prepareStatement(setTable(SQL_ADD_ENTRIES, list)); + stmt.setString(1, hostname); + stmt.setLong(2, id); + if (id2 != 0) + stmt.setLong(3, id2); + int rows = stmt.executeUpdate(); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + //if (_log.shouldDebug()) _log.debug("addEntry(m) took " + (System.currentTimeMillis() - start)); + } + + /** + * @param alt 0 to remove + */ + private void updateEntry(String list, String hostname, long entry, long alt) throws SQLException { + //long start = System.currentTimeMillis(); + DestEntry rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(setTable(SQL_UPDATE_ENTRIES, list)); + stmt.setString(1, hostname); + stmt.setLong(2, entry); + stmt.setLong(3, alt); + int rows = stmt.executeUpdate(); + verifyUpdated(rows); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + //if (_log.shouldDebug()) _log.debug("updateEntry took " + (System.currentTimeMillis() - start)); + } + + //// deleters //// + + public DestEntry removeEntry(String list, String hostname) throws IOException { + //long start = System.currentTimeMillis(); + list = internal(list); + DestEntry rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(setTable(SQL_DELETE_ENTRIES, list)); + stmt.setString(1, hostname); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + long id = rs.getLong(1); + if (id == 0) + throw new SQLException("No entry??"); + rv = removeEntry(id); + if (rv == null) + throw new SQLException("No entry?? " + id); + long altid = rs.getLong(2); + if (altid != 0) { + // add the alt to the rv + DestEntry alt = removeEntry(altid); + if (alt == null) + throw new SQLException("No alt entry?? " + altid); + rv.destList = new ArrayList(2); + rv.destList.add(rv.dest); + rv.destList.add(alt.dest); + rv.propsList = new ArrayList(2); + rv.propsList.add(rv.props); + rv.propsList.add(alt.props); + } + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + if (rv != null) { + deleteReverse(rv.dest.getHash(), hostname); + if (rv.destList != null && rv.destList.size() > 1) + deleteReverse(rv.destList.get(1).getHash(), hostname); + + } + //if (_log.shouldDebug()) _log.debug("removeEntry(name) took " + (System.currentTimeMillis() - start)); + return rv; + } + + + //// reverse //// + + public List getReverse(Hash h) throws IOException { + return getReverse(h, true); + } + + /** + * @param checkForward true for external use, false for internal + */ + private List getReverse(Hash h, boolean checkForward) throws IOException { + long idx = DataHelper.fromLong8(h.getData(), 0); + PreparedStatement stmt = null; + List rv = new ArrayList(); + try { + stmt = _con.prepareStatement(SQL_GET_REVERSE); + stmt.setLong(1, idx); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + String x = rs.getString(1); + String[] names = DataHelper.split(x, REVERSE_SEPARATOR); + rv.addAll(Arrays.asList(names)); + if (rv.isEmpty() && _log.shouldWarn()) + _log.warn("Empty getReverse() for " + h + " idx " + idx + " contains:'" + x + "'"); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + if (checkForward && !rv.isEmpty()) { + // now do the forward lookups to verify + for (Iterator iter = rv.iterator(); iter.hasNext(); ) { + String host = iter.next(); + List ld = getDests(host); + if (ld != null) { + boolean found = false; + for (Destination d : ld) { + if (d.calculateHash().equals(h)) { + found = true; + break; + } + } + if (!found) { + _log.warn("hash collision idx " + idx + " host " + host); + iter.remove(); + } + } else { + _log.warn("host gone, was still in reverse idx " + idx + " host " + host); + iter.remove(); + } + } + } + if (!rv.isEmpty()) + return rv; + return null; + } + + // unlike in BFNS, only getReverse() is public; we manage all + // updates internally + + private void addReverse(Hash h, String hostname) throws IOException { + List rev = getReverse(h, false); + if (rev != null) { + if (!rev.contains(hostname)) { + rev.add(hostname); + updateReverse(h, rev); + } + return; + } + long idx = DataHelper.fromLong8(h.getData(), 0); + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_SET_REVERSE); + stmt.setLong(1, idx); + stmt.setString(2, hostname); + int rows = stmt.executeUpdate(); + verifyUpdated(rows); + } catch (SQLException se) { + if (_log.shouldWarn()) + _log.warn("FAIL addReverse() " + hostname + " to " + h + " idx " + idx + " existing: " + getReverse(h)); + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + } + + private void updateReverse(Hash h, List hostnames) throws IOException { + StringBuilder buf = new StringBuilder(); + int sz = hostnames.size(); + for (int i = 0; i < sz; i++) { + buf.append(hostnames.get(i)); + if (i != sz - 1) + buf.append(REVERSE_SEPARATOR); + } + long idx = DataHelper.fromLong8(h.getData(), 0); + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_UPDATE_REVERSE); + stmt.setString(1, buf.toString()); + stmt.setLong(2, idx); + int rows = stmt.executeUpdate(); + verifyUpdated(rows); + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + } + + private void deleteReverse(Hash h, String hostname) throws IOException { + List rev = getReverse(h, false); + if (rev != null) { + if (rev.remove(hostname)) { + if (rev.isEmpty()) + deleteReverse(h); + else + updateReverse(h, rev); + } + } + } + + private void deleteReverse(Hash h) throws IOException { + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_DELETE_REVERSE); + long idx = DataHelper.fromLong8(h.getData(), 0); + stmt.setLong(1, idx); + int rows = stmt.executeUpdate(); + verifyUpdated(rows); + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + } + + // Entry table operators based on entry ID - private below here // + + private Destination getDest(long id) throws SQLException { + Destination rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_GET_DEST); + stmt.setLong(1, id); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + byte[] d = rs.getBytes(1); + Destination dest = new Destination(); + dest.fromByteArray(d); + rv = dest; + } + } catch (DataFormatException dfe) { + throw new SQLException("bad dest", dfe); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + private DestEntry getEntry(long id) throws SQLException { + DestEntry rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_GET_ENTRY); + stmt.setLong(1, id); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + byte[] d = rs.getBytes(1); + Destination dest = new Destination(); + dest.fromByteArray(d); + rv = new DestEntry(); + rv.dest = dest; + String a = rs.getString(2); + String m = rs.getString(3); + String n = rs.getString(4); + String s = rs.getString(5); + String v = rs.getString(6); + Properties props = new Properties(); + if (a != null) + props.setProperty("a", a); + if (m != null) + props.setProperty("m", m); + if (n != null) + props.setProperty("notes", n); + if (s != null) + props.setProperty("s", s); + if (v != null) + props.setProperty("v", v); + rv.props = props; + } + } catch (DataFormatException dfe) { + throw new SQLException("bad dest", dfe); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + /** + * Must not exist + * @return id + */ + private long setEntry(Destination dest, Properties props) throws SQLException { + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_SET_ENTRY); + stmt.setBytes(1, dest.toByteArray()); + String a = props.getProperty("a"); + String m = props.getProperty("m"); + String n = props.getProperty("notes"); + String s = props.getProperty("s"); + String v = props.getProperty("v"); + if (a != null) + stmt.setLong(2, Long.parseLong(a)); + if (m != null) + stmt.setLong(3, Long.parseLong(m)); + if (n != null && n.length() > 0) + stmt.setString(4, n); + if (s != null) + stmt.setString(5, s); + stmt.setBoolean(6, Boolean.valueOf(v)); + ResultSet rs = stmt.executeQuery(); + if (!rs.next()) + throw new SQLException("no result?"); + return rs.getLong(1); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + } + + /** + * Must exist + */ + private void updateEntry(long id, Properties props) throws SQLException { + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_UPDATE_ENTRY); + String a = props.getProperty("a"); + String m = props.getProperty("m"); + String n = props.getProperty("notes"); + String s = props.getProperty("s"); + String v = props.getProperty("v"); + if (a != null) + stmt.setLong(1, Long.parseLong(a)); + if (m != null) + stmt.setLong(2, Long.parseLong(m)); + if (n != null && n.length() > 0) + stmt.setString(3, n); + else + stmt.setNull(3, Types.VARCHAR); + if (s != null) + stmt.setString(4, s); + else + stmt.setNull(4, Types.VARCHAR); + stmt.setBoolean(5, Boolean.valueOf(v)); + stmt.setLong(6, id); + int rows = stmt.executeUpdate(); + verifyUpdated(rows); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + } + + private DestEntry removeEntry(long id) throws SQLException { + DestEntry rv = null; + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(SQL_DELETE_ENTRY); + stmt.setLong(1, id); + ResultSet rs = stmt.executeQuery(); + if (rs.next()) { + byte[] d = rs.getBytes(1); + Destination dest = new Destination(); + dest.fromByteArray(d); + rv = new DestEntry(); + rv.dest = dest; + String a = rs.getString(2); + String m = rs.getString(3); + String n = rs.getString(4); + String s = rs.getString(5); + String v = rs.getString(6); + Properties props = new Properties(); + if (a != null) + props.setProperty("a", a); + if (m != null) + props.setProperty("m", m); + if (n != null) + props.setProperty("notes", n); + if (s != null) + props.setProperty("s", s); + if (v != null) + props.setProperty("v", v); + rv.props = props; + } + } catch (DataFormatException dfe) { + throw new SQLException("bad dest", dfe); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + return rv; + } + + /// pragmas /// + + private void configure() throws IOException { + long start = System.currentTimeMillis(); + PreparedStatement stmt = null; + try { + // this one is persistent and is set in the DDL, but doesn't hurt + stmt = _con.prepareStatement(PRAGMA_SYNCH); + stmt.execute(); + try { stmt.close(); } catch (SQLException se) {} + stmt = _con.prepareStatement(PRAGMA_JOURNAL); + stmt.execute(); + if (getSize("hosts.txt") > 0) { + // if we already imported, + // shrink the checkpoint window so we don't use excessive disk + try { stmt.close(); } catch (SQLException se) {} + stmt = _con.prepareStatement(PRAGMA_AUTOCHECKPOINT); + stmt.execute(); + } + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + if (_log.shouldDebug()) _log.debug("configure took " + (System.currentTimeMillis() - start)); + } + + public void checkpoint() throws IOException { + long start = System.currentTimeMillis(); + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(PRAGMA_CHECKPOINT); + stmt.execute(); + try { stmt.close(); } catch (SQLException se) {} + // after the import, naming service will call this, + // shrink the checkpoint window + stmt = _con.prepareStatement(PRAGMA_AUTOCHECKPOINT); + stmt.execute(); + } catch (SQLException se) { + handleSQLException(se); + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + if (_log.shouldDebug()) _log.debug("checkpoint took " + (System.currentTimeMillis() - start)); + } + + /// helpers below here /// + + + /** + * Call after any UPDATE or DELETE call + */ + private boolean verifyUpdated(int rows) throws SQLException { + if (rows != 1) + //throw new SQLException("UPDATE/DELETE failed, rows changed: " + rows); + _log.warn("UPDATE/DELETE failed, rows changed: " + rows); + return rows == 1; + } + + private void handleSQLException(SQLException se) throws IOException { + if (_log.shouldLog(Log.WARN)) + _log.warn("fail", se); + throw new IOException("SQL failure", se); + } + + private String internal(String external) { + String rv = _internal.get(external); + return rv != null ? rv : external; + } + + private static String setTable(String query, String table) { + // TODO escape + return query.replace("??", '"' + table + '"'); + } +} diff --git a/src/java/net/i2p/client/naming/sqldb/DBInit.java b/src/java/net/i2p/client/naming/sqldb/DBInit.java new file mode 100644 index 0000000..ccb1eec --- /dev/null +++ b/src/java/net/i2p/client/naming/sqldb/DBInit.java @@ -0,0 +1,143 @@ +package net.i2p.client.naming.sqldb; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import net.i2p.I2PAppContext; +import net.i2p.util.Log; + + +/** + * Initialize the database, creating and updating to the latest version if necessary + * + * Adapted from syndie (public domain) + */ +class DBInit { + + private final Connection _con; + private final Log _log; + + private static final String DDL_PREFIX = "/net/i2p/client/naming/sqldb/resources/ddl_update"; + private static final String DDL_SUFFIX = ".txt"; + + public DBInit(I2PAppContext ctx, Connection dbConn) { + _con = dbConn; + _log = ctx.logManager().getLog(DBInit.class); + } + + /** + * Initialize the DB, update to latest version if necessary + */ + public void initDB() throws SQLException { + int version = checkDBVersion(); + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Known DB version: " + version); + if (version < 0) { + if (_log.shouldLog(Log.INFO)) + _log.info("Building the database..."); + } + int updates = getDBUpdateCount(); + if (updates <= 0) + throw new SQLException("No DDL 0"); + for (int i = version + 1; i < updates; i++) { + if (i >= version) { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Updating database version " + i + " to " + (i+1)); + updateDB(i); + } + } + } + + private int checkDBVersion() { + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement("SELECT value FROM info WHERE key = 'version'"); + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + int rv = rs.getInt(1); + if (!rs.wasNull()) + return rv; + } + return -1; + } catch (SQLException se) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Unable to check the database version (does not exist?)", se); + return -1; + } finally { + if (stmt != null) try { stmt.close(); } catch (SQLException se) {} + } + } + + private int getDBUpdateCount() { + int updates = 0; + while (true) { + InputStream in = getClass().getResourceAsStream(DDL_PREFIX + updates + DDL_SUFFIX); + if (in != null) { + updates++; + try { in.close(); } catch (IOException ioe) {} + } else { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("There were " + updates + " database updates known"); + return updates; + } + } + } + + /** + * Create a new DB with the version 1 ddl_update0.txt, or + * Update from oldVersion to oldVersion + 1 using ddl_update{oldVersion}.txt + */ + private void updateDB(int oldVersion) throws SQLException { + StringBuilder cmdBuf = new StringBuilder(); + BufferedReader r = null; + try { + InputStream in = getClass().getResourceAsStream(DDL_PREFIX + oldVersion + DDL_SUFFIX); + if (in != null) { + r = new BufferedReader(new InputStreamReader(in)); + String line = null; + while ( (line = r.readLine()) != null) { + line = line.trim(); + if (line.startsWith("//") || line.startsWith("--")) + continue; + cmdBuf.append(' ').append(line); + if (line.endsWith(";")) { + exec(cmdBuf.toString()); + cmdBuf.setLength(0); + } + } + r.close(); + r = null; + } else { + throw new SQLException("Cannot load update for database version " + oldVersion); + } + } catch (IOException ioe) { + if (_log.shouldLog(Log.ERROR)) + _log.error("Error reading the db script", ioe); + throw new SQLException(ioe.toString()); + } catch (SQLException se) { + if (_log.shouldLog(Log.ERROR)) + _log.error("Error building the db, failed on statement: " + cmdBuf, se); + throw se; + } finally { + if (r != null) try { r.close(); } catch (IOException ioe) {} + } + } + + private void exec(String cmd) throws SQLException { + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Exec [" + cmd + "]"); + PreparedStatement stmt = null; + try { + stmt = _con.prepareStatement(cmd); + stmt.execute(); + } finally { + if (stmt != null) stmt.close(); + } + } +} diff --git a/src/java/net/i2p/client/naming/sqldb/SQLNamingService.java b/src/java/net/i2p/client/naming/sqldb/SQLNamingService.java new file mode 100644 index 0000000..4439446 --- /dev/null +++ b/src/java/net/i2p/client/naming/sqldb/SQLNamingService.java @@ -0,0 +1,1428 @@ +/* + * free (adj.): unencumbered; not under the control of others + * Written by mihi in 2004 and released into the public domain + * with no warranty of any kind, either expressed or implied. + * It probably won't make your computer catch on fire, or eat + * your children, but it might. Use at your own risk. + */ +package net.i2p.client.naming.sqldb; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TreeMap; + +import net.i2p.I2PAppContext; +import net.i2p.client.naming.DummyNamingService; +import net.i2p.client.naming.HostsTxtNamingService; +import net.i2p.client.naming.NamingService; +import net.i2p.client.naming.NamingServiceListener; +import net.i2p.client.naming.SingleFileNamingService; +import net.i2p.crypto.SigType; +import net.i2p.data.DataFormatException; +import net.i2p.data.DataHelper; +import net.i2p.data.Destination; +import net.i2p.data.Hash; +import net.i2p.util.LHMCache; +import net.i2p.util.Log; +import net.i2p.util.SecureFileOutputStream; +import net.i2p.util.SystemVersion; +import net.i2p.util.VersionComparator; + + +/** + * A naming service using a sqlite3 database. + * + * See resources/ddl/ddl_update0.txt for the schema. + * + * @since xxx + */ +public class SQLNamingService extends DummyNamingService { + + private final DBClient _dbc; + private final List _lists; + private final List _invalid; + private final Map _negativeCache; + private volatile boolean _isClosed; + private final boolean _readOnly; + + private static final String HOSTS_DB = "hostsdb.sqlite3"; + private static final String FALLBACK_LIST = "hosts.txt"; + private static final String PROP_FORCE = "i2p.naming.sql.writeInAppContext"; + + private static final String PROP_VERSION = "version"; + private static final String PROP_CREATED = "created"; + private static final String PROP_UPGRADED = "upgraded"; + private static final String VERSION = "1"; + + private static final String PROP_ADDED = "a"; + // See susidns + //private static final String PROP_MODDED = "m"; + private static final String PROP_SOURCE = "s"; + // See susidns + //private static final String PROP_VALIDATED = "v"; + + private static final String DUMMY = ""; + private static final int NEGATIVE_CACHE_SIZE = 32; + private static final int MAX_VALUE_LENGTH = 4096; + private static final int MAX_DESTS_PER_HOST = 2; + + /** + * Opens the database at hostsdb.sqlite3 or creates a new + * one and imports entries from hosts.txt, userhosts.txt, and privatehosts.txt. + * + * If not in router context, the database will be opened read-only + * unless the property i2p.naming.blockfile.writeInAppContext is true. + * Not designed for multiple instantiations or simultaneous use by multple JVMs. + * + * @throws RuntimeException on fatal error + */ + public SQLNamingService(I2PAppContext context) { + super(context); + _lists = new ArrayList(); + _invalid = new ArrayList(); + _negativeCache = new LHMCache(NEGATIVE_CACHE_SIZE); + DBClient dbc = null; + boolean readOnly = false; + File f = new File(_context.getRouterDir(), HOSTS_DB); + if (f.exists()) { + try { + // closing a BlockFile does not close the underlying file, + // so we must create and retain a RAF so we may close it later + + // *** Open readonly if not in router context (unless forced) + readOnly = (!f.canWrite()) || + ((!context.isRouterContext()) && (!context.getBooleanProperty(PROP_FORCE))); + dbc = initExisting(f); + if (readOnly && context.isRouterContext()) + _log.logAlways(Log.WARN, "Read-only hosts database in router context"); + } catch (IOException ioe) { + File corrupt = new File(_context.getRouterDir(), HOSTS_DB + '.' + System.currentTimeMillis() + ".corrupt"); + _log.log(Log.CRIT, "Corrupt, unsupported version, or unreadable database " + + f + ", moving to " + corrupt + + " and creating new database", ioe); + boolean success = f.renameTo(corrupt); + if (!success) + _log.log(Log.CRIT, "Failed to move corrupt database " + f + " to " + corrupt); + } + } + boolean shouldImport = false; + if (dbc == null) { + try { + dbc = initNew(f); + SecureFileOutputStream.setPerms(f); + shouldImport = true; + } catch (IOException ioe) { + _log.log(Log.CRIT, "Failed to initialize database", ioe); + throw new RuntimeException(ioe); + } + readOnly = false; + } + _dbc = dbc; + _readOnly = readOnly; + _context.addShutdownTask(new Shutdown()); + if (shouldImport && !readOnly) { + try { + importHosts(); + } catch (IOException ioe) { + _log.log(Log.CRIT, "Failed to import hosts to new database", ioe); + } + } + } + + /** + * Create a new database and initialize it from the local files + * privatehosts.txt, userhosts.txt, and hosts.txt, + * creating a table in the database for each. + */ + private DBClient initNew(File f) throws IOException { + long start = _context.clock().now(); + try { + DBClient rv = new DBClient(_context, f); + rv.connect(); + if (_log.shouldLog(Log.INFO)) + _log.info("DB init took " + DataHelper.formatDuration(_context.clock().now() - start)); + return rv; + } catch (RuntimeException e) { + _log.error("Failed to initialize database", e); + throw new IOException(e.toString()); + } + } + + private void importHosts() throws IOException { + long start = _context.clock().now(); + try { + + NamingService ns = _context.namingService(); + String type = ns.getName(); + String listc = _context.getProperty(HostsTxtNamingService.PROP_HOSTS_FILE, + HostsTxtNamingService.DEFAULT_HOSTS_FILE); + List lists = getFilenames(listc); + _log.info("Found naming service " + type + " with lists " + listc); + int total = 0; + if (type.equals("BlockfileNamingService")) { + Properties props = new Properties(); + Properties opts = new Properties(); + List optslist = new ArrayList(2); + for (String list : lists) { + int count = 0; + _lists.add(list); + props.clear(); + props.setProperty("list", list); + // use getNames() so we don't load the whole thing into memory + // however, + // BFNS.getNames() broken before 0.9.62 + Set names = ns.getNames(props); + _log.warn("Found " + names.size() + " in " + list); + //Map entries = ns.getEntries(props); + //_log.warn("Found " + entries.size() + " in entry map"); + for (String name : names) { + optslist.clear(); + List dests = ns.lookupAll(name, props, optslist); + if (!dests.isEmpty()) { + optslist.get(0).setProperty("list", list); + if (dests.size() > 1) { + optslist.get(1).setProperty("list", list); + } + put(name, dests, optslist, false); + count++; + //if (_log.shouldDebug()) + // _log.debug("Imported " + name + " with props " + optslist.get(0)); + } else { + _log.logAlways(Log.WARN, "Unable to import entry for " + name + + " from list " + list + "???"); + } + } + total += count; + _log.logAlways(Log.INFO, "Migrated " + count + " hosts from blockflie list " + list + " to new hosts database"); + } + } else if (type.equals("HostsTxtNamingService")) { + for (String hostsfile : lists) { + int count = 0; + _lists.add(hostsfile); + File file = new File(_context.getRouterDir(), hostsfile); + if ((!file.exists()) || !(file.canRead())) + continue; + BufferedReader in = null; + String sourceMsg = "Imported from " + hostsfile + " file"; + try { + in = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"), 16*1024); + String line = null; + while ( (line = in.readLine()) != null) { + if (line.startsWith("#")) + continue; + int split = line.indexOf('='); + if (split <= 0) + continue; + String key = line.substring(0, split).toLowerCase(Locale.US); + if (line.indexOf('#') > 0) { // trim off any end of line comment + line = line.substring(0, line.indexOf('#')).trim(); + if (line.length() < split + 1) + continue; + } + String b64 = line.substring(split+1).trim(); + Destination d = lookupBase64(b64); + if (d != null) { + addEntry(hostsfile, key, d, sourceMsg); + count++; + if (_log.shouldLog(Log.INFO)) + _log.info("Imported " + key); + } else { + _log.logAlways(Log.WARN, "Unable to import entry for " + key + + " from file " + file + " - bad Base 64: " + b64); + } + } + } catch (IOException ioe) { + _log.error("Failed to read hosts from " + file, ioe); + } finally { + if (in != null) try { in.close(); } catch (IOException ioe) {} + } + total += count; + _log.logAlways(Log.INFO, "Migrated " + count + " hosts from " + file + " to new hosts database"); + + if (_log.shouldLog(Log.INFO)) + _log.info("DB import took " + DataHelper.formatDuration(_context.clock().now() - start)); + _dbc.checkpoint(); + } + } else { + _log.logAlways(Log.WARN, "Unable to import entries from naming service " + type); + } + if (total <= 0) + _log.logAlways(Log.WARN, "No hosts.txt files found, Initialized hosts database with zero entries"); + } catch (RuntimeException e) { + _log.error("Failed to initialize database", e); + throw new IOException(e.toString()); + } + } + + /** + * Read the info block of an existing database. + */ + private DBClient initExisting(File f) throws IOException { + long start = _context.clock().now(); + try { + DBClient rv = new DBClient(_context, f); + rv.connect(); + Properties info = rv.getInfo(); + + _lists.addAll(rv.getLists()); + if (_lists.isEmpty()) + throw new IOException("No lists"); + long createdOn = 0; + String created = info.getProperty(PROP_CREATED); + if (created != null) { + try { + createdOn = Long.parseLong(created); + } catch (NumberFormatException nfe) {} + } + + String version = info.getProperty(PROP_VERSION); + if (version == null) + throw new IOException("No version"); + if (VersionComparator.comp(version, VERSION) > 0) + throw new IOException("Database version is " + version + + " but this implementation only supports versions 1-" + VERSION + + " Did you downgrade I2P??"); + if (_log.shouldLog(Log.INFO)) + _log.info("Found database version " + version + + " created " + (new Date(createdOn)).toString() + + " containing lists: " + _lists); + if (_log.shouldLog(Log.INFO)) + _log.info("DB init took " + DataHelper.formatDuration(_context.clock().now() - start)); + return rv; + } catch (RuntimeException e) { + _log.error("Failed to initialize database", e); + throw new IOException(e.toString()); + } + } + + /** + * Caller must synchronize + * @return entry or null, or throws ioe + */ + private DestEntry getEntry(String listname, String key) throws IOException { + return _dbc.getEntry(listname, key); + } + + /** + * Caller must synchronize + * @param source may be null + */ + private void addEntry(String listname, String key, Destination dest, String source) throws IOException { + try { + Properties props = new Properties(); + props.setProperty(PROP_ADDED, Long.toString(_context.clock().now())); + if (source != null) + props.setProperty(PROP_SOURCE, source); + _dbc.addEntry(listname, key, dest, props); + } catch (IOException ioe) { + _log.error("DB add error", ioe); + // delete index?? + throw ioe; + } catch (RuntimeException e) { + _log.error("DB add error", e); + throw new IOException(e.toString()); + } + } + + private static List getFilenames(String list) { + StringTokenizer tok = new StringTokenizer(list, ","); + List rv = new ArrayList(tok.countTokens()); + while (tok.hasMoreTokens()) + rv.add(tok.nextToken()); + return rv; + } + + ///// Reverse index methods + + /** + * Caller must synchronize. + * Returns null without exception on error (logs only). + * + * @return all found if more than one + */ + private List getReverseEntries(Hash hash) { + try { + return _dbc.getReverse(hash); + } catch (IOException ioe) { + return null; + } + } + + ////////// Start NamingService API + + /* + * + * Will strip a "www." prefix and retry if lookup fails + * + * @param hostname upper/lower case ok + * @param options If non-null and contains the key "list", lookup in + * that list only, otherwise all lists + */ + @Override + public Destination lookup(String hostname, Properties lookupOptions, Properties storedOptions) { + Destination rv = lookup2(hostname, lookupOptions, storedOptions); + if (rv == null) { + // if hostname starts with "www.", strip and try again + // but not for www.i2p + hostname = hostname.toLowerCase(Locale.US); + if (hostname.startsWith("www.") && hostname.length() > 7) { + hostname = hostname.substring(4); + rv = lookup2(hostname, lookupOptions, storedOptions); + } + } + return rv; + } + + /* + * Single dest version. + * + * @param lookupOptions If non-null and contains the key "list", lookup in + * that list only, otherwise all lists + */ + private Destination lookup2(String hostname, Properties lookupOptions, Properties storedOptions) { + String listname = null; + if (lookupOptions != null) + listname = lookupOptions.getProperty("list"); + + Destination d = null; + // only use cache if we aren't retreiving options or specifying the list + if (listname == null && storedOptions == null) { + d = super.lookup(hostname, null, null); + if (d != null) + return d; + // Base32 failed? + if (hostname.length() == BASE32_HASH_LENGTH + 8 && hostname.toLowerCase(Locale.US).endsWith(".b32.i2p")) + return null; + } + + String key = hostname.toLowerCase(Locale.US); + synchronized(_negativeCache) { + if (_negativeCache.get(key) != null) + return null; + } + synchronized(_dbc) { + if (_isClosed) + return null; + for (String list : _lists) { + if (listname != null && !list.equals(listname)) + continue; + try { + DestEntry de = getEntry(list, key); + if (de != null) { + if (!validate(key, de, listname)) + continue; + d = de.dest; + if (storedOptions != null && de.props != null) + storedOptions.putAll(de.props); + break; + } + } catch (IOException ioe) { + break; + } + } + deleteInvalid(); + } + if (d != null) { + putCache(hostname, d); + } else { + synchronized(_negativeCache) { + _negativeCache.put(key, DUMMY); + } + } + return d; + } + + /* + * Multiple dests version. + * DB MUST be version 4. + * + * @param lookupOptions If non-null and contains the key "list", lookup in + * that list only, otherwise all lists + */ + private List lookupAll2(String hostname, Properties lookupOptions, List storedOptions) { + // only use cache for b32 + if (hostname.length() == BASE32_HASH_LENGTH + 8 && hostname.toLowerCase(Locale.US).endsWith(".b32.i2p")) { + Destination d = super.lookup(hostname, null, null); + if (d != null) { + if (storedOptions != null) + storedOptions.add(null); + return Collections.singletonList(d); + } + // Base32 failed? + return null; + } + String key = hostname.toLowerCase(Locale.US); + synchronized(_negativeCache) { + if (_negativeCache.get(key) != null) + return null; + } + String listname = null; + if (lookupOptions != null) + listname = lookupOptions.getProperty("list"); + + List rv = null; + synchronized(_dbc) { + if (_isClosed) + return null; + for (String list : _lists) { + if (listname != null && !list.equals(listname)) + continue; + try { + DestEntry de = getEntry(list, key); + if (de != null) { + if (!validate(key, de, listname)) + continue; + if (de.destList != null) { + rv = de.destList; + if (storedOptions != null) + storedOptions.addAll(de.propsList); + } else { + rv = Collections.singletonList(de.dest); + if (storedOptions != null) + storedOptions.add(de.props); + } + break; + } + } catch (IOException ioe) { + break; + } + } + deleteInvalid(); + } + if (rv != null) { + putCache(hostname, rv.get(0)); + } else { + synchronized(_negativeCache) { + _negativeCache.put(key, DUMMY); + } + } + return rv; + } + + /** + * @param options If non-null and contains the key "list", add to that list + * (default "hosts.txt") + * Use the key "s" for the source + */ + @Override + public boolean put(String hostname, Destination d, Properties options) { + return put(hostname, d, options, false); + } + + /** + * @param options If non-null and contains the key "list", add to that list + * (default "hosts.txt") + * Use the key "s" for the source. + * Key "a" will be added with the current time, unless + * "a" is present in options. + */ + @Override + public boolean putIfAbsent(String hostname, Destination d, Properties options) { + return put(hostname, d, options, true); + } + + /** + * Single dest version + * This does not prevent adding b32. Caller must check. + * + * @param checkExisting if true, fail if entry already exists + */ + private boolean put(String hostname, Destination d, Properties options, boolean checkExisting) { + if (_readOnly) { + _log.error("Add entry failed, read-only hosts database"); + return false; + } + String key = hostname.toLowerCase(Locale.US); + synchronized(_negativeCache) { + _negativeCache.remove(key); + } + String listname = FALLBACK_LIST; + Properties props = new Properties(); + props.setProperty(PROP_ADDED, Long.toString(_context.clock().now())); + if (options != null) { + props.putAll(options); + String list = options.getProperty("list"); + if (list != null) { + listname = list; + props.remove("list"); + } + } + synchronized(_dbc) { + if (_isClosed) + return false; + try { + boolean changed = (checkExisting || !_listeners.isEmpty()) && _dbc.getDest(key) != null; + if (changed && checkExisting) + return false; + _dbc.addEntry(listname, key, d, props); + if (changed) { + removeCache(hostname); + } + for (NamingServiceListener nsl : _listeners) { + if (changed) + nsl.entryChanged(this, hostname, d, options); + else + nsl.entryAdded(this, hostname, d, options); + } + return true; + } catch (IOException ioe) { + _log.error("DB add error", ioe); + return false; + } catch (RuntimeException re) { + _log.error("DB add error", re); + return false; + } + } + } + + /** + * Multiple dests version. + * This does not prevent adding b32. Caller must check. + * + * @param propsList may be null, or entries may be null + * @param checkExisting if true, fail if entry already exists + */ + private boolean put(String hostname, List dests, List propsList, boolean checkExisting) { + int sz = dests.size(); + if (sz <= 0) + throw new IllegalArgumentException(); + if (sz == 1) + return put(hostname, dests.get(0), propsList != null ? propsList.get(0) : null, checkExisting); + if (_readOnly) { + _log.error("Add entry failed, read-only hosts database"); + return false; + } + String key = hostname.toLowerCase(Locale.US); + synchronized(_negativeCache) { + _negativeCache.remove(key); + } + String listname = FALLBACK_LIST; + String date = Long.toString(_context.clock().now()); + List outProps = new ArrayList(propsList.size()); + for (Properties options : propsList) { + Properties props = new Properties(); + props.setProperty(PROP_ADDED, date); + if (options != null) { + props.putAll(options); + String list = options.getProperty("list"); + if (list != null) { + listname = list; + props.remove("list"); + } + } + outProps.add(props); + } + synchronized(_dbc) { + if (_isClosed) + return false; + try { + boolean changed = (checkExisting || !_listeners.isEmpty()) && _dbc.getDest(listname, key) != null; + if (changed && checkExisting) + return false; + _dbc.addEntry(listname, key, dests, outProps); + if (changed) { + removeCache(hostname); + } + for (int i = 0; i < dests.size(); i++) { + Destination d = dests.get(i); + Properties options = propsList.get(i); + for (NamingServiceListener nsl : _listeners) { + if (changed) + nsl.entryChanged(this, hostname, d, options); + else + nsl.entryAdded(this, hostname, d, options); + } + } + return true; + } catch (IOException ioe) { + _log.error("DB add error", ioe); + return false; + } catch (RuntimeException re) { + _log.error("DB add error", re); + return false; + } + } + } + + /** + * @param options If non-null and contains the key "list", remove + * from that list (default "hosts.txt", NOT all lists) + */ + @Override + public boolean remove(String hostname, Properties options) { + if (_readOnly) { + _log.error("Remove entry failed, read-only hosts database"); + return false; + } + String key = hostname.toLowerCase(Locale.US); + String listname = FALLBACK_LIST; + if (options != null) { + String list = options.getProperty("list"); + if (list != null) { + listname = list; + } + } + synchronized(_dbc) { + if (_isClosed) + return false; + try { + DestEntry removed = _dbc.removeEntry(listname, key); + boolean rv = removed != null; + if (rv) { + removeCache(hostname); + for (NamingServiceListener nsl : _listeners) { + nsl.entryRemoved(this, key); + } + } + return rv; + } catch (IOException ioe) { + _log.error("DB remove error", ioe); + return false; + } catch (RuntimeException re) { + _log.error("DB remove error", re); + return false; + } + } + } + + /** + * @param options If non-null and contains the key "list", get + * from that list (default "hosts.txt", NOT all lists) + * Key "skip": skip that many entries + * Key "limit": max number to return + * Key "search": return only those matching substring + * Key "startsWith": return only those starting with + * ("[0-9]" allowed) + * Key "beginWith": start here in the iteration + * Don't use both startsWith and beginWith. + * Search, startsWith, and beginWith values must be lower case. + */ + @Override + public Map getEntries(Properties options) { + String listname = FALLBACK_LIST; + String search = null; + String startsWith = null; + String beginWith = null; + int limit = Integer.MAX_VALUE; + int skip = 0; + if (options != null) { + String ln = options.getProperty("list"); + if (ln != null) + listname = ln; + search = options.getProperty("search"); + startsWith = options.getProperty("startsWith"); + beginWith = options.getProperty("beginWith"); + if (beginWith == null && startsWith != null) { + if (startsWith.equals("[0-9]")) + beginWith = "0"; + else + beginWith = startsWith; + } + String lim = options.getProperty("limit"); + try { + limit = Integer.parseInt(lim); + } catch (NumberFormatException nfe) {} + String sk = options.getProperty("skip"); + try { + skip = Integer.parseInt(sk); + } catch (NumberFormatException nfe) {} + } + if (_log.shouldLog(Log.DEBUG)) + _log.debug("Searching " + listname + " beginning with " + beginWith + + " starting with " + startsWith + " search string " + search + + " limit=" + limit + " skip=" + skip); + synchronized(_dbc) { + if (_isClosed) + return Collections.emptyMap(); + try { + return _dbc.getDests(listname, search, beginWith, limit, skip); + } catch (IOException ioe) { + _log.error("DB lookup error", ioe); + return Collections.emptyMap(); + } catch (RuntimeException re) { + _log.error("DB lookup error", re); + return Collections.emptyMap(); + } finally { + deleteInvalid(); + } + } + } + + /** + * @param options If non-null and contains the key "list", get + * from that list (default "hosts.txt", NOT all lists) + * Key "skip": skip that many entries + * Key "limit": max number to return + * Key "search": return only those matching substring + * Key "startsWith": return only those starting with + * ("[0-9]" allowed) + * Key "beginWith": start here in the iteration + * Don't use both startsWith and beginWith. + * Search, startsWith, and beginWith values must be lower case. + */ + @Override + public Map getBase64Entries(Properties options) { + String listname = FALLBACK_LIST; + String search = null; + String startsWith = null; + String beginWith = null; + int limit = Integer.MAX_VALUE; + int skip = 0; + if (options != null) { + String ln = options.getProperty("list"); + if (ln != null) + listname = ln; + search = options.getProperty("search"); + startsWith = options.getProperty("startsWith"); + beginWith = options.getProperty("beginWith"); + if (beginWith == null && startsWith != null) { + if (startsWith.equals("[0-9]")) + beginWith = "0"; + else + beginWith = startsWith; + } + String lim = options.getProperty("limit"); + try { + limit = Integer.parseInt(lim); + } catch (NumberFormatException nfe) {} + String sk = options.getProperty("skip"); + try { + skip = Integer.parseInt(sk); + } catch (NumberFormatException nfe) {} + } + synchronized(_dbc) { + if (_isClosed) + return Collections.emptyMap(); + try { + Map dests = _dbc.getDests(listname, search, beginWith, limit, skip); + if (dests == null) + return Collections.emptyMap(); + Map rv = new TreeMap(); + for (Map.Entry e : dests.entrySet()) { + rv.put(e.getKey(), e.getValue().toBase64()); + } + return rv; + } catch (IOException ioe) { + _log.error("DB lookup error", ioe); + return Collections.emptyMap(); + } catch (RuntimeException re) { + _log.error("DB lookup error", re); + return Collections.emptyMap(); + } finally { + deleteInvalid(); + } + } + } + + /** + * Export in a hosts.txt format. + * Output is sorted. + * Caller must close writer. + * + * @param options If non-null and contains the key "list", get + * from that list (default "hosts.txt", NOT all lists) + * Key "search": return only those matching substring + * Key "startsWith": return only those starting with + * ("[0-9]" allowed) + * Key "beginWith": start here in the iteration + */ + @Override + public void export(Writer out, Properties options) throws IOException { + String listname = FALLBACK_LIST; + String search = null; + String startsWith = null; + String beginWith = null; + if (options != null) { + String ln = options.getProperty("list"); + if (ln != null) + listname = ln; + search = options.getProperty("search"); + startsWith = options.getProperty("startsWith"); + beginWith = options.getProperty("beginWith"); + if (beginWith == null && startsWith != null) { + if (startsWith.equals("[0-9]")) + beginWith = "0"; + else + beginWith = startsWith; + } + } + out.write("# Address book: "); + out.write(getName()); + out.write(" (" + listname + ')'); + final String nl = System.getProperty("line.separator", "\n"); + out.write(nl); + out.write("# Exported: "); + out.write((new Date()).toString()); + out.write(nl); + synchronized(_dbc) { + if (_isClosed) + return; + try { + Map entries = _dbc.getEntries(listname, search, beginWith); + if (entries == null) + entries = Collections.emptyMap(); + Map rv = new TreeMap(); + int cnt = 0; + for (Map.Entry e : entries.entrySet()) { + String key = e.getKey(); + DestEntry de = e.getValue(); + if (!validate(key, de, listname)) + continue; + int dsz = de.destList != null ? de.destList.size() : 1; + // new non-DSA dest is put first, so put in reverse + // order so importers will see the older dest first + for (int i = dsz - 1; i >= 0; i--) { + Properties p; + Destination d; + if (i == 0) { + p = de.props; + d = de.dest; + } else { + p = de.propsList.get(i); + d = de.destList.get(i); + } + out.write("# "); + out.write(key); + out.write(": "); + out.write(d.toBase32()); + out.write(nl); + out.write(key); + out.write('='); + out.write(d.toBase64()); + if (p != null) + SingleFileNamingService.writeOptions(p, out); + out.write(nl); + cnt++; + } + } + if (beginWith != null || search != null) { + if (cnt <= 0) { + out.write("# No entries"); + out.write(nl); + return; + } + if (cnt > 1) { + out.write("# " + cnt + " entries"); + out.write(nl); + } + } + } catch (RuntimeException re) { + throw new IOException("DB lookup error", re); + } finally { + deleteInvalid(); + } + } + } + + /** + * @param options If non-null and contains the key "list", get + * from that list (default "hosts.txt", NOT all lists) + * Key "skip": skip that many entries + * Key "limit": max number to return + * Key "search": return only those matching substring + * Key "startsWith": return only those starting with + * ("[0-9]" allowed) + * Key "beginWith": start here in the iteration + * Don't use both startsWith and beginWith. + * Search, startsWith, and beginWith values must be lower case. + */ + @Override + public Set getNames(Properties options) { + String listname = FALLBACK_LIST; + String search = null; + String startsWith = null; + String beginWith = null; + int limit = Integer.MAX_VALUE; + int skip = 0; + if (options != null) { + String ln = options.getProperty("list"); + if (ln != null) + listname = ln; + search = options.getProperty("search"); + startsWith = options.getProperty("startsWith"); + beginWith = options.getProperty("beginWith"); + if (beginWith == null && startsWith != null) { + if (startsWith.equals("[0-9]")) + beginWith = "0"; + else + beginWith = startsWith; + } + String lim = options.getProperty("limit"); + try { + limit = Integer.parseInt(lim); + } catch (NumberFormatException nfe) {} + String sk = options.getProperty("skip"); + try { + skip = Integer.parseInt(sk); + } catch (NumberFormatException nfe) {} + } + synchronized(_dbc) { + if (_isClosed) + return Collections.emptySet(); + try { + return _dbc.getNames(listname, search, beginWith, limit, skip); + } catch (IOException ioe) { + _log.error("DB lookup error", ioe); + return Collections.emptySet(); + } catch (RuntimeException re) { + _log.error("DB lookup error", re); + return Collections.emptySet(); + } + } + } + + /** + * @param options ignored + */ + @Override + public String reverseLookup(Destination d, Properties options) { + return reverseLookup(d.calculateHash()); + } + + @Override + public String reverseLookup(Hash h) { + List ls; + synchronized(_dbc) { + if (_isClosed) + return null; + ls = getReverseEntries(h); + } + return (ls != null) ? ls.get(0) : null; + } + + /** + * @param options ignored + */ + @Override + public List reverseLookupAll(Destination d, Properties options) { + return reverseLookupAll(d.calculateHash()); + } + + @Override + public List reverseLookupAll(Hash h) { + synchronized(_dbc) { + if (_isClosed) + return null; + return getReverseEntries(h); + } + } + + /** + * @param options If non-null and contains the key "list", return the + * size of that list (default "hosts.txt", NOT all lists) + */ + @Override + public int size(Properties options) { + String listname = FALLBACK_LIST; + if (options != null) { + String list = options.getProperty("list"); + if (list != null) { + listname = list; + } + } + synchronized(_dbc) { + if (_isClosed) + return 0; + try { + return _dbc.getSize(listname); + } catch (IOException ioe) { + _log.error("DB size error", ioe); + return 0; + } catch (RuntimeException re) { + _log.error("DB size error", re); + return 0; + } + } + } + + public void shutdown() { + close(); + } + + ////////// End NamingService API + + //// Begin new API for multiple Destinations + + /** + * Return all of the entries found in the first list found, or in the list + * specified in lookupOptions. Does not aggregate all destinations found + * in all lists. + * + * If storedOptions is non-null, it must be a List that supports null entries. + * If the returned value (the List of Destinations) is non-null, + * the same number of Properties objects will be added to storedOptions. + * If no properties were found for a given Destination, the corresponding + * entry in the storedOptions list will be null. + * + * @param lookupOptions input parameter, NamingService-specific, may be null + * @param storedOptions output parameter, NamingService-specific, any stored properties will be added if non-null + * @return non-empty List of Destinations, or null if nothing found + */ + @Override + public List lookupAll(String hostname, Properties lookupOptions, List storedOptions) { + List rv = lookupAll2(hostname, lookupOptions, storedOptions); + if (rv == null) { + // if hostname starts with "www.", strip and try again + // but not for www.i2p + hostname = hostname.toLowerCase(Locale.US); + if (hostname.startsWith("www.") && hostname.length() > 7) { + hostname = hostname.substring(4); + rv = lookupAll2(hostname, lookupOptions, storedOptions); + } + } + // we sort the destinations in addDestination(), + // which is a lot easier than sorting them here + return rv; + } + + /** + * Add a Destination to an existing hostname's entry in the addressbook. + * + * This does not prevent adding b32. Caller must check. + * + * @param options NamingService-specific, may be null + * @return success + */ + @Override + public boolean addDestination(String hostname, Destination d, Properties options) { + List storedOptions = new ArrayList(4); + synchronized(_dbc) { + // We use lookupAll2(), not lookupAll(), because if hostname starts with www., + // we do not want to read in from the + // non-www hostname and then copy it to a new www hostname. + List dests = lookupAll2(hostname, options, storedOptions); + if (dests == null) + return put(hostname, d, options, false); + if (dests.contains(d)) + return false; + if (dests.size() >= MAX_DESTS_PER_HOST) + return false; + List newDests = new ArrayList(dests.size() + 1); + newDests.addAll(dests); + // TODO better sort by sigtype preference. + // For now, non-DSA at the front, DSA at the end + SigType type = d.getSigningPublicKey().getType(); + if (type != SigType.DSA_SHA1 && type.isAvailable()) { + newDests.add(0, d); + storedOptions.add(0, options); + } else { + newDests.add(d); + storedOptions.add(options); + } + return put(hostname, newDests, storedOptions, false); + } + } + + /** + * Remove a hostname's entry only if it contains the Destination d. + * If the NamingService supports multiple Destinations per hostname, + * and this is the only Destination, removes the entire entry. + * If aditional Destinations remain, it only removes the + * specified Destination from the entry. + * + * @param options NamingService-specific, may be null + * @return true if entry containing d was successfully removed. + */ + @Override + public boolean remove(String hostname, Destination d, Properties options) { + List storedOptions = new ArrayList(4); + synchronized(_dbc) { + // We use lookupAll2(), not lookupAll(), because if hostname starts with www., + // we do not want to read in from the + // non-www hostname and then copy it to a new www hostname. + List dests = lookupAll2(hostname, options, storedOptions); + if (dests == null) + return false; + for (int i = 0; i < dests.size(); i++) { + Destination dd = dests.get(i); + if (dd.equals(d)) { + // Found it. Remove and return. + if (dests.size() == 1) + return remove(hostname, options); + List newDests = new ArrayList(dests.size() - 1); + for (int j = 0; j < dests.size(); j++) { + if (j != i) + newDests.add(dests.get(j)); + } + storedOptions.remove(i); + if (options != null) { + String list = options.getProperty("list"); + if (list != null) + storedOptions.get(0).setProperty("list", list); + } + return put(hostname, newDests, storedOptions, false); + } + } + } + return false; + } + + //// End new API for multiple Destinations + + /** + * Continuously validate anything we read in. + * Queue anything invalid to be removed at the end of the operation. + * Caller must sync! + * @return valid + */ + private boolean validate(String key, DestEntry de, String listname) { + if (key == null) + return false; + // de.props may be null + // publickey check is a quick proxy to detect dest deserialization failure + boolean rv = key.length() > 0 && + de != null && + de.dest != null && + de.dest.getPublicKey() != null; + if (rv && de.destList != null) { + // additional checks for multi-dest + rv = de.propsList != null && + de.destList.size() == de.propsList.size() && + !de.destList.contains(null); + } + if ((!rv) && (!_readOnly)) + _invalid.add(new InvalidEntry(key, listname)); + return rv; + } + + /** + * Remove and log all invalid entries queued by validate() + * while scanning in lookup() or getEntries(). + * We delete in the order detected. + * Caller must sync! + */ + private void deleteInvalid() { + if (_invalid.isEmpty()) + return; + _log.error("Removing " + _invalid.size() + " corrupt entries from database"); + for (InvalidEntry ie : _invalid) { + String key = ie.key; + String list = ie.list; + try { + // this will often return null since it was corrupt + boolean success = _dbc.removeEntry(list, key) != null; + if (success) + _log.error("Removed corrupt \"" + key + "\" from database " + list); + else + _log.error("May have Failed to remove corrupt \"" + key + "\" from database " + list); + } catch (RuntimeException re) { + _log.error("Error while removing corrupt \"" + key + "\" from database " + list, re); + } catch (IOException ioe) { + _log.error("Error while removing corrput \"" + key + "\" from database " + list, ioe); + } + } + _invalid.clear(); + } + + private void close() { + synchronized(_dbc) { + try { + _dbc.close(); + } catch (Exception e) { + if (_log.shouldLog(Log.WARN)) + _log.warn("Error closing", e); + } + _isClosed = true; + } + synchronized(_negativeCache) { + _negativeCache.clear(); + } + clearCache(); + } + + private class Shutdown implements Runnable { + public void run() { + close(); + } + } + + /** + * A DestEntry contains Properties and a Destination, + * and is serialized in that order. + */ + static class DestEntry { + /** May be null. + * If more than one dest, contains the first props. + */ + public Properties props; + + /** May not be null. + * If more than one dest, contains the first dest. + */ + public Destination dest; + + /** May be null - v4 only - same size as destList - may contain null entries + * Only non-null if more than one dest. + * First entry always equal to props. + */ + public List propsList; + + /** May be null - v4 only - same size as propsList + * Only non-null if more than one dest. + * First entry always equal to dest. + */ + public List destList; + + @Override + public String toString() { + return "DestEntry (" + DataHelper.toString(props) + + ") " + dest.toString(); + } + } + + /** + * Used to store entries that need deleting + */ + private static class InvalidEntry { + public final String key; + public final String list; + + public InvalidEntry(String k, String l) { + key = k; + list = l; + } + } + + private static final String PROP_IMPL = "i2p.naming.impl"; + private static final String DEFAULT_IMPL = "net.i2p.router.naming.BlockfileNamingService"; + + /** + * SQLNamingService [force] + * force = force writable + */ + public static void main(String[] args) throws Exception { + Properties ctxProps = new Properties(); + if (args.length > 0 && args[0].equals("force")) + ctxProps.setProperty(PROP_FORCE, "true"); + // to force import from BFNS even in app ctx + ctxProps.setProperty(PROP_IMPL, DEFAULT_IMPL); + I2PAppContext ctx = new I2PAppContext(ctxProps); + SQLNamingService sqlns = new SQLNamingService(ctx); + Properties sprops = new Properties(); + String lname = "privatehosts.txt"; + sprops.setProperty("list", lname); + System.out.println("List " + lname + " contains " + sqlns.size(sprops)); + lname = "userhosts.txt"; + sprops.setProperty("list", lname); + System.out.println("List " + lname + " contains " + sqlns.size(sprops)); + lname = "hosts.txt"; + sprops.setProperty("list", lname); + System.out.println("List " + lname + " contains " + sqlns.size(sprops)); + +/**** +*/ + List names = null; + Properties props = new Properties(); + try { + DataHelper.loadProps(props, new File("hosts.txt"), true); + names = new ArrayList(props.stringPropertyNames()); + Collections.shuffle(names); + } catch (IOException ioe) { + System.out.println("No hosts.txt to test with"); + sqlns.close(); + return; + } + + System.out.println("size() reports " + sqlns.size()); + System.out.println("getNames() returns " + sqlns.getNames().size()); + + System.out.println("Testing with " + names.size() + " hostnames"); + int found = 0; + int notfound = 0; + int rfound = 0; + int rnotfound = 0; + long start = System.currentTimeMillis(); + for (String name : names) { + Destination dest = sqlns.lookup(name); + if (dest != null) { + found++; + String reverse = sqlns.reverseLookup(dest); + if (reverse != null) + rfound++; + else + rnotfound++; + } else { + notfound++; + } + } + System.out.println("SQLNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start)); + System.out.println("found " + found + " notfound " + notfound); + System.out.println("reverse found " + rfound + " notfound " + rnotfound); + + //if (true) return; + + System.out.println("Removing all " + names.size() + " hostnames"); + found = 0; + notfound = 0; + Collections.shuffle(names); + start = System.currentTimeMillis(); + for (String name : names) { + if (sqlns.remove(name)) + found++; + else + notfound++; + } + System.out.println("SQLNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start)); + System.out.println("removed " + found + " not removed " + notfound); + + System.out.println("Adding back " + names.size() + " hostnames"); + found = 0; + notfound = 0; + Collections.shuffle(names); + start = System.currentTimeMillis(); + for (String name : names) { + try { + if (sqlns.put(name, new Destination(props.getProperty(name)))) + found++; + else + notfound++; + } catch (DataFormatException dfe) {} + } + System.out.println("SQLNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start)); + System.out.println("Added " + found + " not added " + notfound); + System.out.println("size() reports " + sqlns.size()); + + + //sqlns.dumpDB(); +/* +****/ + sqlns.close(); + ctx.logManager().flush(); + System.out.flush(); +/**** + if (true) return; + + HostsTxtNamingService htns = new HostsTxtNamingService(I2PAppContext.getGlobalContext()); + found = 0; + notfound = 0; + start = System.currentTimeMillis(); + for (String name : names) { + Destination dest = htns.lookup(name); + if (dest != null) + found++; + else + notfound++; + } + System.out.println("HTNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start)); + System.out.println("found " + found + " notfound " + notfound); +****/ + } +}