retry failed hosts after one hour

TODO.md

@@ -32,6 +32,10 @@ For ease of deployment for new users, and so that users do not need to run a sep
 
 Basically any non-gui non-cli user interface
 
+##### Metadata editing and search
+
+To enable parsing of metadata from known file types and the user editing it or adding manual metadata
+
 ### Small Items
 
 * Detect if router is dead and show warning or exit
@@ -39,4 +43,3 @@ Basically any non-gui non-cli user interface
 * Download file sequentially
 * Unsharing of files
 * Multiple-selection download, Ctrl-A
-* Automatic sharing of new files in shared directories (more like medium item)

cli/src/main/groovy/com/muwire/cli/Cli.groovy

@@ -35,7 +35,7 @@ class Cli {
         
         Core core 
         try {
-            core = new Core(props, home, "0.2.6")
+            core = new Core(props, home, "0.2.9")
         } catch (Exception bad) {
             bad.printStackTrace(System.out)
             println "Failed to initialize core, exiting"
@@ -73,7 +73,7 @@ class Cli {
         
         Timer timer = new Timer("status-printer", true)
         timer.schedule({
-            println "Connections $connectionsListener.connections Uploads $uploadsListener.uploads Shared $sharedListener.shared"
+            println String.valueOf(new Date()) + " Connections $connectionsListener.connections Uploads $uploadsListener.uploads Shared $sharedListener.shared"
         } as TimerTask, 60000, 60000)
         
         def latch = new CountDownLatch(1)
@@ -119,11 +119,11 @@ class Cli {
         volatile int uploads
         public void onUploadEvent(UploadEvent e) {
             uploads++
-            println "Starting upload of ${e.uploader.file.getName()} to ${e.uploader.request.downloader.getHumanReadableName()}"
+            println String.valueOf(new Date()) + " Starting upload of ${e.uploader.file.getName()} to ${e.uploader.request.downloader.getHumanReadableName()}"
         }
         public void onUploadFinishedEvent(UploadFinishedEvent e) {
             uploads--
-            println "Finished upload of ${e.uploader.file.getName()} to ${e.uploader.request.downloader.getHumanReadableName()}"
+            println String.valueOf(new Date()) + " Finished upload of ${e.uploader.file.getName()} to ${e.uploader.request.downloader.getHumanReadableName()}"
         }
     }
     

cli/src/main/groovy/com/muwire/cli/CliDownloader.groovy

@@ -53,7 +53,7 @@ class CliDownloader {
 
         Core core
         try {
-            core = new Core(props, home, "0.2.6")
+            core = new Core(props, home, "0.2.9")
         } catch (Exception bad) {
             bad.printStackTrace(System.out)
             println "Failed to initialize core, exiting"

cli/src/main/groovy/com/muwire/cli/FileList.groovy

@@ -0,0 +1,23 @@
+package com.muwire.cli
+
+import com.muwire.core.util.DataUtil
+
+import groovy.json.JsonSlurper
+import net.i2p.data.Base64
+
+class FileList {
+    public static void main(String [] args) {
+        if (args.length < 1) {
+            println "pass files.json as argument"
+            System.exit(1)
+        }
+        
+        def slurper = new JsonSlurper()
+        File filesJson = new File(args[0])
+        filesJson.eachLine { 
+            def json = slurper.parseText(it)
+            String name = DataUtil.readi18nString(Base64.decode(json.file))
+            println "$name,$json.length,$json.pieceSize,$json.infoHash"
+        }
+    }
+}

core/src/main/groovy/com/muwire/core/Core.groovy

@@ -72,6 +72,7 @@ public class Core {
     private final HasherService hasherService
     private final DownloadManager downloadManager
     private final DirectoryWatcher directoryWatcher
+    final FileManager fileManager
         
     public Core(MuWireSettings props, File home, String myVersion) {
         this.home = home		
@@ -155,7 +156,7 @@ public class Core {
 		
 		
 		log.info "initializing file manager"
-		FileManager fileManager = new FileManager(eventBus, props)
+		fileManager = new FileManager(eventBus, props)
 		eventBus.register(FileHashedEvent.class, fileManager)
 		eventBus.register(FileLoadedEvent.class, fileManager)
 		eventBus.register(FileDownloadedEvent.class, fileManager)
@@ -276,7 +277,7 @@ public class Core {
             }
         }
         
-        Core core = new Core(props, home, "0.2.6")
+        Core core = new Core(props, home, "0.2.9")
         core.startServices()
         
         // ... at the end, sleep or execute script

core/src/main/groovy/com/muwire/core/MuWireSettings.groovy

@@ -30,7 +30,7 @@ class MuWireSettings {
         nickname = props.getProperty("nickname","MuWireUser")
         downloadLocation = new File((String)props.getProperty("downloadLocation", 
             System.getProperty("user.home")))
-        downloadRetryInterval = Integer.parseInt(props.getProperty("downloadRetryInterval","15"))
+        downloadRetryInterval = Integer.parseInt(props.getProperty("downloadRetryInterval","5"))
         updateCheckInterval = Integer.parseInt(props.getProperty("updateCheckInterval","36"))
         shareDownloadedFiles = Boolean.parseBoolean(props.getProperty("shareDownloadedFiles","true"))
         

core/src/main/groovy/com/muwire/core/download/DownloadManager.groovy

@@ -58,6 +58,8 @@ public class DownloadManager {
         e.result.each { 
             destinations.add(it.sender.destination)
         }
+        destinations.addAll(e.sources)
+        destinations.remove(me.destination)
         
         def downloader = new Downloader(eventBus, this, me, e.target, size,
             infohash, pieceSize, connector, destinations,

core/src/main/groovy/com/muwire/core/download/DownloadSession.groovy

@@ -85,7 +85,7 @@ class DownloadSession {
             if (code.startsWith("404 ")) {
                 log.warning("file not found")
                 endpoint.close()
-                return
+                return false
             }
                 
             if (code.startsWith("416 ")) {

core/src/main/groovy/com/muwire/core/download/Downloader.groovy

@@ -21,6 +21,7 @@ import com.muwire.core.files.FileDownloadedEvent
 
 import groovy.util.logging.Log
 import net.i2p.data.Destination
+import net.i2p.util.ConcurrentHashSet
 
 @Log
 public class Downloader {
@@ -49,6 +50,7 @@ public class Downloader {
     private final File incompleteFile
     final int pieceSizePow2
     private final Map<Destination, DownloadWorker> activeWorkers = new ConcurrentHashMap<>()
+    private final Set<Destination> successfulDestinations = new ConcurrentHashSet<>()
     
     
     private volatile boolean cancelled
@@ -249,6 +251,7 @@ public class Downloader {
                     requestPerformed = currentSession.request()
                     if (!requestPerformed)
                         break
+                    successfulDestinations.add(endpoint.destination)
                     writePieces()
                 }
             } catch (Exception bad) {
@@ -268,7 +271,7 @@ public class Downloader {
                     }
                     eventBus.publish(
                         new FileDownloadedEvent(
-                            downloadedFile : new DownloadedFile(file, getInfoHash(), pieceSizePow2, Collections.emptySet()),
+                            downloadedFile : new DownloadedFile(file, getInfoHash(), pieceSizePow2, successfulDestinations),
                         downloader : Downloader.this))
                             
                 } 

core/src/main/groovy/com/muwire/core/download/UIDownloadEvent.groovy

@@ -3,8 +3,11 @@ package com.muwire.core.download
 import com.muwire.core.Event
 import com.muwire.core.search.UIResultEvent
 
+import net.i2p.data.Destination
+
 class UIDownloadEvent extends Event {
     
     UIResultEvent[] result
+    Set<Destination> sources
     File target
 }

core/src/main/groovy/com/muwire/core/files/DirectoryWatcher.groovy

@@ -5,6 +5,8 @@ import java.nio.file.FileSystems
 import java.nio.file.Path
 import java.nio.file.Paths
 import static java.nio.file.StandardWatchEventKinds.*
+
+import java.nio.file.ClosedWatchServiceException
 import java.nio.file.WatchEvent
 import java.nio.file.WatchKey
 import java.nio.file.WatchService
@@ -79,7 +81,7 @@ class DirectoryWatcher {
                 }
                 key.reset()
             }
-        } catch (InterruptedException e) {
+        } catch (InterruptedException|ClosedWatchServiceException e) {
             if (!shutdown)
                 throw e
         }
@@ -91,6 +93,8 @@ class DirectoryWatcher {
         log.fine("created entry $f")
         if (f.isDirectory())
             f.toPath().register(watchService, kinds)
+        else
+            waitingFiles.put(f, System.currentTimeMillis())
     }
         
     private void processModified(Path parent, Path path) {

core/src/main/groovy/com/muwire/core/hostcache/Host.groovy

@@ -5,9 +5,11 @@ import net.i2p.data.Destination
 class Host {
 
 	private static final int MAX_FAILURES = 3
+    private static final int CLEAR_INTERVAL = 60 * 60 * 1000
 	
 	final Destination destination
 	int failures,successes
+    long lastAttempt
 	
 	public Host(Destination destination) {
 		this.destination = destination
@@ -16,11 +18,13 @@ class Host {
 	synchronized void onConnect() {
 		failures = 0
 		successes++
+        lastAttempt = System.currentTimeMillis()
 	}
 	
 	synchronized void onFailure() {
 		failures++
 		successes = 0
+        lastAttempt = System.currentTimeMillis()
 	}
 	
 	synchronized boolean isFailed() {
@@ -34,4 +38,8 @@ class Host {
     synchronized void clearFailures() {
         failures = 0
     }
+    
+    synchronized void canTryAgain() {
+        System.currentTimeMillis() - lastAttempt > CLEAR_INTERVAL
+    }
 }

core/src/main/groovy/com/muwire/core/hostcache/HostCache.groovy

@@ -109,6 +109,8 @@ class HostCache extends Service {
 				Host host = new Host(dest)
 				host.failures = Integer.valueOf(String.valueOf(entry.failures))
 				host.successes = Integer.valueOf(String.valueOf(entry.successes))
+                if (entry.lastAttempt != null)
+                    host.lastAttempt = entry.lastAttempt
 				if (allowHost(host))
 					hosts.put(dest, host)
 			}
@@ -118,7 +120,7 @@ class HostCache extends Service {
 	}
 	
 	private boolean allowHost(Host host) {
-		if (host.isFailed())
+		if (host.isFailed() && !host.canTryAgain())
 			return false
 		if (host.destination == myself)
 			return false
@@ -143,6 +145,7 @@ class HostCache extends Service {
 					map.destination = dest.toBase64()
 					map.failures = host.failures
 					map.successes = host.successes
+                    map.lastAttempt = host.lastAttempt
 					def json = JsonOutput.toJson(map)
 					writer.println json
 				}

core/src/main/groovy/com/muwire/core/search/ResultsParser.groovy

@@ -1,5 +1,7 @@
 package com.muwire.core.search
 
+import java.util.stream.Collectors
+
 import javax.naming.directory.InvalidSearchControlsException
 
 import com.muwire.core.InfoHash
@@ -7,6 +9,7 @@ import com.muwire.core.Persona
 import com.muwire.core.util.DataUtil
 
 import net.i2p.data.Base64
+import net.i2p.data.Destination
 
 class ResultsParser {
     public static UIResultEvent parse(Persona p, UUID uuid, def json) throws InvalidSearchResultException {
@@ -58,6 +61,7 @@ class ResultsParser {
                  size : size,
                  infohash : parsedIH,
                  pieceSize : pieceSize,
+                 sources : Collections.emptySet(),
                  uuid : uuid)
         } catch (Exception e) {
             throw new InvalidSearchResultException("parsing search result failed",e)
@@ -82,11 +86,17 @@ class ResultsParser {
             if (infoHash.length != InfoHash.SIZE)
                 throw new InvalidSearchResultException("invalid infohash size $infoHash.length")
             int pieceSize = json.pieceSize
+            
+            Set<Destination> sources = Collections.emptySet()
+            if (json.sources != null) 
+                sources = json.sources.stream().map({new Destination(it)}).collect(Collectors.toSet())
+            
             return new UIResultEvent( sender : p,
                 name : name,
                 size : size,
                 infohash : new InfoHash(infoHash),
                 pieceSize : pieceSize,
+                sources : sources,
                 uuid: uuid)
         } catch (Exception e) {
             throw new InvalidSearchResultException("parsing search result failed",e)

core/src/main/groovy/com/muwire/core/search/ResultsSender.groovy

@@ -11,7 +11,9 @@ import java.util.concurrent.Executor
 import java.util.concurrent.Executors
 import java.util.concurrent.ThreadFactory
 import java.util.concurrent.atomic.AtomicInteger
+import java.util.stream.Collectors
 
+import com.muwire.core.DownloadedFile
 import com.muwire.core.EventBus
 import com.muwire.core.InfoHash
 
@@ -54,12 +56,16 @@ class ResultsSender {
                 int pieceSize = it.getPieceSize()
                 if (pieceSize == 0)
                     pieceSize = FileHasher.getPieceSize(length)
+                Set<Destination> suggested = Collections.emptySet()
+                if (it instanceof DownloadedFile) 
+                    suggested = it.sources
                 def uiResultEvent = new UIResultEvent( sender : me,
                     name : it.getFile().getName(),
                     size : length,
                     infohash : it.getInfoHash(),
                     pieceSize : pieceSize,
-                    uuid : uuid
+                    uuid : uuid,
+                    sources : suggested
                     )
                     eventBus.publish(uiResultEvent)
             }
@@ -110,6 +116,10 @@ class ResultsSender {
                         }
                         obj.hashList = hashListB64
                     }
+                    
+                    if (it instanceof DownloadedFile)
+                        obj.sources = it.sources.stream().map({dest -> dest.toBase64()}).collect(Collectors.toSet())
+                    
                     def json = jsonOutput.toJson(obj)
                     os.writeShort((short)json.length())
                     os.write(json.getBytes(StandardCharsets.US_ASCII))

core/src/main/groovy/com/muwire/core/search/UIResultEvent.groovy

@@ -4,8 +4,11 @@ import com.muwire.core.Event
 import com.muwire.core.InfoHash
 import com.muwire.core.Persona
 
+import net.i2p.data.Destination
+
 class UIResultEvent extends Event {
     Persona sender
+    Set<Destination> sources
     UUID uuid
     String name
     long size

gradle.properties

@@ -1,5 +1,8 @@
 group         = com.muwire
-version       = 0.2.6
+version       = 0.2.9
 groovyVersion = 2.4.15
 slf4jVersion  = 1.7.25
 spockVersion  = 1.1-groovy-2.4
+
+sourceCompatibility=1.8
+targetCompatibility=1.8

gui/griffon-app/controllers/com/muwire/gui/MainFrameController.groovy

@@ -129,8 +129,9 @@ class MainFrameController {
         def group = selected.getClientProperty("mvc-group")
         
         def resultsBucket = group.model.hashBucket[result.infohash]
+        def sources = group.model.sourcesBucket[result.infohash]
          
-        core.eventBus.publish(new UIDownloadEvent(result : resultsBucket, target : file))
+        core.eventBus.publish(new UIDownloadEvent(result : resultsBucket, sources: sources, target : file))
     }
     
     @ControllerAction

gui/griffon-app/models/com/muwire/gui/SearchTabModel.groovy

@@ -23,6 +23,7 @@ class SearchTabModel {
     String uuid
     def results = []
     def hashBucket = [:]
+    def sourcesBucket = [:]
     
     
     void mvcGroupInit(Map<String, String> args) {
@@ -37,7 +38,7 @@ class SearchTabModel {
     
     void handleResult(UIResultEvent e) {
         if (uiSettings.excludeLocalResult &&
-            e.sender == core.me)
+            core.fileManager.rootToFiles.containsKey(e.infohash))
             return
         runInsideUIAsync {
             def bucket = hashBucket.get(e.infohash)
@@ -46,6 +47,13 @@ class SearchTabModel {
                 hashBucket[e.infohash] = bucket
             }
             bucket << e
+            
+            Set sourceBucket = sourcesBucket.get(e.infohash)
+            if (sourceBucket == null) {
+                sourceBucket = new HashSet()
+                sourcesBucket.put(e.infohash, sourceBucket)
+            }
+            sourceBucket.addAll(e.sources)
                             
             results << e
             JTable table = builder.getVariable("results-table")
@@ -56,13 +64,22 @@ class SearchTabModel {
     void handleResultBatch(UIResultEvent[] batch) {
         runInsideUIAsync {
             batch.each { 
-                if (uiSettings.excludeLocalResult && it.sender == core.me)
+                if (uiSettings.excludeLocalResult && 
+                    core.fileManager.rootToFiles.containsKey(it.infohash))
                     return
                 def bucket = hashBucket.get(it.infohash)
                 if (bucket == null) {
                     bucket = []
                     hashBucket[it.infohash] = bucket
                 }
+                
+                Set sourceBucket = sourcesBucket.get(it.infohash)
+                if (sourceBucket == null) {
+                    sourceBucket = new HashSet()
+                    sourcesBucket.put(it.infohash, sourceBucket)
+                }
+                sourceBucket.addAll(it.sources)
+                
                 bucket << it
                 results << it
             }

gui/griffon-app/views/com/muwire/gui/MainFrameView.groovy

@@ -1,6 +1,7 @@
 package com.muwire.gui
 
 import griffon.core.artifact.GriffonView
+import griffon.core.env.Metadata
 import griffon.inject.MVCMember
 import griffon.metadata.ArtifactProviderFor
 import net.i2p.data.Base64
@@ -37,6 +38,7 @@ import java.awt.event.MouseEvent
 import java.nio.charset.StandardCharsets
 
 import javax.annotation.Nonnull
+import javax.inject.Inject
 
 @ArtifactProviderFor(GriffonView)
 class MainFrameView {
@@ -45,6 +47,8 @@ class MainFrameView {
     @MVCMember @Nonnull
     MainFrameModel model
     
+    @Inject Metadata metadata
+    
     def downloadsTable
     def lastDownloadSortEvent
     def lastSharedSortEvent
@@ -54,7 +58,7 @@ class MainFrameView {
         builder.with {
             application(size : [1024,768], id: 'main-frame',
             locationRelativeTo : null,
-            title: application.configuration['application.title'],
+            title: application.configuration['application.title'] + " " + metadata["application.version"],
             iconImage:   imageIcon('/griffon-icon-48x48.png').image,
             iconImages: [imageIcon('/griffon-icon-48x48.png').image,
                 imageIcon('/griffon-icon-32x32.png').image,

gui/griffon-app/views/com/muwire/gui/SearchTabView.groovy

@@ -47,8 +47,9 @@ class SearchTabView {
                 resultsTable = table(id : "results-table", autoCreateRowSorter : true) {
                     tableModel(list: model.results) {
                         closureColumn(header: "Name", preferredWidth: 350, type: String, read : {row -> row.name.replace('<','_')})
-                        closureColumn(header: "Size", preferredWidth: 50, type: Long, read : {row -> row.size})
-                        closureColumn(header: "Sources", preferredWidth: 10, type : Integer, read : { row -> model.hashBucket[row.infohash].size()})
+                        closureColumn(header: "Size", preferredWidth: 20, type: Long, read : {row -> row.size})
+                        closureColumn(header: "Direct Sources", preferredWidth: 50, type : Integer, read : { row -> model.hashBucket[row.infohash].size()})
+                        closureColumn(header: "Possible Sources", preferredWidth : 50, type : Integer, read : {row -> model.sourcesBucket[row.infohash].size()})
                         closureColumn(header: "Sender", preferredWidth: 170, type: String, read : {row -> row.sender.getHumanReadableName()})
                         closureColumn(header: "Trust", preferredWidth: 50, type: String, read : {row ->
                           model.core.trustService.getLevel(row.sender.destination).toString()