File caching (#59)

* Make response handling more modular

* Remove shared state of the BlockPuller tasks

* Implement dispatching errors at the file downloading

* Remove unneeded locking

* Count sums as long

* Fix error caused by using notify without syncrhonized

* Add TODOs

* BlockPuller: Add timeout and concurrency limit

* Cache file blocks on disk during downloading

* Read downloaded files block by block

* Add function to delete temp data in bulk

* Implement deleting temp data at download failure

* Fix error in progress reporting

* Fix canceling

* Fix handling timeouts at block requests

* Add file caching

The files were already cached (saved locally), but it was not tried to use them. This names files by their hash to ensure that no old file version is used. The side effect of this is that equal files are only downloaded and saved once.

This closes #43

What's missing:

- delete local files which are no longer provided by the server
- delete local files which were not used for a long time
- delete all local files when the menu item to clear the cache is used

* Fix file caching failure when using the SyncthingProvider

* Run auto reformat

* Use status object for the block puller download progress

* Improve file downloading by the SyncthingProvider

* Fix NullPointerException when getting totalFileSize

* Add additional check to cancel on race condition

* Check integrity at IndexHandler.getFileInfoAndBlockByPath()

* Use FileInfo.checkBlocks() for the integrity check

app/src/main/kotlin/net/syncthing/lite/dialogs/downloadfile/DownloadFileDialogFragment.kt

@@ -78,7 +78,7 @@ class DownloadFileDialogFragment : DialogFragment() {
                                 Intent(Intent.ACTION_VIEW)
                                         .setDataAndType(
                                                 FileProvider.getUriForFile(context!!, "net.syncthing.lite.fileprovider", status.file),
-                                                MimeTypeMap.getSingleton().getMimeTypeFromExtension(FilenameUtils.getExtension(status.file.name))
+                                                MimeTypeMap.getSingleton().getMimeTypeFromExtension(FilenameUtils.getExtension(fileSpec.fileName))
                                         )
                                         .newTask()
                                         .addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION)

app/src/main/kotlin/net/syncthing/lite/dialogs/downloadfile/DownloadFileDialogViewModel.kt

@@ -42,7 +42,7 @@ class DownloadFileDialogViewModel : ViewModel() {
                 )!!
 
                 val task = DownloadFileTask(
-                        externalCacheDir = externalCacheDir,
+                        fileStorageDirectory = externalCacheDir,
                         syncthingClient = syncthingClient,
                         fileInfo = fileInfo,
                         onProgress = { status ->

app/src/main/kotlin/net/syncthing/lite/library/DownloadFilePath.kt

@@ -0,0 +1,9 @@
+package net.syncthing.lite.library
+
+import java.io.File
+
+data class DownloadFilePath (val baseDirectory: File, val fileHash: String) {
+    val filesDirectory = File(baseDirectory, fileHash.substring(0, 2))
+    val targetFile = File(filesDirectory, fileHash.substring(2))
+    val tempFile = File(filesDirectory, fileHash.substring(2) + "_temp")
+}

app/src/main/kotlin/net/syncthing/lite/library/DownloadFileTask.kt

@@ -14,7 +14,7 @@ import org.apache.commons.io.FileUtils
 import java.io.File
 import java.io.IOException
 
-class DownloadFileTask(private val externalCacheDir: File,
+class DownloadFileTask(private val fileStorageDirectory: File,
                        syncthingClient: SyncthingClient,
                        private val fileInfo: FileInfo,
                        private val onProgress: (status: BlockPullerStatus) -> Unit,
@@ -58,20 +58,41 @@ class DownloadFileTask(private val externalCacheDir: File,
     private var doneListenerCalled = false
 
     init {
+        val file = DownloadFilePath(fileStorageDirectory, fileInfo.hash!!)
+
         launch {
+            if (file.targetFile.exists()) {
+                if (BuildConfig.DEBUG) {
+                    Log.d(TAG, "there is already a file")
+                }
+
+                callComplete(file.targetFile)
+            }
+
             syncthingClient.getBlockPuller(fileInfo.folder, { blockPuller ->
                 val job = launch {
                     try {
+                        if (!file.filesDirectory.isDirectory) {
+                            if (!file.filesDirectory.mkdirs()) {
+                                throw IOException("could not create output directory")
+                            }
+                        }
+
+                        // download the file to a temp location
                         val inputStream = blockPuller.pullFileCoroutine(fileInfo, this@DownloadFileTask::callProgress)
 
-                        val outputFile = File("$externalCacheDir/${fileInfo.folder}/${fileInfo.path}")
-                        FileUtils.copyInputStreamToFile(inputStream, outputFile)
+                        try {
+                            FileUtils.copyInputStreamToFile(inputStream, file.tempFile)
+                            file.tempFile.renameTo(file.targetFile)
+                        } finally {
+                            file.tempFile.delete()
+                        }
 
                         if (BuildConfig.DEBUG) {
                             Log.i(TAG, "Downloaded file $fileInfo")
                         }
 
-                        callComplete(outputFile)
+                        callComplete(file.targetFile)
                     } catch (e: Exception) {
                         callError(e)
 

app/src/main/kotlin/net/syncthing/lite/library/SyncthingProvider.kt

@@ -96,8 +96,8 @@ class SyncthingProvider : DocumentsProvider() {
     override fun openDocument(documentId: String, mode: String, signal: CancellationSignal?):
             ParcelFileDescriptor {
         Log.d(Tag, "openDocument($documentId, $mode, $signal)")
-        val fileInfo = FileInfo(folder = getFolderIdForDocId(documentId),
-                path = getPathForDocId(documentId), type = FileInfo.FileType.FILE)
+        val fileInfo = getIndexBrowser(getFolderIdForDocId(documentId))
+                .getFileInfoByAbsolutePath(getPathForDocId(documentId))
         val accessMode = ParcelFileDescriptor.parseMode(mode)
         if (accessMode != ParcelFileDescriptor.MODE_READ_ONLY) {
             throw NotImplementedError()

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/BlockPuller.kt

@@ -61,6 +61,12 @@ class BlockPuller internal constructor(private val connectionHandler: Connection
         logger.info("pulling file = {}", fileBlocks)
         NetworkUtils.assertProtocol(connectionHandler.hasFolder(fileBlocks.folder), { "supplied connection handler $connectionHandler will not share folder ${fileBlocks.folder}" })
 
+        // the file could have changed since the caller read it
+        // this would save the file using a wrong name, so throw here
+        if (fileBlocks.hash != fileInfo.hash) {
+            throw IllegalStateException("the current file entry hash does not match the hash of the provided one")
+        }
+
         val blockTempIdByHash = Collections.synchronizedMap(HashMap<String, String>())
 
         var status = BlockPullerStatus(

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/IndexHandler.kt

@@ -19,6 +19,7 @@ import net.syncthing.java.core.configuration.Configuration
 import net.syncthing.java.core.interfaces.IndexRepository
 import net.syncthing.java.core.interfaces.Sequencer
 import net.syncthing.java.core.interfaces.TempRepository
+import net.syncthing.java.core.utils.BlockUtils
 import net.syncthing.java.core.utils.NetworkUtils
 import net.syncthing.java.core.utils.awaitTerminationSafe
 import net.syncthing.java.core.utils.submitLogging
@@ -234,6 +235,9 @@ class IndexHandler(private val configuration: Configuration, val indexRepository
             assert(fileInfo.isFile())
             val fileBlocks = indexRepository.findFileBlocks(folder, path)
             checkNotNull(fileBlocks, {"file blocks not found for file info = $fileInfo"})
+
+            FileInfo.checkBlocks(fileInfo, fileBlocks!!)
+
             Pair.of(fileInfo, fileBlocks)
         }
     }