Update whatsnew

This closes https://github.com/syncthing/syncthing-lite/issues/106

Releasing.md

@@ -0,0 +1,8 @@
+# Releasing
+
+- do tests
+- update translations using ``tx pull -a -af`` (as extra merge request or branch for the case it does not build correctly)
+- update the version name and version code of the app
+- update the changelog at [app/src/main/play/en-GB/whatsnew](https://github.com/syncthing/syncthing-lite/blob/master/app/src/main/play/en-GB/whatsnew)
+- create a tag/ release in GitHub with an changelog; The tag name should be the version number
+- F-Droid picks up the release by the tag; additonally, the tag triggers a CI build which uploads the generated APK to Google Play

Roadmap.md

@@ -0,0 +1,22 @@
+# Roadmap
+
+## What should happen
+
+- fixing bugs and crashs
+  - just create a issue WITH a detailed crash report (not: it does not work)
+  - search if there is an other issue for it before creating a new one
+- showing more details in the UI (<https://github.com/syncthing/syncthing-lite/issues/44>)
+- add option to manually select the IP address of an device (<https://github.com/syncthing/syncthing-lite/issues/25>)
+- allow custom discovery servers or disabling device discovery (<https://github.com/syncthing/syncthing-lite/issues/105>)
+- downloading all files of an folder (<https://github.com/syncthing/syncthing-lite/issues/34>)
+- selective folder sharing (<https://github.com/syncthing/syncthing-lite/issues/17>)
+- better server offline handling (<https://github.com/syncthing/syncthing-lite/issues/63>)
+- file uploading support (it currently does not work) <https://github.com/syncthing/syncthing-lite/issues/70>
+
+## What could happen
+
+- thumbnails (<https://github.com/syncthing/syncthing-lite/issues/37>)
+
+## What will not happen
+
+- additional encryption within the App (see <https://github.com/syncthing/syncthing-lite/issues/85>)

app/build.gradle

@@ -19,8 +19,8 @@ android {
         applicationId "net.syncthing.lite"
         minSdkVersion 21
         targetSdkVersion 26
-        versionCode 14
-        versionName "0.3.4"
+        versionCode 17
+        versionName "0.3.7"
         multiDexEnabled true
         playAccountConfig = playAccountConfigs.defaultAccountConfig
     }
@@ -89,11 +89,9 @@ dependencies {
      */
     implementation(project(':syncthing-client')) {
         exclude group: 'commons-logging', module: 'commons-logging'
-        exclude group: 'org.apache.httpcomponents', module: 'httpclient'
         exclude group: 'org.slf4j'
         exclude group: 'ch.qos.logback'
     }
-    implementation 'org.apache.httpcomponents:httpclient-android:4.3.5.1'
 
     implementation 'sk.baka.slf4j:slf4j-handroid:1.7.26'
     implementation 'com.google.zxing:android-integration:3.3.0'

app/proguard-rules.pro

@@ -28,6 +28,9 @@
   volatile <fields>;
 }
 
+# fix detecting the main dispatcher
+-keepnames class kotlinx.coroutines.internal.MainDispatcherFactory {}
+
 # disable warnings
 -dontwarn com.google.protobuf.UnsafeUtil
 -dontwarn com.google.protobuf.UnsafeUtil$1

app/src/main/AndroidManifest.xml

@@ -4,6 +4,7 @@
     <uses-permission android:name="android.permission.INTERNET"/>
 
     <application
+        android:name=".android.Application"
         android:allowBackup="true"
         android:icon="@mipmap/ic_launcher"
         android:label="@string/app_name"

app/src/main/kotlin/net/syncthing/lite/activities/FolderBrowserActivity.kt

@@ -4,16 +4,15 @@ import android.app.Activity
 import android.content.Intent
 import android.databinding.DataBindingUtil
 import android.os.Bundle
-import android.util.Log
-import kotlinx.coroutines.CompletableDeferred
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.GlobalScope
-import kotlinx.coroutines.launch
-import net.syncthing.java.bep.IndexBrowser
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.ConflatedBroadcastChannel
+import kotlinx.coroutines.channels.consumeEach
+import net.syncthing.java.bep.index.browser.DirectoryContentListing
+import net.syncthing.java.bep.index.browser.DirectoryListing
+import net.syncthing.java.bep.index.browser.DirectoryNotFoundListing
+import net.syncthing.java.bep.index.browser.IndexBrowser
 import net.syncthing.java.core.beans.FileInfo
-import net.syncthing.java.core.beans.FolderInfo
 import net.syncthing.java.core.utils.PathUtils
-import net.syncthing.lite.BuildConfig
 import net.syncthing.lite.R
 import net.syncthing.lite.adapters.FolderContentsAdapter
 import net.syncthing.lite.adapters.FolderContentsListener
@@ -22,30 +21,43 @@ import net.syncthing.lite.dialogs.FileMenuDialogFragment
 import net.syncthing.lite.dialogs.FileUploadDialog
 import net.syncthing.lite.dialogs.ReconnectIssueDialogFragment
 import net.syncthing.lite.dialogs.downloadfile.DownloadFileDialogFragment
-import org.jetbrains.anko.doAsync
 
 class FolderBrowserActivity : SyncthingActivity() {
 
     companion object {
-
-        private const val TAG = "FolderBrowserActivity"
         private const val REQUEST_SELECT_UPLOAD_FILE = 171
-
+        private const val STATUS_PATH = "path"
         const val EXTRA_FOLDER_NAME = "folder_name"
     }
 
-    private lateinit var binding: ActivityFolderBrowserBinding
-    private lateinit var indexBrowser: IndexBrowser
-    private val adapter = FolderContentsAdapter()
+    private lateinit var folder: String
+
+    private val path = ConflatedBroadcastChannel<String>()
+    private val listing = ConflatedBroadcastChannel<DirectoryListing?>(null)
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
-        binding = DataBindingUtil.setContentView(this, R.layout.activity_folder_browser)
-        binding.mainListViewUploadHereButton.setOnClickListener { showUploadHereDialog() }
+
+        val binding: ActivityFolderBrowserBinding = DataBindingUtil.setContentView(this, R.layout.activity_folder_browser)
+        val adapter = FolderContentsAdapter()
+
         binding.listView.adapter = adapter
+        binding.mainListViewUploadHereButton.setOnClickListener {
+            startActivityForResult(
+                    Intent(Intent.ACTION_OPEN_DOCUMENT).apply {
+                        addCategory(Intent.CATEGORY_OPENABLE)
+                        type = "*/*"
+                    },
+                    REQUEST_SELECT_UPLOAD_FILE
+            )
+        }
         adapter.listener = object: FolderContentsListener {
             override fun onItemClicked(fileInfo: FileInfo) {
-                navigateToFolder(fileInfo)
+                if (fileInfo.isDirectory()) {
+                    path.offer(fileInfo.path)
+                } else {
+                    DownloadFileDialogFragment.newInstance(fileInfo).show(supportFragmentManager)
+                }
             }
 
             override fun onItemLongClicked(fileInfo: FileInfo): Boolean {
@@ -58,112 +70,94 @@ class FolderBrowserActivity : SyncthingActivity() {
                 }
             }
         }
-        val folder = intent.getStringExtra(EXTRA_FOLDER_NAME)
-        libraryHandler?.syncthingClient {
-            indexBrowser = it.indexHandler.newIndexBrowser(folder, true, true)
-            indexBrowser.setOnFolderChangedListener(this::onFolderChanged)
-        }
 
         ReconnectIssueDialogFragment.showIfNeeded(this)
+
+        folder = intent.getStringExtra(EXTRA_FOLDER_NAME)
+        path.offer(if (savedInstanceState == null) IndexBrowser.ROOT_PATH else savedInstanceState.getString(STATUS_PATH))
+
+        launch {
+            var job = Job()
+
+            path.consumeEach { path ->
+                job.cancel()
+                job = Job()
+
+                binding.listView.scrollToPosition(0)
+
+                listing.send(null)
+
+                async(job) {
+                    libraryHandler.libraryManager.streamDirectoryListing(folder, path).consumeEach {
+                        listing.send(it)
+                    }
+                }
+            }
+        }
+
+        launch {
+            listing.openSubscription().consumeEach { listing ->
+                if (listing == null) {
+                    binding.isLoading = true
+                } else {
+                    supportActionBar?.title = if (PathUtils.isRoot(listing.path)) folder else PathUtils.getFileName(listing.path)
+                    binding.isLoading = false
+                    adapter.data = if (listing is DirectoryContentListing)
+                        listing.entries.sortedWith(IndexBrowser.sortAlphabeticallyDirectoriesFirst)
+                    else
+                        emptyList()
+                }
+            }
+        }
     }
 
-    override fun onDestroy() {
-        super.onDestroy()
-        Thread {
-            indexBrowser.setOnFolderChangedListener(null)
-            indexBrowser.close()
-        }.start()
+    override fun onSaveInstanceState(outState: Bundle) {
+        super.onSaveInstanceState(outState)
+
+        outState.putString(STATUS_PATH, path.value)
+    }
+
+    private fun goUp(): Boolean {
+        val currentListing = listing.value
+
+        val parentPath = when (currentListing) {
+            is DirectoryContentListing -> currentListing.parentEntry?.path
+            is DirectoryNotFoundListing -> currentListing.theoreticalParentPath
+            else -> null
+        }
+
+        return if (parentPath == null) {
+            false
+        } else {
+            path.offer(parentPath)
+
+            true
+        }
     }
 
     override fun onBackPressed() {
-        //click item '0', ie '..' (go to parent)
-        navigateToFolder(adapter.data[0])
+        if (!goUp()) {
+            super.onBackPressed()
+        }
     }
 
     override fun onActivityResult(requestCode: Int, resultCode: Int, intent: Intent?) {
         if (requestCode == REQUEST_SELECT_UPLOAD_FILE && resultCode == Activity.RESULT_OK) {
-            libraryHandler?.syncthingClient { syncthingClient ->
+            libraryHandler.syncthingClient { syncthingClient ->
                 GlobalScope.launch (Dispatchers.Main) {
                     // FIXME: it would be better if the dialog would use the library handler
-                    FileUploadDialog(this@FolderBrowserActivity, syncthingClient, intent!!.data,
-                            indexBrowser.folder, indexBrowser.currentPath,
-                            { showFolderListView(indexBrowser.currentPath) }).show()
+                    FileUploadDialog(
+                            this@FolderBrowserActivity,
+                            syncthingClient,
+                            intent!!.data,
+                            folder,
+                            path.value,
+                            { /* nothing to do on success */ }
+                    ).show()
                 }
             }
         } else {
             super.onActivityResult(requestCode, resultCode, intent)
         }
     }
-
-    private fun showFolderListView(path: String) {
-        indexBrowser.navigateToNearestPath(path)
-        navigateToFolder(indexBrowser.currentPathInfo())
-    }
-
-    private fun navigateToFolder(fileInfo: FileInfo) {
-        Log.d(TAG, "navigate to path = '" + fileInfo.path + "' from path = '" + indexBrowser.currentPath + "'")
-        if (indexBrowser.isRoot() && PathUtils.isParent(fileInfo.path)) {
-            finish()
-        } else {
-            if (fileInfo.isDirectory()) {
-                doAsync {
-                    indexBrowser.navigateTo(fileInfo)
-                }
-
-                Log.d(TAG, "load folder cache bg")
-                binding.isLoading = true
-            } else {
-                if (BuildConfig.DEBUG) {
-                    Log.i(TAG, "pulling file = " + fileInfo)
-                }
-
-                DownloadFileDialogFragment.newInstance(fileInfo).show(supportFragmentManager)
-            }
-        }
-    }
-
-    private fun onFolderChanged() {
-        GlobalScope.launch {
-            val list = indexBrowser.listFiles()
-
-            Log.i("navigateToFolder", "list for path = '" + indexBrowser.currentPath + "' list = " + list.size + " records")
-            Log.d("navigateToFolder", "list for path = '" + indexBrowser.currentPath + "' list = " + list)
-            assert(!list.isEmpty())//list must contain at least the 'parent' path
-
-            val title = if (indexBrowser.isRoot()) {
-                val result = CompletableDeferred<String?>()
-
-                libraryHandler.folderBrowser {
-                    result.complete(it.getFolderInfo(indexBrowser.folder)?.label)
-                }
-
-                result.await()
-            } else {
-                indexBrowser.currentPathInfo().fileName
-            }
-
-            runOnUiThread {
-                binding.isLoading = false
-                adapter.data = list
-                binding.listView.scrollToPosition(0)
-                supportActionBar?.title = title
-            }
-        }
-    }
-
-    private fun updateFolderListView() {
-        showFolderListView(indexBrowser.currentPath)
-    }
-
-    private fun showUploadHereDialog() {
-        val intent = Intent(Intent.ACTION_OPEN_DOCUMENT)
-        intent.addCategory(Intent.CATEGORY_OPENABLE)
-        intent.type = "*/*"
-        startActivityForResult(intent, REQUEST_SELECT_UPLOAD_FILE)
-    }
-
-    override fun onIndexUpdateComplete(folderInfo: FolderInfo) {
-        super.onIndexUpdateComplete(folderInfo)
-        updateFolderListView()
-    }
 }

app/src/main/kotlin/net/syncthing/lite/activities/IntroActivity.kt

@@ -100,18 +100,18 @@ class IntroActivity : AppIntro() {
 
         override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
             binding = DataBindingUtil.inflate(inflater, R.layout.fragment_intro_two, container, false)
-            binding.enterDeviceId!!.scanQrCode.setOnClickListener {
+            binding.enterDeviceId.scanQrCode.setOnClickListener {
                 FragmentIntentIntegrator(this@IntroFragmentTwo).initiateScan()
             }
-            binding.enterDeviceId!!.scanQrCode.setImageResource(R.drawable.ic_qr_code_white_24dp)
+            binding.enterDeviceId.scanQrCode.setImageResource(R.drawable.ic_qr_code_white_24dp)
             return binding.root
         }
 
         override fun onActivityResult(requestCode: Int, resultCode: Int, intent: Intent?) {
             val scanResult = IntentIntegrator.parseActivityResult(requestCode, resultCode, intent)
             if (scanResult?.contents != null && scanResult.contents.isNotBlank()) {
-                binding.enterDeviceId!!.deviceId.setText(scanResult.contents)
-                binding.enterDeviceId!!.deviceIdHolder.isErrorEnabled = false
+                binding.enterDeviceId.deviceId.setText(scanResult.contents)
+                binding.enterDeviceId.deviceIdHolder.isErrorEnabled = false
             }
         }
 
@@ -121,11 +121,11 @@ class IntroActivity : AppIntro() {
          */
         fun isDeviceIdValid(): Boolean {
             return try {
-                val deviceId = binding.enterDeviceId!!.deviceId.text.toString()
+                val deviceId = binding.enterDeviceId.deviceId.text.toString()
                 Util.importDeviceId(libraryHandler, context, deviceId, { })
                 true
             } catch (e: IOException) {
-                binding.enterDeviceId!!.deviceId.error = getString(R.string.invalid_device_id)
+                binding.enterDeviceId.deviceId.error = getString(R.string.invalid_device_id)
                 false
             }
         }

app/src/main/kotlin/net/syncthing/lite/activities/SyncthingActivity.kt

@@ -4,22 +4,18 @@ import android.app.AlertDialog
 import android.databinding.DataBindingUtil
 import android.os.Bundle
 import android.support.design.widget.Snackbar
-import android.support.v7.app.AppCompatActivity
 import android.view.LayoutInflater
-import net.syncthing.java.core.beans.FolderInfo
 import net.syncthing.lite.BuildConfig
 import net.syncthing.lite.R
+import net.syncthing.lite.async.CoroutineActivity
 import net.syncthing.lite.databinding.DialogLoadingBinding
 import net.syncthing.lite.library.LibraryHandler
-import org.jetbrains.anko.contentView
 import org.slf4j.impl.HandroidLoggerAdapter
 
-abstract class SyncthingActivity : AppCompatActivity() {
+abstract class SyncthingActivity : CoroutineActivity() {
     val libraryHandler: LibraryHandler by lazy {
         LibraryHandler(
-                context = this@SyncthingActivity,
-                onIndexUpdateProgressListener = this::onIndexUpdateProgress,
-                onIndexUpdateCompleteListener = this::onIndexUpdateComplete
+                context = this@SyncthingActivity
         )
     }
     private var loadingDialog: AlertDialog? = null
@@ -58,19 +54,6 @@ abstract class SyncthingActivity : AppCompatActivity() {
         loadingDialog?.dismiss()
     }
 
-    open fun onIndexUpdateProgress(folderInfo: FolderInfo, percentage: Int) {
-        val message = getString(R.string.index_update_progress_label, folderInfo.label, percentage)
-        snackBar?.setText(message) ?: run {
-            snackBar = Snackbar.make(contentView!!, message, Snackbar.LENGTH_INDEFINITE)
-            snackBar?.show()
-        }
-    }
-
-    open fun onIndexUpdateComplete(folderInfo: FolderInfo) {
-        snackBar?.dismiss()
-        snackBar = null
-    }
-
     open fun onLibraryLoaded() {
         // nothing to do
     }

app/src/main/kotlin/net/syncthing/lite/adapters/FoldersListAdapter.kt

@@ -2,8 +2,10 @@ package net.syncthing.lite.adapters
 
 import android.support.v7.widget.RecyclerView
 import android.text.format.DateUtils
+import android.util.Log
 import android.view.LayoutInflater
 import android.view.ViewGroup
+import net.syncthing.java.bep.folder.FolderStatus
 import net.syncthing.java.core.beans.FolderInfo
 import net.syncthing.java.core.beans.FolderStats
 import net.syncthing.lite.R
@@ -11,7 +13,7 @@ import net.syncthing.lite.databinding.ListviewFolderBinding
 import kotlin.properties.Delegates
 
 class FoldersListAdapter: RecyclerView.Adapter<FolderListViewHolder>() {
-    var data: List<Pair<FolderInfo, FolderStats>> by Delegates.observable(listOf()) {
+    var data: List<FolderStatus> by Delegates.observable(listOf()) {
         _, _, _ -> notifyDataSetChanged()
     }
 
@@ -22,7 +24,7 @@ class FoldersListAdapter: RecyclerView.Adapter<FolderListViewHolder>() {
     }
 
     override fun getItemCount() = data.size
-    override fun getItemId(position: Int) = data[position].first.folderId.hashCode().toLong()
+    override fun getItemId(position: Int) = data[position].info.folderId.hashCode().toLong()
 
     override fun onCreateViewHolder(parent: ViewGroup, viewType: Int) = FolderListViewHolder (
             ListviewFolderBinding.inflate(
@@ -32,15 +34,23 @@ class FoldersListAdapter: RecyclerView.Adapter<FolderListViewHolder>() {
 
     override fun onBindViewHolder(holder: FolderListViewHolder, position: Int) {
         val binding = holder.binding
-        val (folderInfo, folderStats) = data[position]
+        val item = data[position]
+        val (folderInfo, folderStats) = item
         val context = holder.itemView.context
 
+        Log.d("FolderListAdapter", "$item")
+
         binding.folderName = context.getString(R.string.folder_label_format, folderInfo.label, folderInfo.folderId)
 
         binding.lastModification = context.getString(R.string.last_modified_time,
                 DateUtils.getRelativeDateTimeString(context, folderStats.lastUpdate.time, DateUtils.MINUTE_IN_MILLIS, DateUtils.WEEK_IN_MILLIS, 0))
 
-        binding.info = context.getString(R.string.folder_content_info, folderStats.describeSize(), folderStats.fileCount, folderStats.dirCount)
+        binding.info = context.getString(R.string.folder_content_info, folderStats.sizeDescription, folderStats.fileCount, folderStats.dirCount)
+
+        binding.info2 = if (item.missingIndexUpdates == 0L)
+            null
+        else
+            context.getString(R.string.pending_index_updates, item.missingIndexUpdates)
 
         binding.root.setOnClickListener {
             listener?.onFolderClicked(folderInfo, folderStats)
@@ -52,4 +62,4 @@ class FolderListViewHolder(val binding: ListviewFolderBinding): RecyclerView.Vie
 
 interface FolderListAdapterListener {
     fun onFolderClicked(folderInfo: FolderInfo, folderStats: FolderStats)
-}
+}

app/src/main/kotlin/net/syncthing/lite/android/Application.kt

@@ -0,0 +1,64 @@
+package net.syncthing.lite.android
+
+import android.app.Application
+import android.content.ClipData
+import android.content.ClipboardManager
+import android.content.Context
+import android.os.Handler
+import android.os.Looper
+import android.util.Log
+import net.syncthing.lite.BuildConfig
+import org.jetbrains.anko.defaultSharedPreferences
+import java.io.PrintWriter
+import java.io.StringWriter
+
+class Application: Application() {
+    companion object {
+        private const val LOG_TAG = "Application"
+        private const val PREF_ENABLE_CRASH_HANDLER = "crash_handler"
+        private val handler = Handler(Looper.getMainLooper())
+    }
+
+    override fun onCreate() {
+        super.onCreate()
+
+        val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
+        val defaultHandler = Thread.getDefaultUncaughtExceptionHandler()
+        val mainThread = Thread.currentThread()
+
+        if (defaultHandler == null) {
+            Log.w(LOG_TAG, "could not get default crash handler")
+        }
+
+        fun handleCrash(ex: Throwable) {
+            Log.w(LOG_TAG, "app crashed", ex)
+
+            val enableCustomCrashHandling = defaultSharedPreferences.getBoolean(PREF_ENABLE_CRASH_HANDLER, false)
+
+            if (enableCustomCrashHandling) {
+                clipboard.primaryClip = ClipData.newPlainText(
+                        "stacktrace",
+                        StringWriter().apply {
+                            append("Version: ").append(BuildConfig.VERSION_NAME).append('\n')
+                            append(Log.getStackTraceString(ex)).append('\n')
+                            ex.printStackTrace(PrintWriter(this))
+                        }.buffer.toString()
+                )
+            }
+
+            if (defaultHandler != null) {
+                defaultHandler.uncaughtException(mainThread, ex)
+            } else {
+                System.exit(1)
+            }
+        }
+
+        Thread.setDefaultUncaughtExceptionHandler { thread, ex ->
+            if (Looper.getMainLooper() === Looper.myLooper()) {
+                handleCrash(ex)
+            } else {
+                handler.post { handleCrash(ex) }
+            }
+        }
+    }
+}

app/src/main/kotlin/net/syncthing/lite/async/CoroutineActivity.kt

@@ -0,0 +1,19 @@
+package net.syncthing.lite.async
+
+import android.support.v7.app.AppCompatActivity
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlin.coroutines.CoroutineContext
+
+abstract class CoroutineActivity: AppCompatActivity(), CoroutineScope {
+    val job = Job()
+
+    override val coroutineContext: CoroutineContext
+        get() = job + Dispatchers.Main
+
+    override fun onDestroy() {
+        super.onDestroy()
+        job.cancel()
+    }
+}

app/src/main/kotlin/net/syncthing/lite/async/CoroutineFragment.kt

@@ -0,0 +1,19 @@
+package net.syncthing.lite.async
+
+import android.support.v4.app.Fragment
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlin.coroutines.CoroutineContext
+
+abstract class CoroutineFragment: Fragment(), CoroutineScope {
+    val job = Job()
+
+    override val coroutineContext: CoroutineContext
+        get() = job + Dispatchers.Main
+
+    override fun onDestroy() {
+        super.onDestroy()
+        job.cancel()
+    }
+}

app/src/main/kotlin/net/syncthing/lite/fragments/DevicesFragment.kt

@@ -39,12 +39,12 @@ class DevicesFragment : SyncthingFragment() {
 
     override fun onResume() {
         super.onResume()
-        libraryHandler?.syncthingClient { it.addOnConnectionChangedListener { _ -> updateDeviceList() } }
+        libraryHandler.syncthingClient { it.addOnConnectionChangedListener { _ -> updateDeviceList() } }
     }
 
     override fun onPause() {
         super.onPause()
-        libraryHandler?.syncthingClient { it.removeOnConnectionChangedListener{ _ -> updateDeviceList() } }
+        libraryHandler.syncthingClient { it.removeOnConnectionChangedListener{ _ -> updateDeviceList() } }
     }
 
     override fun onLibraryLoaded() {
@@ -61,10 +61,12 @@ class DevicesFragment : SyncthingFragment() {
                         .setTitle(getString(R.string.remove_device_title, deviceInfo.name))
                         .setMessage(getString(R.string.remove_device_message, deviceInfo.deviceId.deviceId.substring(0, 7)))
                         .setPositiveButton(android.R.string.yes) { _, _ ->
-                            libraryHandler?.configuration { config ->
+                            libraryHandler.library { config, syncthingClient, _ ->
                                 config.peers = config.peers.filterNot { it.deviceId == deviceInfo.deviceId }.toSet()
                                 config.persistLater()
                                 updateDeviceList()
+
+                                syncthingClient.disconnectFromRemovedDevices()
                             }
                         }
                         .setNegativeButton(android.R.string.no, null)

app/src/main/kotlin/net/syncthing/lite/fragments/FoldersFragment.kt

@@ -2,12 +2,10 @@ package net.syncthing.lite.fragments
 
 import android.arch.lifecycle.Observer
 import android.os.Bundle
-import android.util.Log
 import android.view.LayoutInflater
 import android.view.View
 import android.view.ViewGroup
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.channels.consumeEach
 import kotlinx.coroutines.launch
 import net.syncthing.java.core.beans.FolderInfo
 import net.syncthing.java.core.beans.FolderStats
@@ -18,49 +16,30 @@ import net.syncthing.lite.databinding.FragmentFoldersBinding
 import org.jetbrains.anko.intentFor
 
 class FoldersFragment : SyncthingFragment() {
+    override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View? {
+        val adapter = FoldersListAdapter()
 
-    private val TAG = "FoldersFragment"
-
-    private lateinit var binding: FragmentFoldersBinding
-
-    override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?,
-                              savedInstanceState: Bundle?): View? {
-        binding = FragmentFoldersBinding.inflate(layoutInflater, container, false)
+        adapter.listener = object : FolderListAdapterListener {
+            override fun onFolderClicked(folderInfo: FolderInfo, folderStats: FolderStats) {
+                startActivity(
+                        activity!!.intentFor<FolderBrowserActivity>(
+                                FolderBrowserActivity.EXTRA_FOLDER_NAME to folderInfo.folderId
+                        )
+                )
+            }
+        }
 
+        val binding = FragmentFoldersBinding.inflate(layoutInflater, container, false)
+        binding.list.adapter = adapter
         libraryHandler.isListeningPortTaken.observe(this, Observer { binding.listeningPortTaken = it })
 
+        launch {
+            libraryHandler.subscribeToFolderStatusList().consumeEach {
+                adapter.data = it
+                binding.isEmpty = it.isEmpty()
+            }
+        }
+
         return binding.root
     }
-
-    override fun onLibraryLoaded() {
-        showAllFoldersListView()
-    }
-
-    private fun showAllFoldersListView() {
-        libraryHandler.folderBrowser { folderBrowser ->
-            val list = folderBrowser.folderInfoAndStatsList()
-
-            GlobalScope.launch (Dispatchers.Main) {
-                Log.i(TAG, "list folders = " + list + " (" + list.size + " records)")
-                val adapter = FoldersListAdapter().apply { data = list }
-                binding.list.adapter = adapter
-                adapter.listener = object : FolderListAdapterListener {
-                    override fun onFolderClicked(folderInfo: FolderInfo, folderStats: FolderStats) {
-                        startActivity(
-                                activity!!.intentFor<FolderBrowserActivity>(
-                                        FolderBrowserActivity.EXTRA_FOLDER_NAME to folderInfo.folderId
-                                )
-                        )
-                    }
-                }
-
-                binding.isEmpty = list.isEmpty()
-            }
-        }
-    }
-
-    override fun onIndexUpdateComplete(folderInfo: FolderInfo) {
-        super.onIndexUpdateComplete(folderInfo)
-        showAllFoldersListView()
-    }
 }

app/src/main/kotlin/net/syncthing/lite/fragments/SettingsFragment.kt

@@ -13,19 +13,26 @@ class SettingsFragment : PreferenceFragmentCompat() {
 
         val localDeviceName = findPreference("local_device_name") as EditTextPreference
         val appVersion      = findPreference("app_version")
+        val forceStop       = findPreference("force_stop")
 
         (activity as SyncthingActivity?)?.let { activity ->
             val versionName = activity.packageManager.getPackageInfo(activity.packageName, 0)?.versionName
             appVersion.summary = versionName
 
-            activity.libraryHandler?.configuration { localDeviceName.text = it.localDeviceName }
+            activity.libraryHandler.configuration { localDeviceName.text = it.localDeviceName }
             localDeviceName.setOnPreferenceChangeListener { _, _ ->
-                activity.libraryHandler?.configuration { conf ->
+                activity.libraryHandler.configuration { conf ->
                     conf.localDeviceName = localDeviceName.text
                     conf.persistLater()
                 }
                 true
             }
         }
+
+        forceStop.setOnPreferenceClickListener {
+            System.exit(0)
+
+            true
+        }
     }
-}
+}

app/src/main/kotlin/net/syncthing/lite/fragments/SyncthingFragment.kt

@@ -1,15 +1,10 @@
 package net.syncthing.lite.fragments
 
-import android.support.v4.app.Fragment
-import net.syncthing.java.core.beans.FolderInfo
+import net.syncthing.lite.async.CoroutineFragment
 import net.syncthing.lite.library.LibraryHandler
 
-abstract class SyncthingFragment : Fragment() {
-    val libraryHandler: LibraryHandler by lazy { LibraryHandler(
-            context = context!!,
-            onIndexUpdateProgressListener = this::onIndexUpdateProgress,
-            onIndexUpdateCompleteListener = this::onIndexUpdateComplete
-    )}
+abstract class SyncthingFragment : CoroutineFragment() {
+    val libraryHandler: LibraryHandler by lazy { LibraryHandler(context = context!!)}
 
     override fun onStart() {
         super.onStart()
@@ -27,8 +22,4 @@ abstract class SyncthingFragment : Fragment() {
     }
 
     open fun onLibraryLoaded() {}
-
-    open fun onIndexUpdateProgress(folderInfo: FolderInfo, percentage: Int) {}
-
-    open fun onIndexUpdateComplete(folderInfo: FolderInfo) {}
-}
+}

app/src/main/kotlin/net/syncthing/lite/library/DownloadFileTask.kt

@@ -72,43 +72,35 @@ class DownloadFileTask(private val fileStorageDirectory: File,
                 return@launch
             }
 
-            syncthingClient.getBlockPuller(fileInfo.folder, { blockPuller ->
-                val job = launch {
-                    try {
-                        if (!file.filesDirectory.isDirectory) {
-                            if (!file.filesDirectory.mkdirs()) {
-                                throw IOException("could not create output directory")
-                            }
-                        }
-
-                        // download the file to a temp location
-                        val inputStream = blockPuller.pullFileCoroutine(fileInfo, this@DownloadFileTask::callProgress)
-
-                        try {
-                            FileUtils.copyInputStreamToFile(inputStream, file.tempFile)
-                            file.tempFile.renameTo(file.targetFile)
-                        } finally {
-                            file.tempFile.delete()
-                        }
-
-                        if (BuildConfig.DEBUG) {
-                            Log.i(TAG, "Downloaded file $fileInfo")
-                        }
-
-                        callComplete(file.targetFile)
-                    } catch (e: Exception) {
-                        callError(e)
-
-                        if (BuildConfig.DEBUG) {
-                            Log.w(TAG, "Failed to download file $fileInfo", e)
-                        }
+            try {
+                if (!file.filesDirectory.isDirectory) {
+                    if (!file.filesDirectory.mkdirs()) {
+                        throw IOException("could not create output directory")
                     }
                 }
 
-                cancellationSignal.setOnCancelListener {
-                    job.cancel()
+                // download the file to a temp location
+                val inputStream = syncthingClient.pullFile(fileInfo, this@DownloadFileTask::callProgress)
+
+                try {
+                    FileUtils.copyInputStreamToFile(inputStream, file.tempFile)
+                    file.tempFile.renameTo(file.targetFile)
+                } finally {
+                    file.tempFile.delete()
                 }
-            }, { callError(IOException("could not get block puller for file")) })
+
+                if (BuildConfig.DEBUG) {
+                    Log.i(TAG, "Downloaded file $fileInfo")
+                }
+
+                callComplete(file.targetFile)
+            } catch (e: Exception) {
+                callError(e)
+
+                if (BuildConfig.DEBUG) {
+                    Log.w(TAG, "Failed to download file $fileInfo", e)
+                }
+            }
         }
     }
 

app/src/main/kotlin/net/syncthing/lite/library/LibraryHandler.kt

@@ -5,16 +5,16 @@ import android.arch.lifecycle.MutableLiveData
 import android.content.Context
 import android.os.Handler
 import android.os.Looper
-import android.util.Log
-import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.channels.BroadcastChannel
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.channels.consumeEach
 import kotlinx.coroutines.launch
-import net.syncthing.java.bep.FolderBrowser
+import net.syncthing.java.bep.folder.FolderBrowser
+import net.syncthing.java.bep.folder.FolderStatus
 import net.syncthing.java.client.SyncthingClient
 import net.syncthing.java.core.beans.DeviceId
-import net.syncthing.java.core.beans.FileInfo
-import net.syncthing.java.core.beans.FolderInfo
-import net.syncthing.java.core.beans.IndexInfo
 import net.syncthing.java.core.configuration.Configuration
 import org.jetbrains.anko.doAsync
 import java.util.concurrent.atomic.AtomicBoolean
@@ -25,18 +25,19 @@ import java.util.concurrent.atomic.AtomicBoolean
  *
  * It's possible to do multiple start and stop cycles with one instance of this class.
  */
-class LibraryHandler(context: Context,
-                     private val onIndexUpdateProgressListener: (FolderInfo, Int) -> Unit = {_, _ -> },
-                     private val onIndexUpdateCompleteListener: (FolderInfo) -> Unit = {}) {
+class LibraryHandler(context: Context) {
 
     companion object {
         private const val TAG = "LibraryHandler"
         private val handler = Handler(Looper.getMainLooper())
     }
 
-    private val libraryManager = DefaultLibraryManager.with(context)
+    val libraryManager = DefaultLibraryManager.with(context)
     private val isStarted = AtomicBoolean(false)
     private val isListeningPortTakenInternal = MutableLiveData<Boolean>().apply { postValue(false) }
+    private val indexUpdateCompleteMessages = BroadcastChannel<String>(capacity = 16)
+    private val folderStatusList = BroadcastChannel<List<FolderStatus>>(capacity = Channel.CONFLATED)
+    private var job: Job = Job()
 
     val isListeningPortTaken: LiveData<Boolean> = isListeningPortTakenInternal
 
@@ -62,9 +63,21 @@ class LibraryHandler(context: Context,
 
             val client = libraryInstance.syncthingClient
 
-            client.indexHandler.registerOnIndexRecordAcquiredListener(this::onIndexRecordAcquired)
-            client.indexHandler.registerOnFullIndexAcquiredListenersListener(this::onRemoteIndexAcquired)
             client.discoveryHandler.registerMessageFromUnknownDeviceListener(internalMessageFromUnknownDeviceListener)
+
+            job = Job()
+
+            GlobalScope.launch (job) {
+                libraryInstance.syncthingClient.indexHandler.subscribeToOnFullIndexAcquiredEvents().consumeEach {
+                    indexUpdateCompleteMessages.send(it)
+                }
+            }
+
+            GlobalScope.launch (job) {
+                libraryInstance.folderBrowser.folderInfoAndStatusStream().consumeEach {
+                    folderStatusList.send(it)
+                }
+            }
         }
     }
 
@@ -73,10 +86,10 @@ class LibraryHandler(context: Context,
             throw IllegalStateException("already stopped")
         }
 
+        job!!.cancel()
+
         syncthingClient {
             try {
-                it.indexHandler.unregisterOnIndexRecordAcquiredListener(this::onIndexRecordAcquired)
-                it.indexHandler.unregisterOnFullIndexAcquiredListenersListener(this::onRemoteIndexAcquired)
                 it.discoveryHandler.unregisterMessageFromUnknownDeviceListener(internalMessageFromUnknownDeviceListener)
             } catch (e: IllegalArgumentException) {
                 // ignored, no idea why this is thrown
@@ -86,22 +99,6 @@ class LibraryHandler(context: Context,
         libraryManager.stopLibraryUsage()
     }
 
-    private fun onIndexRecordAcquired(folderInfo: FolderInfo, newRecords: List<FileInfo>, indexInfo: IndexInfo) {
-        Log.i(TAG, "handleIndexRecordEvent trigger folder list update from index record acquired")
-
-        GlobalScope.launch (Dispatchers.Main) {
-            onIndexUpdateProgressListener(folderInfo, (indexInfo.getCompleted() * 100).toInt())
-        }
-    }
-
-    private fun onRemoteIndexAcquired(folderInfo: FolderInfo) {
-        Log.i(TAG, "handleIndexAcquiredEvent trigger folder list update from index acquired")
-
-        GlobalScope.launch (Dispatchers.Main) {
-            onIndexUpdateCompleteListener(folderInfo)
-        }
-    }
-
     /*
      * The callback is executed asynchronously.
      * As soon as it returns, there is no guarantee about the availability of the library
@@ -139,4 +136,7 @@ class LibraryHandler(context: Context,
     fun unregisterMessageFromUnknownDeviceListener(listener: (DeviceId) -> Unit) {
         messageFromUnknownDeviceListeners.remove(listener)
     }
+
+    fun subscribeToOnFullIndexAcquiredEvents() = indexUpdateCompleteMessages.openSubscription()
+    fun subscribeToFolderStatusList() = folderStatusList.openSubscription()
 }

app/src/main/kotlin/net/syncthing/lite/library/LibraryInstance.kt

@@ -7,6 +7,7 @@ import net.syncthing.java.core.configuration.Configuration
 import net.syncthing.repository.android.SqliteIndexRepository
 import net.syncthing.repository.android.TempDirectoryLocalRepository
 import net.syncthing.repository.android.database.RepositoryDatabase
+import org.jetbrains.anko.defaultSharedPreferences
 import java.io.File
 import java.net.DatagramSocket
 import java.net.InetAddress
@@ -53,12 +54,13 @@ class LibraryInstance (context: Context) {
                     closeDatabaseOnClose = false,
                     clearTempStorageHook = { tempRepository.deleteAllData() }
             ),
-            tempRepository = tempRepository
+            tempRepository = tempRepository,
+            enableDetailedException = context.defaultSharedPreferences.getBoolean("detailed_exception", false)
     )
-    val folderBrowser = syncthingClient.indexHandler.newFolderBrowser()
+    val folderBrowser = syncthingClient.indexHandler.folderBrowser
+    val indexBrowser = syncthingClient.indexHandler.indexBrowser
 
     fun shutdown() {
-        folderBrowser.close()
         syncthingClient.close()
         configuration.persistNow()
     }

app/src/main/kotlin/net/syncthing/lite/library/LibraryManager.kt

@@ -2,8 +2,15 @@ package net.syncthing.lite.library
 
 import android.os.Handler
 import android.os.Looper
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.async
+import kotlinx.coroutines.channels.ConflatedBroadcastChannel
+import kotlinx.coroutines.channels.consumeEach
+import kotlinx.coroutines.channels.produce
 import java.util.concurrent.Executors
-import kotlin.coroutines.experimental.suspendCoroutine
+import kotlin.coroutines.resume
+import kotlin.coroutines.suspendCoroutine
 
 /**
  * This class manages the access to an LibraryInstance
@@ -34,7 +41,7 @@ class LibraryManager (
     // only this Thread should access instance and userCounter
     private val startStopExecutor = Executors.newSingleThreadExecutor()
 
-    private var instance: LibraryInstance? = null
+    private val instanceStream = ConflatedBroadcastChannel<LibraryInstance?>(null)
     private var userCounter = 0
 
     fun startLibraryUsage(callback: (LibraryInstance) -> Unit) {
@@ -42,12 +49,12 @@ class LibraryManager (
             val newUserCounter = ++userCounter
             handler.post { userCounterListener(newUserCounter) }
 
-            if (instance == null) {
-                instance = synchronousInstanceCreator()
+            if (instanceStream.value == null) {
+                instanceStream.offer(synchronousInstanceCreator())
                 handler.post { isRunningListener(true) }
             }
 
-            handler.post { callback(instance!!) }
+            handler.post { callback(instanceStream.value!!) }
         }
     }
 
@@ -59,6 +66,16 @@ class LibraryManager (
         }
     }
 
+    suspend fun <T> withLibrary(action: suspend (LibraryInstance) -> T): T {
+        val instance = startLibraryUsageCoroutine()
+
+        return try {
+            action(instance)
+        } finally {
+            stopLibraryUsage()
+        }
+    }
+
     fun stopLibraryUsage() {
         startStopExecutor.submit {
             val newUserCounter = --userCounter
@@ -76,8 +93,8 @@ class LibraryManager (
     fun shutdownIfThereAreZeroUsers(listener: (wasShutdownPerformed: Boolean) -> Unit = {}) {
         startStopExecutor.submit {
             if (userCounter == 0) {
-                instance?.shutdown()
-                instance = null
+                instanceStream.value?.shutdown()
+                instanceStream.offer(null)
 
                 handler.post { isRunningListener(false) }
                 handler.post { listener(true) }
@@ -86,4 +103,21 @@ class LibraryManager (
             }
         }
     }
+
+    fun streamDirectoryListing(folder: String, path: String) = GlobalScope.produce {
+        var job = Job()
+
+        instanceStream.openSubscription().consumeEach { instance ->
+            job.cancel()
+            job = Job()
+
+            if (instance != null) {
+                async (job) {
+                    instance.indexBrowser.streamDirectoryListing(folder, path).consumeEach {
+                        send(it)
+                    }
+                }
+            }
+        }
+    }
 }

app/src/main/kotlin/net/syncthing/lite/library/SyncthingProvider.kt

@@ -10,33 +10,36 @@ import android.provider.DocumentsProvider
 import android.util.Log
 import kotlinx.coroutines.cancel
 import kotlinx.coroutines.runBlocking
-import net.syncthing.java.bep.IndexBrowser
+import net.syncthing.java.bep.index.browser.DirectoryContentListing
+import net.syncthing.java.bep.index.browser.DirectoryNotFoundListing
 import net.syncthing.java.core.beans.FileInfo
 import net.syncthing.java.core.beans.FolderInfo
-import net.syncthing.java.core.beans.FolderStats
 import net.syncthing.lite.R
 import java.io.FileNotFoundException
 import java.net.URLConnection
-import java.util.concurrent.CountDownLatch
 
 class SyncthingProvider : DocumentsProvider() {
 
     companion object {
         private const val Tag = "SyncthingProvider"
+
         private val DefaultRootProjection = arrayOf(
                 Root.COLUMN_ROOT_ID,
                 Root.COLUMN_FLAGS,
                 Root.COLUMN_TITLE,
                 Root.COLUMN_SUMMARY,
                 Root.COLUMN_DOCUMENT_ID,
-                Root.COLUMN_ICON)
+                Root.COLUMN_ICON
+        )
+
         private val DefaultDocumentProjection = arrayOf(
                 Document.COLUMN_DOCUMENT_ID,
                 Document.COLUMN_DISPLAY_NAME,
                 Document.COLUMN_SIZE,
                 Document.COLUMN_MIME_TYPE,
                 Document.COLUMN_LAST_MODIFIED,
-                Document.COLUMN_FLAGS)
+                Document.COLUMN_FLAGS
+        )
     }
 
     override fun onCreate(): Boolean {
@@ -45,96 +48,122 @@ class SyncthingProvider : DocumentsProvider() {
     }
 
     // this instance is not started -> it connects and disconnects on demand
-    private val libraryHandler: LibraryHandler by lazy { LibraryHandler(context) }
     private val libraryManager: LibraryManager by lazy { DefaultLibraryManager.with(context) }
 
     override fun queryRoots(projection: Array<String>?): Cursor {
         Log.d(Tag, "queryRoots($projection)")
-        val latch = CountDownLatch(1)
-        var folders: List<Pair<FolderInfo, FolderStats>>? = null
-        libraryHandler.folderBrowser { folderBrowser ->
-            folders = folderBrowser.folderInfoAndStatsList()
-            latch.countDown()
-        }
-        latch.await()
 
-        val result = MatrixCursor(projection ?: DefaultRootProjection)
-        folders!!.forEach { folder ->
-            val row = result.newRow()
-            row.add(Root.COLUMN_ROOT_ID, folder.first.folderId)
-            row.add(Root.COLUMN_SUMMARY, folder.first.label)
-            row.add(Root.COLUMN_FLAGS, 0)
-            row.add(Root.COLUMN_TITLE, context.getString(R.string.app_name))
-            row.add(Root.COLUMN_DOCUMENT_ID, getDocIdForFile(folder.first))
-            row.add(Root.COLUMN_ICON, R.mipmap.ic_launcher)
+        return runBlocking {
+            libraryManager.withLibrary { instance ->
+                MatrixCursor(projection ?: DefaultRootProjection).apply {
+                    instance.folderBrowser.folderInfoAndStatusList().forEach { folder ->
+                        newRow().apply {
+                            add(Root.COLUMN_ROOT_ID, folder.info.folderId)
+                            add(Root.COLUMN_SUMMARY, folder.info.label)
+                            add(Root.COLUMN_FLAGS, 0)
+                            add(Root.COLUMN_TITLE, context.getString(R.string.app_name))
+                            add(Root.COLUMN_DOCUMENT_ID, getDocIdForFile(folder.info))
+                            add(Root.COLUMN_ICON, R.mipmap.ic_launcher)
+                        }
+                    }
+                }
+            }
         }
-        return result
     }
 
     override fun queryChildDocuments(parentDocumentId: String, projection: Array<String>?,
                                      sortOrder: String?): Cursor {
         Log.d(Tag, "queryChildDocuments($parentDocumentId, $projection, $sortOrder)")
-        val result = MatrixCursor(projection ?: DefaultDocumentProjection)
-        getIndexBrowser(getFolderIdForDocId(parentDocumentId))
-                .listFiles(getPathForDocId(parentDocumentId))
-                .forEach { fileInfo ->
-                    includeFile(result, fileInfo)
+
+        return runBlocking {
+            libraryManager.withLibrary { instance ->
+                val listing = instance.indexBrowser.getDirectoryListing(
+                        folder = getFolderIdForDocId(parentDocumentId),
+                        path = getPathForDocId(parentDocumentId)
+                )
+
+                when (listing) {
+                    is DirectoryNotFoundListing -> throw FileNotFoundException()
+                    is DirectoryContentListing -> {
+                        val result = MatrixCursor(projection ?: DefaultDocumentProjection)
+
+                        listing.entries.forEach { entry ->
+                            includeFile(result, entry)
+                        }
+
+                        result
+                    }
                 }
-        return result
+            }
+        }
     }
 
     override fun queryDocument(documentId: String, projection: Array<String>?): Cursor {
         Log.d(Tag, "queryDocument($documentId, $projection)")
-        val result = MatrixCursor(projection ?: DefaultDocumentProjection)
-        val fileInfo = getIndexBrowser(getFolderIdForDocId(documentId))
-                .getFileInfoByAbsolutePath(getPathForDocId(documentId))
-        includeFile(result, fileInfo)
-        return result
+
+        return runBlocking {
+            libraryManager.withLibrary {  instance ->
+                val fileInfo = instance.indexBrowser.getFileInfoByAbsolutePathAllowNull(
+                        folder = getFolderIdForDocId(documentId),
+                        path = getPathForDocId(documentId)
+                ) ?: throw FileNotFoundException()
+
+                MatrixCursor(projection ?: DefaultDocumentProjection).apply {
+                    includeFile(this, fileInfo)
+                }
+            }
+        }
     }
 
     @Throws(FileNotFoundException::class)
     override fun openDocument(documentId: String, mode: String, signal: CancellationSignal?):
             ParcelFileDescriptor {
         Log.d(Tag, "openDocument($documentId, $mode, $signal)")
-        val fileInfo = getIndexBrowser(getFolderIdForDocId(documentId))
-                .getFileInfoByAbsolutePath(getPathForDocId(documentId))
+
         val accessMode = ParcelFileDescriptor.parseMode(mode)
+
         if (accessMode != ParcelFileDescriptor.MODE_READ_ONLY) {
             throw NotImplementedError()
         }
 
-        val outputFile = runBlocking {
-            signal?.setOnCancelListener {
-                this.coroutineContext.cancel()
-            }
+        return runBlocking {
+            libraryManager.withLibrary { instance ->
+                val fileInfo = instance.indexBrowser.getFileInfoByAbsolutePathAllowNull(
+                        folder = getFolderIdForDocId(documentId),
+                        path = getPathForDocId(documentId)
+                ) ?: throw FileNotFoundException()
 
-            val libraryInstance = libraryManager.startLibraryUsageCoroutine()
+                signal?.setOnCancelListener {
+                    this.coroutineContext.cancel()
+                }
 
-            try {
-                DownloadFileTask.downloadFileCoroutine(
+                val outputFile = DownloadFileTask.downloadFileCoroutine(
                         externalCacheDir = context.externalCacheDir,
-                        syncthingClient = libraryInstance.syncthingClient,
+                        syncthingClient = instance.syncthingClient,
                         fileInfo = fileInfo,
                         onProgress = { /* ignore the progress */ }
                 )
-            } finally {
-                libraryManager.stopLibraryUsage()
+
+                ParcelFileDescriptor.open(outputFile, ParcelFileDescriptor.MODE_READ_ONLY)
             }
         }
-
-        return ParcelFileDescriptor.open(outputFile, ParcelFileDescriptor.MODE_READ_ONLY)
     }
 
     private fun includeFile(result: MatrixCursor, fileInfo: FileInfo) {
-        val row = result.newRow()
-        row.add(Document.COLUMN_DOCUMENT_ID, getDocIdForFile(fileInfo))
-        row.add(Document.COLUMN_DISPLAY_NAME, fileInfo.fileName)
-        row.add(Document.COLUMN_SIZE, fileInfo.size)
-        val mime = if (fileInfo.isDirectory()) Document.MIME_TYPE_DIR
-        else URLConnection.guessContentTypeFromName(fileInfo.fileName)
-        row.add(Document.COLUMN_MIME_TYPE, mime)
-        row.add(Document.COLUMN_LAST_MODIFIED, fileInfo.lastModified)
-        row.add(Document.COLUMN_FLAGS, 0)
+        result.newRow().apply {
+            add(Document.COLUMN_DOCUMENT_ID, getDocIdForFile(fileInfo))
+            add(Document.COLUMN_DISPLAY_NAME, fileInfo.fileName)
+            add(Document.COLUMN_SIZE, fileInfo.size)
+            add(
+                    Document.COLUMN_MIME_TYPE,
+                    if (fileInfo.isDirectory())
+                        Document.MIME_TYPE_DIR
+                    else
+                        URLConnection.guessContentTypeFromName(fileInfo.fileName)
+            )
+            add(Document.COLUMN_LAST_MODIFIED, fileInfo.lastModified)
+            add(Document.COLUMN_FLAGS, 0)
+        }
     }
 
     private fun getFolderIdForDocId(docId: String) = docId.split(":")[0]
@@ -144,15 +173,4 @@ class SyncthingProvider : DocumentsProvider() {
     private fun getDocIdForFile(folderInfo: FolderInfo) = folderInfo.folderId + ":"
 
     private fun getDocIdForFile(fileInfo: FileInfo) = fileInfo.folder + ":" + fileInfo.path
-
-    private fun getIndexBrowser(folderId: String): IndexBrowser {
-        val latch = CountDownLatch(1)
-        var indexBrowser: IndexBrowser? = null
-        libraryHandler.syncthingClient {
-            indexBrowser = it.indexHandler.newIndexBrowser(folderId)
-            latch.countDown()
-        }
-        latch.await()
-        return indexBrowser!!
-    }
 }

app/src/main/kotlin/net/syncthing/lite/library/UploadFileTask.kt

@@ -5,6 +5,8 @@ import android.net.Uri
 import android.os.Handler
 import android.os.Looper
 import android.util.Log
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.launch
 import net.syncthing.java.bep.BlockPusher
 import net.syncthing.java.client.SyncthingClient
 import net.syncthing.java.core.utils.PathUtils
@@ -31,22 +33,28 @@ class UploadFileTask(context: Context, syncthingClient: SyncthingClient,
 
     init {
         Log.i(TAG, "Uploading file $localFile to folder $syncthingFolder:$syncthingPath")
-        syncthingClient.getBlockPusher(syncthingFolder, { blockPusher ->
-            val observer = blockPusher.pushFile(uploadStream, syncthingFolder, syncthingPath)
 
-            handler.post { onProgress(observer) }
+        GlobalScope.launch {
+            try {
+                val blockPusher = syncthingClient.getBlockPusher(folderId = syncthingFolder)
+                val observer = blockPusher.pushFile(uploadStream, syncthingFolder, syncthingPath)
 
-            while (!observer.isCompleted()) {
-                if (isCancelled)
-                    return@getBlockPusher
-
-                observer.waitForProgressUpdate()
-                Log.i(TAG, "upload progress = ${observer.progressPercentage()}%")
                 handler.post { onProgress(observer) }
+
+                while (!observer.isCompleted()) {
+                    if (isCancelled)
+                        return@launch
+
+                    observer.waitForProgressUpdate()
+                    Log.i(TAG, "upload progress = ${observer.progressPercentage()}%")
+                    handler.post { onProgress(observer) }
+                }
+                IOUtils.closeQuietly(uploadStream)
+                handler.post { onComplete() }
+            } catch (ex: Exception) {
+                handler.post { onError() }
             }
-            IOUtils.closeQuietly(uploadStream)
-            handler.post { onComplete() }
-        }, { handler.post { onError() } })
+        }
     }
 
     fun cancel() {

app/src/main/kotlin/net/syncthing/lite/utils/Util.kt

@@ -44,10 +44,11 @@ object Util {
     fun importDeviceId(libraryHandler: LibraryHandler?, context: Context?, deviceId: String,
                        onComplete: () -> Unit) {
         val deviceId2 = DeviceId(deviceId.toUpperCase(Locale.US))
-        libraryHandler?.configuration { configuration ->
+        libraryHandler?.library { configuration, syncthingClient, _ ->
             if (!configuration.peerIds.contains(deviceId2)) {
                 configuration.peers = configuration.peers + DeviceInfo(deviceId2, null)
                 configuration.persistLater()
+                syncthingClient.connectToNewlyAddedDevices()
                 GlobalScope.launch (Dispatchers.Main) {
                     context?.toast(context.getString(R.string.device_import_success, deviceId2.shortId))
                     onComplete()

app/src/main/play/en-GB/whatsnew

@@ -1,6 +1 @@
-- add option to export files
-- send correct file names to apps by which files are opened
-- adaptive icon
-- updated translations
-- validate discovery servers
-- bugfixes
+- fix crash after launch in release builds

app/src/main/res/layout/listview_folder.xml

@@ -13,9 +13,17 @@
         <variable
             name="info"
             type="String" />
+
+        <variable
+            name="info2"
+            type="String" />
+
+        <import type="android.view.View" />
+        <import type="android.text.TextUtils" />
     </data>
 
-    <RelativeLayout
+    <LinearLayout
+        android:orientation="vertical"
         android:background="?selectableItemBackground"
         android:layout_width="match_parent"
         android:layout_height="wrap_content"
@@ -29,13 +37,9 @@
             android:text="@{folderName}"
             android:id="@+id/folder_name_view"
             android:layout_width="match_parent"
-            android:layout_alignParentEnd="true"
             android:layout_height="wrap_content"
             android:ellipsize="end"
             android:maxLines="1"
-            android:layout_alignParentTop="true"
-            android:gravity="top"
-            android:textAlignment="gravity"
             android:textSize="20sp"
             android:textStyle="bold"/>
 
@@ -47,9 +51,7 @@
             android:layout_height="wrap_content"
             android:ellipsize="end"
             android:maxLines="1"
-            android:layout_below="@id/folder_name_view"
-            android:textSize="14sp"
-            android:layout_alignParentStart="true" />
+            android:textSize="14sp" />
 
         <TextView
             tools:text="Additional information"
@@ -59,11 +61,18 @@
             android:layout_height="wrap_content"
             android:ellipsize="end"
             android:maxLines="1"
-            android:layout_below="@id/folder_lastmod_info"
-            android:textSize="14sp"
-            android:layout_alignParentStart="true"  />
+            android:textSize="14sp" />
 
+        <TextView
+            android:visibility="@{TextUtils.isEmpty(info2) ? View.GONE : View.VISIBLE}"
+            tools:text="Index Update Progress"
+            android:text="@{info2}"
+            android:layout_width="match_parent"
+            android:layout_height="wrap_content"
+            android:ellipsize="end"
+            android:maxLines="1"
+            android:textSize="14sp" />
 
-    </RelativeLayout>
+    </LinearLayout>
 
-</layout>
+</layout>

app/src/main/res/values/strings.xml

@@ -42,6 +42,15 @@
     <string name="settings_local_device_summary">The name that other devices will see for this device</string>
     <string name="settings_shutdown_delay_title">Shutdown delay</string>
     <string name="settings_shutdown_delay_summary">Time before shuting down the Syncthing client after its last usage</string>
+    <string name="settings_crash_handler_title">Custom Crash-Handler</string>
+    <string name="settings_crash_handler_summary">Copy the error message to the clipboard when the App crashes</string>
+    <string name="settings_detailed_exception_title">Enable more detailed crash reports</string>
+    <string name="settings_detailed_exception_summary">
+        This could leak private data.
+        You should only use it with the custom crash handler and should not send a crash report without review.
+        Changes of this need an App restart (use force stop to be safe).
+    </string>
+    <string name="settings_force_stop">Force stop this App</string>
     <string name="device_id_dialog_title">Enter Device ID</string>
     <string name="settings_shutdown_delay_10_seconds">10 seconds</string>
     <string name="settings_shutdown_delay_30_seconds">30 seconds</string>
@@ -54,4 +63,5 @@
         This does not apply to local discovery connections.
     </string>
     <string name="dialog_file_save_as">Save as</string>
+    <string name="pending_index_updates">%d index updates pending</string>
 </resources>

app/src/main/res/xml/preferences.xml

@@ -24,10 +24,24 @@
 
         -->
 
+        <CheckBoxPreference
+            android:key="crash_handler"
+            android:title="@string/settings_crash_handler_title"
+            android:summary="@string/settings_crash_handler_summary" />
+
+        <CheckBoxPreference
+            android:key="detailed_exception"
+            android:title="@string/settings_detailed_exception_title"
+            android:summary="@string/settings_detailed_exception_summary" />
+
         <Preference
             android:key="app_version"
             android:title="@string/settings_app_version_title"/>
 
+        <Preference
+            android:key="force_stop"
+            android:title="@string/settings_force_stop" />
+
     </PreferenceCategory>
 
 </PreferenceScreen>

prepare-release.bash

@@ -1,54 +0,0 @@
-#!/bin/bash
-
-set -e
-
-NEW_VERSION_NAME=$1
-OLD_VERSION_NAME=$(grep "versionName" "app/build.gradle" | awk '{print $2}' | tr -d "\"")
-if [[ -z ${NEW_VERSION_NAME} ]]
-then
-    echo "New version name is empty. Please set a new version. Current version: $OLD_VERSION_NAME"
-    exit
-fi
-
-echo "
-
-Updating Translations
------------------------------
-"
-tx push -s
-# Force push/pull to make sure this is executed. Apparently tx only compares timestamps, not file
-# contents. So if a file was `touch`ed, it won't be updated by default.
-tx pull -a -f
-git add -A "app/src/main/res/values-*/strings.xml"
-if ! git diff --cached --exit-code;
-then
-    git commit -m "Imported translations"
-fi
-
-echo "
-
-Updating Version
------------------------------
-"
-OLD_VERSION_CODE=$(grep "versionCode" "app/build.gradle" -m 1 | awk '{print $2}')
-NEW_VERSION_CODE=$(($OLD_VERSION_CODE + 1))
-sed -i "s/versionCode $OLD_VERSION_CODE/versionCode $NEW_VERSION_CODE/" "app/build.gradle"
-sed -i "s/versionName \"$OLD_VERSION_NAME\"/versionName \"$NEW_VERSION_NAME\"/" "app/build.gradle"
-
-LIBRARY_NAME="com.github.Nutomic:syncthing-java"
-sed -i "s/$LIBRARY_NAME:$OLD_VERSION_NAME/$LIBRARY_NAME:$NEW_VERSION_NAME/" "app/build.gradle"
-
-git add "app/build.gradle"
-git commit -m "Version $NEW_VERSION_NAME"
-git tag ${NEW_VERSION_NAME}
-
-echo "
-
-Running Lint
------------------------------
-"
-./gradlew clean lintVitalRelease
-
-echo "
-Update ready.
-"

publish-release.bash

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-version=$(git describe --tags)
-regex='^[0-9]+\.[0-9]+\.[0-9]+$'
-if [[ ! ${version} =~ $regex ]]
-then
-    echo "Current commit is not a release"
-    exit;
-fi
-
-echo "
-
-Pushing to Github
------------------------------
-"
-git push
-git push --tags
-
-echo "
-
-Push to Google Play
------------------------------
-"
-
-read -s -p "Enter signing password: " password
-
-SIGNING_PASSWORD=${password} ./gradlew assembleRelease
-
-# Upload apk and listing to Google Play
-SIGNING_PASSWORD=${password} ./gradlew publishRelease
-
-echo "
-
-Release published!
-"

settings.gradle

@@ -1 +1 @@
-include ':app', ':syncthing-repository-android', ':syncthing-repository-default', ':syncthing-relay-client', ':syncthing-bep', ':syncthing-core', ':syncthing-client', ':syncthing-discovery', ':syncthing-client-cli', ':syncthing-http-relay-client'
+include ':app', ':syncthing-repository-android', ':syncthing-repository-default', ':syncthing-relay-client', ':syncthing-bep', ':syncthing-core', ':syncthing-client', ':syncthing-discovery', ':syncthing-client-cli'

syncthing-bep/build.gradle

@@ -6,7 +6,6 @@ dependencies {
     compile fileTree(dir: 'libs', include: ['*.jar'])
     compile project(':syncthing-core')
     compile project(':syncthing-relay-client')
-    compile project(':syncthing-http-relay-client')
     compile "net.jpountz.lz4:lz4:1.3.0"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
     implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.0.0'

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/BlockPuller.kt

@@ -17,58 +17,45 @@ package net.syncthing.java.bep
 import com.google.protobuf.ByteString
 import kotlinx.coroutines.*
 import kotlinx.coroutines.channels.Channel
-import net.syncthing.java.bep.BlockExchangeProtos.ErrorCode
-import net.syncthing.java.bep.BlockExchangeProtos.Request
+import net.syncthing.java.bep.connectionactor.ConnectionActorWrapper
+import net.syncthing.java.bep.index.IndexHandler
 import net.syncthing.java.bep.utils.longSumBy
 import net.syncthing.java.core.beans.BlockInfo
 import net.syncthing.java.core.beans.FileBlocks
 import net.syncthing.java.core.beans.FileInfo
 import net.syncthing.java.core.interfaces.TempRepository
-import net.syncthing.java.core.utils.NetworkUtils
 import org.bouncycastle.util.encoders.Hex
 import org.slf4j.LoggerFactory
-import java.io.ByteArrayInputStream
-import java.io.IOException
-import java.io.InputStream
-import java.io.SequenceInputStream
+import java.io.*
 import java.security.MessageDigest
 import java.util.*
 import kotlin.collections.HashMap
-import kotlin.coroutines.resume
-
-class BlockPuller internal constructor(private val connectionHandler: ConnectionHandler,
-                                       private val indexHandler: IndexHandler,
-                                       private val responseHandler: ResponseHandler,
-                                       private val tempRepository: TempRepository) {
 
+object BlockPuller {
     private val logger = LoggerFactory.getLogger(javaClass)
 
-    fun pullFileSync(
+    suspend fun pullFile(
             fileInfo: FileInfo,
-            progressListener: (status: BlockPullerStatus) -> Unit = {  }
+            progressListener: (status: BlockPullerStatus) -> Unit = {  },
+            connections: List<ConnectionActorWrapper>,
+            indexHandler: IndexHandler,
+            tempRepository: TempRepository
     ): InputStream {
-        return runBlocking {
-            pullFileCoroutine(fileInfo, progressListener)
+        val connectionHelper = MultiConnectionHelper(connections) {
+            it.hasFolder(fileInfo.folder)
         }
-    }
 
-    suspend fun pullFileCoroutine(
-            fileInfo: FileInfo,
-            progressListener: (status: BlockPullerStatus) -> Unit = {  }
-    ): InputStream {
-        val fileBlocks = indexHandler.waitForRemoteIndexAcquired(connectionHandler)
-                .getFileInfoAndBlocksByPath(fileInfo.folder, fileInfo.path)
-                ?.value
-                ?: throw IOException("file not found in local index for folder = ${fileInfo.folder} path = ${fileInfo.path}")
-        logger.info("pulling file = {}", fileBlocks)
-        NetworkUtils.assertProtocol(connectionHandler.hasFolder(fileBlocks.folder), { "supplied connection handler $connectionHandler will not share folder ${fileBlocks.folder}" })
+        // fail early if there is no matching connection
+        connectionHelper.pickConnection()
+
+        val (newFileInfo, fileBlocks) = indexHandler.getFileInfoAndBlocksByPath(fileInfo.folder, fileInfo.path) ?: throw FileNotFoundException()
 
-        // the file could have changed since the caller read it
-        // this would save the file using a wrong name, so throw here
         if (fileBlocks.hash != fileInfo.hash) {
             throw IllegalStateException("the current file entry hash does not match the hash of the provided one")
         }
 
+        logger.info("pulling file = {}", fileBlocks)
+
         val blockTempIdByHash = Collections.synchronizedMap(HashMap<String, String>())
 
         var status = BlockPullerStatus(
@@ -77,6 +64,47 @@ class BlockPuller internal constructor(private val connectionHandler: Connection
                 totalFileSize = fileBlocks.size
         )
 
+        suspend fun pullBlock(fileBlocks: FileBlocks, block: BlockInfo, timeoutInMillis: Long, connectionActorWrapper: ConnectionActorWrapper): ByteArray {
+            logger.debug("sent message for block, hash = {}", block.hash)
+
+            val response =
+                    withTimeout(timeoutInMillis) {
+                        try {
+                            connectionActorWrapper.sendRequest(
+                                    BlockExchangeProtos.Request.newBuilder()
+                                            .setFolder(fileBlocks.folder)
+                                            .setName(fileBlocks.path)
+                                            .setOffset(block.offset)
+                                            .setSize(block.size)
+                                            .setHash(ByteString.copyFrom(Hex.decode(block.hash)))
+                                            .buildPartial()
+                            )
+                        } catch (ex: TimeoutCancellationException) {
+                            // It seems like the TimeoutCancellationException
+                            // is handled differently so that the timeout is ignored.
+                            // Due to that, it's converted to an IOException.
+
+                            throw IOException("timeout during requesting block")
+                        }
+                    }
+
+            if (response.code != BlockExchangeProtos.ErrorCode.NO_ERROR) {
+                // the server does not have/ want to provide this file -> don't ask him again
+                connectionHelper.disableConnection(connectionActorWrapper)
+
+                throw IOException("received error response ${response.code}")
+            }
+
+            val data = response.data.toByteArray()
+            val hash = Hex.toHexString(MessageDigest.getInstance("SHA-256").digest(data))
+
+            if (hash != block.hash) {
+                throw IllegalStateException("expected block with hash ${block.hash}, but got block with hash $hash")
+            }
+
+            return data
+        }
+
         try {
             val reportProgressLock = Object()
 
@@ -96,9 +124,31 @@ class BlockPuller internal constructor(private val connectionHandler: Connection
                 repeat(4 /* 4 blocks per time */) { workerNumber ->
                     async {
                         for (block in pipe) {
-                            logger.debug("request block with hash = {} from worker {}", block.hash, workerNumber)
+                            logger.debug("message block with hash = {} from worker {}", block.hash, workerNumber)
 
-                            val blockContent = pullBlock(fileBlocks, block, 1000 * 60 /* 60 seconds timeout per block */)
+                            lateinit var blockContent: ByteArray
+
+                            val attempts = 0..4
+
+                            for (attempt in attempts) {
+                                try {
+                                    blockContent = pullBlock(fileBlocks, block, 1000 * 60 /* 60 seconds timeout per block */, connectionHelper.pickConnection())
+
+                                    break
+                                } catch (ex: IOException) {
+                                    if (attempt == attempts.last) {
+                                        throw ex
+                                    } else {
+                                        // will retry after a pause
+                                        // 0: 300 ms after the first attempt
+                                        // 1: 1200 ms after the second attempt
+                                        // 2: 2700 ms after the third attempt
+                                        // 3: 4800 ms after the third attempt
+                                        // total: 9000 ms
+                                        delay((attempt + 1) * (attempt + 1) * 300L)
+                                    }
+                                }
+                            }
 
                             blockTempIdByHash[block.hash] = tempRepository.pushTempData(blockContent)
 
@@ -140,57 +190,6 @@ class BlockPuller internal constructor(private val connectionHandler: Connection
             throw ex
         }
     }
-
-    private suspend fun pullBlock(fileBlocks: FileBlocks, block: BlockInfo, timeoutInMillis: Long): ByteArray {
-        logger.debug("sent request for block, hash = {}", block.hash)
-
-        val response =
-                withTimeout(timeoutInMillis) {
-                    try {
-                        doRequest(
-                                Request.newBuilder()
-                                        .setFolder(fileBlocks.folder)
-                                        .setName(fileBlocks.path)
-                                        .setOffset(block.offset)
-                                        .setSize(block.size)
-                                        .setHash(ByteString.copyFrom(Hex.decode(block.hash)))
-                        )
-                    } catch (ex: TimeoutCancellationException) {
-                        // It seems like the TimeoutCancellationException
-                        // is handled differently so that the timeout is ignored.
-                        // Due to that, it's converted to an IOException.
-
-                        throw IOException("timeout during requesting block")
-                    }
-                }
-
-        NetworkUtils.assertProtocol(response.code == ErrorCode.NO_ERROR) {
-            "received error response, code = ${response.code}"
-        }
-
-        val data = response.data.toByteArray()
-        val hash = Hex.toHexString(MessageDigest.getInstance("SHA-256").digest(data))
-
-        if (hash != block.hash) {
-            throw IllegalStateException("expected block with hash ${block.hash}, but got block with hash $hash")
-        }
-
-        return data
-    }
-
-    private suspend fun doRequest(request: Request.Builder): BlockExchangeProtos.Response {
-        return suspendCancellableCoroutine { continuation ->
-            val requestId = responseHandler.registerListener { response ->
-                continuation.resume(response)
-            }
-
-            connectionHandler.sendMessage(
-                    request
-                            .setId(requestId)
-                            .build()
-            )
-        }
-    }
 }
 
 data class BlockPullerStatus(

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/BlockPusher.kt

@@ -1,5 +1,6 @@
-/* 
+/*
  * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
  *
  * This Java file is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -14,14 +15,24 @@
 package net.syncthing.java.bep
 
 import com.google.protobuf.ByteString
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.async
+import kotlinx.coroutines.channels.consumeEach
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
 import net.syncthing.java.bep.BlockExchangeProtos.Vector
-import net.syncthing.java.core.beans.*
+import net.syncthing.java.bep.connectionactor.ConnectionActorWrapper
+import net.syncthing.java.bep.index.FolderStatsUpdateCollector
+import net.syncthing.java.bep.index.IndexElementProcessor
+import net.syncthing.java.bep.index.IndexHandler
+import net.syncthing.java.bep.index.IndexMessageProcessor
+import net.syncthing.java.core.beans.BlockInfo
+import net.syncthing.java.core.beans.DeviceId
 import net.syncthing.java.core.beans.FileInfo.Version
+import net.syncthing.java.core.beans.FolderStats
 import net.syncthing.java.core.utils.BlockUtils
 import net.syncthing.java.core.utils.NetworkUtils
-import net.syncthing.java.core.utils.submitLogging
 import org.apache.commons.io.IOUtils
-import org.apache.commons.lang3.tuple.Pair
 import org.bouncycastle.util.encoders.Hex
 import org.slf4j.LoggerFactory
 import java.io.Closeable
@@ -31,37 +42,36 @@ import java.nio.ByteBuffer
 import java.security.MessageDigest
 import java.util.*
 import java.util.concurrent.ConcurrentHashMap
-import java.util.concurrent.ExecutionException
 import java.util.concurrent.Executors
-import java.util.concurrent.Future
 import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicReference
 
-class BlockPusher internal constructor(private val localDeviceId: DeviceId,
-                                       private val connectionHandler: ConnectionHandler,
-                                       private val indexHandler: IndexHandler) {
+// TODO: refactor this
+class BlockPusher(private val localDeviceId: DeviceId,
+                  private val connectionHandler: ConnectionActorWrapper,
+                  private val indexHandler: IndexHandler,
+                  private val requestHandlerRegistry: RequestHandlerRegistry) {
 
     private val logger = LoggerFactory.getLogger(javaClass)
 
-
-    fun pushDelete(folderId: String, targetPath: String): IndexEditObserver {
-        val fileInfo = indexHandler.waitForRemoteIndexAcquired(connectionHandler).getFileInfoByPath(folderId, targetPath)!!
+    suspend fun pushDelete(folderId: String, targetPath: String): BlockExchangeProtos.IndexUpdate {
+        val fileInfo = indexHandler.waitForRemoteIndexAcquiredWithTimeout(connectionHandler).getFileInfoByPath(folderId, targetPath)!!
         NetworkUtils.assertProtocol(connectionHandler.hasFolder(fileInfo.folder), {"supplied connection handler $connectionHandler will not share folder ${fileInfo.folder}"})
-        return IndexEditObserver(sendIndexUpdate(folderId, BlockExchangeProtos.FileInfo.newBuilder()
+        return sendIndexUpdate(folderId, BlockExchangeProtos.FileInfo.newBuilder()
                 .setName(targetPath)
                 .setType(BlockExchangeProtos.FileInfoType.valueOf(fileInfo.type.name))
-                .setDeleted(true), fileInfo.versionList))
+                .setDeleted(true), fileInfo.versionList)
     }
 
-    fun pushDir(folder: String, path: String): IndexEditObserver {
+    suspend fun pushDir(folder: String, path: String): BlockExchangeProtos.IndexUpdate {
         NetworkUtils.assertProtocol(connectionHandler.hasFolder(folder), {"supplied connection handler $connectionHandler will not share folder $folder"})
-        return IndexEditObserver(sendIndexUpdate(folder, BlockExchangeProtos.FileInfo.newBuilder()
+        return sendIndexUpdate(folder, BlockExchangeProtos.FileInfo.newBuilder()
                 .setName(path)
-                .setType(BlockExchangeProtos.FileInfoType.DIRECTORY), null))
+                .setType(BlockExchangeProtos.FileInfoType.DIRECTORY), null)
     }
 
-    fun pushFile(inputStream: InputStream, folderId: String, targetPath: String): FileUploadObserver {
-        val fileInfo = indexHandler.waitForRemoteIndexAcquired(connectionHandler).getFileInfoByPath(folderId, targetPath)
+    suspend fun pushFile(inputStream: InputStream, folderId: String, targetPath: String): FileUploadObserver {
+        val fileInfo = indexHandler.waitForRemoteIndexAcquiredWithTimeout(connectionHandler).getFileInfoByPath(folderId, targetPath)
         NetworkUtils.assertProtocol(connectionHandler.hasFolder(folderId), {"supplied connection handler $connectionHandler will not share folder $folderId"})
         assert(fileInfo == null || fileInfo.folder == folderId)
         assert(fileInfo == null || fileInfo.path == targetPath)
@@ -72,56 +82,53 @@ class BlockPusher internal constructor(private val localDeviceId: DeviceId,
         val uploadError = AtomicReference<Exception>()
         val isCompleted = AtomicBoolean(false)
         val updateLock = Object()
-        val listener = {request: BlockExchangeProtos.Request ->
-            if (request.folder == folderId && request.name == targetPath) {
+        val requestFilter = RequestHandlerFilter(
+                deviceId = connectionHandler.deviceId,
+                folderId = folderId,
+                path = targetPath
+        )
+
+        requestHandlerRegistry.registerListener(requestFilter) { request ->
+            GlobalScope.async {
                 val hash = Hex.toHexString(request.hash.toByteArray())
                 logger.debug("handling block request = {}:{}-{} ({})", request.name, request.offset, request.size, hash)
                 val data = dataSource.getBlock(request.offset, request.size, hash)
-                val future = connectionHandler.sendMessage(BlockExchangeProtos.Response.newBuilder()
+
+                sentBlocks.add(hash)
+                synchronized(updateLock) {
+                    updateLock.notifyAll()
+                }
+
+                BlockExchangeProtos.Response.newBuilder()
                         .setCode(BlockExchangeProtos.ErrorCode.NO_ERROR)
                         .setData(ByteString.copyFrom(data))
                         .setId(request.id)
-                        .build())
-                monitoringProcessExecutorService.submitLogging {
-                    try {
-                        future.get()
-                        sentBlocks.add(hash)
-                        synchronized(updateLock) {
-                            updateLock.notifyAll()
-                        }
-                        //TODO retry on error, register error and throw on watcher
-                    } catch (ex: InterruptedException) {
-                        //return and do nothing
-                    } catch (ex: ExecutionException) {
-                        uploadError.set(ex)
-                        synchronized(updateLock) {
-                            updateLock.notifyAll()
-                        }
-                    }
-                }
+                        .build()
             }
         }
-        connectionHandler.registerOnRequestMessageReceivedListeners(listener)
+
         logger.debug("send index update for file = {}", targetPath)
-        val indexListener = { folderInfo: FolderInfo, newRecords: List<FileInfo>, indexInfo: IndexInfo ->
-            if (folderInfo.folderId == folderId) {
-                for (fileInfo2 in newRecords) {
-                    if (fileInfo2.path == targetPath && fileInfo2.hash == dataSource.getHash()) { //TODO check not invalid
-                        //                                sentBlocks.addAll(dataSource.getHashes());
-                        isCompleted.set(true)
-                        synchronized(updateLock) {
-                            updateLock.notifyAll()
+        val indexListenerStream = indexHandler.subscribeToOnIndexRecordAcquiredEvents()
+        GlobalScope.launch {
+            indexListenerStream.consumeEach { (indexFolderId, newRecords, _) ->
+                if (indexFolderId == folderId) {
+                    for (fileInfo2 in newRecords) {
+                        if (fileInfo2.path == targetPath && fileInfo2.hash == dataSource.getHash()) { //TODO check not invalid
+                            //                                sentBlocks.addAll(dataSource.getHashes());
+                            isCompleted.set(true)
+                            synchronized(updateLock) {
+                                updateLock.notifyAll()
+                            }
                         }
                     }
                 }
             }
         }
-        indexHandler.registerOnIndexRecordAcquiredListener(indexListener)
         val indexUpdate = sendIndexUpdate(folderId, BlockExchangeProtos.FileInfo.newBuilder()
                 .setName(targetPath)
                 .setSize(fileSize)
                 .setType(BlockExchangeProtos.FileInfoType.FILE)
-                .addAllBlocks(dataSource.blocks), fileInfo?.versionList).right
+                .addAllBlocks(dataSource.blocks), fileInfo?.versionList)
         return object : FileUploadObserver() {
 
             override fun progressPercentage() = if (isCompleted.get()) 100 else (sentBlocks.size.toFloat() / dataSource.getHashes().size).toInt()
@@ -132,9 +139,27 @@ class BlockPusher internal constructor(private val localDeviceId: DeviceId,
             override fun close() {
                 logger.debug("closing upload process")
                 monitoringProcessExecutorService.shutdown()
-                indexHandler.unregisterOnIndexRecordAcquiredListener(indexListener)
-                connectionHandler.unregisterOnRequestMessageReceivedListeners(listener)
-                val fileInfo1 = indexHandler.pushRecord(indexUpdate.folder, indexUpdate.filesList.single())
+                indexListenerStream.cancel()
+                requestHandlerRegistry.unregisterListener(requestFilter)
+                val (fileInfo1, folderStatsUpdate) = indexHandler.indexRepository.runInTransaction {
+                    val folderStatsUpdateCollector = FolderStatsUpdateCollector(folderId)
+
+                    // TODO: notify the IndexBrowsers again (as it was earlier)
+                    val fileInfo = IndexElementProcessor.pushRecord(
+                            it,
+                            indexUpdate.folder,
+                            indexUpdate.filesList.single(),
+                            folderStatsUpdateCollector,
+                            it.findFileInfo(folderId, indexUpdate.filesList.single().name)
+                    )
+
+                    IndexMessageProcessor.handleFolderStatsUpdate(it, folderStatsUpdateCollector)
+                    val folderStatsUpdate = it.findFolderStats(folderId) ?: FolderStats.createDummy(folderId)
+
+                    fileInfo to folderStatsUpdate
+                }
+
+                runBlocking { indexHandler.sendFolderStatsUpdate(folderStatsUpdate) }
                 logger.info("sent file info record = {}", fileInfo1)
             }
 
@@ -152,10 +177,10 @@ class BlockPusher internal constructor(private val localDeviceId: DeviceId,
         }
     }
 
-    private fun sendIndexUpdate(folderId: String, fileInfoBuilder: BlockExchangeProtos.FileInfo.Builder,
-                                oldVersions: Iterable<Version>?): Pair<Future<*>, BlockExchangeProtos.IndexUpdate> {
+    private suspend fun sendIndexUpdate(folderId: String, fileInfoBuilder: BlockExchangeProtos.FileInfo.Builder,
+                                        oldVersions: Iterable<Version>?): BlockExchangeProtos.IndexUpdate {
         run {
-            val nextSequence = indexHandler.sequencer().nextSequence()
+            val nextSequence = indexHandler.getNextSequenceNumber()
             val list = oldVersions ?: emptyList()
             logger.debug("version list = {}", list)
             val id = ByteBuffer.wrap(localDeviceId.toHashData()).long
@@ -182,7 +207,10 @@ class BlockPusher internal constructor(private val localDeviceId: DeviceId,
                 .addFiles(fileInfo)
                 .build()
         logger.debug("index update = {}", fileInfo)
-        return Pair.of(connectionHandler.sendMessage(indexUpdate), indexUpdate)
+
+        connectionHandler.sendIndexUpdate(indexUpdate)
+
+        return indexUpdate
     }
 
     abstract inner class FileUploadObserver : Closeable {
@@ -203,33 +231,6 @@ class BlockPusher internal constructor(private val localDeviceId: DeviceId,
         }
     }
 
-    inner class IndexEditObserver(private val future: Future<*>, private val indexUpdate: BlockExchangeProtos.IndexUpdate) : Closeable {
-
-        //throw exception if job has errors
-        @Throws(InterruptedException::class, ExecutionException::class)
-        fun isCompleted(): Boolean {
-            return if (future.isDone) {
-                future.get()
-                true
-            } else {
-                false
-            }
-        }
-
-        constructor(pair: Pair<Future<*>, BlockExchangeProtos.IndexUpdate>) : this(pair.left, pair.right)
-
-        @Throws(InterruptedException::class, ExecutionException::class)
-        fun waitForComplete() {
-            future.get()
-        }
-
-        @Throws(IOException::class)
-        override fun close() {
-            indexHandler.pushRecord(indexUpdate.folder, indexUpdate.filesList.single())
-        }
-
-    }
-
     private class DataSource @Throws(IOException::class) constructor(private val inputStream: InputStream) {
 
         var size: Long = 0

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/ConnectionHandler.kt

@@ -1,517 +0,0 @@
-/* 
- * Copyright (C) 2016 Davide Imbriaco
- * Copyright (C) 2018 Jonas Lochmann
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.bep
-
-import com.google.protobuf.ByteString
-import com.google.protobuf.MessageLite
-import net.jpountz.lz4.LZ4Factory
-import net.syncthing.java.bep.BlockExchangeProtos.*
-import net.syncthing.java.client.protocol.rp.RelayClient
-import net.syncthing.java.core.beans.DeviceAddress
-import net.syncthing.java.core.beans.DeviceId
-import net.syncthing.java.core.beans.DeviceInfo
-import net.syncthing.java.core.beans.FolderInfo
-import net.syncthing.java.core.configuration.Configuration
-import net.syncthing.java.core.interfaces.TempRepository
-import net.syncthing.java.core.security.KeystoreHandler
-import net.syncthing.java.core.utils.NetworkUtils
-import net.syncthing.java.core.utils.submitLogging
-import net.syncthing.java.httprelay.HttpRelayClient
-import org.apache.commons.io.IOUtils
-import org.apache.commons.lang3.tuple.Pair
-import org.slf4j.LoggerFactory
-import java.io.Closeable
-import java.io.DataInputStream
-import java.io.DataOutputStream
-import java.io.IOException
-import java.lang.reflect.InvocationTargetException
-import java.nio.ByteBuffer
-import java.security.cert.CertificateException
-import java.util.concurrent.ConcurrentHashMap
-import java.util.concurrent.Executors
-import java.util.concurrent.Future
-import java.util.concurrent.TimeUnit
-import javax.net.ssl.SSLSocket
-
-class ConnectionHandler(private val configuration: Configuration, val address: DeviceAddress,
-                        private val indexHandler: IndexHandler,
-                        private val tempRepository: TempRepository,
-                        private val onNewFolderSharedListener: (ConnectionHandler, FolderInfo) -> Unit,
-                        private val onConnectionChangedListener: (ConnectionHandler) -> Unit) : Closeable {
-
-    private val logger = LoggerFactory.getLogger(javaClass)
-
-    private val outExecutorService = Executors.newSingleThreadExecutor()
-    private val inExecutorService = Executors.newSingleThreadExecutor()
-    private val messageProcessingService = Executors.newCachedThreadPool()
-    private val periodicExecutorService = Executors.newSingleThreadScheduledExecutor()
-    private lateinit var socket: SSLSocket
-    private var inputStream: DataInputStream? = null
-    private var outputStream: DataOutputStream? = null
-    private var lastActive = Long.MIN_VALUE
-    internal var clusterConfigInfo: ClusterConfigInfo? = null
-        private set
-    private val clusterConfigWaitingLock = Object()
-    private val responseHandler = ResponseHandler()
-    private val blockPuller = BlockPuller(this, indexHandler, responseHandler, tempRepository)
-    private val blockPusher = BlockPusher(configuration.localDeviceId, this, indexHandler)
-    private val onRequestMessageReceivedListeners = mutableSetOf<(Request) -> Unit>()
-    private var isClosed = false
-    var isConnected = false
-        private set
-
-    fun deviceId(): DeviceId = address.deviceId()
-
-    private fun checkNotClosed() {
-        NetworkUtils.assertProtocol(!isClosed, {"connection $this closed"})
-    }
-
-    internal fun registerOnRequestMessageReceivedListeners(listener: (Request) -> Unit) {
-        onRequestMessageReceivedListeners.add(listener)
-    }
-
-    internal fun unregisterOnRequestMessageReceivedListeners(listener: (Request) -> Unit) {
-        assert(onRequestMessageReceivedListeners.contains(listener))
-        onRequestMessageReceivedListeners.remove(listener)
-    }
-
-    @Throws(IOException::class, KeystoreHandler.CryptoException::class)
-    fun connect(): ConnectionHandler {
-        checkNotClosed()
-        assert(!isConnected, {"already connected!"})
-        logger.info("connecting to {}", address.address)
-
-        val keystoreHandler = KeystoreHandler.Loader().loadKeystore(configuration)
-
-        socket = when (address.getType()) {
-            DeviceAddress.AddressType.TCP -> {
-                logger.debug("opening tcp ssl connection")
-                keystoreHandler.createSocket(address.getSocketAddress(), KeystoreHandler.BEP)
-            }
-            DeviceAddress.AddressType.RELAY -> {
-                logger.debug("opening relay connection")
-                keystoreHandler.wrapSocket(RelayClient(configuration).openRelayConnection(address), KeystoreHandler.BEP)
-            }
-            DeviceAddress.AddressType.HTTP_RELAY, DeviceAddress.AddressType.HTTPS_RELAY -> {
-                logger.debug("opening http relay connection")
-                keystoreHandler.wrapSocket(HttpRelayClient().openRelayConnection(address), KeystoreHandler.BEP)
-            }
-            else -> throw UnsupportedOperationException("unsupported address type = " + address.getType())
-        }
-        inputStream = DataInputStream(socket.inputStream)
-        outputStream = DataOutputStream(socket.outputStream)
-
-        sendHelloMessage(BlockExchangeProtos.Hello.newBuilder()
-                .setClientName(configuration.clientName)
-                .setClientVersion(configuration.clientVersion)
-                .setDeviceName(configuration.localDeviceName)
-                .build().toByteArray())
-        markActivityOnSocket()
-
-        receiveHelloMessage()
-        try {
-            KeystoreHandler.assertSocketCertificateValid(socket, address.deviceId())
-        } catch (e: CertificateException) {
-            throw IOException(e)
-        }
-
-        run {
-            val clusterConfigBuilder = ClusterConfig.newBuilder()
-            for (folder in configuration.folders) {
-                val folderBuilder = Folder.newBuilder()
-                        .setId(folder.folderId)
-                        .setLabel(folder.label)
-                run {
-                    //our device
-                    val deviceBuilder = Device.newBuilder()
-                            .setId(ByteString.copyFrom(configuration.localDeviceId.toHashData()))
-                            .setIndexId(indexHandler.sequencer().indexId())
-                            .setMaxSequence(indexHandler.sequencer().currentSequence())
-                    folderBuilder.addDevices(deviceBuilder)
-                }
-                run {
-                    //other device
-                    val deviceBuilder = Device.newBuilder()
-                            .setId(ByteString.copyFrom(DeviceId(address.deviceId).toHashData()))
-                    val indexSequenceInfo = indexHandler.indexRepository.findIndexInfoByDeviceAndFolder(address.deviceId(), folder.folderId)
-                    indexSequenceInfo?.let {
-                        deviceBuilder
-                                .setIndexId(indexSequenceInfo.indexId)
-                                .setMaxSequence(indexSequenceInfo.localSequence)
-                        logger.info("send delta index info device = {} index = {} max (local) sequence = {}",
-                                indexSequenceInfo.deviceId,
-                                indexSequenceInfo.indexId,
-                                indexSequenceInfo.localSequence)
-                    }
-                    folderBuilder.addDevices(deviceBuilder)
-                }
-                clusterConfigBuilder.addFolders(folderBuilder)
-                //TODO other devices??
-            }
-            sendMessage(clusterConfigBuilder.build())
-        }
-        synchronized(clusterConfigWaitingLock) {
-            startMessageListenerService()
-            while (clusterConfigInfo == null && !isClosed) {
-                logger.debug("wait for cluster config")
-                try {
-                    clusterConfigWaitingLock.wait()
-                } catch (e: InterruptedException) {
-                    throw IOException(e)
-                }
-            }
-            if (clusterConfigInfo == null) {
-                throw IOException("unable to retrieve cluster config from peer!")
-            }
-        }
-        for (folder in configuration.folders) {
-            if (hasFolder(folder.folderId)) {
-                sendIndexMessage(folder.folderId)
-            }
-        }
-        periodicExecutorService.scheduleWithFixedDelay({ this.sendPing() }, 90, 90, TimeUnit.SECONDS)
-        isConnected = true
-        onConnectionChangedListener(this)
-        return this
-    }
-
-    fun getBlockPuller(): BlockPuller {
-        return blockPuller
-    }
-
-    fun getBlockPusher(): BlockPusher {
-        return blockPusher
-    }
-
-    private fun sendIndexMessage(folderId: String) {
-        sendMessage(Index.newBuilder()
-                .setFolder(folderId)
-                .build())
-    }
-
-    fun closeBg() {
-        Thread { close() }.start()
-    }
-
-    /**
-     * Receive hello message and save device name to configuration.
-     */
-    @Throws(IOException::class)
-    private fun receiveHelloMessage() {
-        val magic = inputStream!!.readInt()
-        NetworkUtils.assertProtocol(magic == MAGIC, {"magic mismatch, expected $MAGIC, got $magic"})
-        val length = inputStream!!.readShort().toInt()
-        NetworkUtils.assertProtocol(length > 0, {"invalid lenght, must be >0, got $length"})
-        val buffer = ByteArray(length)
-        inputStream!!.readFully(buffer)
-        val hello = BlockExchangeProtos.Hello.parseFrom(buffer)
-        logger.info("Received hello message, deviceName=${hello.deviceName}, clientName=${hello.clientName}, clientVersion=${hello.clientVersion}")
-        configuration.peers = configuration.peers.map { peer ->
-                if (peer.deviceId == deviceId()) {
-                    DeviceInfo(deviceId(), hello.deviceName)
-                } else {
-                    peer
-                }
-            }.toSet()
-        configuration.persistLater()
-    }
-
-    private fun sendHelloMessage(payload: ByteArray): Future<*> {
-        return outExecutorService.submitLogging {
-            try {
-                logger.debug("Sending hello message")
-                val header = ByteBuffer.allocate(6)
-                header.putInt(MAGIC)
-                header.putShort(payload.size.toShort())
-                outputStream!!.write(header.array())
-                outputStream!!.write(payload)
-                outputStream!!.flush()
-            } catch (ex: IOException) {
-                if (outExecutorService.isShutdown) {
-                    return@submitLogging
-                }
-                logger.error("error writing to output stream", ex)
-                closeBg()
-            }
-        }
-    }
-
-    private fun sendPing(): Future<*> {
-        return sendMessage(Ping.newBuilder().build())
-    }
-
-    private fun markActivityOnSocket() {
-        lastActive = System.currentTimeMillis()
-    }
-
-    @Throws(IOException::class)
-    private fun receiveMessage(): Pair<BlockExchangeProtos.MessageType, MessageLite> {
-        var headerLength = inputStream!!.readShort().toInt()
-        while (headerLength == 0) {
-            logger.warn("got headerLength == 0, skipping short")
-            headerLength = inputStream!!.readShort().toInt()
-        }
-        markActivityOnSocket()
-        NetworkUtils.assertProtocol(headerLength > 0, {"invalid lenght, must be >0, got $headerLength"})
-        val headerBuffer = ByteArray(headerLength)
-        inputStream!!.readFully(headerBuffer)
-        val header = BlockExchangeProtos.Header.parseFrom(headerBuffer)
-        var messageLength = 0
-        while (messageLength == 0) {
-            logger.warn("received readInt() == 0, expecting 'bep message header length' (int >0), ignoring (keepalive?)")
-            messageLength = inputStream!!.readInt()
-        }
-        NetworkUtils.assertProtocol(messageLength >= 0, {"invalid lenght, must be >=0, got $messageLength"})
-        var messageBuffer = ByteArray(messageLength)
-        inputStream!!.readFully(messageBuffer)
-        markActivityOnSocket()
-        if (header.compression == BlockExchangeProtos.MessageCompression.LZ4) {
-            val uncompressedLength = ByteBuffer.wrap(messageBuffer).int
-            messageBuffer = LZ4Factory.fastestInstance().fastDecompressor().decompress(messageBuffer, 4, uncompressedLength)
-        }
-        val messageTypeInfo = messageTypesByProtoMessageType[header.type]
-        NetworkUtils.assertProtocol(messageTypeInfo != null, {"unsupported message type = ${header.type}"})
-        try {
-            val message = messageTypeInfo!!.parseFrom(messageBuffer)
-            return Pair.of(header.type, message)
-        } catch (e: Exception) {
-            when (e) {
-                is IllegalAccessException, is IllegalArgumentException, is InvocationTargetException, is NoSuchMethodException, is SecurityException ->
-                    throw IOException(e)
-                else -> throw e
-            }
-        }
-    }
-
-    internal fun sendMessage(message: MessageLite): Future<*> {
-        checkNotClosed()
-        val messageTypeInfo = messageTypesByJavaClass[message.javaClass]
-        messageTypeInfo!!
-        val header = BlockExchangeProtos.Header.newBuilder()
-                .setCompression(BlockExchangeProtos.MessageCompression.NONE)
-                // invert map
-                .setType(messageTypeInfo.protoMessageType)
-                .build()
-        val headerData = header.toByteArray()
-        val messageData = message.toByteArray() //TODO compression
-        return outExecutorService.submit<Any> {
-            try {
-                logger.debug("sending message type = {} {}", header.type, getIdForMessage(message))
-                markActivityOnSocket()
-                outputStream!!.writeShort(headerData.size)
-                outputStream!!.write(headerData)
-                outputStream!!.writeInt(messageData.size)//with compression, check this
-                outputStream!!.write(messageData)
-                outputStream!!.flush()
-                markActivityOnSocket()
-            } catch (ex: IOException) {
-                if (!outExecutorService.isShutdown) {
-                    logger.error("error writing to output stream", ex)
-                    closeBg()
-                }
-                throw ex
-            }
-
-            null
-        }
-    }
-
-    override fun close() {
-        if (!isClosed) {
-            sendMessage(Close.getDefaultInstance())
-            isClosed = true
-            isConnected = false
-            periodicExecutorService.shutdown()
-            outExecutorService.shutdown()
-            inExecutorService.shutdown()
-            messageProcessingService.shutdown()
-            assert(onRequestMessageReceivedListeners.isEmpty())
-            if (outputStream != null) {
-                IOUtils.closeQuietly(outputStream)
-                outputStream = null
-            }
-            if (inputStream != null) {
-                IOUtils.closeQuietly(inputStream)
-                inputStream = null
-            }
-            try {
-              IOUtils.closeQuietly(socket)
-            } catch (ex: Exception) {
-              // ignore this
-              // this can throw an exception if socket was not yet initialized/ set
-              // as Kotlin does an check about this, the closeQuietly does not catch it
-            }
-            logger.info("closed connection {}", address)
-            synchronized(clusterConfigWaitingLock) {
-                clusterConfigWaitingLock.notifyAll()
-            }
-            onConnectionChangedListener(this)
-            try {
-                periodicExecutorService.awaitTermination(2, TimeUnit.SECONDS)
-                outExecutorService.awaitTermination(2, TimeUnit.SECONDS)
-                inExecutorService.awaitTermination(2, TimeUnit.SECONDS)
-                messageProcessingService.awaitTermination(2, TimeUnit.SECONDS)
-            } catch (ex: InterruptedException) {
-                logger.warn("", ex)
-            }
-
-        }
-    }
-
-    /**
-     * return time elapsed since last activity on socket, inputStream millis
-     *
-     * @return
-     */
-    fun getLastActive(): Long {
-        return System.currentTimeMillis() - lastActive
-    }
-
-    private fun startMessageListenerService() {
-        inExecutorService.submitLogging {
-            try {
-                while (!Thread.interrupted()) {
-                    val message = receiveMessage()
-                    messageProcessingService.submitLogging {
-                        logger.debug("received message type = {} {}", message.left, getIdForMessage(message.right))
-                        when (message.left) {
-                            BlockExchangeProtos.MessageType.INDEX -> {
-                                val index = message.value as Index
-                                indexHandler.handleIndexMessageReceivedEvent(index.folder, index.filesList, this)
-                            }
-                            BlockExchangeProtos.MessageType.INDEX_UPDATE -> {
-                                val update = message.value as IndexUpdate
-                                indexHandler.handleIndexMessageReceivedEvent(update.folder, update.filesList, this)
-                            }
-                            BlockExchangeProtos.MessageType.REQUEST -> {
-                                onRequestMessageReceivedListeners.forEach { it(message.value as Request) }
-                            }
-                            BlockExchangeProtos.MessageType.RESPONSE -> {
-                                responseHandler.handleResponse(message.value as Response)
-                            }
-                            BlockExchangeProtos.MessageType.PING -> logger.debug("ping message received")
-                            BlockExchangeProtos.MessageType.CLOSE -> {
-                                val close = message.value as BlockExchangeProtos.Close
-                                logger.info("received close message, reason=${close.reason}")
-                                closeBg()
-                            }
-                            BlockExchangeProtos.MessageType.CLUSTER_CONFIG -> {
-                                NetworkUtils.assertProtocol(clusterConfigInfo == null, {"received cluster config message twice!"})
-                                clusterConfigInfo = ClusterConfigInfo()
-                                val clusterConfig = message.value as ClusterConfig
-                                for (folder in clusterConfig.foldersList ?: emptyList()) {
-                                    val folderInfo = ClusterConfigFolderInfo(folder.id, folder.label)
-                                    val devicesById = (folder.devicesList ?: emptyList())
-                                            .associateBy { input ->
-                                                DeviceId.fromHashData(input.id!!.toByteArray())
-                                            }
-                                    val otherDevice = devicesById[address.deviceId()]
-                                    val ourDevice = devicesById[configuration.localDeviceId]
-                                    if (otherDevice != null) {
-                                        folderInfo.isAnnounced = true
-                                    }
-                                    if (ourDevice != null) {
-                                        folderInfo.isShared = true
-                                        logger.info("folder shared from device = {} folder = {}", address.deviceId, folderInfo)
-                                        val folderIds = configuration.folders.map { it.folderId }
-                                        if (!folderIds.contains(folderInfo.folderId)) {
-                                            val fi = FolderInfo(folderInfo.folderId, folderInfo.label)
-                                            configuration.folders = configuration.folders + fi
-                                            onNewFolderSharedListener(this, fi)
-                                            logger.info("new folder shared = {}", folderInfo)
-                                        }
-                                    } else {
-                                        logger.info("folder not shared from device = {} folder = {}", address.deviceId, folderInfo)
-                                    }
-                                    clusterConfigInfo!!.putFolderInfo(folderInfo)
-                                }
-                                configuration.persistLater()
-                                indexHandler.handleClusterConfigMessageProcessedEvent(clusterConfig)
-                                synchronized(clusterConfigWaitingLock) {
-                                    clusterConfigWaitingLock.notifyAll()
-                                }
-                            }
-                        }
-                    }
-                }
-            } catch (ex: IOException) {
-                if (inExecutorService.isShutdown) {
-                    return@submitLogging
-                }
-                logger.error("error receiving message", ex)
-                closeBg()
-            }
-        }
-    }
-
-    override fun toString(): String {
-        return "ConnectionHandler{" + "address=" + address + ", lastActive=" + getLastActive() / 1000.0 + "secs ago}"
-    }
-
-    internal inner class ClusterConfigInfo {
-
-        private val folderInfoById = ConcurrentHashMap<String, ClusterConfigFolderInfo>()
-
-        fun getSharedFolders(): Set<String> = folderInfoById.values.filter { it.isShared }.map { it.folderId }.toSet()
-
-        fun putFolderInfo(folderInfo: ClusterConfigFolderInfo) {
-            folderInfoById[folderInfo.folderId] = folderInfo
-        }
-
-    }
-
-    fun hasFolder(folder: String): Boolean {
-        return clusterConfigInfo!!.getSharedFolders().contains(folder)
-    }
-
-    companion object {
-
-        private const val MAGIC = 0x2EA7D90B
-
-        private val messageTypes = listOf(
-                MessageTypeInfo(MessageType.CLOSE, Close::class.java) { Close.parseFrom(it) },
-                MessageTypeInfo(MessageType.CLUSTER_CONFIG, ClusterConfig::class.java) { ClusterConfig.parseFrom(it) },
-                MessageTypeInfo(MessageType.DOWNLOAD_PROGRESS, DownloadProgress::class.java) { DownloadProgress.parseFrom(it) },
-                MessageTypeInfo(MessageType.INDEX, Index::class.java) { Index.parseFrom(it) },
-                MessageTypeInfo(MessageType.INDEX_UPDATE, IndexUpdate::class.java) { IndexUpdate.parseFrom(it) },
-                MessageTypeInfo(MessageType.PING, Ping::class.java) { Ping.parseFrom(it) },
-                MessageTypeInfo(MessageType.REQUEST, Request::class.java) { Request.parseFrom(it) },
-                MessageTypeInfo(MessageType.RESPONSE, Response::class.java) { Response.parseFrom(it) }
-        )
-
-        private val messageTypesByProtoMessageType = messageTypes.map { it.protoMessageType to it }.toMap()
-        private val messageTypesByJavaClass = messageTypes.map { it.javaClass to it }.toMap()
-
-        /**
-         * get id for message bean/instance, for log tracking
-         *
-         * @param message
-         * @return id for message bean
-         */
-        private fun getIdForMessage(message: MessageLite): String {
-            return when (message) {
-                is Request -> Integer.toString(message.id)
-                is Response -> Integer.toString(message.id)
-                else -> Integer.toString(Math.abs(message.hashCode()))
-            }
-        }
-    }
-
-    data class MessageTypeInfo(
-            val protoMessageType: MessageType,
-            val javaClass: Class<out MessageLite>,
-            val parseFrom: (data: ByteArray) -> MessageLite
-    )
-}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/FolderBrowser.kt

@@ -1,58 +0,0 @@
-/* 
- * Copyright (C) 2016 Davide Imbriaco
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.bep
-
-import net.syncthing.java.core.beans.FolderInfo
-import net.syncthing.java.core.beans.FolderStats
-import net.syncthing.java.core.interfaces.IndexRepository
-import java.io.Closeable
-
-class FolderBrowser internal constructor(private val indexHandler: IndexHandler) : Closeable {
-    private val folderStatsCache = mutableMapOf<String, FolderStats>()
-    private val indexRepositoryEventListener = { event: IndexRepository.FolderStatsUpdatedEvent ->
-        addFolderStats(event.getFolderStats())
-    }
-
-    fun folderInfoAndStatsList(): List<Pair<FolderInfo, FolderStats>> =
-            indexHandler.folderInfoList()
-                    .map { folderInfo -> Pair(folderInfo, getFolderStats(folderInfo.folderId)) }
-                    .sortedBy { it.first.label }
-
-    init {
-        indexHandler.indexRepository.setOnFolderStatsUpdatedListener(indexRepositoryEventListener)
-        addFolderStats(indexHandler.indexRepository.findAllFolderStats())
-    }
-
-    private fun addFolderStats(folderStatsList: List<FolderStats>) {
-        for (folderStats in folderStatsList) {
-            folderStatsCache.put(folderStats.folderId, folderStats)
-        }
-    }
-
-    fun getFolderStats(folder: String): FolderStats {
-        return folderStatsCache[folder] ?: let {
-            FolderStats.Builder()
-                    .setFolder(folder)
-                    .build()
-        }
-    }
-
-    fun getFolderInfo(folder: String): FolderInfo? {
-        return indexHandler.getFolderInfo(folder)
-    }
-
-    override fun close() {
-        indexHandler.indexRepository.setOnFolderStatsUpdatedListener(null)
-    }
-}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/IndexBrowser.kt

@@ -1,184 +0,0 @@
-/* 
- * Copyright (C) 2016 Davide Imbriaco
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.bep
-
-import net.syncthing.java.core.beans.FileInfo
-import net.syncthing.java.core.interfaces.IndexRepository
-import net.syncthing.java.core.utils.PathUtils
-import net.syncthing.java.core.utils.awaitTerminationSafe
-import net.syncthing.java.core.utils.submitLogging
-import org.apache.commons.lang3.StringUtils
-import org.slf4j.LoggerFactory
-import java.io.Closeable
-import java.util.*
-import java.util.concurrent.Executors
-
-class IndexBrowser internal constructor(private val indexRepository: IndexRepository, private val indexHandler: IndexHandler,
-                                       val folder: String, private val includeParentInList: Boolean = false,
-                                       private val allowParentInRoot: Boolean = false, ordering: Comparator<FileInfo>?) : Closeable {
-
-    private fun isParent(fileInfo: FileInfo) = PathUtils.isParent(fileInfo.path)
-
-    val ALPHA_ASC_DIR_FIRST: Comparator<FileInfo> =
-            compareBy<FileInfo>({!isParent(it)}, {!it.isDirectory()})
-                    .thenBy { it.fileName.toLowerCase() }
-    val LAST_MOD_DESC: Comparator<FileInfo> =
-            compareBy<FileInfo>({!isParent(it)}, {it.lastModified})
-                    .thenBy { it.fileName.toLowerCase() }
-
-    private val ordering = ordering ?: ALPHA_ASC_DIR_FIRST
-    private val logger = LoggerFactory.getLogger(javaClass)
-
-    var currentPath: String = PathUtils.ROOT_PATH
-        private set
-    private val PARENT_FILE_INFO: FileInfo
-    private val ROOT_FILE_INFO: FileInfo
-    private val executorService = Executors.newSingleThreadScheduledExecutor()
-    private val preloadJobs = mutableSetOf<String>()
-    private val preloadJobsLock = Any()
-    private var mOnPathChangedListener: (() -> Unit)? = null
-
-    private fun isCacheReady(): Boolean {
-        synchronized(preloadJobsLock) {
-            return preloadJobs.isEmpty()
-        }
-    }
-
-    internal fun onIndexChangedevent(folder: String, newRecord: FileInfo) {
-        if (folder == this.folder) {
-            preloadFileInfoForCurrentPath()
-        }
-    }
-
-    fun currentPathInfo(): FileInfo = getFileInfoByAbsolutePath(currentPath)
-
-    fun currentPathFileName(): String? = PathUtils.getFileName(currentPath)
-
-    fun isRoot(): Boolean = PathUtils.isRoot(currentPath)
-
-    init {
-        assert(folder.isNotEmpty())
-        PARENT_FILE_INFO = FileInfo(folder = folder, type = FileInfo.FileType.DIRECTORY, path = PathUtils.PARENT_PATH)
-        ROOT_FILE_INFO = FileInfo(folder = folder, type = FileInfo.FileType.DIRECTORY, path = PathUtils.ROOT_PATH)
-        navigateToAbsolutePath(PathUtils.ROOT_PATH)
-    }
-
-    fun setOnFolderChangedListener(onPathChangedListener: (() -> Unit)?) {
-        mOnPathChangedListener = onPathChangedListener
-    }
-
-    private fun preloadFileInfoForCurrentPath() {
-        logger.debug("trigger preload for folder = '{}'", folder)
-        synchronized(preloadJobsLock) {
-            currentPath.let<String, Any> { currentPath ->
-                if (preloadJobs.contains(currentPath)) {
-                    preloadJobs.remove(currentPath)
-                    preloadJobs.add(currentPath) ///add last
-                } else {
-                    preloadJobs.add(currentPath)
-                    executorService.submitLogging(object : Runnable {
-
-                        override fun run() {
-
-                            val preloadPath =
-                                    synchronized(preloadJobsLock) {
-                                        assert(!preloadJobs.isEmpty())
-                                        preloadJobs.last() //pop last job
-                                    }
-
-                            logger.info("folder preload BEGIN for folder = '{}' path = '{}'", folder, preloadPath)
-                            getFileInfoByAbsolutePath(preloadPath)
-                            if (!PathUtils.isRoot(preloadPath)) {
-                                val parent = PathUtils.getParentPath(preloadPath)
-                                getFileInfoByAbsolutePath(parent)
-                                listFiles(parent)
-                            }
-                            for (record in listFiles(preloadPath)) {
-                                if (record.path == PARENT_FILE_INFO.path && record.isDirectory()) {
-                                    listFiles(record.path)
-                                }
-                            }
-                            logger.info("folder preload END for folder = '{}' path = '{}'", folder, preloadPath)
-                            synchronized(preloadJobsLock) {
-                                preloadJobs.remove(preloadPath)
-                                if (isCacheReady()) {
-                                    logger.info("cache ready, notify listeners")
-                                    mOnPathChangedListener?.invoke()
-                                } else {
-                                    logger.info("still {} job[s] left in cache loader", preloadJobs.size)
-                                    executorService.submitLogging(this)
-                                }
-                            }
-                        }
-                    })
-                }
-            }
-        }
-    }
-
-    fun listFiles(path: String = currentPath): List<FileInfo> {
-        logger.debug("doListFiles for path = '{}' BEGIN", path)
-        val list = ArrayList(indexRepository.findNotDeletedFilesByFolderAndParent(folder, path))
-        logger.debug("doListFiles for path = '{}' : {} records loaded)", path, list.size)
-        if (includeParentInList && (!PathUtils.isRoot(path) || allowParentInRoot)) {
-            list.add(0, PARENT_FILE_INFO)
-        }
-        return list.sortedWith(ordering)
-    }
-
-    fun getFileInfoByAbsolutePath(path: String): FileInfo {
-        return if (PathUtils.isRoot(path)) {
-            ROOT_FILE_INFO
-        } else {
-            logger.debug("doGetFileInfoByAbsolutePath for path = '{}' BEGIN", path)
-            val fileInfo = indexRepository.findNotDeletedFileInfo(folder, path) ?: error("file not found for path = $path")
-            logger.debug("doGetFileInfoByAbsolutePath for path = '{}' END", path)
-            fileInfo
-        }
-    }
-
-    fun navigateTo(fileInfo: FileInfo) {
-        assert(fileInfo.isDirectory())
-        assert(fileInfo.folder == folder)
-        return if (fileInfo.path == PARENT_FILE_INFO.path)
-            navigateToAbsolutePath(PathUtils.getParentPath(currentPath))
-        else
-            navigateToAbsolutePath(fileInfo.path)
-    }
-
-    fun navigateToNearestPath(oldPath: String) {
-        if (!StringUtils.isBlank(oldPath)) {
-            navigateToAbsolutePath(oldPath)
-        }
-    }
-
-    private fun navigateToAbsolutePath(newPath: String) {
-        if (PathUtils.isRoot(newPath)) {
-            currentPath = PathUtils.ROOT_PATH
-        } else {
-            val fileInfo = getFileInfoByAbsolutePath(newPath)
-            assert(fileInfo.isDirectory(), {"cannot navigate to path ${fileInfo.path}: not a directory"})
-            currentPath = fileInfo.path
-        }
-        logger.info("navigate to path = '{}'", currentPath)
-        preloadFileInfoForCurrentPath()
-    }
-
-    override fun close() {
-        logger.info("closing")
-        indexHandler.unregisterIndexBrowser(this)
-        executorService.shutdown()
-        executorService.awaitTerminationSafe()
-    }
-}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/IndexHandler.kt

@@ -1,452 +0,0 @@
-/* 
- * Copyright (C) 2016 Davide Imbriaco
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.bep
-
-import net.syncthing.java.core.beans.*
-import net.syncthing.java.core.beans.FileInfo.Version
-import net.syncthing.java.core.configuration.Configuration
-import net.syncthing.java.core.interfaces.IndexRepository
-import net.syncthing.java.core.interfaces.Sequencer
-import net.syncthing.java.core.interfaces.TempRepository
-import net.syncthing.java.core.utils.NetworkUtils
-import net.syncthing.java.core.utils.awaitTerminationSafe
-import net.syncthing.java.core.utils.submitLogging
-import org.apache.commons.lang3.tuple.Pair
-import org.apache.http.util.TextUtils
-import org.bouncycastle.util.encoders.Hex
-import org.slf4j.LoggerFactory
-import java.io.Closeable
-import java.io.IOException
-import java.util.*
-import java.util.concurrent.Executors
-
-class IndexHandler(private val configuration: Configuration, val indexRepository: IndexRepository,
-                   private val tempRepository: TempRepository) : Closeable {
-    private val logger = LoggerFactory.getLogger(javaClass)
-    private val folderInfoByFolder = mutableMapOf<String, FolderInfo>()
-    private val indexMessageProcessor = IndexMessageProcessor()
-    private var lastIndexActivity: Long = 0
-    private val writeAccessLock = Object()
-    private val indexWaitLock = Object()
-    private val indexBrowsers = mutableSetOf<IndexBrowser>()
-    private val onIndexRecordAcquiredListeners = mutableSetOf<(FolderInfo, List<FileInfo>, IndexInfo) -> Unit>()
-    private val onFullIndexAcquiredListeners = mutableSetOf<(FolderInfo) -> Unit>()
-
-    private fun lastActive(): Long = System.currentTimeMillis() - lastIndexActivity
-
-    fun sequencer(): Sequencer = indexRepository.getSequencer()
-
-    fun folderList(): List<String> = folderInfoByFolder.keys.toList()
-
-    fun folderInfoList(): List<FolderInfo> = folderInfoByFolder.values.toList()
-
-    private fun markActive() {
-        lastIndexActivity = System.currentTimeMillis()
-    }
-
-    fun registerOnIndexRecordAcquiredListener(listener: (FolderInfo, List<FileInfo>, IndexInfo) -> Unit) {
-        onIndexRecordAcquiredListeners.add(listener)
-    }
-
-    fun unregisterOnIndexRecordAcquiredListener(listener: (FolderInfo, List<FileInfo>, IndexInfo) -> Unit) {
-        assert(onIndexRecordAcquiredListeners.contains(listener))
-        onIndexRecordAcquiredListeners.remove(listener)
-    }
-
-    fun registerOnFullIndexAcquiredListenersListener(listener: (FolderInfo) -> Unit) {
-        onFullIndexAcquiredListeners.add(listener)
-    }
-
-    fun unregisterOnFullIndexAcquiredListenersListener(listener: (FolderInfo) -> Unit) {
-        assert(onFullIndexAcquiredListeners.contains(listener))
-        onFullIndexAcquiredListeners.remove(listener)
-    }
-
-    init {
-        loadFolderInfoFromConfig()
-    }
-
-    private fun loadFolderInfoFromConfig() {
-        synchronized(writeAccessLock) {
-            for (folderInfo in configuration.folders) {
-                folderInfoByFolder.put(folderInfo.folderId, folderInfo) //TODO reference 'folder info' repository
-            }
-        }
-    }
-
-    @Synchronized
-    fun clearIndex() {
-        synchronized(writeAccessLock) {
-            indexRepository.clearIndex()
-            folderInfoByFolder.clear()
-            loadFolderInfoFromConfig()
-        }
-    }
-
-    internal fun isRemoteIndexAcquired(clusterConfigInfo: ConnectionHandler.ClusterConfigInfo, peerDeviceId: DeviceId): Boolean {
-        var ready = true
-        for (folder in clusterConfigInfo.getSharedFolders()) {
-            val indexSequenceInfo = indexRepository.findIndexInfoByDeviceAndFolder(peerDeviceId, folder)
-            if (indexSequenceInfo == null || indexSequenceInfo.localSequence < indexSequenceInfo.maxSequence) {
-                logger.debug("waiting for index on folder = {} sequenceInfo = {}", folder, indexSequenceInfo)
-                ready = false
-            }
-        }
-        return ready
-    }
-
-    @Throws(InterruptedException::class)
-    fun waitForRemoteIndexAcquired(connectionHandler: ConnectionHandler, timeoutSecs: Long? = null): IndexHandler {
-        val timeoutMillis = (timeoutSecs ?: DEFAULT_INDEX_TIMEOUT) * 1000
-        synchronized(indexWaitLock) {
-            while (!isRemoteIndexAcquired(connectionHandler.clusterConfigInfo!!, connectionHandler.deviceId())) {
-                indexWaitLock.wait(timeoutMillis)
-                NetworkUtils.assertProtocol(connectionHandler.getLastActive() < timeoutMillis || lastActive() < timeoutMillis,
-                        {"unable to acquire index from connection $connectionHandler, timeout reached!"})
-            }
-        }
-        logger.debug("acquired all indexes on connection {}", connectionHandler)
-        return this
-    }
-
-    fun handleClusterConfigMessageProcessedEvent(clusterConfig: BlockExchangeProtos.ClusterConfig) {
-        synchronized(writeAccessLock) {
-            for (folderRecord in clusterConfig.foldersList) {
-                val folder = folderRecord.id
-                val folderInfo = updateFolderInfo(folder, folderRecord.label)
-                logger.debug("acquired folder info from cluster config = {}", folderInfo)
-                for (deviceRecord in folderRecord.devicesList) {
-                    val deviceId = DeviceId.fromHashData(deviceRecord.id.toByteArray())
-                    if (deviceRecord.hasIndexId() && deviceRecord.hasMaxSequence()) {
-                        val folderIndexInfo = updateIndexInfo(folder, deviceId, deviceRecord.indexId, deviceRecord.maxSequence, null)
-                        logger.debug("acquired folder index info from cluster config = {}", folderIndexInfo)
-                    }
-                }
-            }
-        }
-    }
-
-    fun handleIndexMessageReceivedEvent(folderId: String, filesList: List<BlockExchangeProtos.FileInfo>, connectionHandler: ConnectionHandler) {
-        indexMessageProcessor.handleIndexMessageReceivedEvent(folderId, filesList, connectionHandler)
-    }
-
-    fun pushRecord(folder: String, bepFileInfo: BlockExchangeProtos.FileInfo): FileInfo? {
-        var fileBlocks: FileBlocks? = null
-        val builder = FileInfo.Builder()
-                .setFolder(folder)
-                .setPath(bepFileInfo.name)
-                .setLastModified(Date(bepFileInfo.modifiedS * 1000 + bepFileInfo.modifiedNs / 1000000))
-                .setVersionList((if (bepFileInfo.hasVersion()) bepFileInfo.version.countersList else null ?: emptyList()).map { record -> Version(record.id, record.value) })
-                .setDeleted(bepFileInfo.deleted)
-        when (bepFileInfo.type) {
-            BlockExchangeProtos.FileInfoType.FILE -> {
-                fileBlocks = FileBlocks(folder, builder.getPath()!!, ((bepFileInfo.blocksList ?: emptyList())).map { record ->
-                    BlockInfo(record.offset, record.size, Hex.toHexString(record.hash.toByteArray()))
-                })
-                builder
-                        .setTypeFile()
-                        .setHash(fileBlocks.hash)
-                        .setSize(bepFileInfo.size)
-            }
-            BlockExchangeProtos.FileInfoType.DIRECTORY -> builder.setTypeDir()
-            else -> {
-                logger.warn("unsupported file type = {}, discarding file info", bepFileInfo.type)
-                return null
-            }
-        }
-        return addRecord(builder.build(), fileBlocks)
-    }
-
-    private fun updateIndexInfo(folder: String, deviceId: DeviceId, indexId: Long?, maxSequence: Long?, localSequence: Long?): IndexInfo {
-        synchronized(writeAccessLock) {
-            var indexSequenceInfo = indexRepository.findIndexInfoByDeviceAndFolder(deviceId, folder)
-            var shouldUpdate = false
-            val builder: IndexInfo.Builder
-            if (indexSequenceInfo == null) {
-                shouldUpdate = true
-                assert(indexId != null, {"index sequence info not found, and supplied null index id (folder = $folder, device = $deviceId)"})
-                builder = IndexInfo.newBuilder()
-                        .setFolder(folder)
-                        .setDeviceId(deviceId.deviceId)
-                        .setIndexId(indexId!!)
-                        .setLocalSequence(0)
-                        .setMaxSequence(-1)
-            } else {
-                builder = indexSequenceInfo.copyBuilder()
-            }
-            if (indexId != null && indexId != builder.getIndexId()) {
-                shouldUpdate = true
-                builder.setIndexId(indexId)
-            }
-            if (maxSequence != null && maxSequence > builder.getMaxSequence()) {
-                shouldUpdate = true
-                builder.setMaxSequence(maxSequence)
-            }
-            if (localSequence != null && localSequence > builder.getLocalSequence()) {
-                shouldUpdate = true
-                builder.setLocalSequence(localSequence)
-            }
-            if (shouldUpdate) {
-                indexSequenceInfo = builder.build()
-                indexRepository.updateIndexInfo(indexSequenceInfo)
-            }
-            return indexSequenceInfo!!
-        }
-    }
-
-    private fun addRecord(record: FileInfo, fileBlocks: FileBlocks?): FileInfo? {
-        synchronized(writeAccessLock) {
-            val lastModified = indexRepository.findFileInfoLastModified(record.folder, record.path)
-            return if (lastModified != null && record.lastModified < lastModified) {
-                logger.trace("discarding record = {}, modified before local record", record)
-                null
-            } else {
-                indexRepository.updateFileInfo(record, fileBlocks)
-                logger.trace("loaded new record = {}", record)
-                indexBrowsers.forEach {
-                    it.onIndexChangedevent(record.folder, record)
-                }
-                record
-            }
-        }
-    }
-
-    fun getFileInfoByPath(folder: String, path: String): FileInfo? {
-        return indexRepository.findFileInfo(folder, path)
-    }
-
-    fun getFileInfoAndBlocksByPath(folder: String, path: String): Pair<FileInfo, FileBlocks>? {
-        val fileInfo = getFileInfoByPath(folder, path)
-        return if (fileInfo == null) {
-            null
-        } else {
-            assert(fileInfo.isFile())
-            val fileBlocks = indexRepository.findFileBlocks(folder, path)
-            checkNotNull(fileBlocks, {"file blocks not found for file info = $fileInfo"})
-
-            FileInfo.checkBlocks(fileInfo, fileBlocks!!)
-
-            Pair.of(fileInfo, fileBlocks)
-        }
-    }
-
-    private fun updateFolderInfo(folder: String, label: String?): FolderInfo {
-        var folderInfo: FolderInfo? = folderInfoByFolder[folder]
-        if (folderInfo == null || !TextUtils.isEmpty(label)) {
-            folderInfo = FolderInfo(folder, label)
-            folderInfoByFolder.put(folderInfo.folderId, folderInfo)
-        }
-        return folderInfo
-    }
-
-    fun getFolderInfo(folder: String): FolderInfo? {
-        return folderInfoByFolder[folder]
-    }
-
-    fun getIndexInfo(device: DeviceId, folder: String): IndexInfo? {
-        return indexRepository.findIndexInfoByDeviceAndFolder(device, folder)
-    }
-
-    fun newFolderBrowser(): FolderBrowser {
-        return FolderBrowser(this)
-    }
-
-    fun newIndexBrowser(folder: String, includeParentInList: Boolean = false, allowParentInRoot: Boolean = false,
-                        ordering: Comparator<FileInfo>? = null): IndexBrowser {
-        val indexBrowser = IndexBrowser(indexRepository, this, folder, includeParentInList, allowParentInRoot, ordering)
-        indexBrowsers.add(indexBrowser)
-        return indexBrowser
-    }
-
-    internal fun unregisterIndexBrowser(indexBrowser: IndexBrowser) {
-        assert(indexBrowsers.contains(indexBrowser))
-        indexBrowsers.remove(indexBrowser)
-    }
-
-    override fun close() {
-        assert(indexBrowsers.isEmpty())
-        assert(onIndexRecordAcquiredListeners.isEmpty())
-        assert(onFullIndexAcquiredListeners.isEmpty())
-        indexMessageProcessor.stop()
-    }
-
-    private inner class IndexMessageProcessor {
-
-        private val executorService = Executors.newSingleThreadExecutor()
-        private var queuedMessages = 0
-        private var queuedRecords: Long = 0
-        //        private long lastRecordProcessingTime = 0;
-        //        , delay = 0;
-        //        private boolean addProcessingDelayForInterface = true;
-        //        private final int MIN_DELAY = 0, MAX_DELAY = 5000, MAX_RECORD_PER_PROCESS = 16, DELAY_FACTOR = 1;
-        private var startTime: Long? = null
-
-        fun handleIndexMessageReceivedEvent(folderId: String, filesList: List<BlockExchangeProtos.FileInfo>, connectionHandler: ConnectionHandler) {
-            logger.info("received index message event, preparing (queued records = {} event record count = {})", queuedRecords, filesList.size)
-            markActive()
-            val clusterConfigInfo = connectionHandler.clusterConfigInfo
-            val peerDeviceId = connectionHandler.deviceId()
-            //            List<BlockExchangeProtos.FileInfo> fileList = event.getFilesList();
-            //            for (int index = 0; index < fileList.size(); index += MAX_RECORD_PER_PROCESS) {
-            //                BlockExchangeProtos.IndexUpdate data = BlockExchangeProtos.IndexUpdate.newBuilder()
-            //                    .addAllFiles(Iterables.limit(Iterables.skip(fileList, index), MAX_RECORD_PER_PROCESS))
-            //                    .setFolder(event.getFolder())
-            //                    .build();
-            //                if (queuedMessages > 0) {
-            //                    storeAndProcessBg(data, clusterConfigInfo, peerDeviceId);
-            //                } else {
-            //                    processBg(data, clusterConfigInfo, peerDeviceId);
-            //                }
-            //            }
-            val data = BlockExchangeProtos.IndexUpdate.newBuilder()
-                    .addAllFiles(filesList)
-                    .setFolder(folderId)
-                    .build()
-            if (queuedMessages > 0) {
-                storeAndProcessBg(data, clusterConfigInfo, peerDeviceId)
-            } else {
-                processBg(data, clusterConfigInfo, peerDeviceId)
-            }
-        }
-
-        private fun processBg(data: BlockExchangeProtos.IndexUpdate, clusterConfigInfo: ConnectionHandler.ClusterConfigInfo?, peerDeviceId: DeviceId) {
-            logger.debug("received index message event, queuing for processing")
-            queuedMessages++
-            queuedRecords += data.filesCount.toLong()
-            executorService.submitLogging(object : ProcessingRunnable() {
-                override fun runProcess() {
-                    doHandleIndexMessageReceivedEvent(data, clusterConfigInfo, peerDeviceId)
-                }
-            })
-        }
-
-        private fun storeAndProcessBg(data: BlockExchangeProtos.IndexUpdate, clusterConfigInfo: ConnectionHandler.ClusterConfigInfo?, peerDeviceId: DeviceId) {
-            val key = tempRepository.pushTempData(data.toByteArray())
-            logger.debug("received index message event, stored to temp record {}, queuing for processing", key)
-            queuedMessages++
-            queuedRecords += data.filesCount.toLong()
-            executorService.submitLogging(object : ProcessingRunnable() {
-                override fun runProcess() {
-                    try {
-                        doHandleIndexMessageReceivedEvent(key, clusterConfigInfo, peerDeviceId)
-                    } catch (ex: IOException) {
-                        logger.error("error processing index message", ex)
-                    }
-
-                }
-
-            })
-        }
-
-        private abstract inner class ProcessingRunnable : Runnable {
-
-            override fun run() {
-                startTime = System.currentTimeMillis()
-                runProcess()
-                queuedMessages--
-                //                lastRecordProcessingTime = stopwatch.elapsed(TimeUnit.MILLISECONDS) - delay;
-                //                logger.info("processed a bunch of records, {}*{} remaining", queuedMessages, MAX_RECORD_PER_PROCESS);
-                //                logger.debug("processed index message in {} secs", lastRecordProcessingTime / 1000d);
-                startTime = null
-            }
-
-            protected abstract fun runProcess()
-
-            //        private boolean isVersionOlderThanSequence(BlockExchangeProtos.FileInfo fileInfo, long localSequence) {
-            //            long fileSequence = fileInfo.getSequence();
-            //            //TODO should we check last version instead of sequence? verify
-            //            return fileSequence < localSequence;
-            //        }
-            @Throws(IOException::class)
-            protected fun doHandleIndexMessageReceivedEvent(key: String, clusterConfigInfo: ConnectionHandler.ClusterConfigInfo?, peerDeviceId: DeviceId) {
-                logger.debug("processing index message event from temp record {}", key)
-                markActive()
-                val data = tempRepository.popTempData(key)
-                val message = BlockExchangeProtos.IndexUpdate.parseFrom(data)
-                doHandleIndexMessageReceivedEvent(message, clusterConfigInfo, peerDeviceId)
-            }
-
-            protected fun doHandleIndexMessageReceivedEvent(message: BlockExchangeProtos.IndexUpdate, clusterConfigInfo: ConnectionHandler.ClusterConfigInfo?, peerDeviceId: DeviceId) {
-                //            synchronized (writeAccessLock) {
-                //                if (addProcessingDelayForInterface) {
-                //                    delay = Math.min(MAX_DELAY, Math.max(MIN_DELAY, lastRecordProcessingTime * DELAY_FACTOR));
-                //                    logger.info("add delay of {} secs before processing index message (to allow UI to process)", delay / 1000d);
-                //                    try {
-                //                        Thread.sleep(delay);
-                //                    } catch (InterruptedException ex) {
-                //                        logger.warn("interrupted", ex);
-                //                    }
-                //                } else {
-                //                    delay = 0;
-                //                }
-                logger.info("processing index message with {} records (queue size: messages = {} records = {})", message.filesCount, queuedMessages, queuedRecords)
-                //            String deviceId = connectionHandler.getDeviceId();
-                val folderId = message.folder
-                var sequence: Long = -1
-                val newRecords = mutableListOf<FileInfo>()
-                //                IndexInfo oldIndexInfo = indexRepository.findIndexInfoByDeviceAndFolder(deviceId, folder);
-                //            Stopwatch stopwatch = Stopwatch.createStarted();
-                logger.debug("processing {} index records for folder {}", message.filesList.size, folderId)
-                for (fileInfo in message.filesList) {
-                    markActive()
-                    //                    if (oldIndexInfo != null && isVersionOlderThanSequence(fileInfo, oldIndexInfo.getLocalSequence())) {
-                    //                        logger.trace("skipping file {}, version older than sequence {}", fileInfo, oldIndexInfo.getLocalSequence());
-                    //                    } else {
-                    val newRecord = pushRecord(folderId, fileInfo)
-                    if (newRecord != null) {
-                        newRecords.add(newRecord)
-                    }
-                    sequence = Math.max(fileInfo.sequence, sequence)
-                    markActive()
-                    //                    }
-                }
-                val newIndexInfo = updateIndexInfo(folderId, peerDeviceId, null, null, sequence)
-                val elap = System.currentTimeMillis() - startTime!!
-                queuedRecords -= message.filesCount.toLong()
-                logger.info("processed {} index records, acquired {} ({} secs, {} record/sec)", message.filesCount, newRecords.size, elap / 1000.0, Math.round(message.filesCount / (elap / 1000.0) * 100) / 100.0)
-                if (logger.isInfoEnabled && newRecords.size <= 10) {
-                    for (fileInfo in newRecords) {
-                        logger.info("acquired record = {}", fileInfo)
-                    }
-                }
-                val folderInfo = folderInfoByFolder[folderId]
-                if (!newRecords.isEmpty()) {
-                    onIndexRecordAcquiredListeners.forEach { it(folderInfo!!, newRecords, newIndexInfo) }
-                }
-                logger.debug("index info = {}", newIndexInfo)
-                if (isRemoteIndexAcquired(clusterConfigInfo!!, peerDeviceId)) {
-                    logger.debug("index acquired")
-                    onFullIndexAcquiredListeners.forEach { it(folderInfo!!)}
-                }
-                //                IndexHandler.this.notifyAll();
-                markActive()
-                synchronized(indexWaitLock) {
-                    indexWaitLock.notifyAll()
-                }
-            }
-        }
-
-        fun stop() {
-            logger.info("stopping index record processor")
-            executorService.shutdown()
-            executorService.awaitTerminationSafe()
-        }
-
-    }
-
-    companion object {
-
-        private const val DEFAULT_INDEX_TIMEOUT: Long = 30
-    }
-}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/MultiConnectionHelper.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep
+
+import net.syncthing.java.bep.connectionactor.ConnectionActorWrapper
+import java.io.IOException
+import java.util.*
+
+class MultiConnectionHelper (
+        initialConnections: List<ConnectionActorWrapper>,
+        private val connectionFilter: (ConnectionActorWrapper) -> Boolean
+) {
+    companion object {
+        private val random = Random()
+    }
+
+    private val usableConnections = initialConnections.toMutableList()
+
+    fun pickConnection(): ConnectionActorWrapper {
+        val possibleConnections = synchronized(usableConnections) {
+            usableConnections.filter { it.isConnected and connectionFilter(it) }
+        }
+
+        if (possibleConnections.isEmpty()) {
+            throw IOException("no matching connection is available")
+        } else if (possibleConnections.size == 1) {
+            return possibleConnections.first()
+        } else {
+            return possibleConnections[random.nextInt(possibleConnections.size)]
+        }
+    }
+
+    fun disableConnection(wrapper: ConnectionActorWrapper) {
+        synchronized(usableConnections) {
+            usableConnections.remove(wrapper)
+        }
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/RequestHandlerRegistry.kt

@@ -0,0 +1,54 @@
+package net.syncthing.java.bep
+
+import kotlinx.coroutines.Deferred
+import net.syncthing.java.core.beans.DeviceId
+import java.io.IOException
+
+class RequestHandlerRegistry {
+    private val listeners = mutableMapOf<RequestHandlerFilter, (BlockExchangeProtos.Request) -> Deferred<BlockExchangeProtos.Response>>()
+
+    suspend fun handleRequest(source: DeviceId, request: BlockExchangeProtos.Request): BlockExchangeProtos.Response {
+        val rule = RequestHandlerFilter(
+                deviceId = source,
+                folderId = request.folder,
+                path = request.name
+        )
+
+        val matchingListener = synchronized(listeners) {
+            listeners[rule]
+        }
+
+        if (matchingListener != null) {
+            return matchingListener(request).await()
+        } else {
+            return BlockExchangeProtos.Response.newBuilder()
+                    .setId(request.id)
+                    .setCode(BlockExchangeProtos.ErrorCode.GENERIC)
+                    .build()
+        }
+    }
+
+    fun registerListener(filter: RequestHandlerFilter, listener: (BlockExchangeProtos.Request) -> Deferred<BlockExchangeProtos.Response>) {
+        synchronized(listeners) {
+            val oldListener = listeners[filter]
+
+            if (oldListener != null) {
+                throw IOException("there is already an listener for this filter")
+            }
+
+            listeners[filter] = listener
+        }
+    }
+
+    fun unregisterListener(filter: RequestHandlerFilter) {
+        synchronized(listeners) {
+            listeners.remove(filter)
+        }
+    }
+}
+
+data class RequestHandlerFilter(
+        val deviceId: DeviceId,
+        val folderId: String,
+        val path: String
+)

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/ResponseHandler.kt

@@ -1,50 +0,0 @@
-/*
- * Copyright (C) 2018 Jonas Lochmann
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.bep
-
-import org.slf4j.LoggerFactory
-import java.util.*
-import java.util.concurrent.atomic.AtomicInteger
-import kotlin.collections.HashMap
-
-class ResponseHandler {
-    companion object {
-        private val logger = LoggerFactory.getLogger(ResponseHandler::class.java)
-    }
-
-    private val responseListeners = Collections.synchronizedMap(HashMap<Int, (BlockExchangeProtos.Response) -> Unit>())
-    private val nextRequestId = AtomicInteger(0)
-
-    fun registerListener(listener: (BlockExchangeProtos.Response) -> Unit): Int {
-        val requestId = nextRequestId.getAndIncrement()
-
-        responseListeners[requestId] = listener
-
-        return requestId
-    }
-
-    fun unregisterListener(requestId: Int) {
-        responseListeners.remove(requestId)
-    }
-
-    fun handleResponse(response: BlockExchangeProtos.Response) {
-        val listener = responseListeners.remove(response.id)
-
-        if (listener != null) {
-            listener(response)
-        } else {
-            logger.warn("received response for {} without associated handler", response.id)
-        }
-    }
-}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/ClusterConfigHandler.kt

@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import com.google.protobuf.ByteString
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.bep.index.IndexHandler
+import net.syncthing.java.core.beans.DeviceId
+import net.syncthing.java.core.beans.FolderInfo
+import net.syncthing.java.core.configuration.Configuration
+import org.slf4j.LoggerFactory
+
+object ClusterConfigHandler {
+    private val logger = LoggerFactory.getLogger(ClusterConfigHandler::class.java)
+
+    fun buildClusterConfig(
+            configuration: Configuration,
+            indexHandler: IndexHandler,
+            deviceId: DeviceId
+    ): BlockExchangeProtos.ClusterConfig {
+        val builder = BlockExchangeProtos.ClusterConfig.newBuilder()
+
+        indexHandler.indexRepository.runInTransaction { indexTransaction ->
+            for (folder in configuration.folders) {
+                val folderBuilder = BlockExchangeProtos.Folder.newBuilder()
+                        .setId(folder.folderId)
+                        .setLabel(folder.label)
+
+                // add this device
+                folderBuilder.addDevices(
+                        BlockExchangeProtos.Device.newBuilder()
+                                .setId(ByteString.copyFrom(configuration.localDeviceId.toHashData()))
+                                .setIndexId(indexTransaction.getSequencer().indexId())
+                                .setMaxSequence(indexTransaction.getSequencer().currentSequence())
+                )
+
+                // add other device
+                val indexSequenceInfo = indexTransaction.findIndexInfoByDeviceAndFolder(deviceId, folder.folderId)
+
+                folderBuilder.addDevices(
+                        BlockExchangeProtos.Device.newBuilder()
+                                .setId(ByteString.copyFrom(deviceId.toHashData()))
+                                .apply {
+                                    indexSequenceInfo?.let {
+                                        setIndexId(indexSequenceInfo.indexId)
+                                        setMaxSequence(indexSequenceInfo.localSequence)
+
+                                        logger.info("send delta index info device = {} index = {} max (local) sequence = {}",
+                                                indexSequenceInfo.deviceId,
+                                                indexSequenceInfo.indexId,
+                                                indexSequenceInfo.localSequence)
+                                    }
+                                }
+                )
+
+                builder.addFolders(folderBuilder)
+
+                // TODO: add the other devices to the cluster config
+            }
+        }
+
+        return builder.build()
+    }
+
+    // TODO: understand this
+    internal suspend fun handleReceivedClusterConfig(
+            clusterConfig: BlockExchangeProtos.ClusterConfig,
+            configuration: Configuration,
+            otherDeviceId: DeviceId,
+            indexHandler: IndexHandler
+    ): ClusterConfigInfo {
+        val folderInfoList = mutableListOf<ClusterConfigFolderInfo>()
+        val newSharedFolders = mutableListOf<FolderInfo>()
+
+        for (folder in clusterConfig.foldersList ?: emptyList()) {
+            var folderInfo = ClusterConfigFolderInfo(folder.id, folder.label)
+            val devicesById = (folder.devicesList ?: emptyList())
+                    .associateBy { input ->
+                        DeviceId.fromHashData(input.id!!.toByteArray())
+                    }
+            val otherDevice = devicesById[otherDeviceId]
+            val ourDevice = devicesById[configuration.localDeviceId]
+            if (otherDevice != null) {
+                folderInfo = folderInfo.copy(isAnnounced = true)
+            }
+            if (ourDevice != null) {
+                folderInfo = folderInfo.copy(isShared = true)
+                logger.info("folder shared from device = {} folder = {}", otherDeviceId, folderInfo)
+
+                val newFolderInfo = FolderInfo(folderInfo.folderId, folderInfo.label)
+
+                val oldFolderEntry = configuration.folders.find { it.folderId == folderInfo.folderId }
+
+                if (oldFolderEntry == null) {
+                    configuration.folders = configuration.folders + newFolderInfo
+                    newSharedFolders.add(newFolderInfo)
+                    logger.info("new folder shared = {}", folderInfo)
+                } else {
+                    if (oldFolderEntry != newFolderInfo) {
+                        configuration.folders = configuration.folders.filter { it != oldFolderEntry }.toSet() + setOf(newFolderInfo)
+                    }
+                }
+            } else {
+                logger.info("folder not shared from device = {} folder = {}", otherDeviceId, folderInfo)
+            }
+
+            folderInfoList.add(folderInfo)
+        }
+        configuration.persistLater()
+        indexHandler.handleClusterConfigMessageProcessedEvent(clusterConfig)
+
+        return ClusterConfigInfo(folderInfoList, newSharedFolders)
+    }
+}
+
+class ClusterConfigInfo (val folderInfo: List<ClusterConfigFolderInfo>, val newSharedFolders: List<FolderInfo>) {
+    companion object {
+        val dummy = ClusterConfigInfo(folderInfo = emptyList(), newSharedFolders = emptyList())
+    }
+
+    val folderInfoById = folderInfo.associateBy { it.folderId }
+    val sharedFolderIds: Set<String> by lazy {
+        folderInfo.filter { it.isShared }.map { it.folderId }.toSet()
+    }
+}
+
+data class ClusterConfigFolderInfo(
+        val folderId: String,
+        val label: String = folderId,
+        val isAnnounced: Boolean = false,
+        val isShared: Boolean = false
+) {
+    init {
+        assert(folderId.isNotEmpty())
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/ConnectionAction.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import kotlinx.coroutines.CompletableDeferred
+import net.syncthing.java.bep.BlockExchangeProtos
+
+sealed class ConnectionAction
+object CloseConnectionAction: ConnectionAction()
+class SendRequestConnectionAction(
+        val request: BlockExchangeProtos.Request,
+        val completableDeferred: CompletableDeferred<BlockExchangeProtos.Response>
+): ConnectionAction()
+class ConfirmIsConnectedAction(val completableDeferred: CompletableDeferred<ClusterConfigInfo>): ConnectionAction()
+class SendIndexUpdateAction(
+        val message: BlockExchangeProtos.IndexUpdate,
+        val completableDeferred: CompletableDeferred<Unit?>
+): ConnectionAction()

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/ConnectionActorGenerator.kt

@@ -0,0 +1,204 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.*
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.bep.index.IndexHandler
+import net.syncthing.java.core.beans.DeviceAddress
+import net.syncthing.java.core.configuration.Configuration
+import org.slf4j.LoggerFactory
+import java.io.IOException
+
+object ConnectionActorGenerator {
+    private val closed = Channel<ConnectionAction>().apply { cancel() }
+    private val logger = LoggerFactory.getLogger(ConnectionActorGenerator::class.java)
+
+    private fun deviceAddressesGenerator(deviceAddress: ReceiveChannel<DeviceAddress>) = GlobalScope.produce<List<DeviceAddress>> (capacity = Channel.CONFLATED) {
+        val addresses = mutableMapOf<String, DeviceAddress>()
+
+        deviceAddress.consumeEach { address ->
+            val isNew = addresses[address.address] == null
+
+            addresses[address.address] = address
+
+            if (isNew) {
+                send(
+                        addresses.values.sortedBy { it.score }
+                )
+            }
+        }
+    }
+
+    private fun <T> waitForFirstValue(source: ReceiveChannel<T>, time: Long) = GlobalScope.produce<T> {
+        source.consume {
+            val firstValue = source.receive()
+            var lastValue = firstValue
+
+            try {
+                withTimeout(time) {
+                    while (true) {
+                        lastValue = source.receive()
+                    }
+                }
+
+                throw IllegalStateException()
+            } catch (ex: TimeoutCancellationException) {
+                // this is expected here
+            }
+
+            send(lastValue)
+
+            // other values without delay
+            for (value in source) {
+                send(value)
+            }
+        }
+    }
+
+    fun generateConnectionActors(
+            deviceAddress: ReceiveChannel<DeviceAddress>,
+            configuration: Configuration,
+            indexHandler: IndexHandler,
+            requestHandler: (BlockExchangeProtos.Request) -> Deferred<BlockExchangeProtos.Response>
+    ) = generateConnectionActorsFromDeviceAddressList(
+            deviceAddressSource = waitForFirstValue(
+                    source = deviceAddressesGenerator(deviceAddress),
+                    time = 1000
+            ),
+            configuration = configuration,
+            indexHandler = indexHandler,
+            requestHandler = requestHandler
+    )
+
+    fun generateConnectionActorsFromDeviceAddressList(
+            deviceAddressSource: ReceiveChannel<List<DeviceAddress>>,
+            configuration: Configuration,
+            indexHandler: IndexHandler,
+            requestHandler: (BlockExchangeProtos.Request) -> Deferred<BlockExchangeProtos.Response>
+    ) = GlobalScope.produce<Pair<SendChannel<ConnectionAction>, ClusterConfigInfo>> {
+        var currentActor: SendChannel<ConnectionAction> = closed
+        var currentDeviceAddress: DeviceAddress? = null
+
+        suspend fun closeCurrent() {
+            if (currentActor != closed) {
+                currentActor.close()
+                currentActor = closed
+                send(currentActor to ClusterConfigInfo.dummy)
+            }
+        }
+
+        suspend fun tryConnectingToAddressHandleBaseErrors(deviceAddress: DeviceAddress) = try {
+            val newActor = ConnectionActor.createInstance(deviceAddress, configuration, indexHandler, requestHandler)
+            val clusterConfig = ConnectionActorUtil.waitUntilConnected(newActor)
+
+            newActor to clusterConfig
+        } catch (ex: Exception) {
+            logger.warn("failed to connect to $deviceAddress", ex)
+
+            when (ex) {
+                is IOException -> {/* expected -> ignore */}
+                is InterruptedException -> {/* expected -> ignore */}
+                else -> throw ex
+            }
+
+            null
+        }
+
+        suspend fun dispatchConnection(
+                connection: SendChannel<ConnectionAction>,
+                clusterConfig: ClusterConfigInfo,
+                deviceAddress: DeviceAddress
+        ) {
+            currentActor = connection
+            currentDeviceAddress = deviceAddress
+
+            send(connection to clusterConfig)
+        }
+
+        suspend fun tryConnectingToAddress(deviceAddress: DeviceAddress): Boolean {
+            closeCurrent()
+
+            var connection = tryConnectingToAddressHandleBaseErrors(deviceAddress) ?: return false
+
+            if (connection.second.newSharedFolders.isNotEmpty()) {
+                logger.debug("connected to $deviceAddress with new folders -> reconnect")
+                // reconnect to send new cluster config
+                connection.first.close()
+                connection = tryConnectingToAddressHandleBaseErrors(deviceAddress) ?: return false
+            }
+
+            logger.debug("connected to $deviceAddress")
+
+            dispatchConnection(connection.first, connection.second, deviceAddress)
+
+            return true
+        }
+
+        fun isConnected() = !currentActor.isClosedForSend
+
+        invokeOnClose {
+            currentActor.close()
+        }
+
+        val reconnectTicker = ticker(delayMillis = 30 * 1000, initialDelayMillis = 0)
+
+        deviceAddressSource.consume {
+            var lastDeviceAddressList: List<DeviceAddress> = emptyList()
+
+            while (true) {
+                if (isConnected()) {
+                    lastDeviceAddressList = deviceAddressSource.poll() ?: lastDeviceAddressList
+
+                    if (lastDeviceAddressList.isNotEmpty()) {
+                        if (reconnectTicker.poll() != null) {
+                            if (currentDeviceAddress != lastDeviceAddressList.first()) {
+                                val oldDeviceAddress = currentDeviceAddress!!
+
+                                if (!tryConnectingToAddress(lastDeviceAddressList.first())) {
+                                    tryConnectingToAddress(oldDeviceAddress)
+                                }
+                            }
+                        }
+                    } else {
+                        closeCurrent()
+                    }
+
+                    delay(500)  // don't take too much CPU
+                } else /* is not connected */ {
+                    // get the new list version if there is any
+                    lastDeviceAddressList = deviceAddressSource.poll() ?: lastDeviceAddressList
+
+                    // try all addresses
+                    for (address in lastDeviceAddressList) {
+                        if (tryConnectingToAddress(address)) {
+                            break
+                        }
+                    }
+
+                    // reset countdown before trying other connection if it would be time now
+                    // this does not reset if it has not counted down the whole time yet
+                    reconnectTicker.poll()
+
+                    // wait for new device address list but not more than 15 seconds before the next iteration
+                    lastDeviceAddressList = withTimeoutOrNull(15 * 1000) {
+                        deviceAddressSource.receive()
+                    } ?: lastDeviceAddressList
+                }
+            }
+        }
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/ConnectionActorUtil.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.channels.SendChannel
+import net.syncthing.java.bep.BlockExchangeProtos
+
+object ConnectionActorUtil {
+    suspend fun waitUntilConnected(actor: SendChannel<ConnectionAction>): ClusterConfigInfo {
+        val deferred = CompletableDeferred<ClusterConfigInfo>()
+
+        actor.send(ConfirmIsConnectedAction(deferred))
+        actor.invokeOnClose { deferred.cancel() }
+
+        return deferred.await()
+    }
+
+    suspend fun sendRequest(request: BlockExchangeProtos.Request, actor: SendChannel<ConnectionAction>): BlockExchangeProtos.Response {
+        val deferred = CompletableDeferred<BlockExchangeProtos.Response>()
+
+        actor.send(SendRequestConnectionAction(request, deferred))
+
+        return deferred.await()
+    }
+
+    suspend fun sendIndexUpdate(update: BlockExchangeProtos.IndexUpdate, actor: SendChannel<ConnectionAction>) {
+        val deferred = CompletableDeferred<Unit?>()
+
+        actor.send(SendIndexUpdateAction(update, deferred))
+
+        deferred.await()
+    }
+
+    suspend fun disconnect(actor: SendChannel<ConnectionAction>) {
+        actor.send(CloseConnectionAction)
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/ConnectionActorWrapper.kt

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.ReceiveChannel
+import kotlinx.coroutines.channels.SendChannel
+import kotlinx.coroutines.channels.consumeEach
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.core.beans.DeviceId
+import java.io.IOException
+
+class ConnectionActorWrapper (
+        private val source: ReceiveChannel<Pair<SendChannel<ConnectionAction>, ClusterConfigInfo>>,
+        val deviceId: DeviceId,
+        val connectivityChangeListener: () -> Unit
+) {
+    private val job = Job()
+
+    private var currentConnectionActor: SendChannel<ConnectionAction>? = null
+    private var clusterConfigInfo: ClusterConfigInfo? = null
+
+    var isConnected = false
+        get() = currentConnectionActor?.isClosedForSend == false
+
+    init {
+        GlobalScope.launch (job) {
+            source.consumeEach { (connectionActor, clusterConfig) ->
+                currentConnectionActor = connectionActor
+                clusterConfigInfo = clusterConfig
+            }
+        }
+
+        // this is a very simple solution but it does its job
+        GlobalScope.launch (job) {
+            var previousConnected = false
+
+            while (isActive) {
+                val nowConnected = isConnected
+
+                if (previousConnected != nowConnected) {
+                    previousConnected = nowConnected
+
+                    connectivityChangeListener()
+                }
+
+                delay(200)
+            }
+        }
+    }
+
+    suspend fun sendRequest(request: BlockExchangeProtos.Request) = ConnectionActorUtil.sendRequest(
+            request,
+            currentConnectionActor ?: throw IOException("not connected")
+    )
+
+    suspend fun sendIndexUpdate(update: BlockExchangeProtos.IndexUpdate) = ConnectionActorUtil.sendIndexUpdate(
+            update,
+            currentConnectionActor ?: throw IOException("not connected")
+    )
+
+    fun hasFolder(folderId: String) = clusterConfigInfo?.sharedFolderIds?.contains(folderId) ?: false
+
+    fun getClusterConfig() = clusterConfigInfo ?: throw IOException("not connected")
+
+    fun shutdown() {
+        job.cancel()
+    }
+
+    // this triggers a disconnection
+    // the ConnectionActorGenerator will reconnect soon
+    fun reconnect() {
+        val actor = currentConnectionActor
+
+        GlobalScope.launch {
+            if (actor != null) {
+                ConnectionActorUtil.disconnect(actor)
+            }
+        }
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/ConnectionConstants.kt

@@ -1,5 +1,6 @@
-/* 
+/*
  * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
  *
  * This Java file is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -11,13 +12,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package net.syncthing.java.bep
-
-internal data class ClusterConfigFolderInfo(val folderId: String, var label: String = folderId,
-                                            var isAnnounced: Boolean = false, var isShared: Boolean = false) {
-
-    init {
-        assert(folderId.isNotEmpty())
-    }
+package net.syncthing.java.bep.connectionactor
 
+object ConnectionConstants {
+    const val MAGIC = 0x2EA7D90B
 }

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/HelloMessageHandler.kt

@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.core.beans.DeviceId
+import net.syncthing.java.core.beans.DeviceInfo
+import net.syncthing.java.core.configuration.Configuration
+import net.syncthing.java.core.utils.NetworkUtils
+import org.slf4j.LoggerFactory
+import java.io.DataInputStream
+import java.io.DataOutputStream
+import java.nio.ByteBuffer
+
+object HelloMessageHandler {
+    private val logger = LoggerFactory.getLogger(HelloMessageHandler::class.java)
+
+    fun sendHelloMessage(configuration: Configuration, outputStream: DataOutputStream) {
+        sendHelloMessage(
+                BlockExchangeProtos.Hello.newBuilder()
+                        .setClientName(configuration.clientName)
+                        .setClientVersion(configuration.clientVersion)
+                        .setDeviceName(configuration.localDeviceName)
+                        .build(),
+                outputStream
+        )
+    }
+
+    private fun sendHelloMessage(message: BlockExchangeProtos.Hello, outputStream: DataOutputStream) {
+        sendHelloMessage(message.toByteArray(), outputStream)
+    }
+
+    private fun sendHelloMessage(payload: ByteArray, outputStream: DataOutputStream) {
+        logger.debug("Sending hello message")
+
+        outputStream.apply {
+            write(
+                    ByteBuffer.allocate(6).apply {
+                        putInt(ConnectionConstants.MAGIC)
+                        putShort(payload.size.toShort())
+                    }.array()
+            )
+            write(payload)
+            flush()
+        }
+    }
+
+    fun receiveHelloMessage(
+            inputStream: DataInputStream
+    ): BlockExchangeProtos.Hello {
+        val magic = inputStream.readInt()
+        NetworkUtils.assertProtocol(magic == ConnectionConstants.MAGIC) {"magic mismatch, got $magic"}
+
+        val length = inputStream.readShort().toInt()
+        NetworkUtils.assertProtocol(length > 0) {"invalid length, must be > 0, got $length"}
+
+        return BlockExchangeProtos.Hello.parseFrom(
+                ByteArray(length).apply {
+                    inputStream.readFully(this)
+                }
+        )
+    }
+
+    fun processHelloMessage(
+            hello: BlockExchangeProtos.Hello,
+            configuration: Configuration,
+            deviceId: DeviceId
+    ) {
+        logger.info("Received hello message, deviceName=${hello.deviceName}, clientName=${hello.clientName}, clientVersion=${hello.clientVersion}")
+
+        // update the local device name
+        // TODO: this could need some locking
+        configuration.peers = configuration.peers.map { peer ->
+            if (peer.deviceId == deviceId) {
+                DeviceInfo(deviceId, hello.deviceName)
+            } else {
+                peer
+            }
+        }.toSet()
+
+        configuration.persistLater()
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/Instance.kt

@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import com.google.protobuf.MessageLite
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.channels.SendChannel
+import kotlinx.coroutines.channels.consumeEach
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.bep.index.IndexHandler
+import net.syncthing.java.core.beans.DeviceAddress
+import net.syncthing.java.core.configuration.Configuration
+import net.syncthing.java.core.security.KeystoreHandler
+import java.io.DataInputStream
+import java.io.DataOutputStream
+import java.io.IOException
+import java.util.*
+
+object ConnectionActor {
+    fun createInstance(
+            address: DeviceAddress,
+            configuration: Configuration,
+            indexHandler: IndexHandler,
+            requestHandler: (BlockExchangeProtos.Request) -> Deferred<BlockExchangeProtos.Response>
+    ): SendChannel<ConnectionAction> {
+        val channel = Channel<ConnectionAction>(Channel.RENDEZVOUS)
+
+        GlobalScope.async (Dispatchers.IO) {
+            OpenConnection.openSocketConnection(address, configuration).use { socket ->
+                val inputStream = DataInputStream(socket.inputStream)
+                val outputStream = DataOutputStream(socket.outputStream)
+
+                val helloMessage = coroutineScope {
+                    async { HelloMessageHandler.sendHelloMessage(configuration, outputStream) }
+                    async { HelloMessageHandler.receiveHelloMessage(inputStream) }.await()
+                }
+
+                // the hello message exchange should happen before the certificate validation
+                KeystoreHandler.assertSocketCertificateValid(socket, address.deviceId)
+
+                // now (after the validation) use the content of the hello message
+                HelloMessageHandler.processHelloMessage(helloMessage, configuration, address.deviceId)
+
+                // helpers for messages
+                val sendPostAuthMessageLock = Mutex()
+                val receivePostAuthMessageLock = Mutex()
+
+                suspend fun sendPostAuthMessage(message: MessageLite) = sendPostAuthMessageLock.withLock {
+                    PostAuthenticationMessageHandler.sendMessage(outputStream, message, markActivityOnSocket = {})
+                }
+
+                suspend fun receivePostAuthMessage() = receivePostAuthMessageLock.withLock {
+                    PostAuthenticationMessageHandler.receiveMessage(inputStream, markActivityOnSocket = {})
+                }
+
+                // cluster config exchange
+                val clusterConfig = coroutineScope {
+                    launch { sendPostAuthMessage(ClusterConfigHandler.buildClusterConfig(configuration, indexHandler, address.deviceId)) }
+                    async { receivePostAuthMessage() }.await()
+                }.second
+
+                if (!(clusterConfig is BlockExchangeProtos.ClusterConfig)) {
+                    throw IOException("first message was not a cluster config message")
+                }
+
+                val clusterConfigInfo = ClusterConfigHandler.handleReceivedClusterConfig(
+                        clusterConfig = clusterConfig,
+                        configuration = configuration,
+                        otherDeviceId = address.deviceId,
+                        indexHandler = indexHandler
+                )
+
+                fun hasFolder(folder: String) = clusterConfigInfo.sharedFolderIds.contains(folder)
+
+                val messageListeners = Collections.synchronizedMap(mutableMapOf<Int, CompletableDeferred<BlockExchangeProtos.Response>>())
+
+                try {
+                    launch {
+                        while (isActive) {
+                            val message = receivePostAuthMessage().second
+
+                            when (message) {
+                                is BlockExchangeProtos.Response -> {
+                                    val listener = messageListeners.remove(message.id)
+                                    listener
+                                            ?: throw IOException("got response ${message.id} but there is no response listener")
+                                    listener.complete(message)
+                                }
+                                is BlockExchangeProtos.Index -> {
+                                    indexHandler.handleIndexMessageReceivedEvent(
+                                            folderId = message.folder,
+                                            filesList = message.filesList,
+                                            clusterConfigInfo = clusterConfigInfo,
+                                            peerDeviceId = address.deviceId
+                                    )
+                                }
+                                is BlockExchangeProtos.IndexUpdate -> {
+                                    indexHandler.handleIndexMessageReceivedEvent(
+                                            folderId = message.folder,
+                                            filesList = message.filesList,
+                                            clusterConfigInfo = clusterConfigInfo,
+                                            peerDeviceId = address.deviceId
+                                    )
+                                }
+                                is BlockExchangeProtos.Request -> {
+                                    launch {
+                                        val response = requestHandler(message).await()
+
+                                        try {
+                                            sendPostAuthMessage(response)
+                                        } catch (ex: IOException) {
+                                            // the connection was closed in the time between - ignore it
+                                        }
+                                    }
+                                }
+                                is BlockExchangeProtos.Ping -> { /* nothing to do */
+                                }
+                                is BlockExchangeProtos.ClusterConfig -> throw IOException("received cluster config twice")
+                                is BlockExchangeProtos.Close -> socket.close()
+                                else -> throw IOException("unsupported message type ${message.javaClass}")
+                            }
+                        }
+                    }
+
+                    // send index messages - TODO: Why?
+                    for (folder in configuration.folders) {
+                        if (hasFolder(folder.folderId)) {
+                            sendPostAuthMessage(
+                                    BlockExchangeProtos.Index.newBuilder()
+                                            .setFolder(folder.folderId)
+                                            .build()
+                            )
+                        }
+                    }
+
+                    launch {
+                        // send ping all 90 seconds
+                        // TODO: only send when there were no messages for 90 seconds
+
+                        while (isActive) {
+                            delay(90 * 1000)
+
+                            launch { sendPostAuthMessage(BlockExchangeProtos.Ping.getDefaultInstance()) }
+                        }
+                    }
+
+                    var nextRequestId = 0
+
+                    channel.consumeEach { action ->
+                        when (action) {
+                            CloseConnectionAction -> throw InterruptedException()
+                            is SendRequestConnectionAction -> {
+                                val requestId = nextRequestId++
+
+                                messageListeners[requestId] = action.completableDeferred
+
+                                // async to allow handling the next action faster
+                                async {
+                                    try {
+                                        sendPostAuthMessage(
+                                                action.request.toBuilder()
+                                                        .setId(requestId)
+                                                        .build()
+                                        )
+                                    } catch (ex: Exception) {
+                                        action.completableDeferred.cancel(ex)
+                                    }
+                                }
+                            }
+                            is ConfirmIsConnectedAction -> {
+                                action.completableDeferred.complete(clusterConfigInfo)
+
+                                // otherwise, Kotlin would warn that the return
+                                // type does not match to the other branches
+                                null
+                            }
+                            is SendIndexUpdateAction -> {
+                                async {
+                                    try {
+                                        sendPostAuthMessage(action.message)
+                                    } catch (ex: Exception) {
+                                        action.completableDeferred.cancel(ex)
+                                    }
+                                }
+                            }
+                        }.let { /* prevents compiling if one action is not handled */ }
+                    }
+                } finally {
+                    // send close message
+                    withContext(NonCancellable) {
+                        if (socket.isConnected) {
+                            sendPostAuthMessage(BlockExchangeProtos.Close.getDefaultInstance())
+                        }
+                    }
+
+                    // cancel all pending listeners
+                    messageListeners.values.forEach { it.cancel() }
+                }
+            }
+        }.invokeOnCompletion { ex ->
+            if (ex != null) {
+                channel.cancel(ex)
+            } else {
+                channel.cancel()
+            }
+        }
+
+        return channel
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/MessageTypes.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import com.google.protobuf.MessageLite
+import net.syncthing.java.bep.BlockExchangeProtos
+
+object MessageTypes {
+    val messageTypes = listOf(
+            MessageTypeInfo(BlockExchangeProtos.MessageType.CLOSE, BlockExchangeProtos.Close::class.java) { BlockExchangeProtos.Close.parseFrom(it) },
+            MessageTypeInfo(BlockExchangeProtos.MessageType.CLUSTER_CONFIG, BlockExchangeProtos.ClusterConfig::class.java) { BlockExchangeProtos.ClusterConfig.parseFrom(it) },
+            MessageTypeInfo(BlockExchangeProtos.MessageType.DOWNLOAD_PROGRESS, BlockExchangeProtos.DownloadProgress::class.java) { BlockExchangeProtos.DownloadProgress.parseFrom(it) },
+            MessageTypeInfo(BlockExchangeProtos.MessageType.INDEX, BlockExchangeProtos.Index::class.java) { BlockExchangeProtos.Index.parseFrom(it) },
+            MessageTypeInfo(BlockExchangeProtos.MessageType.INDEX_UPDATE, BlockExchangeProtos.IndexUpdate::class.java) { BlockExchangeProtos.IndexUpdate.parseFrom(it) },
+            MessageTypeInfo(BlockExchangeProtos.MessageType.PING, BlockExchangeProtos.Ping::class.java) { BlockExchangeProtos.Ping.parseFrom(it) },
+            MessageTypeInfo(BlockExchangeProtos.MessageType.REQUEST, BlockExchangeProtos.Request::class.java) { BlockExchangeProtos.Request.parseFrom(it) },
+            MessageTypeInfo(BlockExchangeProtos.MessageType.RESPONSE, BlockExchangeProtos.Response::class.java) { BlockExchangeProtos.Response.parseFrom(it) }
+    )
+
+    val messageTypesByProtoMessageType = messageTypes.map { it.protoMessageType to it }.toMap()
+    val messageTypesByJavaClass = messageTypes.map { it.javaClass to it }.toMap()
+
+    fun getIdForMessage(message: MessageLite) = when (message) {
+        is BlockExchangeProtos.Request -> Integer.toString(message.id)
+        is BlockExchangeProtos.Response -> Integer.toString(message.id)
+        else -> Integer.toString(Math.abs(message.hashCode()))
+    }
+}
+
+data class MessageTypeInfo(
+        val protoMessageType: BlockExchangeProtos.MessageType,
+        val javaClass: Class<out MessageLite>,
+        val parseFrom: (data: ByteArray) -> MessageLite
+)

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/OpenConnection.kt

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import net.syncthing.java.client.protocol.rp.RelayClient
+import net.syncthing.java.core.beans.DeviceAddress
+import net.syncthing.java.core.configuration.Configuration
+import net.syncthing.java.core.security.KeystoreHandler
+import org.slf4j.LoggerFactory
+import javax.net.ssl.SSLSocket
+
+object OpenConnection {
+    private val logger = LoggerFactory.getLogger(OpenConnection::class.java)
+
+    fun openSocketConnection(
+            address: DeviceAddress,
+            configuration: Configuration
+    ): SSLSocket {
+        val keystoreHandler = KeystoreHandler.Loader().loadKeystore(configuration)
+
+        return when (address.type) {
+            DeviceAddress.AddressType.TCP -> {
+                logger.debug("opening tcp ssl connection")
+                keystoreHandler.createSocket(address.getSocketAddress())
+            }
+            DeviceAddress.AddressType.RELAY -> {
+                logger.debug("opening relay connection")
+                keystoreHandler.wrapSocket(RelayClient(configuration).openRelayConnection(address))
+            }
+            else -> throw UnsupportedOperationException("unsupported address type ${address.type}")
+        }
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/connectionactor/PostAuthenticationMessageHandler.kt

@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.connectionactor
+
+import com.google.protobuf.MessageLite
+import net.jpountz.lz4.LZ4Factory
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.core.utils.NetworkUtils
+import org.slf4j.LoggerFactory
+import java.io.DataInputStream
+import java.io.DataOutputStream
+import java.io.IOException
+import java.lang.reflect.InvocationTargetException
+import java.nio.ByteBuffer
+
+object PostAuthenticationMessageHandler {
+    private val logger = LoggerFactory.getLogger(PostAuthenticationMessageHandler::class.java)
+
+    fun sendMessage(
+            outputStream: DataOutputStream,
+            message: MessageLite,
+            markActivityOnSocket: () -> Unit
+    ) {
+        val messageTypeInfo = MessageTypes.messageTypesByJavaClass[message.javaClass]!!
+        val header = BlockExchangeProtos.Header.newBuilder()
+                .setCompression(BlockExchangeProtos.MessageCompression.NONE)
+                .setType(messageTypeInfo.protoMessageType)
+                .build()
+        val headerData = header.toByteArray()
+        val messageData = message.toByteArray() //TODO support compression
+
+        logger.debug("sending message type = {} {}", header.type, MessageTypes.getIdForMessage(message))
+        markActivityOnSocket()
+
+        outputStream.apply {
+            writeShort(headerData.size)
+            write(headerData)
+            writeInt(messageData.size)
+            write(messageData)
+            flush()
+        }
+
+        markActivityOnSocket()
+    }
+
+    fun receiveMessage(
+            inputStream: DataInputStream,
+            markActivityOnSocket: () -> Unit
+    ): Pair<BlockExchangeProtos.MessageType, MessageLite> {
+        val header = BlockExchangeProtos.Header.parseFrom(readHeader(
+                inputStream = inputStream,
+                retryReadingLength = true,
+                markActivityOnSocket = markActivityOnSocket
+        ))
+
+        var messageBuffer = readMessage(
+                inputStream = inputStream,
+                retryReadingLength = true,
+                markActivityOnSocket = markActivityOnSocket
+        )
+
+        if (header.compression == BlockExchangeProtos.MessageCompression.LZ4) {
+            val uncompressedLength = ByteBuffer.wrap(messageBuffer).int
+            messageBuffer = LZ4Factory.fastestInstance().fastDecompressor().decompress(messageBuffer, 4, uncompressedLength)
+        }
+
+        val messageTypeInfo = MessageTypes.messageTypesByProtoMessageType[header.type]
+        NetworkUtils.assertProtocol(messageTypeInfo != null) {"unsupported message type = ${header.type}"}
+
+        try {
+            return header.type to messageTypeInfo!!.parseFrom(messageBuffer)
+        } catch (e: Exception) {
+            when (e) {
+                is IllegalAccessException, is IllegalArgumentException, is InvocationTargetException, is NoSuchMethodException, is SecurityException ->
+                    throw IOException(e)
+                else -> throw e
+            }
+        }
+    }
+
+    private fun readHeader(
+            inputStream: DataInputStream,
+            markActivityOnSocket: () -> Unit,
+            retryReadingLength: Boolean
+    ): ByteArray {
+        var headerLength = inputStream.readShort().toInt()
+
+        // TODO: what is this good for?
+        if (retryReadingLength) {
+            while (headerLength == 0) {
+                logger.warn("got headerLength == 0, skipping short")
+                headerLength = inputStream.readShort().toInt()
+            }
+        }
+
+        markActivityOnSocket()
+
+        NetworkUtils.assertProtocol(headerLength > 0) {"invalid length, must be > 0, got $headerLength"}
+
+        return ByteArray(headerLength).apply {
+            inputStream.readFully(this)
+        }
+    }
+
+    private fun readMessage(
+            inputStream: DataInputStream,
+            markActivityOnSocket: () -> Unit,
+            retryReadingLength: Boolean
+    ): ByteArray {
+        var messageLength = inputStream.readInt()
+
+        // TODO: what is this good for?
+        if (retryReadingLength) {
+            while (messageLength == 0) {
+                logger.warn("received readInt() == 0, expecting 'bep message header length' (int >0), ignoring (keepalive?)")
+                messageLength = inputStream.readInt()
+            }
+        }
+
+        NetworkUtils.assertProtocol(messageLength >= 0) {"invalid length, must be >= 0, got $messageLength"}
+
+        val messageBuffer = ByteArray(messageLength)
+        inputStream.readFully(messageBuffer)
+        markActivityOnSocket()
+
+        return messageBuffer
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/folder/FolderBrowser.kt

@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.folder
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.ConflatedBroadcastChannel
+import kotlinx.coroutines.channels.consumeEach
+import kotlinx.coroutines.channels.first
+import kotlinx.coroutines.channels.produce
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import net.syncthing.java.bep.index.IndexHandler
+import net.syncthing.java.core.beans.FolderStats
+import net.syncthing.java.core.configuration.Configuration
+import java.io.Closeable
+
+class FolderBrowser internal constructor(private val indexHandler: IndexHandler, private val configuration: Configuration) : Closeable {
+    private val job = Job()
+    private val foldersStatus = ConflatedBroadcastChannel<Map<String, FolderStatus>>()
+
+    init {
+        GlobalScope.launch (job) {
+            // get initial status
+            val currentFolderStats = mutableMapOf<String, FolderStats>()
+
+            var currentIndexInfo = withContext(Dispatchers.IO) {
+                indexHandler.indexRepository.runInTransaction { indexTransaction ->
+                    configuration.folders.map { it.folderId }.forEach { folderId ->
+                        currentFolderStats[folderId] = indexTransaction.findFolderStats(folderId) ?: FolderStats.createDummy(folderId)
+                    }
+
+                    indexTransaction.findAllIndexInfos().groupBy { it.folderId }.toMutableMap()
+                }
+            }
+
+            // send status
+            suspend fun dispatch() {
+                foldersStatus.send(
+                        configuration.folders.map { info ->
+                            FolderStatus(
+                                    info = info,
+                                    stats = currentFolderStats[info.folderId] ?: FolderStats.createDummy(info.folderId),
+                                    indexInfo = currentIndexInfo[info.folderId] ?: emptyList()
+                            )
+                        }.associateBy { it.info.folderId }
+                )
+            }
+
+            dispatch()
+
+            // handle changes
+            val updateLock = Mutex()
+
+            async {
+                indexHandler.subscribeFolderStatsUpdatedEvents().consumeEach { folderStats ->
+                    updateLock.withLock {
+                        currentFolderStats[folderStats.folderId] = folderStats
+
+                        dispatch()
+                    }
+                }
+            }
+
+            async {
+                indexHandler.subscribeToOnIndexRecordAcquiredEvents().consumeEach { event ->
+                    updateLock.withLock {
+                        val oldList = currentIndexInfo[event.folderId] ?: emptyList()
+                        val newList = oldList.filter { it.deviceId != event.indexInfo.deviceId } + event.indexInfo
+                        currentIndexInfo[event.folderId] = newList
+
+                        dispatch()
+                    }
+                }
+            }
+        }
+    }
+
+    fun folderInfoAndStatusStream() = GlobalScope.produce {
+        foldersStatus.openSubscription().consumeEach { folderStats ->
+            send(
+                    folderStats
+                            .values
+                            .sortedBy { it.info.label }
+            )
+        }
+    }
+
+    suspend fun folderInfoAndStatusList(): List<FolderStatus> = folderInfoAndStatusStream().first()
+
+    suspend fun getFolderStatus(folder: String): FolderStatus {
+        return getFolderStatus(folder, foldersStatus.openSubscription().first())
+    }
+
+    fun getFolderStatusSync(folder: String) = runBlocking { getFolderStatus(folder) }
+
+    private fun getFolderStatus(folder: String, folderStatus: Map<String, FolderStatus>) = folderStatus[folder] ?: FolderStatus.createDummy(folder)
+
+    override fun close() {
+        job.cancel()
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/folder/FolderStatus.kt

@@ -0,0 +1,29 @@
+package net.syncthing.java.bep.folder
+
+import net.syncthing.java.bep.utils.longMaxBy
+import net.syncthing.java.core.beans.FolderInfo
+import net.syncthing.java.core.beans.FolderStats
+import net.syncthing.java.core.beans.IndexInfo
+
+data class FolderStatus(
+        val info: FolderInfo,
+        val stats: FolderStats,
+        val indexInfo: List<IndexInfo>
+) {
+    companion object {
+        fun createDummy(folder: String) = FolderStatus(
+                info = FolderInfo(folder, folder),
+                stats = FolderStats.createDummy(folder),
+                indexInfo = emptyList()
+        )
+    }
+
+    val missingIndexUpdates: Long by lazy {
+        Math.max(
+                0,
+                indexInfo.longMaxBy ({ it.maxSequence }, 0) -
+                        indexInfo.longMaxBy ({ it.localSequence }, 0)
+
+        )
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/FolderStatsUpdateCollector.kt

@@ -0,0 +1,17 @@
+package net.syncthing.java.bep.index
+
+import java.util.*
+
+class FolderStatsUpdateCollector (val folderId: String) {
+    var deltaFileCount = 0L
+    var deltaDirCount = 0L
+    var deltaSize = 0L
+    var lastModified = Date(0)
+
+    fun isEmpty() = (
+            deltaFileCount == 0L &&
+                    deltaDirCount == 0L &&
+                    deltaSize == 0L &&
+                    lastModified.time == 0L
+            )
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/IndexElementProcessor.kt

@@ -0,0 +1,173 @@
+package net.syncthing.java.bep.index
+
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.core.beans.BlockInfo
+import net.syncthing.java.core.beans.FileBlocks
+import net.syncthing.java.core.beans.FileInfo
+import net.syncthing.java.core.interfaces.IndexTransaction
+import org.bouncycastle.util.encoders.Hex
+import org.slf4j.LoggerFactory
+import java.io.IOException
+import java.util.*
+
+object IndexElementProcessor {
+    private val logger = LoggerFactory.getLogger(IndexElementProcessor::class.java)
+
+    fun pushRecords(
+            transaction: IndexTransaction,
+            folder: String,
+            updates: List<BlockExchangeProtos.FileInfo>,
+            oldRecords: Map<String, FileInfo>,
+            folderStatsUpdateCollector: FolderStatsUpdateCollector,
+            enableDetailedException: Boolean
+    ): List<FileInfo> {
+        // this always keeps the last version per path
+        val filesToProcess = updates
+                .sortedBy { it.sequence }
+                .reversed()
+                .distinctBy { it.name /* this is the whole path */ }
+                .reversed()
+
+        val preparedUpdates = filesToProcess.mapNotNull {
+            try {
+                prepareUpdate(folder, it)
+            } catch (ex: Exception) {
+                if (enableDetailedException) {
+                    throw IOException("error processing index update: ${it.name}", ex)
+                } else {
+                    throw ex
+                }
+            }
+        }
+
+        val updatesToApply = preparedUpdates.filter { shouldUpdateRecord(oldRecords[it.first.path], it.first) }
+
+        transaction.updateFileInfoAndBlocks(
+                fileInfos = updatesToApply.map { it.first },
+                fileBlocks = updatesToApply.mapNotNull { it.second }
+        )
+
+        for ((newRecord) in updatesToApply) {
+            updateFolderStatsCollector(oldRecords[newRecord.path], newRecord, folderStatsUpdateCollector)
+        }
+
+        return updatesToApply.map { it.first }
+    }
+
+    fun pushRecord(
+            transaction: IndexTransaction,
+            folder: String,
+            bepFileInfo: BlockExchangeProtos.FileInfo,
+            folderStatsUpdateCollector: FolderStatsUpdateCollector,
+            oldRecord: FileInfo?
+    ): FileInfo? {
+        val update = prepareUpdate(folder, bepFileInfo)
+
+        return if (update != null) {
+            addRecord(
+                    transaction = transaction,
+                    newRecord = update.first,
+                    fileBlocks = update.second,
+                    folderStatsUpdateCollector = folderStatsUpdateCollector,
+                    oldRecord = oldRecord
+            )
+        } else {
+            null
+        }
+    }
+
+    private fun prepareUpdate(
+            folder: String,
+            bepFileInfo: BlockExchangeProtos.FileInfo
+    ): Pair<FileInfo, FileBlocks?>? {
+        val builder = FileInfo.Builder()
+                .setFolder(folder)
+                .setPath(bepFileInfo.name)
+                .setLastModified(Date(bepFileInfo.modifiedS * 1000 + bepFileInfo.modifiedNs / 1000000))
+                .setVersionList((if (bepFileInfo.hasVersion()) bepFileInfo.version.countersList else null ?: emptyList()).map { record -> FileInfo.Version(record.id, record.value) })
+                .setDeleted(bepFileInfo.deleted)
+
+        var fileBlocks: FileBlocks? = null
+
+        when (bepFileInfo.type) {
+            BlockExchangeProtos.FileInfoType.FILE -> {
+                fileBlocks = FileBlocks(folder, builder.getPath()!!, ((bepFileInfo.blocksList ?: emptyList())).map { record ->
+                    BlockInfo(record.offset, record.size, Hex.toHexString(record.hash.toByteArray()))
+                })
+                builder
+                        .setTypeFile()
+                        .setHash(fileBlocks.hash)
+                        .setSize(bepFileInfo.size)
+            }
+            BlockExchangeProtos.FileInfoType.DIRECTORY -> builder.setTypeDir()
+            else -> {
+                logger.warn("unsupported file type = {}, discarding file info", bepFileInfo.type)
+                return null
+            }
+        }
+
+        return builder.build() to fileBlocks
+    }
+
+    private fun shouldUpdateRecord(
+            oldRecord: FileInfo?,
+            newRecord: FileInfo
+    ) = oldRecord == null || newRecord.lastModified >= oldRecord.lastModified
+
+    private fun addRecord(
+            transaction: IndexTransaction,
+            newRecord: FileInfo,
+            oldRecord: FileInfo?,
+            fileBlocks: FileBlocks?,
+            folderStatsUpdateCollector: FolderStatsUpdateCollector
+    ): FileInfo? {
+        return if (shouldUpdateRecord(oldRecord, newRecord)) {
+            logger.trace("discarding record = {}, modified before local record", newRecord)
+            null
+        } else {
+            logger.trace("loaded new record = {}", newRecord)
+
+            transaction.updateFileInfo(newRecord, fileBlocks)
+            updateFolderStatsCollector(oldRecord, newRecord, folderStatsUpdateCollector)
+
+            newRecord
+        }
+    }
+
+    private fun updateFolderStatsCollector(
+            oldRecord: FileInfo?,
+            newRecord: FileInfo,
+            folderStatsUpdateCollector: FolderStatsUpdateCollector
+    ) {
+        val oldMissing = oldRecord == null || oldRecord.isDeleted
+        val newMissing = newRecord.isDeleted
+        val oldSizeMissing = oldMissing || !oldRecord!!.isFile()
+        val newSizeMissing = newMissing || !newRecord.isFile()
+
+        if (!oldSizeMissing) {
+            folderStatsUpdateCollector.deltaSize -= oldRecord!!.size!!
+        }
+
+        if (!newSizeMissing) {
+            folderStatsUpdateCollector.deltaSize += newRecord.size!!
+        }
+
+        if (!oldMissing) {
+            if (oldRecord!!.isFile()) {
+                folderStatsUpdateCollector.deltaFileCount--
+            } else if (oldRecord.isDirectory()) {
+                folderStatsUpdateCollector.deltaDirCount--
+            }
+        }
+
+        if (!newMissing) {
+            if (newRecord.isFile()) {
+                folderStatsUpdateCollector.deltaFileCount++
+            } else if (newRecord.isDirectory()) {
+                folderStatsUpdateCollector.deltaDirCount++
+            }
+        }
+
+        folderStatsUpdateCollector.lastModified = newRecord.lastModified
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/IndexHandler.kt

@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.index
+
+import kotlinx.coroutines.channels.BroadcastChannel
+import kotlinx.coroutines.channels.consume
+import kotlinx.coroutines.withTimeoutOrNull
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.bep.connectionactor.ClusterConfigInfo
+import net.syncthing.java.bep.connectionactor.ConnectionActorWrapper
+import net.syncthing.java.bep.folder.FolderBrowser
+import net.syncthing.java.bep.index.browser.IndexBrowser
+import net.syncthing.java.core.beans.*
+import net.syncthing.java.core.configuration.Configuration
+import net.syncthing.java.core.interfaces.IndexRepository
+import net.syncthing.java.core.interfaces.IndexTransaction
+import net.syncthing.java.core.interfaces.TempRepository
+import org.apache.commons.lang3.tuple.Pair
+import org.slf4j.LoggerFactory
+import java.io.Closeable
+import java.io.IOException
+
+data class IndexRecordAcquiredEvent(val folderId: String, val files: List<FileInfo>, val indexInfo: IndexInfo)
+
+class IndexHandler(
+        configuration: Configuration,
+        val indexRepository: IndexRepository,
+        tempRepository: TempRepository,
+        enableDetailedException: Boolean
+) : Closeable {
+    private val logger = LoggerFactory.getLogger(javaClass)
+    private val onIndexRecordAcquiredEvents = BroadcastChannel<IndexRecordAcquiredEvent>(capacity = 16)
+    private val onFullIndexAcquiredEvents = BroadcastChannel<String>(capacity = 16)
+    private val onFolderStatsUpdatedEvents = BroadcastChannel<FolderStats>(capacity = 16)
+
+    private val indexMessageProcessor = IndexMessageQueueProcessor(
+            indexRepository = indexRepository,
+            tempRepository = tempRepository,
+            isRemoteIndexAcquired = ::isRemoteIndexAcquired,
+            onIndexRecordAcquiredEvents = onIndexRecordAcquiredEvents,
+            onFullIndexAcquiredEvents = onFullIndexAcquiredEvents,
+            onFolderStatsUpdatedEvents = onFolderStatsUpdatedEvents,
+            enableDetailedException = enableDetailedException
+    )
+
+    fun subscribeToOnFullIndexAcquiredEvents() = onFullIndexAcquiredEvents.openSubscription()
+    fun subscribeToOnIndexRecordAcquiredEvents() = onIndexRecordAcquiredEvents.openSubscription()
+    fun subscribeFolderStatsUpdatedEvents() = onFolderStatsUpdatedEvents.openSubscription()
+
+    fun getNextSequenceNumber() = indexRepository.runInTransaction { it.getSequencer().nextSequence() }
+
+    fun clearIndex() {
+        indexRepository.runInTransaction { it.clearIndex() }
+    }
+
+    private fun isRemoteIndexAcquiredWithoutTransaction(clusterConfigInfo: ClusterConfigInfo, peerDeviceId: DeviceId): Boolean {
+        return indexRepository.runInTransaction { transaction -> isRemoteIndexAcquired(clusterConfigInfo, peerDeviceId, transaction) }
+    }
+
+    private fun isRemoteIndexAcquired(clusterConfigInfo: ClusterConfigInfo, peerDeviceId: DeviceId, transaction: IndexTransaction): Boolean {
+        return clusterConfigInfo.sharedFolderIds.find { sharedFolderId ->
+            // try to find one folder which is not yet ready
+            val indexSequenceInfo = transaction.findIndexInfoByDeviceAndFolder(peerDeviceId, sharedFolderId)
+
+            indexSequenceInfo == null || indexSequenceInfo.localSequence < indexSequenceInfo.maxSequence
+        } == null
+    }
+
+    // the old implementation kept waiting when index updates were still happening, but waiting 30 seconds should be enough
+    suspend fun waitForRemoteIndexAcquiredWithTimeout(connectionHandler: ConnectionActorWrapper, timeoutSecs: Long? = null): IndexHandler {
+        val timeoutMillis = (timeoutSecs ?: DEFAULT_INDEX_TIMEOUT) * 1000
+
+        val ok = withTimeoutOrNull(timeoutMillis) {
+            waitForRemoteIndexAcquiredWithoutTimeout(connectionHandler)
+
+            true
+        } ?: false
+
+        if (!ok) {
+            throw IOException("unable to acquire index from connection $connectionHandler, timeout reached!")
+        }
+
+        return this
+    }
+
+    suspend fun waitForRemoteIndexAcquiredWithoutTimeout(connectionHandler: ConnectionActorWrapper) {
+        val events = onFullIndexAcquiredEvents.openSubscription()
+
+        events.consume {
+            fun isDone() = isRemoteIndexAcquiredWithoutTransaction(connectionHandler.getClusterConfig(), connectionHandler.deviceId)
+
+            if (isDone()) {
+                return
+            }
+
+            for (event in events) {
+                if (isDone()) {
+                    return
+                }
+            }
+        }
+    }
+
+    suspend fun handleClusterConfigMessageProcessedEvent(clusterConfig: BlockExchangeProtos.ClusterConfig) {
+        val updatedIndexInfos = indexRepository.runInTransaction { transaction ->
+            val updatedIndexInfos = mutableListOf<IndexInfo>()
+
+            for (folderRecord in clusterConfig.foldersList) {
+                val folder = folderRecord.id
+                logger.debug("acquired folder info from cluster config = {}", folder)
+                for (deviceRecord in folderRecord.devicesList) {
+                    val deviceId = DeviceId.fromHashData(deviceRecord.id.toByteArray())
+                    if (deviceRecord.hasIndexId() && deviceRecord.hasMaxSequence()) {
+                        val folderIndexInfo = UpdateIndexInfo.updateIndexInfo(transaction, folder, deviceId, deviceRecord.indexId, deviceRecord.maxSequence, null)
+                        logger.debug("acquired folder index info from cluster config = {}", folderIndexInfo)
+                        updatedIndexInfos.add(folderIndexInfo)
+                    }
+                }
+            }
+
+            updatedIndexInfos
+        }
+
+        updatedIndexInfos.forEach {
+            onIndexRecordAcquiredEvents.send(
+                    IndexRecordAcquiredEvent(
+                            folderId = it.folderId,
+                            indexInfo = it,
+                            files = emptyList()
+                    )
+            )
+        }
+    }
+
+    internal suspend fun handleIndexMessageReceivedEvent(folderId: String, filesList: List<BlockExchangeProtos.FileInfo>, clusterConfigInfo: ClusterConfigInfo, peerDeviceId: DeviceId) {
+        indexMessageProcessor.handleIndexMessageReceivedEvent(folderId, filesList, clusterConfigInfo, peerDeviceId)
+    }
+
+    fun getFileInfoByPath(folder: String, path: String): FileInfo? {
+        return indexRepository.runInTransaction { it.findFileInfo(folder, path) }
+    }
+
+    fun getFileInfoAndBlocksByPath(folder: String, path: String): Pair<FileInfo, FileBlocks>? {
+        return indexRepository.runInTransaction { transaction ->
+            val fileInfo = transaction.findFileInfo(folder, path)
+
+            if (fileInfo == null) {
+                null
+            } else {
+                val fileBlocks = transaction.findFileBlocks(folder, path)
+
+                assert(fileInfo.isFile())
+                checkNotNull(fileBlocks) {"file blocks not found for file info = $fileInfo"}
+
+                FileInfo.checkBlocks(fileInfo, fileBlocks)
+
+                Pair.of(fileInfo, fileBlocks)
+            }
+        }
+    }
+
+    val folderBrowser = FolderBrowser(this, configuration)
+    val indexBrowser = IndexBrowser(indexRepository, this)
+
+    suspend fun sendFolderStatsUpdate(event: FolderStats) {
+        onFolderStatsUpdatedEvents.send(event)
+    }
+
+    override fun close() {
+        onIndexRecordAcquiredEvents.close()
+        onFullIndexAcquiredEvents.close()
+        indexMessageProcessor.stop()
+    }
+
+    companion object {
+
+        private const val DEFAULT_INDEX_TIMEOUT: Long = 30
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/IndexMessageProcessor.kt

@@ -0,0 +1,64 @@
+package net.syncthing.java.bep.index
+
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.core.beans.DeviceId
+import net.syncthing.java.core.beans.FileInfo
+import net.syncthing.java.core.beans.FolderStats
+import net.syncthing.java.core.beans.IndexInfo
+import net.syncthing.java.core.interfaces.IndexTransaction
+import org.slf4j.LoggerFactory
+
+object IndexMessageProcessor {
+    private val logger = LoggerFactory.getLogger(IndexMessageProcessor::class.java)
+
+    fun doHandleIndexMessageReceivedEvent(
+            message: BlockExchangeProtos.IndexUpdate,
+            peerDeviceId: DeviceId,
+            transaction: IndexTransaction,
+            enableDetailedException: Boolean
+    ): Result {
+        val folderId = message.folder
+
+        logger.debug("processing {} index records for folder {}", message.filesList.size, folderId)
+
+        val oldRecords = transaction.findFileInfo(folderId, message.filesList.map { it.name })
+        val folderStatsUpdateCollector = FolderStatsUpdateCollector(message.folder)
+
+        val newRecords = IndexElementProcessor.pushRecords(
+                transaction = transaction,
+                oldRecords = oldRecords,
+                folder = folderId,
+                folderStatsUpdateCollector = folderStatsUpdateCollector,
+                updates = message.filesList,
+                enableDetailedException = enableDetailedException
+        )
+
+        var sequence: Long = -1
+
+        for (newRecord in message.filesList) {
+            sequence = Math.max(newRecord.sequence, sequence)
+        }
+
+        handleFolderStatsUpdate(transaction, folderStatsUpdateCollector)
+
+        val newIndexInfo = UpdateIndexInfo.updateIndexInfo(transaction, folderId, peerDeviceId, null, null, sequence)
+
+        return Result(newIndexInfo, newRecords.toList(), transaction.findFolderStats(folderId) ?: FolderStats.createDummy(folderId))
+    }
+
+    fun handleFolderStatsUpdate(transaction: IndexTransaction, folderStatsUpdateCollector: FolderStatsUpdateCollector) {
+        if (folderStatsUpdateCollector.isEmpty()) {
+            return
+        }
+
+        transaction.updateOrInsertFolderStats(
+                folder = folderStatsUpdateCollector.folderId,
+                deltaSize = folderStatsUpdateCollector.deltaSize,
+                deltaFileCount = folderStatsUpdateCollector.deltaFileCount,
+                deltaDirCount = folderStatsUpdateCollector.deltaDirCount,
+                lastUpdate = folderStatsUpdateCollector.lastModified
+        )
+    }
+
+    data class Result(val newIndexInfo: IndexInfo, val updatedFiles: List<FileInfo>, val newFolderStats: FolderStats)
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/IndexMessageQueueProcessor.kt

@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.index
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.channels.BroadcastChannel
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.channels.consumeEach
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.bep.connectionactor.ClusterConfigInfo
+import net.syncthing.java.core.beans.DeviceId
+import net.syncthing.java.core.beans.FolderStats
+import net.syncthing.java.core.interfaces.IndexRepository
+import net.syncthing.java.core.interfaces.IndexTransaction
+import net.syncthing.java.core.interfaces.TempRepository
+import org.slf4j.LoggerFactory
+
+class IndexMessageQueueProcessor (
+        private val indexRepository: IndexRepository,
+        private val tempRepository: TempRepository,
+        private val onIndexRecordAcquiredEvents: BroadcastChannel<IndexRecordAcquiredEvent>,
+        private val onFullIndexAcquiredEvents: BroadcastChannel<String>,
+        private val onFolderStatsUpdatedEvents: BroadcastChannel<FolderStats>,
+        private val isRemoteIndexAcquired: (ClusterConfigInfo, DeviceId, IndexTransaction) -> Boolean,
+        private val enableDetailedException: Boolean
+) {
+    private data class IndexUpdateAction(val update: BlockExchangeProtos.IndexUpdate, val clusterConfigInfo: ClusterConfigInfo, val peerDeviceId: DeviceId)
+    private data class StoredIndexUpdateAction(val updateId: String, val clusterConfigInfo: ClusterConfigInfo, val peerDeviceId: DeviceId)
+
+    companion object {
+        private val logger = LoggerFactory.getLogger(IndexMessageQueueProcessor::class.java)
+        private const val BATCH_SIZE = 128
+    }
+
+    private val job = Job()
+    private val indexUpdateIncomingLock = Mutex()
+    private val indexUpdateProcessStoredQueue = Channel<StoredIndexUpdateAction>(capacity = Channel.UNLIMITED)
+    private val indexUpdateProcessingQueue = Channel<IndexUpdateAction>(capacity = Channel.RENDEZVOUS)
+
+    suspend fun handleIndexMessageReceivedEvent(folderId: String, filesList: List<BlockExchangeProtos.FileInfo>, clusterConfigInfo: ClusterConfigInfo, peerDeviceId: DeviceId) {
+        filesList.chunked(BATCH_SIZE).forEach { chunck ->
+            handleIndexMessageReceivedEventWithoutChuncking(folderId, chunck, clusterConfigInfo, peerDeviceId)
+        }
+    }
+
+    suspend fun handleIndexMessageReceivedEventWithoutChuncking(folderId: String, filesList: List<BlockExchangeProtos.FileInfo>, clusterConfigInfo: ClusterConfigInfo, peerDeviceId: DeviceId) {
+        indexUpdateIncomingLock.withLock {
+            logger.info("received index message event, preparing")
+
+            val data = BlockExchangeProtos.IndexUpdate.newBuilder()
+                    .addAllFiles(filesList)
+                    .setFolder(folderId)
+                    .build()
+
+            if (indexUpdateProcessingQueue.offer(IndexUpdateAction(data, clusterConfigInfo, peerDeviceId))) {
+                // message is beeing processed now
+            } else {
+                val key = tempRepository.pushTempData(data.toByteArray())
+
+                logger.debug("received index message event, stored to temp record {}, queuing for processing", key)
+                indexUpdateProcessStoredQueue.send(StoredIndexUpdateAction(key, clusterConfigInfo, peerDeviceId))
+            }
+        }
+    }
+
+    init {
+        GlobalScope.launch(Dispatchers.IO + job) {
+            indexUpdateProcessingQueue.consumeEach {
+                doHandleIndexMessageReceivedEvent(it)
+            }
+        }
+
+        GlobalScope.launch(Dispatchers.IO + job) {
+            indexUpdateProcessStoredQueue.consumeEach { action ->
+                logger.debug("processing index message event from temp record {}", action.updateId)
+
+                val data = tempRepository.popTempData(action.updateId)
+                val message = BlockExchangeProtos.IndexUpdate.parseFrom(data)
+
+                indexUpdateProcessingQueue.send(IndexUpdateAction(
+                        message,
+                        action.clusterConfigInfo,
+                        action.peerDeviceId
+                ))
+            }
+        }
+    }
+
+    private suspend fun doHandleIndexMessageReceivedEvent(action: IndexUpdateAction) {
+        val (message, clusterConfigInfo, peerDeviceId) = action
+
+        logger.info("processing index message with {} records", message.filesCount)
+
+        val (indexResult, wasIndexAcquired) = indexRepository.runInTransaction { indexTransaction ->
+            val wasIndexAcquiredBefore = isRemoteIndexAcquired(clusterConfigInfo, peerDeviceId, indexTransaction)
+
+            val startTime = System.currentTimeMillis()
+
+            val indexResult = IndexMessageProcessor.doHandleIndexMessageReceivedEvent(
+                    message = message,
+                    peerDeviceId = peerDeviceId,
+                    transaction = indexTransaction,
+                    enableDetailedException = enableDetailedException
+            )
+
+            val endTime = System.currentTimeMillis()
+
+            logger.info("processed {} index records, acquired {} in ${endTime - startTime} ms", message.filesCount, indexResult.updatedFiles.size)
+
+            logger.debug("index info = {}", indexResult.newIndexInfo)
+
+            indexResult to ((!wasIndexAcquiredBefore) && isRemoteIndexAcquired(clusterConfigInfo, peerDeviceId, indexTransaction))
+        }
+
+        if (indexResult.updatedFiles.isNotEmpty()) {
+            onIndexRecordAcquiredEvents.send(IndexRecordAcquiredEvent(message.folder, indexResult.updatedFiles, indexResult.newIndexInfo))
+        }
+
+        onFolderStatsUpdatedEvents.send(indexResult.newFolderStats)
+
+        if (wasIndexAcquired) {
+            logger.debug("index acquired")
+            onFullIndexAcquiredEvents.send(message.folder)
+        }
+    }
+
+    fun stop() {
+        logger.info("stopping index record processor")
+        job.cancel()
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/UpdateIndexInfo.kt

@@ -0,0 +1,50 @@
+package net.syncthing.java.bep.index
+
+import net.syncthing.java.core.beans.DeviceId
+import net.syncthing.java.core.beans.IndexInfo
+import net.syncthing.java.core.interfaces.IndexTransaction
+
+object UpdateIndexInfo {
+    fun updateIndexInfo(
+            transaction: IndexTransaction,
+            folder: String,
+            deviceId: DeviceId,
+            indexId: Long?,
+            maxSequence: Long?,
+            localSequence: Long?
+    ): IndexInfo {
+        val oldIndexSequenceInfo = transaction.findIndexInfoByDeviceAndFolder(deviceId, folder)
+
+        var newIndexSequenceInfo = oldIndexSequenceInfo ?: kotlin.run {
+            assert(indexId != null) {
+                "index sequence info not found, and supplied null index id (folder = $folder, device = $deviceId)"
+            }
+
+            IndexInfo(
+                    folderId = folder,
+                    deviceId = deviceId.deviceId,
+                    indexId = indexId!!,
+                    localSequence = 0,
+                    maxSequence = -1
+            )
+        }
+
+        if (indexId != null && indexId != newIndexSequenceInfo.indexId) {
+            newIndexSequenceInfo = newIndexSequenceInfo.copy(indexId = indexId)
+        }
+
+        if (maxSequence != null && maxSequence > newIndexSequenceInfo.maxSequence) {
+            newIndexSequenceInfo = newIndexSequenceInfo.copy(maxSequence = maxSequence)
+        }
+
+        if (localSequence != null && localSequence > newIndexSequenceInfo.localSequence) {
+            newIndexSequenceInfo = newIndexSequenceInfo.copy(localSequence = localSequence)
+        }
+
+        if (oldIndexSequenceInfo != newIndexSequenceInfo) {
+            transaction.updateIndexInfo(newIndexSequenceInfo)
+        }
+
+        return newIndexSequenceInfo
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/browser/DirectoryListing.kt

@@ -0,0 +1,25 @@
+package net.syncthing.java.bep.index.browser
+
+import net.syncthing.java.core.beans.FileInfo
+import net.syncthing.java.core.utils.PathUtils
+
+sealed class DirectoryListing {
+    abstract val folder: String
+    abstract val path: String
+}
+
+data class DirectoryContentListing(
+        val directoryInfo: FileInfo,
+        val parentEntry: FileInfo?,
+        val entries: List<FileInfo>
+): DirectoryListing() {
+    override val folder = directoryInfo.folder
+    override val path = directoryInfo.path
+}
+
+data class DirectoryNotFoundListing(
+        override val folder: String,
+        override val path: String
+): DirectoryListing() {
+    val theoreticalParentPath = if (PathUtils.isRoot(path)) null else PathUtils.getParentPath(path)
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/index/browser/IndexBrowser.kt

@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.bep.index.browser
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.channels.consume
+import kotlinx.coroutines.channels.produce
+import kotlinx.coroutines.withContext
+import net.syncthing.java.bep.index.IndexHandler
+import net.syncthing.java.core.beans.FileInfo
+import net.syncthing.java.core.interfaces.IndexRepository
+import net.syncthing.java.core.interfaces.IndexTransaction
+import net.syncthing.java.core.utils.PathUtils
+import java.util.*
+
+class IndexBrowser internal constructor(
+        private val indexRepository: IndexRepository,
+        private val indexHandler: IndexHandler
+) {
+    companion object {
+        val sortAlphabeticallyDirectoriesFirst: Comparator<FileInfo> =
+                compareBy<FileInfo>({!isParent(it) }, {!it.isDirectory()})
+                        .thenBy { it.fileName.toLowerCase() }
+
+        val sortByLastModification: Comparator<FileInfo> =
+                compareBy<FileInfo>({!isParent(it) }, {it.lastModified})
+                        .thenBy { it.fileName.toLowerCase() }
+
+        private fun isParent(fileInfo: FileInfo) = PathUtils.isParent(fileInfo.path)
+
+        fun getPathFileName(path: String) = PathUtils.getFileName(path)
+
+        const val ROOT_PATH = PathUtils.ROOT_PATH
+    }
+
+    fun getDirectoryListing(folder: String, path: String): DirectoryListing = indexRepository.runInTransaction { indexTransaction ->
+        val entries = indexTransaction.findNotDeletedFilesByFolderAndParent(folder, path)
+        val parentPath = if (PathUtils.isRoot(path)) null else PathUtils.getParentPath(path)
+        val parentEntry = if (PathUtils.isRoot(path)) null else getFileInfoByPathAllowNull(folder, PathUtils.getParentPath(path), indexTransaction)
+        val directoryInfo = getFileInfoByPathAllowNull(folder, path, indexTransaction)
+
+        if ((parentPath != null && parentEntry == null) || directoryInfo == null || directoryInfo.type != FileInfo.FileType.DIRECTORY) {
+            DirectoryNotFoundListing(folder, path)
+        } else {
+            DirectoryContentListing(
+                    entries = entries,
+                    parentEntry = parentEntry,
+                    directoryInfo = directoryInfo
+            )
+        }
+    }
+
+    fun streamDirectoryListing(folder: String, path: String) = GlobalScope.produce {
+        indexHandler.subscribeToOnIndexRecordAcquiredEvents().consume {
+            val directoryName = PathUtils.getFileName(path)
+            val parentPath = if (PathUtils.isRoot(path)) null else PathUtils.getParentPath(path)
+            val parentDirectoryName = if (parentPath != null) PathUtils.getFileName(parentPath) else null
+            val parentParentPath = if (parentPath == null || PathUtils.isRoot(parentPath)) null else PathUtils.getParentPath(parentPath)
+
+            // get the initial state
+            var (entries, parentEntry, directoryInfo) = withContext (Dispatchers.IO) {
+                indexRepository.runInTransaction { indexTransaction ->
+                    val entries = indexTransaction.findNotDeletedFilesByFolderAndParent(folder, path)
+                    val parentEntry = if (PathUtils.isRoot(path)) null else getFileInfoByPathAllowNull(folder, PathUtils.getParentPath(path), indexTransaction)
+                    val directoryInfo = getFileInfoByPathAllowNull(folder, path, indexTransaction)
+
+                    Triple(entries, parentEntry, directoryInfo)
+                }
+            }
+
+            var previousStatus: DirectoryListing? = null
+
+            suspend fun dispatch() {
+                // let Kotlin understand that the value does not change during running this
+                val directoryInfo = directoryInfo
+
+                val newStatus = if ((parentPath != null && parentEntry == null) || directoryInfo == null || directoryInfo.type != FileInfo.FileType.DIRECTORY) {
+                    DirectoryNotFoundListing(folder, path)
+                } else {
+                    DirectoryContentListing(
+                            entries = entries,
+                            parentEntry = parentEntry,
+                            directoryInfo = directoryInfo
+                    )
+                }
+
+                if (newStatus != previousStatus) {
+                    previousStatus = newStatus
+                    send(newStatus)
+                }
+            }
+
+            dispatch()
+
+            // handle updates
+            for (event in this) {
+                var hadChanges = false
+
+                if (event.folderId == folder) {
+                    event.files.forEach { fileUpdate ->
+                        // entry change
+                        if (fileUpdate.parent == path) {
+                            hadChanges = true
+
+                            entries = entries.filter { it.fileName != fileUpdate.fileName }
+
+                            if (!fileUpdate.isDeleted) {
+                                entries += listOf(fileUpdate)
+                            }
+                        }
+
+                        // handle directory info changes
+                        if (fileUpdate.parent == parentPath && fileUpdate.fileName == directoryName) {
+                            directoryInfo = if (fileUpdate.isDeleted) null else fileUpdate
+                            hadChanges = true
+                        }
+
+                        // handle parent directory info changes
+                        if (fileUpdate.parent == parentParentPath && fileUpdate.fileName == parentDirectoryName) {
+                            parentEntry = if (fileUpdate.isDeleted) null else fileUpdate
+                            hadChanges = true
+                        }
+                    }
+                }
+
+                if (hadChanges) {
+                    dispatch()
+                }
+            }
+        }
+    }
+
+    fun getFileInfoByAbsolutePath(folder: String, path: String): FileInfo = getFileInfoByAbsolutePathAllowNull(folder, path)
+            ?: error("file not found for path = $path")
+
+    fun getFileInfoByAbsolutePathAllowNull(folder: String, path: String): FileInfo? {
+        return if (PathUtils.isRoot(path)) {
+            FileInfo(folder = folder, type = FileInfo.FileType.DIRECTORY, path = PathUtils.ROOT_PATH)
+        } else {
+            indexRepository.runInTransaction { it.findNotDeletedFileInfo(folder, path) }
+        }
+    }
+
+    fun getFileInfoByPath(folder: String, path: String, transaction: IndexTransaction) = getFileInfoByPathAllowNull(folder, path, transaction)
+            ?: error("file not found for path = $path")
+
+    fun getFileInfoByPathAllowNull(folder: String, path: String, transaction: IndexTransaction): FileInfo? {
+        return if (PathUtils.isRoot(path)) {
+            FileInfo(folder = folder, type = FileInfo.FileType.DIRECTORY, path = PathUtils.ROOT_PATH)
+        } else {
+            transaction.findNotDeletedFileInfo(folder, path)
+        }
+    }
+}

syncthing-bep/src/main/kotlin/net/syncthing/java/bep/utils/MaxBy.kt

@@ -0,0 +1,11 @@
+package net.syncthing.java.bep.utils
+
+inline fun <T> Iterable<T>.longMaxBy(selector: (T) -> Long, defaultValue: Long): Long {
+    var max = defaultValue
+
+    this.forEach {
+        max = Math.max(max, selector(it))
+    }
+
+    return max
+}

syncthing-client-cli/build.gradle

@@ -7,6 +7,7 @@ dependencies {
     compile project(':syncthing-repository-default')
     compile "commons-cli:commons-cli:1.4"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.0.0'
 }
 
 run {

syncthing-client-cli/src/main/kotlin/net/syncthing/java/client/cli/Main.kt

@@ -13,6 +13,10 @@
  */
 package net.syncthing.java.client.cli
 
+import kotlinx.coroutines.channels.consume
+import kotlinx.coroutines.runBlocking
+import net.syncthing.java.bep.index.browser.DirectoryContentListing
+import net.syncthing.java.bep.index.browser.IndexBrowser
 import net.syncthing.java.client.SyncthingClient
 import net.syncthing.java.core.beans.DeviceId
 import net.syncthing.java.core.beans.DeviceInfo
@@ -25,7 +29,6 @@ import org.slf4j.LoggerFactory
 import java.io.File
 import java.io.FileInputStream
 import java.io.IOException
-import java.util.concurrent.CountDownLatch
 
 class Main(private val commandLine: CommandLine) {
 
@@ -91,28 +94,23 @@ class Main(private val commandLine: CommandLine) {
                 System.out.println("file path = $folderAndPath")
                 val folder = folderAndPath.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[0]
                 val path = folderAndPath.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[1]
-                val latch = CountDownLatch(1)
                 val fileInfo = FileInfo(folder = folder, path = path, type = FileInfo.FileType.FILE)
-                syncthingClient.getBlockPuller(folder, { blockPuller ->
-                    try {
-                        val inputStream = blockPuller.pullFileSync(fileInfo)
-                        val fileName = syncthingClient.indexHandler.getFileInfoByPath(folder, path)!!.fileName
-                        val file  =
-                                if (commandLine.hasOption("o")) {
-                                    val param = File(commandLine.getOptionValue("o"))
-                                    if (param.isDirectory) File(param, fileName) else param
-                                } else {
-                                    File(fileName)
-                                }
-                        FileUtils.copyInputStreamToFile(inputStream, file)
-                        System.out.println("saved file to = $file.absolutePath")
-                    } catch (e: InterruptedException) {
-                        logger.warn("", e)
-                    } catch (e: IOException) {
-                        logger.warn("", e)
+                try {
+                    val inputStream = syncthingClient.pullFileSync(fileInfo)
+                    val fileName = syncthingClient.indexHandler.getFileInfoByPath(folder, path)!!.fileName
+                    val file = if (commandLine.hasOption("o")) {
+                        val param = File(commandLine.getOptionValue("o"))
+                        if (param.isDirectory) File(param, fileName) else param
+                    } else {
+                        File(fileName)
                     }
-                }, { logger.warn("Failed to pull file") })
-                latch.await()
+                    FileUtils.copyInputStreamToFile(inputStream, file)
+                    System.out.println("saved file to = $file.absolutePath")
+                } catch (e: InterruptedException) {
+                    logger.warn("", e)
+                } catch (e: IOException) {
+                    logger.warn("", e)
+                }
             }
             "P" -> {
                 var path = option.value
@@ -121,21 +119,20 @@ class Main(private val commandLine: CommandLine) {
                 System.out.println("file path = $path")
                 val folder = path.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[0]
                 path = path.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[1]
-                val latch = CountDownLatch(1)
-                syncthingClient.getBlockPusher(folder, { blockPusher ->
-                    val observer = blockPusher.pushFile(FileInputStream(file), folder, path)
-                    while (!observer.isCompleted()) {
-                        try {
-                            observer.waitForProgressUpdate()
-                        } catch (e: InterruptedException) {
-                            logger.warn("", e)
-                        }
+                val blockPusher = syncthingClient.getBlockPusher(folder)
 
-                        System.out.println("upload progress ${observer.progressPercentage()}%")
+                val observer = runBlocking {
+                    blockPusher.pushFile(FileInputStream(file), folder, path)
+                }
+                while (!observer.isCompleted()) {
+                    try {
+                        observer.waitForProgressUpdate()
+                    } catch (e: InterruptedException) {
+                        logger.warn("", e)
                     }
-                    latch.countDown()
-                }, { logger.warn("Failed to upload file") })
-                latch.await()
+
+                    System.out.println("upload progress ${observer.progressPercentage()}%")
+                }
                 System.out.println("uploaded file to network")
             }
             "D" -> {
@@ -143,17 +140,16 @@ class Main(private val commandLine: CommandLine) {
                 val folder = path.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[0]
                 path = path.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[1]
                 System.out.println("delete path = $path")
-                val latch = CountDownLatch(1)
-                syncthingClient.getBlockPusher(folder, { blockPusher ->
-                    try {
-                        blockPusher.pushDelete(folder, path).waitForComplete()
-                    } catch (e: InterruptedException) {
-                        logger.warn("", e)
-                    }
+                try {
+                    val blockPusher = syncthingClient.getBlockPusher(folder)
 
-                    latch.countDown()
-                }, { System.out.println("Failed to delete path") })
-                latch.await()
+                    runBlocking {
+                        blockPusher.pushDelete(folder, path)
+                    }
+                } catch (e: InterruptedException) {
+                    logger.warn("", e)
+                    System.out.println("Failed to delete path")
+                }
                 System.out.println("deleted path")
             }
             "M" -> {
@@ -161,47 +157,48 @@ class Main(private val commandLine: CommandLine) {
                 val folder = path.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[0]
                 path = path.split(":".toRegex()).dropLastWhile({ it.isEmpty() }).toTypedArray()[1]
                 System.out.println("dir path = $path")
-                val latch = CountDownLatch(1)
-                syncthingClient.getBlockPusher(folder, { blockPusher ->
-                    try {
-                        blockPusher.pushDir(folder, path).waitForComplete()
-                    } catch (e: InterruptedException) {
-                        logger.warn("", e)
-                    }
+                try {
+                    val blockPusher = syncthingClient.getBlockPusher(folder)
 
-                    latch.countDown()
-                }, { System.out.println("Failed to push directory") })
-                latch.await()
+                    runBlocking {
+                        blockPusher.pushDir(folder, path)
+                    }
+                } catch (e: InterruptedException) {
+                    System.out.println("Failed to push directory")
+                    logger.warn("", e)
+                }
                 System.out.println("uploaded dir to network")
             }
             "L" -> {
-                waitForIndexUpdate(syncthingClient, configuration)
-                for (folder in syncthingClient.indexHandler.folderList()) {
-                    syncthingClient.indexHandler.newIndexBrowser(folder).use { indexBrowser ->
-                        System.out.println("list folder = ${indexBrowser.folder}")
-                        for (fileInfo in indexBrowser.listFiles()) {
+                waitForIndexUpdate(syncthingClient)
+                for (folder in configuration.folders) {
+                    System.out.println("list folder = ${folder}")
+                    val listing = syncthingClient.indexHandler.indexBrowser.getDirectoryListing(folder.folderId, IndexBrowser.ROOT_PATH)
+
+                    if (listing is DirectoryContentListing) {
+                        for (fileInfo in listing.entries) {
                             System.out.println("${fileInfo.type.name.substring(0, 1)}\t${fileInfo.describeSize()}\t${fileInfo.path}")
                         }
                     }
                 }
             }
             "I" -> {
-                waitForIndexUpdate(syncthingClient, configuration)
+                waitForIndexUpdate(syncthingClient)
                 val folderInfo = StringBuilder()
-                for (folder in syncthingClient.indexHandler.folderList()) {
+                for (folder in configuration.folders) {
                     folderInfo.append("\nfolder info: ")
-                            .append(syncthingClient.indexHandler.getFolderInfo(folder))
+                            .append(folder)
                     folderInfo.append("\nfolder stats: ")
-                            .append(syncthingClient.indexHandler.newFolderBrowser().getFolderStats(folder).dumpInfo())
+                            .append(syncthingClient.indexHandler.folderBrowser.getFolderStatusSync(folder.folderId).stats.infoDump)
                             .append("\n")
                 }
                 System.out.println("folders:\n$folderInfo\n")
             }
             "l" -> {
                 var folderInfo = ""
-                for (folder in syncthingClient.indexHandler.folderList()) {
-                    folderInfo += "\nfolder info: " + syncthingClient.indexHandler.getFolderInfo(folder)
-                    folderInfo += "\nfolder stats: " + syncthingClient.indexHandler.newFolderBrowser().getFolderStats(folder).dumpInfo() + "\n"
+                for (folder in configuration.folders) {
+                    folderInfo += "\nfolder info: " + folder
+                    folderInfo += "\nfolder stats: " + syncthingClient.indexHandler.folderBrowser.getFolderStatusSync(folder.folderId).stats.infoDump + "\n"
                 }
                 System.out.println("folders:\n$folderInfo\n")
             }
@@ -217,11 +214,12 @@ class Main(private val commandLine: CommandLine) {
     }
 
     @Throws(InterruptedException::class)
-    private fun waitForIndexUpdate(client: SyncthingClient, configuration: Configuration) {
-        val latch = CountDownLatch(configuration.peers.size)
-        client.indexHandler.registerOnFullIndexAcquiredListenersListener {
-            latch.countDown()
+    private fun waitForIndexUpdate(client: SyncthingClient) {
+        // FIXME: what happens if the index update happened already?
+        runBlocking {
+            client.indexHandler.subscribeToOnFullIndexAcquiredEvents().consume {
+                this.receive()  // wait until there is one event
+            }
         }
-        latch.await()
     }
 }

syncthing-client/build.gradle

@@ -6,4 +6,5 @@ dependencies {
     compile project(':syncthing-bep')
     compile project(':syncthing-discovery')
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.0.0'
 }

syncthing-client/src/main/kotlin/net/syncthing/java/client/Connections.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.client
+
+import net.syncthing.java.bep.connectionactor.ConnectionActorWrapper
+import net.syncthing.java.core.beans.DeviceId
+
+class Connections (val generate: (DeviceId) -> ConnectionActorWrapper) {
+    private val map = mutableMapOf<DeviceId, ConnectionActorWrapper>()
+
+    fun getByDeviceId(deviceId: DeviceId): ConnectionActorWrapper {
+        return synchronized(map) {
+            val oldEntry = map[deviceId]
+
+            if (oldEntry != null) {
+                return oldEntry
+            } else {
+                val newEntry = generate(deviceId)
+
+                map[deviceId] = newEntry
+
+                return newEntry
+            }
+        }
+    }
+
+    fun shutdown() {
+        synchronized(map) {
+            map.values.forEach { it.shutdown() }
+        }
+    }
+
+    fun reconnectAllConnections() {
+        synchronized(map) {
+            map.values.forEach { it.reconnect() }
+        }
+    }
+}

syncthing-client/src/main/kotlin/net/syncthing/java/client/SyncthingClient.kt

@@ -13,57 +13,68 @@
  */
 package net.syncthing.java.client
 
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.async
+import kotlinx.coroutines.runBlocking
 import net.syncthing.java.bep.BlockPuller
+import net.syncthing.java.bep.BlockPullerStatus
 import net.syncthing.java.bep.BlockPusher
-import net.syncthing.java.bep.ConnectionHandler
-import net.syncthing.java.bep.IndexHandler
-import net.syncthing.java.core.beans.DeviceAddress
+import net.syncthing.java.bep.RequestHandlerRegistry
+import net.syncthing.java.bep.connectionactor.ConnectionActorGenerator
+import net.syncthing.java.bep.connectionactor.ConnectionActorWrapper
+import net.syncthing.java.bep.index.IndexHandler
 import net.syncthing.java.core.beans.DeviceId
-import net.syncthing.java.core.beans.DeviceInfo
+import net.syncthing.java.core.beans.FileInfo
 import net.syncthing.java.core.configuration.Configuration
 import net.syncthing.java.core.interfaces.IndexRepository
 import net.syncthing.java.core.interfaces.TempRepository
-import net.syncthing.java.core.security.KeystoreHandler
-import net.syncthing.java.core.utils.awaitTerminationSafe
 import net.syncthing.java.discovery.DiscoveryHandler
-import org.slf4j.LoggerFactory
 import java.io.Closeable
-import java.io.IOException
-import java.util.Collections
-import java.util.TreeSet
-import java.util.concurrent.Executors
-import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicBoolean
-import kotlin.collections.ArrayList
-import kotlin.collections.HashMap
+import java.io.InputStream
+import java.util.*
 
 class SyncthingClient(
         private val configuration: Configuration,
         private val repository: IndexRepository,
-        private val tempRepository: TempRepository
+        private val tempRepository: TempRepository,
+        enableDetailedException: Boolean = false
 ) : Closeable {
-
-    private val logger = LoggerFactory.getLogger(javaClass)
-    val discoveryHandler: DiscoveryHandler
-    val indexHandler: IndexHandler
-    private val connections = Collections.synchronizedSet(createConnectionsSet())
-    private val connectByDeviceIdLocks = Collections.synchronizedMap(HashMap<DeviceId, Object>())
+    val indexHandler = IndexHandler(configuration, repository, tempRepository, enableDetailedException)
+    val discoveryHandler = DiscoveryHandler(configuration)
     private val onConnectionChangedListeners = Collections.synchronizedList(mutableListOf<(DeviceId) -> Unit>())
-    private var connectDevicesScheduler = Executors.newSingleThreadScheduledExecutor()
 
-    private fun createConnectionsSet() = TreeSet<ConnectionHandler>(compareBy { it.address.score })
-
-    init {
-        indexHandler = IndexHandler(configuration, repository, tempRepository)
-        discoveryHandler = DiscoveryHandler(configuration)
-        connectDevicesScheduler.scheduleAtFixedRate(this::updateIndexFromPeers, 0, 15, TimeUnit.SECONDS)
-    }
+    private val requestHandlerRegistry = RequestHandlerRegistry()
+    private val connections = Connections(
+            generate = { deviceId ->
+                ConnectionActorWrapper(
+                        source = ConnectionActorGenerator.generateConnectionActors(
+                                deviceAddress = discoveryHandler.devicesAddressesManager.getDeviceAddressManager(deviceId).streamCurrentDeviceAddresses(),
+                                requestHandler = { request ->
+                                    GlobalScope.async {
+                                        requestHandlerRegistry.handleRequest(
+                                                source = deviceId,
+                                                request = request
+                                        )
+                                    }
+                                },
+                                indexHandler = indexHandler,
+                                configuration = configuration
+                        ),
+                        deviceId = deviceId,
+                        connectivityChangeListener = {
+                            synchronized(onConnectionChangedListeners) {
+                                onConnectionChangedListeners.forEach { it(deviceId) }
+                            }
+                        }
+                )
+            }
+    )
 
     fun clearCacheAndIndex() {
         indexHandler.clearIndex()
         configuration.folders = emptySet()
         configuration.persistLater()
-        updateIndexFromPeers()
+        connections.reconnectAllConnections()
     }
 
     fun addOnConnectionChangedListener(listener: (DeviceId) -> Unit) {
@@ -75,158 +86,61 @@ class SyncthingClient(
         onConnectionChangedListeners.remove(listener)
     }
 
-    @Throws(IOException::class, KeystoreHandler.CryptoException::class)
-    private fun openConnection(deviceAddress: DeviceAddress): ConnectionHandler {
-        logger.debug("Connecting to ${deviceAddress.deviceId}, active connections: ${connections.map { it.deviceId().deviceId }}")
-        val connectionHandler = ConnectionHandler(
-                configuration, deviceAddress, indexHandler, tempRepository, { connectionHandler, _ ->
-                    connectionHandler.close()
-                    openConnection(deviceAddress)
-                },
-                {connection ->
-                    if (!connection.isConnected) {
-                        connections.remove(connection)
-                    }
-                    onConnectionChangedListeners.forEach { it(connection.deviceId()) }
-                })
+    private fun getConnections() = configuration.peerIds.map { connections.getByDeviceId(it) }
 
-        try {
-          connectionHandler.connect()
-        } catch (ex: Exception) {
-          connectionHandler.closeBg()
-
-          throw ex
-        }
-
-        connections.add(connectionHandler)
-
-        return connectionHandler
+    init {
+        discoveryHandler.newDeviceAddressSupplier() // starts the discovery
+        getConnections()
     }
 
-    /**
-     * Takes discovered addresses from [[DiscoveryHandler]] and connects to devices.
-     *
-     * We need to make sure that we are only connecting once to each device.
-     */
-    private fun getPeerConnections(listener: (connection: ConnectionHandler) -> Unit, completeListener: () -> Unit) {
-        // create an copy to prevent dispatching an action two times
-        val connectionsWhichWereDispatched = createConnectionsSet()
-
-        synchronized (connections) {
-          connectionsWhichWereDispatched.addAll(connections)
-        }
-
-        connectionsWhichWereDispatched.forEach { listener(it) }
-
-        discoveryHandler.newDeviceAddressSupplier()
-                .takeWhile { it != null }
-                .filterNotNull()
-                .groupBy { it.deviceId() }
-                .filterNot { it.value.isEmpty() }
-                .forEach { (deviceId, addresses) ->
-                    // create an lock per device id to prevent multiple connections to one device
-
-                    synchronized (connectByDeviceIdLocks) {
-                      if (connectByDeviceIdLocks[deviceId] == null) {
-                        connectByDeviceIdLocks[deviceId] = Object()
-                      }
-                    }
-
-                    synchronized (connectByDeviceIdLocks[deviceId]!!) {
-                      val existingConnection = connections.find { it.deviceId() == deviceId && it.isConnected }
-
-                      if (existingConnection != null) {
-                        connectionsWhichWereDispatched.add(existingConnection)
-                        listener(existingConnection)
-
-                        return@synchronized
-                      }
-
-                      // try to use all addresses
-                      for (address in addresses.distinctBy { it.address }) {
-                        try {
-                          val newConnection = openConnection(address)
-
-                          connectionsWhichWereDispatched.add(newConnection)
-                          listener(newConnection)
-
-                          break  // it worked, no need to try more
-                        } catch (e: IOException) {
-                          logger.warn("error connecting to device = $address", e)
-                        } catch (e: KeystoreHandler.CryptoException) {
-                          logger.warn("error connecting to device = $address", e)
-                        }
-                      }
-                    }
-                }
-
-        // use all connections which were added in the time between and were not added by this function call
-        val newConnectionsBackup = createConnectionsSet()
-
-        synchronized (connections) {
-          newConnectionsBackup.addAll(connections)
-        }
-
-        connectionsWhichWereDispatched.forEach { newConnectionsBackup.remove(it) }
-
-        newConnectionsBackup.forEach { listener(it) }
-
-        completeListener()
+    fun connectToNewlyAddedDevices() {
+        getConnections()
     }
 
-    private fun updateIndexFromPeers() {
-        getPeerConnections({ connection ->
-            try {
-                indexHandler.waitForRemoteIndexAcquired(connection)
-            } catch (ex: InterruptedException) {
-                logger.warn("exception while waiting for index", ex)
-            }
-        }, {})
+    fun disconnectFromRemovedDevices() {
+        // TODO: implement this
     }
 
-    private fun getConnectionForFolder(folder: String, listener: (connection: ConnectionHandler) -> Unit,
-                                       errorListener: () -> Unit) {
-        val isConnected = AtomicBoolean(false)
-        getPeerConnections({ connection ->
-            if (connection.hasFolder(folder) && !isConnected.get()) {
-                listener(connection)
-                isConnected.set(true)
-            }
-        }, {
-            if (!isConnected.get()) {
-                errorListener()
-            }
-        })
+    fun getActiveConnectionsForFolder(folderId: String) = configuration.peerIds
+            .map { connections.getByDeviceId(it) }
+            .filter { it.isConnected && it.hasFolder(folderId) }
+
+    suspend fun pullFile(
+            fileInfo: FileInfo,
+            progressListener: (status: BlockPullerStatus) -> Unit = {  }
+    ): InputStream = BlockPuller.pullFile(
+            fileInfo = fileInfo,
+            progressListener = progressListener,
+            connections = getConnections(),
+            indexHandler = indexHandler,
+            tempRepository = tempRepository
+    )
+
+    fun pullFileSync(fileInfo: FileInfo) = runBlocking { pullFile(fileInfo) }
+
+    fun getBlockPusher(folderId: String): BlockPusher {
+        val connection = getActiveConnectionsForFolder(folderId).first()
+
+        return BlockPusher(
+                localDeviceId = connection.deviceId,
+                connectionHandler = connection,
+                indexHandler = indexHandler,
+                requestHandlerRegistry = requestHandlerRegistry
+        )
     }
 
-    fun getBlockPuller(folderId: String, listener: (BlockPuller) -> Unit, errorListener: () -> Unit) {
-        getConnectionForFolder(folderId, { connection ->
-            listener(connection.getBlockPuller())
-        }, errorListener)
-    }
-
-    fun getBlockPusher(folderId: String, listener: (BlockPusher) -> Unit, errorListener: () -> Unit) {
-        getConnectionForFolder(folderId, { connection ->
-            listener(connection.getBlockPusher())
-        }, errorListener)
-    }
-
-    fun getPeerStatus(): List<DeviceInfo> {
-        return configuration.peers.map { device ->
-            val isConnected = connections.find { it.deviceId() == device.deviceId }?.isConnected ?: false
-            device.copy(isConnected = isConnected)
-        }
+    fun getPeerStatus() = configuration.peers.map { device ->
+        device.copy(
+                isConnected = connections.getByDeviceId(device.deviceId).isConnected
+        )
     }
 
     override fun close() {
-        connectDevicesScheduler.awaitTerminationSafe()
         discoveryHandler.close()
-        // Create copy of list, because it will be modified by handleConnectionClosedEvent(), causing ConcurrentModificationException.
-        ArrayList(connections).forEach{it.close()}
         indexHandler.close()
         repository.close()
         tempRepository.close()
+        connections.shutdown()
         assert(onConnectionChangedListeners.isEmpty())
     }
-
 }

syncthing-core/build.gradle

@@ -9,7 +9,6 @@ dependencies {
     compile "org.slf4j:slf4j-api:1.7.25"
     compile "ch.qos.logback:logback-classic:1.2.3"
     compile "com.google.code.gson:gson:2.8.2"
-    compile "org.apache.httpcomponents:httpclient:4.5.4"
     compile "org.bouncycastle:bcmail-jdk15on:1.59"
     implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
 }

syncthing-core/src/main/kotlin/net/syncthing/java/core/beans/DeviceAddress.kt

@@ -1,4 +1,4 @@
-/* 
+/*
  * Copyright (C) 2016 Davide Imbriaco
  *
  * This Java file is subject to the terms of the Mozilla Public
@@ -18,44 +18,38 @@ import java.net.InetSocketAddress
 import java.net.UnknownHostException
 import java.util.*
 
-/**
- *
- * TODO: this class cant use [[DeviceId]] because [[GlobalDiscoveryHandler.pickAnnounceServers]] uses that field for discovery server URLs.
- */
-class DeviceAddress private constructor(val deviceId: String, private val instanceId: Long?, val address: String, producer: AddressProducer?, score: Int?, lastModified: Date?) {
+// TODO: this should use a data class, but the custom equals prevents it
+class DeviceAddress private constructor(val deviceId: DeviceId, private val instanceId: Long?, val address: String, producer: AddressProducer?, score: Int?, lastModified: Date?) {
     private val producer = producer ?: AddressProducer.UNKNOWN
     val score = score ?: Integer.MAX_VALUE
     private val lastModified = lastModified ?: Date()
 
-    @Deprecated(message = "should use deviceIdObject instead")
-    fun deviceId() = DeviceId(deviceId)
-
-    val deviceIdObject: DeviceId by lazy { DeviceId(deviceId) }
-
     @Throws(UnknownHostException::class)
     private fun getInetAddress(): InetAddress = InetAddress.getByName(address.replaceFirst("^[^:]+://".toRegex(), "").replaceFirst("(:[0-9]+)?(/.*)?$".toRegex(), ""))
 
-    private fun getPort(): Int = if (address.matches("^[a-z]+://[^:]+:([0-9]+).*".toRegex())) {
+    private val port: Int by lazy {
+        if (address.matches("^[a-z]+://[^:]+:([0-9]+).*".toRegex())) {
             Integer.parseInt(address.replaceFirst("^[a-z]+://[^:]+:([0-9]+).*".toRegex(), "$1"))
         } else {
-            DEFAULT_PORT_BY_PROTOCOL[getType()]!!
+            DEFAULT_PORT_BY_PROTOCOL[type]!!
         }
+    }
 
-    fun getType(): AddressType = when {
-        address.isEmpty() -> AddressType.NULL
-        address.startsWith("tcp://") -> AddressType.TCP
-        address.startsWith("relay://") -> AddressType.RELAY
-        address.startsWith("relay-http://") -> AddressType.HTTP_RELAY
-        address.startsWith("relay-https://") -> AddressType.HTTPS_RELAY
-        else -> AddressType.OTHER
+    val type: AddressType by lazy {
+        when {
+            address.isEmpty() -> AddressType.NULL
+            address.startsWith("tcp://") -> AddressType.TCP
+            address.startsWith("relay://") -> AddressType.RELAY
+            else -> AddressType.OTHER
+        }
     }
 
     @Throws(UnknownHostException::class)
-    fun getSocketAddress(): InetSocketAddress = InetSocketAddress(getInetAddress(), getPort())
+    fun getSocketAddress(): InetSocketAddress = InetSocketAddress(getInetAddress(), port)
 
     fun isWorking(): Boolean = score < Integer.MAX_VALUE
 
-    constructor(deviceId: String, address: String) : this(deviceId, null, address, null, null, null)
+    constructor(deviceId: String, address: String) : this(DeviceId(deviceId), null, address, null, null, null)
 
     fun containsUriParamValue(key: String): Boolean {
         return !getUriParam(key).isNullOrEmpty()
@@ -79,7 +73,7 @@ class DeviceAddress private constructor(val deviceId: String, private val instan
     }
 
     enum class AddressType {
-        TCP, RELAY, OTHER, NULL, HTTP_RELAY, HTTPS_RELAY
+        TCP, RELAY, OTHER, NULL
     }
 
     enum class AddressProducer {
@@ -97,18 +91,18 @@ class DeviceAddress private constructor(val deviceId: String, private val instan
         return hash
     }
 
-    override fun equals(obj: Any?): Boolean {
-        if (this === obj) {
+    override fun equals(other: Any?): Boolean {
+        if (this === other) {
             return true
         }
-        if (obj == null) {
+        if (other == null) {
             return false
         }
-        if (javaClass != obj.javaClass) {
+        if (javaClass != other.javaClass) {
             return false
         }
-        val other = obj as DeviceAddress?
-        if (this.deviceId != other!!.deviceId) {
+        other as DeviceAddress
+        if (this.deviceId != other.deviceId) {
             return false
         }
         return this.address == other.address
@@ -120,7 +114,7 @@ class DeviceAddress private constructor(val deviceId: String, private val instan
 
     class Builder {
 
-        private var deviceId: String? = null
+        private var deviceId: DeviceId? = null
         private var instanceId: Long? = null
         private var address: String? = null
         private var producer: AddressProducer? = null
@@ -129,7 +123,7 @@ class DeviceAddress private constructor(val deviceId: String, private val instan
 
         constructor()
 
-        internal constructor(deviceId: String, instanceId: Long?, address: String, producer: AddressProducer, score: Int?, lastModified: Date) {
+        internal constructor(deviceId: DeviceId, instanceId: Long?, address: String, producer: AddressProducer, score: Int?, lastModified: Date) {
             this.deviceId = deviceId
             this.instanceId = instanceId
             this.address = address
@@ -147,11 +141,11 @@ class DeviceAddress private constructor(val deviceId: String, private val instan
             return this
         }
 
-        fun getDeviceId(): String? {
+        fun getDeviceId(): DeviceId? {
             return deviceId
         }
 
-        fun setDeviceId(deviceId: String): Builder {
+        fun setDeviceId(deviceId: DeviceId): Builder {
             this.deviceId = deviceId
             return this
         }
@@ -200,8 +194,7 @@ class DeviceAddress private constructor(val deviceId: String, private val instan
     companion object {
         private val DEFAULT_PORT_BY_PROTOCOL = mapOf(
                 AddressType.TCP to 22000,
-                AddressType.RELAY to 22067,
-                AddressType.HTTP_RELAY to 80,
-                AddressType.HTTPS_RELAY to 443)
+                AddressType.RELAY to 22067
+        )
     }
 }

syncthing-core/src/main/kotlin/net/syncthing/java/core/beans/FolderInfo.kt

@@ -17,7 +17,7 @@ package net.syncthing.java.core.beans
 import com.google.gson.stream.JsonReader
 import com.google.gson.stream.JsonWriter
 
-open class FolderInfo(val folderId: String, label: String? = null) {
+data class FolderInfo(val folderId: String, val label: String) {
     companion object {
         private const val FOLDER_ID = "folderId"
         private const val LABEL = "label"
@@ -43,11 +43,8 @@ open class FolderInfo(val folderId: String, label: String? = null) {
         }
     }
 
-    val label: String
-
     init {
         assert(!folderId.isEmpty())
-        this.label = if (label != null && !label.isEmpty()) label else folderId
     }
 
     override fun toString(): String {

syncthing-core/src/main/kotlin/net/syncthing/java/core/beans/FolderStats.kt

@@ -1,5 +1,6 @@
 /* 
  * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
  *
  * This Java file is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -16,87 +17,22 @@ package net.syncthing.java.core.beans
 import org.apache.commons.io.FileUtils
 import java.util.*
 
-class FolderStats private constructor(val fileCount: Long, val dirCount: Long, val size: Long, val lastUpdate: Date, folder: String, label: String?) : FolderInfo(folder, label) {
-
-    fun getRecordCount(): Long = dirCount + fileCount
-
-    fun describeSize(): String = FileUtils.byteCountToDisplaySize(size)
-
-    fun dumpInfo(): String {
-        return ("folder " + label + " (" + folderId + ") file count = " + fileCount
-                + " dir count = " + dirCount + " folder size = " + describeSize() + " last update = " + lastUpdate)
+data class FolderStats(val fileCount: Long, val dirCount: Long, val size: Long, val lastUpdate: Date, val folderId: String) {
+    companion object {
+        fun createDummy(folderId: String) = FolderStats(
+                fileCount = 0,
+                dirCount = 0,
+                size = 0,
+                lastUpdate = Date(0),
+                folderId = folderId
+        )
     }
 
-    override fun toString(): String {
-        return "FolderStats{folder=$folderId, fileCount=$fileCount, dirCount=$dirCount, size=$size, lastUpdate=$lastUpdate}"
-    }
+    val recordCount: Long = dirCount + fileCount
 
-    fun copyBuilder(): Builder = Builder(fileCount, dirCount, size, folderId, label)
-
-    class Builder {
-
-        private var fileCount: Long = 0
-        private var dirCount: Long = 0
-        private var size: Long = 0
-        private var lastUpdate = Date(0)
-        private var folder: String? = null
-        private var label: String? = null
-
-        constructor()
-
-        constructor(fileCount: Long, dirCount: Long, size: Long, folder: String, label: String) {
-            this.fileCount = fileCount
-            this.dirCount = dirCount
-            this.size = size
-            this.folder = folder
-            this.label = label
-        }
-
-        fun getFileCount(): Long = fileCount
-
-        fun setFileCount(fileCount: Long): Builder {
-            this.fileCount = fileCount
-            return this
-        }
-
-        fun getDirCount(): Long = dirCount
-
-        fun setDirCount(dirCount: Long): Builder {
-            this.dirCount = dirCount
-            return this
-        }
-
-        fun getSize(): Long = size
-
-        fun setSize(size: Long): Builder {
-            this.size = size
-            return this
-        }
-
-        fun getLastUpdate(): Date = lastUpdate
-
-        fun setLastUpdate(lastUpdate: Date): Builder {
-            this.lastUpdate = lastUpdate
-            return this
-        }
-
-        fun getFolder(): String? = folder
-
-        fun setFolder(folder: String): Builder {
-            this.folder = folder
-            return this
-        }
-
-        fun getLabel(): String? = label
-
-        fun setLabel(label: String): Builder {
-            this.label = label
-            return this
-        }
-
-        fun build(): FolderStats {
-            return FolderStats(fileCount, dirCount, size, lastUpdate, folder!!, label)
-        }
+    val sizeDescription: String by lazy { FileUtils.byteCountToDisplaySize(size) }
 
+    val infoDump: String by lazy {
+        "folder $folderId file count = $fileCount dir count = $dirCount folder size = $sizeDescription last update = $lastUpdate"
     }
 }

syncthing-core/src/main/kotlin/net/syncthing/java/core/beans/IndexInfo.kt

@@ -1,5 +1,6 @@
 /* 
  * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
  *
  * This Java file is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -14,98 +15,11 @@
 package net.syncthing.java.core.beans
 
 
-class IndexInfo private constructor(folder: String, val deviceId: String, val indexId: Long, val localSequence: Long, val maxSequence: Long) : FolderInfo(folder) {
+data class IndexInfo(val folderId: String, val deviceId: String, val indexId: Long, val localSequence: Long, val maxSequence: Long) {
 
-    fun getCompleted(): Double = if (maxSequence > 0) localSequence.toDouble() / maxSequence else 0.0
+    val completed: Double = if (maxSequence > 0) localSequence.toDouble() / maxSequence else 0.0
 
     init {
-        assert(!deviceId.isEmpty())
+        assert(deviceId.isNotEmpty())
     }
-
-    fun copyBuilder(): Builder {
-        return Builder(folderId, indexId, deviceId, localSequence, maxSequence)
-    }
-
-    override fun toString(): String {
-        return "FolderIndexInfo{indexId=$indexId, folder=$folderId, deviceId=$deviceId, localSequence=$localSequence, maxSequence=$maxSequence}"
-    }
-
-    class Builder {
-
-        private var indexId: Long = 0
-        private var deviceId: String? = null
-        private var folder: String? = null
-        private var localSequence: Long = 0
-        private var maxSequence: Long = 0
-
-        internal constructor()
-
-        internal constructor(folder: String, indexId: Long, deviceId: String, localSequence: Long, maxSequence: Long) {
-            assert(!folder.isEmpty())
-            assert(!deviceId.isEmpty())
-            this.folder = folder
-            this.indexId = indexId
-            this.deviceId = deviceId
-            this.localSequence = localSequence
-            this.maxSequence = maxSequence
-        }
-
-        fun getIndexId(): Long {
-            return indexId
-        }
-
-        fun getDeviceId(): String? {
-            return deviceId
-        }
-
-        fun getFolder(): String? {
-            return folder
-        }
-
-        fun getLocalSequence(): Long {
-            return localSequence
-        }
-
-        fun getMaxSequence(): Long {
-            return maxSequence
-        }
-
-        fun setIndexId(indexId: Long): Builder {
-            this.indexId = indexId
-            return this
-        }
-
-        fun setDeviceId(deviceId: String): Builder {
-            this.deviceId = deviceId
-            return this
-        }
-
-        fun setFolder(folder: String): Builder {
-            this.folder = folder
-            return this
-        }
-
-        fun setLocalSequence(localSequence: Long): Builder {
-            this.localSequence = localSequence
-            return this
-        }
-
-        fun setMaxSequence(maxSequence: Long): Builder {
-            this.maxSequence = maxSequence
-            return this
-        }
-
-        fun build(): IndexInfo {
-            return IndexInfo(folder!!, deviceId!!, indexId, localSequence, maxSequence)
-        }
-
-    }
-
-    companion object {
-
-        fun newBuilder(): Builder {
-            return Builder()
-        }
-    }
-
 }

syncthing-core/src/main/kotlin/net/syncthing/java/core/interfaces/IndexRepository.kt

@@ -1,5 +1,6 @@
 /* 
  * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
  *
  * This Java file is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -13,46 +14,8 @@
  */
 package net.syncthing.java.core.interfaces
 
-import net.syncthing.java.core.beans.*
 import java.io.Closeable
-import java.util.*
 
 interface IndexRepository: Closeable {
-
-    fun setOnFolderStatsUpdatedListener(listener: ((IndexRepository.FolderStatsUpdatedEvent) -> Unit)?)
-
-    fun getSequencer(): Sequencer
-
-    fun updateIndexInfo(indexInfo: IndexInfo)
-
-    fun findIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo?
-
-    fun findFileInfo(folder: String, path: String): FileInfo?
-
-    fun findFileInfoLastModified(folder: String, path: String): Date?
-
-    fun findNotDeletedFileInfo(folder: String, path: String): FileInfo?
-
-    fun findFileBlocks(folder: String, path: String): FileBlocks?
-
-    fun updateFileInfo(fileInfo: FileInfo, fileBlocks: FileBlocks?)
-
-    fun findNotDeletedFilesByFolderAndParent(folder: String, parentPath: String): List<FileInfo>
-
-    fun clearIndex()
-
-    fun findFolderStats(folder: String): FolderStats?
-
-    fun findAllFolderStats(): List<FolderStats>
-
-    fun findFileInfoBySearchTerm(query: String): List<FileInfo>
-
-    fun countFileInfoBySearchTerm(query: String): Long
-
-    abstract class FolderStatsUpdatedEvent {
-
-        abstract fun getFolderStats(): List<FolderStats>
-
-    }
-
+    fun <T> runInTransaction(action: (IndexTransaction) -> T): T
 }

syncthing-core/src/main/kotlin/net/syncthing/java/core/interfaces/IndexTransaction.kt

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.core.interfaces
+
+import net.syncthing.java.core.beans.*
+import java.util.*
+
+interface IndexTransaction {
+    fun getSequencer(): Sequencer
+
+    fun updateIndexInfo(indexInfo: IndexInfo)
+
+    fun findAllIndexInfos(): List<IndexInfo>
+
+    fun findIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo?
+
+    fun findFileInfo(folder: String, path: String): FileInfo?
+
+    // path to FileInfo
+    fun findFileInfo(folder: String, path: List<String>): Map<String, FileInfo>
+
+    fun findFileInfoLastModified(folder: String, path: String): Date?
+
+    fun findNotDeletedFileInfo(folder: String, path: String): FileInfo?
+
+    fun findFileBlocks(folder: String, path: String): FileBlocks?
+
+    fun updateFileInfo(fileInfo: FileInfo, fileBlocks: FileBlocks?)
+
+    fun updateFileInfoAndBlocks(fileInfos: List<FileInfo>, fileBlocks: List<FileBlocks>)
+
+    fun findNotDeletedFilesByFolderAndParent(folder: String, parentPath: String): List<FileInfo>
+
+    fun clearIndex()
+
+    fun findFolderStats(folder: String): FolderStats?
+
+    fun findAllFolderStats(): List<FolderStats>
+
+    fun updateOrInsertFolderStats(folder: String, deltaFileCount: Long, deltaDirCount: Long, deltaSize: Long, lastUpdate: Date)
+
+    fun findFileInfoBySearchTerm(query: String): List<FileInfo>
+
+    fun countFileInfoBySearchTerm(query: String): Long
+}

syncthing-core/src/main/kotlin/net/syncthing/java/core/security/KeystoreHandler.kt

@@ -77,7 +77,7 @@ class KeystoreHandler private constructor(private val keyStore: KeyStore) {
     }
 
     @Throws(CryptoException::class, IOException::class)
-    private fun wrapSocket(socket: Socket, isServerSocket: Boolean, protocol: String): SSLSocket {
+    private fun wrapSocket(socket: Socket, isServerSocket: Boolean): SSLSocket {
         try {
             logger.debug("wrapping plain socket, server mode = {}", isServerSocket)
             val sslSocket = socketFactory.createSocket(socket, null, socket.port, true) as SSLSocket
@@ -98,7 +98,7 @@ class KeystoreHandler private constructor(private val keyStore: KeyStore) {
     }
 
     @Throws(CryptoException::class, IOException::class)
-    fun createSocket(relaySocketAddress: InetSocketAddress, protocol: String): SSLSocket {
+    fun createSocket(relaySocketAddress: InetSocketAddress): SSLSocket {
         try {
             val socket = socketFactory.createSocket() as SSLSocket
             socket.connect(relaySocketAddress, SOCKET_TIMEOUT)
@@ -115,8 +115,8 @@ class KeystoreHandler private constructor(private val keyStore: KeyStore) {
     }
 
     @Throws(CryptoException::class, IOException::class)
-    fun wrapSocket(relayConnection: RelayConnection, protocol: String): SSLSocket {
-        return wrapSocket(relayConnection.getSocket(), relayConnection.isServerSocket(), protocol)
+    fun wrapSocket(relayConnection: RelayConnection): SSLSocket {
+        return wrapSocket(relayConnection.getSocket(), relayConnection.isServerSocket())
     }
 
     class Loader {
@@ -269,10 +269,15 @@ class KeystoreHandler private constructor(private val keyStore: KeyStore) {
         @Throws(SSLPeerUnverifiedException::class, CertificateException::class)
         fun assertSocketCertificateValid(certificate: Certificate, deviceId: DeviceId) {
             NetworkUtils.assertProtocol(certificate is X509Certificate)
+
             val derData = certificate.encoded
             val deviceIdFromCertificate = derDataToDeviceId(derData)
             logger.trace("remote pem certificate =\n{}", derToPem(derData))
-            NetworkUtils.assertProtocol(deviceIdFromCertificate == deviceId, {"device id mismatch! expected = $deviceId, got = $deviceIdFromCertificate"})
+
+            NetworkUtils.assertProtocol(deviceIdFromCertificate == deviceId) {
+                "device id mismatch! expected = $deviceId, got = $deviceIdFromCertificate"
+            }
+
             logger.debug("remote ssl certificate match deviceId = {}", deviceId)
         }
     }

syncthing-core/src/main/kotlin/net/syncthing/java/core/utils/ExecutorUtils.kt

@@ -1,47 +0,0 @@
-/* 
- * Copyright (C) 2016 Davide Imbriaco
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.core.utils
-
-import org.slf4j.LoggerFactory
-import java.util.concurrent.ExecutorService
-import java.util.concurrent.Future
-import java.util.concurrent.TimeUnit
-
-private val logger = LoggerFactory.getLogger(ExecutorService::class.java)
-
-fun ExecutorService.awaitTerminationSafe() {
-    try {
-        awaitTermination(2, TimeUnit.SECONDS)
-    } catch (ex: InterruptedException) {
-        logger.warn("", ex)
-    }
-}
-
-fun ExecutorService.submitLogging(runnable: Runnable) = submitLogging { runnable.run() }
-
-/**
- * Wrapper method for [[ExecutorService.submit]], which silently swallows exceptions. If an exception is thrown in
- * [[runnable]], logs the exception and force crashes
- */
-fun <T> ExecutorService.submitLogging(runnable: () -> T): Future<T> {
-    return submit<T>({
-        try {
-            runnable()
-        } catch (e: Exception) {
-            logger.error("", e)
-            System.exit(1)
-            null
-        }
-    })
-}

syncthing-core/src/main/kotlin/net/syncthing/java/core/utils/PathUtils.kt

@@ -1,5 +1,6 @@
 /* 
  * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
  *
  * This Java file is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -13,36 +14,90 @@
  */
 package net.syncthing.java.core.utils
 
-import org.apache.commons.io.FilenameUtils
-
 object PathUtils {
-
-    val ROOT_PATH = ""
-    val PATH_SEPARATOR = "/"
-    val PARENT_PATH = ".."
-
-    private fun normalizePath(path: String): String {
-        return FilenameUtils.normalizeNoEndSeparator(path, true).replaceFirst(("^" + PATH_SEPARATOR).toRegex(), "")
-    }
+    const val ROOT_PATH = ""
+    const val PATH_SEPARATOR = "/"
+    const val PATH_SEPARATOR_WIN = "\\"
+    const val PARENT_PATH = ".."
+    const val CURRENT_PATH = "."
 
     fun isRoot(path: String): Boolean {
         return path.isEmpty()
     }
 
+    private fun containsRelativeElements(path: String): Boolean {
+        val pathSegments = path.split(PATH_SEPARATOR)
+
+        return pathSegments.contains(PARENT_PATH) or pathSegments.contains(CURRENT_PATH)
+    }
+
+    private fun isTrimmed(value: String) = value.trim() == value
+    private fun containsWindowsPathSeparator(path: String) = path.contains(PATH_SEPARATOR_WIN)
+    private fun startsWithPathSeperator(path: String) = path.startsWith(PATH_SEPARATOR)
+    private fun isValidPath(path: String) = (!containsRelativeElements(path)) and
+            (!containsWindowsPathSeparator(path)) and
+            path.isNotEmpty() and
+            (!startsWithPathSeperator(path)) and
+            isTrimmed(path)
+
+    private fun containsPathSeparator(file: String) = file.contains(PATH_SEPARATOR) or file.contains(PATH_SEPARATOR_WIN)
+    private fun isFilenameValid(file: String) = file.isNotBlank() and
+            (!containsPathSeparator(file)) and
+            isTrimmed(file)
+
+    private fun assertPathValid(path: String) {
+        if (!isValidPath(path)) {
+            throw IllegalArgumentException("provided path is invalid")
+        }
+    }
+
+    private fun assertFilenameValid(filename: String) {
+        if (!isFilenameValid(filename)) {
+            throw IllegalArgumentException("provided filename is invalid")
+        }
+    }
+
     fun isParent(path: String): Boolean {
         return path == PARENT_PATH
     }
 
     fun getParentPath(path: String): String {
-        assert(!isRoot(path), {"cannot get parent of root path"})
-        return normalizePath(path + PATH_SEPARATOR + PARENT_PATH)
+        assertPathValid(path)
+
+        val pathWithoutSuffix = path.removeSuffix(PATH_SEPARATOR)
+        val previousSeparator = pathWithoutSuffix.lastIndexOf(PATH_SEPARATOR)
+
+        return if (previousSeparator == -1) {
+            ROOT_PATH
+        } else {
+            pathWithoutSuffix.substring(0, previousSeparator)
+        }
     }
 
     fun getFileName(path: String): String {
-        return FilenameUtils.getName(path)
+        if (path.isEmpty()) {
+            // this is required for IndexHandler.ROOT_FILE_INFO
+
+            return ""
+        }
+
+        assertPathValid(path)
+
+        val pathWithoutSuffix = path.removeSuffix(PATH_SEPARATOR)
+        val previousSeparator = pathWithoutSuffix.lastIndexOf(PATH_SEPARATOR)
+
+        return if (previousSeparator == -1) {
+            // the file is in the root directory
+            pathWithoutSuffix
+        } else {
+            pathWithoutSuffix.substring(previousSeparator + 1)
+        }
     }
 
     fun buildPath(dir: String, file: String): String {
-        return normalizePath(dir + PATH_SEPARATOR + file)
+        assertPathValid(dir)
+        assertFilenameValid(file)
+
+        return dir.removeSuffix(PATH_SEPARATOR) + file
     }
 }

syncthing-discovery/src/main/kotlin/net/syncthing/java/discovery/DeviceAddressesManager.kt

@@ -19,16 +19,27 @@ import net.syncthing.java.core.beans.DeviceAddress
 import net.syncthing.java.core.beans.DeviceId
 
 class DeviceAddressesManager (val deviceId: DeviceId) {
+    companion object {
+        private const val MAX_ADDRESSES_PER_TYPE = 16
+    }
+
     private val lock = Object()
     private val deviceAddressesCache = mutableListOf<DeviceAddress>()
     private val listeners = mutableListOf<(DeviceAddress) -> Unit>()
 
     fun putAddress(address: DeviceAddress) {
-        if (address.deviceIdObject != deviceId) {
+        if (address.deviceId != deviceId) {
             throw IllegalArgumentException()
         }
 
         synchronized(lock) {
+            val otherAddressesOfSameType = deviceAddressesCache.filter { it.type == address.type }
+
+            if (otherAddressesOfSameType.size == MAX_ADDRESSES_PER_TYPE) {
+                // forget the oldest one of the same type
+                deviceAddressesCache.remove(otherAddressesOfSameType.first())
+            }
+
             deviceAddressesCache.add(address)
             listeners.forEach { it(address) }
         }

syncthing-discovery/src/main/kotlin/net/syncthing/java/discovery/DiscoveryHandler.kt

@@ -15,6 +15,7 @@
 package net.syncthing.java.discovery
 
 import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.channels.consumeEach
 import kotlinx.coroutines.launch
 import net.syncthing.java.core.beans.DeviceAddress
 import net.syncthing.java.core.beans.DeviceId
@@ -39,7 +40,7 @@ class DiscoveryHandler(private val configuration: Configuration) : Closeable {
     }, { deviceId ->
         onMessageFromUnknownDeviceListeners.forEach { listener -> listener(deviceId) }
     })
-    private val devicesAddressesManager = DevicesAddressesManager()
+    val devicesAddressesManager = DevicesAddressesManager()
     private var isClosed = false
     private val onMessageFromUnknownDeviceListeners = Collections.synchronizedSet(HashSet<(DeviceId) -> Unit>())
 
@@ -74,17 +75,18 @@ class DiscoveryHandler(private val configuration: Configuration) : Closeable {
             val peers = configuration.peerIds
             //do not process address already processed
             list.filter { deviceAddress ->
-                !peers.contains(deviceAddress.deviceIdObject)
+                !peers.contains(deviceAddress.deviceId)
             }
 
-            AddressRanker.pingAddresses(list)
-                    .forEach { putDeviceAddress(it) }
+            AddressRanker.pingAddressesChannel(list).consumeEach {
+                putDeviceAddress(it)
+            }
         }
     }
 
     private fun putDeviceAddress(deviceAddress: DeviceAddress) {
         devicesAddressesManager.getDeviceAddressManager(
-                deviceId = deviceAddress.deviceIdObject
+                deviceId = deviceAddress.deviceId
         ).putAddress(deviceAddress)
     }
 

syncthing-discovery/src/main/kotlin/net/syncthing/java/discovery/protocol/LocalDiscoveryHandler.kt

@@ -14,6 +14,7 @@
  */
 package net.syncthing.java.discovery.protocol
 
+import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.channels.consumeEach
@@ -32,7 +33,7 @@ internal class LocalDiscoveryHandler(private val configuration: Configuration,
     private val job = Job()
 
     fun sendAnnounceMessage() {
-        GlobalScope.launch {
+        GlobalScope.launch (Dispatchers.IO) {
             LocalDiscoveryUtil.sendAnnounceMessage(
                     ownDeviceId = configuration.localDeviceId,
                     instanceId = configuration.instanceId

syncthing-discovery/src/main/kotlin/net/syncthing/java/discovery/protocol/LocalDiscoveryUtil.kt

@@ -77,7 +77,7 @@ object LocalDiscoveryUtil {
                             // discovery announcement is to be used.
                             DeviceAddress.Builder()
                                     .setAddress(address.replaceFirst("tcp://(0.0.0.0|):".toRegex(), "tcp://$sourceAddress:"))
-                                    .setDeviceId(deviceId.deviceId)
+                                    .setDeviceId(deviceId)
                                     .setInstanceId(announce.instanceId)
                                     .setProducer(DeviceAddress.AddressProducer.LOCAL_DISCOVERY)
                                     .build()
@@ -135,7 +135,7 @@ object LocalDiscoveryUtil {
 data class LocalDiscoveryMessage(val deviceId: DeviceId, val addresses: List<DeviceAddress>) {
     init {
         addresses.forEach { address ->
-            if (address.deviceIdObject != deviceId) {
+            if (address.deviceId != deviceId) {
                 throw IllegalArgumentException()
             }
         }

syncthing-discovery/src/main/kotlin/net/syncthing/java/discovery/utils/AddressRanker.kt

@@ -15,6 +15,8 @@
 package net.syncthing.java.discovery.utils
 
 import kotlinx.coroutines.*
+import kotlinx.coroutines.channels.produce
+import kotlinx.coroutines.channels.toList
 import net.syncthing.java.core.beans.DeviceAddress
 import net.syncthing.java.core.beans.DeviceAddress.AddressType
 import org.slf4j.LoggerFactory
@@ -26,49 +28,47 @@ object AddressRanker {
     private const val TCP_CONNECTION_TIMEOUT = 5000
     private val BASE_SCORE_MAP = mapOf(
             AddressType.TCP to 0,
-            AddressType.RELAY to 2000,
-            AddressType.HTTP_RELAY to 1000 * 2000,
-            AddressType.HTTPS_RELAY to 1000 * 2000
+            AddressType.RELAY to 2000
     )
     private val ACCEPTED_ADDRESS_TYPES = BASE_SCORE_MAP.keys
     private val logger = LoggerFactory.getLogger(javaClass)
 
-    suspend fun pingAddresses(sourceAddresses: List<DeviceAddress>) = coroutineScope {
-        addHttpRelays(sourceAddresses)
-                .filter { ACCEPTED_ADDRESS_TYPES.contains(it.getType()) }
-                .toList()   // the following should happen parallel
-                .map {
+    fun pingAddressesChannel(sourceAddresses: List<DeviceAddress>) = GlobalScope.produce<DeviceAddress> {
+        sourceAddresses
+                .filter { ACCEPTED_ADDRESS_TYPES.contains(it.type) }
+                .toList()
+                .map { address ->
                     async {
                         try {
-                            withTimeout(TCP_CONNECTION_TIMEOUT * 2L) {
+                            val addressWithScore = withTimeout(TCP_CONNECTION_TIMEOUT * 2L) {
                                 // this nested async ensures that cancelling/ the timeout has got an effect without delay
                                 GlobalScope.async (Dispatchers.IO) {
-                                    pingAddressSync(it)
+                                    pingAddressSync(address)
                                 }.await()
                             }
+
+                            if (addressWithScore != null) {
+                                send(addressWithScore)
+                            }
                         } catch (ex: Exception) {
                             logger.warn("Failed to ping device", ex)
-
-                            null
                         }
+
+                        null
                     }
                 }
                 .map { it.await() }
-                .filterNotNull()
-                .sortedBy { it.score }
+
+        close()
     }
 
-    private fun getHttpRelays(list: List<DeviceAddress>) = list
-            .asSequence()
-            .filter { address ->
-                address.getType() == AddressType.RELAY && address.containsUriParamValue("httpUrl")
-            }
-            .map { address ->
-                val httpUrl = address.getUriParam("httpUrl")
-                address.copyBuilder().setAddress("relay-" + httpUrl!!).build()
-            }
-
-    private fun addHttpRelays(list: List<DeviceAddress>) = getHttpRelays(list) + list
+    @Deprecated(
+            message = "This is slower than the version which returns the channel",
+            replaceWith = ReplaceWith("pingAddressesChannel")
+    )
+    suspend fun pingAddressesReturnAllResultsAtOnce(sourceAddresses: List<DeviceAddress>) = pingAddressesChannel(sourceAddresses)
+            .toList()
+            .sortedBy { it.score }
 
     private fun pingAddressSync(deviceAddress: DeviceAddress): DeviceAddress? {
         val startTime = System.currentTimeMillis()
@@ -84,7 +84,7 @@ object AddressRanker {
         }
 
         val ping = (System.currentTimeMillis() - startTime).toInt()
-        val baseScore = BASE_SCORE_MAP[deviceAddress.getType()] ?: 0
+        val baseScore = BASE_SCORE_MAP[deviceAddress.type] ?: 0
 
         return deviceAddress.copyBuilder().setScore(ping + baseScore).build()
     }

syncthing-http-relay-client/build.gradle

@@ -1,37 +0,0 @@
-apply plugin: 'java-library'
-apply plugin: 'kotlin'
-apply plugin: 'com.google.protobuf'
-
-dependencies {
-    compile project(':syncthing-relay-client')
-    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
-    implementation "com.google.protobuf:protobuf-lite:$protobuf_lite_version"
-}
-
-protobuf {
-    protoc {
-        artifact = "com.google.protobuf:protoc:3.5.1-1"
-    }
-    plugins {
-        javalite {
-            // The codegen for lite comes as a separate artifact
-            artifact = "com.google.protobuf:protoc-gen-javalite:3.0.0"
-        }
-    }
-    generateProtoTasks {
-        all().each { task ->
-            task.builtins {
-                // In most cases you don't need the full Java output
-                // if you use the lite output.
-                remove java
-            }
-            task.plugins {
-                javalite { }
-            }
-        }
-    }
-}
-
-// Workaround for https://github.com/google/protobuf-gradle-plugin/issues/100
-compileKotlin.dependsOn('generateProto')
-sourceSets.main.kotlin.srcDirs += file("${protobuf.generatedFilesBaseDir}/main/javalite")

syncthing-http-relay-client/src/main/kotlin/net/syncthing/java/httprelay/HttpRelayClient.kt

@@ -1,31 +0,0 @@
-/* 
- * Copyright (C) 2016 Davide Imbriaco
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.httprelay
-
-import net.syncthing.java.core.beans.DeviceAddress
-import net.syncthing.java.core.beans.DeviceAddress.AddressType
-import org.slf4j.LoggerFactory
-
-class HttpRelayClient {
-
-    private val logger = LoggerFactory.getLogger(javaClass)
-
-    fun openRelayConnection(deviceAddress: DeviceAddress): HttpRelayConnection {
-        assert(setOf(AddressType.HTTP_RELAY, AddressType.HTTPS_RELAY).contains(deviceAddress.getType()))
-        val httpRelayServerUrl = deviceAddress.address.replaceFirst("^relay-".toRegex(), "")
-        val deviceId = deviceAddress.deviceId
-        logger.info("open http relay connection, relay url = {}, target device id = {}", httpRelayServerUrl, deviceId)
-        return HttpRelayConnection(httpRelayServerUrl, deviceId)
-    }
-}

syncthing-http-relay-client/src/main/kotlin/net/syncthing/java/httprelay/HttpRelayConnection.kt

@@ -1,304 +0,0 @@
-/* 
- * Copyright (C) 2016 Davide Imbriaco
- *
- * This Java file is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package net.syncthing.java.httprelay
-
-import com.google.protobuf.ByteString
-import net.syncthing.java.core.interfaces.RelayConnection
-import net.syncthing.java.core.utils.NetworkUtils
-import net.syncthing.java.core.utils.submitLogging
-import org.apache.http.HttpStatus
-import org.apache.http.client.methods.HttpPost
-import org.apache.http.entity.ByteArrayEntity
-import org.apache.http.impl.client.HttpClients
-import org.apache.http.util.EntityUtils
-import org.slf4j.LoggerFactory
-import java.io.*
-import java.net.*
-import java.util.concurrent.ExecutionException
-import java.util.concurrent.Executors
-import java.util.concurrent.LinkedBlockingQueue
-import java.util.concurrent.TimeUnit
-
-class HttpRelayConnection internal constructor(private val httpRelayServerUrl: String, deviceId: String) : RelayConnection, Closeable {
-
-    private val logger = LoggerFactory.getLogger(javaClass)
-    private val outgoingExecutorService = Executors.newSingleThreadExecutor()
-    private val incomingExecutorService = Executors.newSingleThreadExecutor()
-    private val flusherStreamService = Executors.newSingleThreadScheduledExecutor()
-    private var peerToRelaySequence: Long = 0
-    private var relayToPeerSequence: Long = 0
-    private val sessionId: String
-    private val incomingDataQueue = LinkedBlockingQueue<ByteArray>()
-    private val socket: Socket
-    private val isServerSocket: Boolean
-    private val inputStream: InputStream
-    private val outputStream: OutputStream
-
-    var isClosed = false
-        private set
-
-    override fun getSocket() = socket
-
-    override fun isServerSocket() = isServerSocket
-
-    init {
-        val serverMessage = sendMessage(HttpRelayProtos.HttpRelayPeerMessage.newBuilder()
-                .setMessageType(HttpRelayProtos.HttpRelayPeerMessageType.CONNECT)
-                .setDeviceId(deviceId))
-        assert(serverMessage.messageType == HttpRelayProtos.HttpRelayServerMessageType.PEER_CONNECTED)
-        assert(!serverMessage.sessionId.isNullOrEmpty())
-        sessionId = serverMessage.sessionId
-        isServerSocket = serverMessage.isServerSocket
-        outputStream = object : OutputStream() {
-
-            private var buffer = ByteArrayOutputStream()
-            private var lastFlush = System.currentTimeMillis()
-
-            init {
-                flusherStreamService.scheduleWithFixedDelay({
-                    if (System.currentTimeMillis() - lastFlush > 1000) {
-                        try {
-                            flush()
-                        } catch (ex: IOException) {
-                            logger.warn("", ex)
-                        }
-
-                    }
-                }, 1, 1, TimeUnit.SECONDS)
-            }
-
-            @Synchronized
-            @Throws(IOException::class)
-            override fun write(i: Int) {
-                NetworkUtils.assertProtocol(!this@HttpRelayConnection.isClosed)
-                buffer.write(i)
-            }
-
-            @Synchronized
-            @Throws(IOException::class)
-            override fun write(bytes: ByteArray, offset: Int, size: Int) {
-                NetworkUtils.assertProtocol(!this@HttpRelayConnection.isClosed)
-                buffer.write(bytes, offset, size)
-            }
-
-            @Synchronized
-            @Throws(IOException::class)
-            override fun flush() {
-                val data = buffer.toByteArray().copyOf().toList()
-                buffer = ByteArrayOutputStream()
-                try {
-                    if (!data.isEmpty()) {
-                        outgoingExecutorService.submit {
-                            sendMessage(HttpRelayProtos.HttpRelayPeerMessage.newBuilder()
-                                    .setMessageType(HttpRelayProtos.HttpRelayPeerMessageType.PEER_TO_RELAY)
-                                    .setSequence(++peerToRelaySequence)
-                                    .setData(data as ByteString))
-                        }.get()
-                    }
-                    lastFlush = System.currentTimeMillis()
-                } catch (ex: InterruptedException) {
-                    logger.error("error", ex)
-                    closeBg()
-                    throw IOException(ex)
-                } catch (ex: ExecutionException) {
-                    logger.error("error", ex)
-                    closeBg()
-                    throw IOException(ex)
-                }
-
-            }
-
-            @Synchronized
-            @Throws(IOException::class)
-            override fun write(bytes: ByteArray) {
-                NetworkUtils.assertProtocol(!this@HttpRelayConnection.isClosed)
-                buffer.write(bytes)
-            }
-
-        }
-        incomingExecutorService.submitLogging {
-            while (!isClosed) {
-                val serverMessage1 =
-                    try {
-                        sendMessage(HttpRelayProtos.HttpRelayPeerMessage.newBuilder().setMessageType(HttpRelayProtos.HttpRelayPeerMessageType.WAIT_FOR_DATA))
-                    } catch (e: IOException) {
-                        logger.warn("Failed to send relay message", e)
-                        return@submitLogging
-                    }
-                if (isClosed) {
-                    return@submitLogging
-                }
-                NetworkUtils.assertProtocol(serverMessage1.messageType == HttpRelayProtos.HttpRelayServerMessageType.RELAY_TO_PEER)
-                NetworkUtils.assertProtocol(serverMessage1.sequence == relayToPeerSequence + 1)
-                if (!serverMessage1.data.isEmpty) {
-                    incomingDataQueue.add(serverMessage1.data.toByteArray())
-                }
-                relayToPeerSequence = serverMessage1.sequence
-            }
-        }
-        inputStream = object : InputStream() {
-
-            private var noMoreData = false
-            private var byteArrayInputStream = ByteArrayInputStream(ByteArray(0))
-
-            @Throws(IOException::class)
-            override fun read(): Int {
-                NetworkUtils.assertProtocol(!this@HttpRelayConnection.isClosed)
-                if (noMoreData) {
-                    return -1
-                }
-                var bite = -1
-                while (bite == -1) {
-                    bite = byteArrayInputStream.read()
-                    try {
-                        val data = incomingDataQueue.poll(1, TimeUnit.SECONDS)
-                        if (data == null) {
-                            //continue
-                        } else if (data.contentEquals(STREAM_CLOSED)) {
-                            noMoreData = true
-                            return -1
-                        } else {
-                            byteArrayInputStream = ByteArrayInputStream(data)
-                        }
-                    } catch (ex: InterruptedException) {
-                        logger.warn("", ex)
-                    }
-
-                }
-                return bite
-            }
-
-        }
-        socket = object : Socket() {
-            override fun isClosed(): Boolean {
-                return this@HttpRelayConnection.isClosed
-            }
-
-            override fun isConnected(): Boolean {
-                return !isClosed
-            }
-
-            @Throws(IOException::class)
-            override fun shutdownOutput() {
-                logger.debug("shutdownOutput")
-                outputStream.flush()
-            }
-
-            @Throws(IOException::class)
-            override fun shutdownInput() {
-                logger.debug("shutdownInput")
-                //do nothing
-            }
-
-            @Synchronized
-            @Throws(IOException::class)
-            override fun close() {
-                logger.debug("received close on socket adapter")
-                this@HttpRelayConnection.close()
-            }
-
-            @Throws(IOException::class)
-            override fun getOutputStream(): OutputStream {
-                return this@HttpRelayConnection.outputStream
-            }
-
-            @Throws(IOException::class)
-            override fun getInputStream(): InputStream {
-                return this@HttpRelayConnection.inputStream
-            }
-
-            @Throws(UnknownHostException::class)
-            override fun getRemoteSocketAddress(): SocketAddress {
-                return InetSocketAddress(inetAddress, port)
-            }
-
-            override fun getPort(): Int {
-                return 22067
-            }
-
-            @Throws(UnknownHostException::class)
-            override fun getInetAddress(): InetAddress {
-                return InetAddress.getByName(URI.create(this@HttpRelayConnection.httpRelayServerUrl).host)
-            }
-
-        }
-    }
-
-    private fun closeBg() {
-
-        Thread { close() }.start()
-    }
-
-    @Throws(IOException::class)
-    private fun sendMessage(peerMessageBuilder: HttpRelayProtos.HttpRelayPeerMessage.Builder): HttpRelayProtos.HttpRelayServerMessage {
-        if (!sessionId.isEmpty()) {
-            peerMessageBuilder.sessionId = sessionId
-        }
-        logger.debug("send http relay peer message = {} session id = {} sequence = {}", peerMessageBuilder.messageType, peerMessageBuilder.sessionId, peerMessageBuilder.sequence)
-        val httpClient = HttpClients.custom()
-                //                .setSSLSocketFactory(new SSLConnectionSocketFactory(new SSLContextBuilder().loadTrustMaterial(null, new TrustSelfSignedStrategy()).build(), SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER))
-                .build()
-        val httpPost = HttpPost(httpRelayServerUrl)
-        httpPost.entity = ByteArrayEntity(peerMessageBuilder.build().toByteArray())
-        val serverMessage = httpClient.execute(httpPost) { response ->
-            NetworkUtils.assertProtocol(response.statusLine.statusCode == HttpStatus.SC_OK, {"http error ${response.statusLine}"})
-            HttpRelayProtos.HttpRelayServerMessage.parseFrom(EntityUtils.toByteArray(response.entity))
-        }
-        logger.debug("received http relay server message = {}", serverMessage.messageType)
-        NetworkUtils.assertProtocol(serverMessage.messageType != HttpRelayProtos.HttpRelayServerMessageType.ERROR, {"server error : ${serverMessage.data.toStringUtf8()}"})
-        return serverMessage
-    }
-
-    override fun close() {
-        if (!isClosed) {
-            isClosed = true
-            logger.info("closing http relay connection {} : {}", httpRelayServerUrl, sessionId)
-            flusherStreamService.shutdown()
-            if (!sessionId.isEmpty()) {
-                try {
-                    outputStream.flush()
-                    sendMessage(HttpRelayProtos.HttpRelayPeerMessage.newBuilder().setMessageType(HttpRelayProtos.HttpRelayPeerMessageType.PEER_CLOSING))
-                } catch (ex: IOException) {
-                    logger.warn("error closing http relay connection", ex)
-                }
-
-            }
-            incomingExecutorService.shutdown()
-            outgoingExecutorService.shutdown()
-            try {
-                incomingExecutorService.awaitTermination(1, TimeUnit.SECONDS)
-            } catch (ex: InterruptedException) {
-                logger.warn("", ex)
-            }
-
-            try {
-                outgoingExecutorService.awaitTermination(1, TimeUnit.SECONDS)
-            } catch (ex: InterruptedException) {
-                logger.warn("", ex)
-            }
-
-            try {
-                flusherStreamService.awaitTermination(1, TimeUnit.SECONDS)
-            } catch (ex: InterruptedException) {
-                logger.warn("", ex)
-            }
-
-            incomingDataQueue.add(STREAM_CLOSED)
-        }
-    }
-
-    companion object {
-        private val STREAM_CLOSED = "STREAM_CLOSED".toByteArray()
-    }
-}

syncthing-http-relay-client/src/main/proto/httpRelayProtos.proto

@@ -1,34 +0,0 @@
-package net.syncthing.java.httprelay;
-
-option optimize_for = LITE_RUNTIME;
-
-message HttpRelayPeerMessage{
-    optional HttpRelayPeerMessageType message_type = 1;
-    optional string session_id = 2;
-    optional string device_id = 3;
-    optional int64 sequence = 4;
-    optional bytes data = 5;
-}
-
-message HttpRelayServerMessage{
-    optional HttpRelayServerMessageType message_type = 1;
-    optional string session_id = 2;
-    optional bool is_server_socket = 3;
-    optional int64 sequence = 4;
-    optional bytes data = 5;
-}
-
-enum HttpRelayPeerMessageType {
-    CONNECT             = 0;
-    PEER_TO_RELAY       = 1;
-    WAIT_FOR_DATA       = 2;
-    PEER_CLOSING        = 3;
-}
-
-enum HttpRelayServerMessageType {
-    PEER_CONNECTED      = 0;
-    DATA_ACCEPTED       = 1;
-    RELAY_TO_PEER       = 2;
-    SERVER_CLOSING      = 3;
-    ERROR               = 4;
-}

syncthing-relay-client/src/main/kotlin/net/syncthing/java/client/protocol/rp/RelayClient.kt

@@ -36,8 +36,8 @@ class RelayClient(configuration: Configuration) {
 
     @Throws(IOException::class, KeystoreHandler.CryptoException::class)
     fun openRelayConnection(address: DeviceAddress): RelayConnection {
-        assert(address.getType() == AddressType.RELAY)
-        val sessionInvitation = getSessionInvitation(address.getSocketAddress(), address.deviceId())
+        assert(address.type == AddressType.RELAY)
+        val sessionInvitation = getSessionInvitation(address.getSocketAddress(), address.deviceId)
         return openConnectionSessionMode(sessionInvitation)
     }
 
@@ -80,7 +80,7 @@ class RelayClient(configuration: Configuration) {
     @Throws(IOException::class, KeystoreHandler.CryptoException::class)
     fun getSessionInvitation(relaySocketAddress: InetSocketAddress, deviceId: DeviceId): SessionInvitation {
         logger.debug("connecting to relay = {} (temporary protocol mode)", relaySocketAddress)
-        keystoreHandler.createSocket(relaySocketAddress, KeystoreHandler.RELAY).use { socket ->
+        keystoreHandler.createSocket(relaySocketAddress).use { socket ->
             RelayDataInputStream(socket.getInputStream()).use { `in` ->
                 RelayDataOutputStream(socket.getOutputStream()).use { out ->
                     run {

syncthing-repository-android/build.gradle

@@ -49,7 +49,6 @@ dependencies {
 
     implementation (project(':syncthing-client')) {
         exclude group: 'commons-logging', module:'commons-logging'
-        exclude group: 'org.apache.httpcomponents', module:'httpclient'
         exclude group: 'org.slf4j'
         exclude group: 'ch.qos.logback'
     }

syncthing-repository-android/src/main/java/net/syncthing/repository/android/SqliteIndexRepository.kt

@@ -1,124 +1,32 @@
 package net.syncthing.repository.android
 
-import net.syncthing.java.core.beans.*
 import net.syncthing.java.core.interfaces.IndexRepository
-import net.syncthing.java.core.interfaces.Sequencer
+import net.syncthing.java.core.interfaces.IndexTransaction
 import net.syncthing.repository.android.database.RepositoryDatabase
-import net.syncthing.repository.android.database.item.*
-import java.util.*
+import java.util.concurrent.Callable
 
 class SqliteIndexRepository(
         private val database: RepositoryDatabase,
         private val closeDatabaseOnClose: Boolean,
         private val clearTempStorageHook: () -> Unit
 ): IndexRepository {
-    private var folderStatsChangeListener: ((IndexRepository.FolderStatsUpdatedEvent) -> Unit)? = null
 
-    // FileInfo
-    override fun findFileInfo(folder: String, path: String) = database.fileInfo().findFileInfo(folder, path)?.native
-    override fun findFileInfoBySearchTerm(query: String) = database.fileInfo().findFileInfoBySearchTerm(query).map { it.native }
-    override fun findFileInfoLastModified(folder: String, path: String): Date? = database.fileInfo().findFileInfoLastModified(folder, path)?.lastModified
-    override fun findNotDeletedFileInfo(folder: String, path: String) = database.fileInfo().findNotDeletedFileInfo(folder, path)?.native
-    override fun findNotDeletedFilesByFolderAndParent(folder: String, parentPath: String) = database.fileInfo().findNotDeletedFilesByFolderAndParent(folder, parentPath).map { it.native }
-    override fun countFileInfoBySearchTerm(query: String) = database.fileInfo().countFileInfoBySearchTerm(query)
+    override fun <T> runInTransaction(action: (IndexTransaction) -> T): T {
+            return database.runInTransaction (object: Callable<T> {
+                override fun call(): T {
+                    val transaction = SqliteTransaction(
+                            database = database,
+                            threadId = Thread.currentThread().id,
+                            clearTempStorageHook = clearTempStorageHook
+                    )
 
-    override fun updateFileInfo(fileInfo: FileInfo, fileBlocks: FileBlocks?) {
-        val newFileInfo = fileInfo
-        val newFileBlocks = fileBlocks
-
-        database.runInTransaction {
-            if (newFileBlocks != null) {
-                FileInfo.checkBlocks(newFileInfo, newFileBlocks)
-
-                database.fileBlocks().mergeBlock(FileBlocksItem.fromNative(newFileBlocks))
-            }
-
-            val oldFileInfo = findFileInfo(newFileInfo.folder, newFileInfo.path)
-
-            database.fileInfo().updateFileInfo(FileInfoItem.fromNative(newFileInfo))
-
-            //update stats
-            var deltaFileCount = 0L
-            var deltaDirCount= 0L
-            var deltaSize = 0L
-            val oldMissing = oldFileInfo == null || oldFileInfo.isDeleted
-            val newMissing = newFileInfo.isDeleted
-            val oldSizeMissing = oldMissing || !oldFileInfo!!.isFile()
-            val newSizeMissing = newMissing || !newFileInfo.isFile()
-            if (!oldSizeMissing) {
-                deltaSize -= oldFileInfo!!.size!!
-            }
-            if (!newSizeMissing) {
-                deltaSize += newFileInfo.size!!
-            }
-            if (!oldMissing) {
-                if (oldFileInfo!!.isFile()) {
-                    deltaFileCount--
-                } else if (oldFileInfo.isDirectory()) {
-                    deltaDirCount--
-                }
-            }
-            if (!newMissing) {
-                if (newFileInfo.isFile()) {
-                    deltaFileCount++
-                } else if (newFileInfo.isDirectory()) {
-                    deltaDirCount++
-                }
-            }
-
-            val newFolderStats = kotlin.run {
-                val updatedRows = database.folderStats().updateFolderStats(
-                        folder = newFileInfo.folder,
-                        deltaDirCount = deltaDirCount,
-                        deltaFileCount = deltaFileCount,
-                        deltaSize = deltaSize,
-                        lastUpdate = newFileInfo.lastModified
-                )
-
-                if (updatedRows == 0L) {
-                    database.folderStats().insertFolderStats(FolderStatsItem(
-                            folder = newFileInfo.folder,
-                            dirCount = deltaDirCount,
-                            fileCount = deltaFileCount,
-                            size = deltaSize,
-                            lastUpdate = newFileInfo.lastModified
-                    ))
-                }
-
-                database.folderStats().getFolderStats(newFileInfo.folder)!!
-            }
-
-            folderStatsChangeListener?.invoke(object : IndexRepository.FolderStatsUpdatedEvent() {
-                override fun getFolderStats(): List<FolderStats> {
-                    return listOf(newFolderStats.native)
+                    return try {
+                        action(transaction)
+                    } finally {
+                        transaction.markFinished()
+                    }
                 }
             })
-        }
-    }
-
-    // FileBlocks
-
-    override fun findFileBlocks(folder: String, path: String) = database.fileBlocks().findFileBlocks(folder, path)?.native
-
-    // FolderStats
-
-    override fun findAllFolderStats() = database.folderStats().findAllFolderStats().map { it.native }
-
-    override fun findFolderStats(folder: String): FolderStats? = database.folderStats().findFolderStats(folder)?.native
-
-    // IndexInfo
-
-    override fun updateIndexInfo(indexInfo: IndexInfo) {
-        database.folderIndexInfo().updateIndexInfo(FolderIndexInfoItem.fromNative(indexInfo))
-    }
-
-    override fun findIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo? = database.folderIndexInfo().findIndexInfoByDeviceAndFolder(deviceId, folder)?.native
-
-    // managment
-
-    override fun clearIndex() {
-        database.clearAllTables()
-        clearTempStorageHook()
     }
 
     override fun close() {
@@ -126,39 +34,4 @@ class SqliteIndexRepository(
             database.close()
         }
     }
-
-    // other
-    private val sequencer = object: Sequencer {
-        fun getDatabaseEntry(): IndexSequenceItem {
-            val entry = database.indexSequence().getItem()
-
-            if (entry != null) {
-                return entry
-            }
-
-            val newEntry = IndexSequenceItem(
-                    indexId = Math.abs(Random().nextLong()) + 1,
-                    currentSequence = Math.abs(Random().nextLong()) + 1
-            )
-
-            database.indexSequence().createItem(newEntry)
-
-            return newEntry
-        }
-
-        override fun indexId() = getDatabaseEntry().indexId
-        override fun currentSequence() = getDatabaseEntry().currentSequence
-
-        override fun nextSequence(): Long {
-            database.indexSequence().incrementSequenceNumber(indexId())
-
-            return currentSequence()
-        }
-    }
-
-    override fun getSequencer() = sequencer
-
-    override fun setOnFolderStatsUpdatedListener(listener: ((IndexRepository.FolderStatsUpdatedEvent) -> Unit)?) {
-        folderStatsChangeListener = listener
-    }
 }

syncthing-repository-android/src/main/java/net/syncthing/repository/android/SqliteTransaction.kt

@@ -0,0 +1,169 @@
+package net.syncthing.repository.android
+
+import net.syncthing.java.core.beans.*
+import net.syncthing.java.core.interfaces.IndexTransaction
+import net.syncthing.java.core.interfaces.Sequencer
+import net.syncthing.repository.android.database.RepositoryDatabase
+import net.syncthing.repository.android.database.item.*
+import java.util.*
+
+class SqliteTransaction(
+        private val database: RepositoryDatabase,
+        private val threadId: Long,
+        private val clearTempStorageHook: () -> Unit
+): IndexTransaction {
+    private var finished = false
+
+    private fun assertAllowed() {
+        if (finished) {
+            throw IllegalStateException("tried to use a transaction which is already done")
+        }
+
+        if (Thread.currentThread().id != threadId) {
+            throw IllegalStateException("tried to access the transaction from an other Thread")
+        }
+    }
+
+    fun markFinished() {
+        finished = true
+    }
+
+    private fun <T> runIfAllowed(block: () -> T): T {
+        assertAllowed()
+
+        return block()
+    }
+
+    // FileInfo
+    override fun findFileInfo(folder: String, path: String) = runIfAllowed {
+        database.fileInfo().findFileInfo(folder, path)?.native
+    }
+
+    override fun findFileInfo(folder: String, path: List<String>): Map<String, FileInfo> = runIfAllowed {
+        database.fileInfo().findFileInfo(folder, path)
+                .map { it.native }
+                .associateBy { it.path }
+    }
+
+    override fun findFileInfoBySearchTerm(query: String) = runIfAllowed {
+        database.fileInfo().findFileInfoBySearchTerm(query).map { it.native }
+    }
+
+    override fun findFileInfoLastModified(folder: String, path: String): Date? = runIfAllowed {
+        database.fileInfo().findFileInfoLastModified(folder, path)?.lastModified
+    }
+
+    override fun findNotDeletedFileInfo(folder: String, path: String) = runIfAllowed {
+        database.fileInfo().findNotDeletedFileInfo(folder, path)?.native
+    }
+
+    override fun findNotDeletedFilesByFolderAndParent(folder: String, parentPath: String) = runIfAllowed {
+        database.fileInfo().findNotDeletedFilesByFolderAndParent(folder, parentPath).map { it.native }
+    }
+
+    override fun countFileInfoBySearchTerm(query: String) = runIfAllowed {
+        database.fileInfo().countFileInfoBySearchTerm(query)
+    }
+
+    override fun updateFileInfo(fileInfo: FileInfo, fileBlocks: FileBlocks?) = runIfAllowed {
+        if (fileBlocks != null) {
+            FileInfo.checkBlocks(fileInfo, fileBlocks)
+
+            database.fileBlocks().mergeBlock(FileBlocksItem.fromNative(fileBlocks))
+        }
+
+        database.fileInfo().updateFileInfo(FileInfoItem.fromNative(fileInfo))
+    }
+
+    override fun updateFileInfoAndBlocks(fileInfos: List<FileInfo>, fileBlocks: List<FileBlocks>) = runIfAllowed {
+        if (fileInfos.isNotEmpty()) {
+            database.fileInfo().updateFileInfo(fileInfos.map { FileInfoItem.fromNative(it) })
+        }
+
+        if (fileBlocks.isNotEmpty()) {
+            database.fileBlocks().mergeBlocks(fileBlocks.map { FileBlocksItem.fromNative(it) })
+        }
+    }
+
+    // FileBlocks
+
+    override fun findFileBlocks(folder: String, path: String) = runIfAllowed {
+        database.fileBlocks().findFileBlocks(folder, path)?.native
+    }
+
+    // FolderStats
+
+    override fun findAllFolderStats() = runIfAllowed {
+        database.folderStats().findAllFolderStats().map { it.native }
+    }
+
+    override fun findFolderStats(folder: String): FolderStats? = runIfAllowed {
+        database.folderStats().findFolderStats(folder)?.native
+    }
+
+    override fun updateOrInsertFolderStats(
+            folder: String,
+            deltaFileCount: Long,
+            deltaDirCount: Long,
+            deltaSize: Long,
+            lastUpdate: Date
+    ) = runIfAllowed {
+        if (database.folderStats().updateFolderStats(folder, deltaFileCount, deltaDirCount, deltaSize, lastUpdate) == 0L) {
+            database.folderStats().insertFolderStats(FolderStatsItem(folder, deltaFileCount, deltaDirCount, lastUpdate, deltaSize))
+        }
+    }
+
+    // IndexInfo
+
+    override fun updateIndexInfo(indexInfo: IndexInfo) = runIfAllowed {
+        database.folderIndexInfo().updateIndexInfo(FolderIndexInfoItem.fromNative(indexInfo))
+    }
+
+    override fun findIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo? = runIfAllowed {
+        database.folderIndexInfo().findIndexInfoByDeviceAndFolder(deviceId, folder)?.native
+    }
+
+    override fun findAllIndexInfos(): List<IndexInfo> = runIfAllowed {
+        database.folderIndexInfo().findAllIndexInfo().map { it.native }
+    }
+
+    // managment
+
+    override fun clearIndex() {
+        runIfAllowed {
+            database.clearAllTables()
+            clearTempStorageHook()
+        }
+    }
+
+    // other
+    private val sequencer = object: Sequencer {
+        private fun getDatabaseEntry(): IndexSequenceItem {
+            val entry = database.indexSequence().getItem()
+
+            if (entry != null) {
+                return entry
+            }
+
+            val newEntry = IndexSequenceItem(
+                    indexId = Math.abs(Random().nextLong()) + 1,
+                    currentSequence = Math.abs(Random().nextLong()) + 1
+            )
+
+            database.indexSequence().createItem(newEntry)
+
+            return newEntry
+        }
+
+        override fun indexId() = runIfAllowed { getDatabaseEntry().indexId }
+        override fun currentSequence() = runIfAllowed { getDatabaseEntry().currentSequence }
+
+        override fun nextSequence(): Long = runIfAllowed {
+            database.indexSequence().incrementSequenceNumber(indexId())
+
+            currentSequence()
+        }
+    }
+
+    override fun getSequencer() = sequencer
+}

syncthing-repository-android/src/main/java/net/syncthing/repository/android/database/dao/FileBlocksDao.kt

@@ -13,4 +13,7 @@ interface FileBlocksDao {
 
     @Insert(onConflict = OnConflictStrategy.REPLACE)
     fun mergeBlock(blocksItem: FileBlocksItem)
+
+    @Insert(onConflict = OnConflictStrategy.REPLACE)
+    fun mergeBlocks(blocksItem: List<FileBlocksItem>)
 }

syncthing-repository-android/src/main/java/net/syncthing/repository/android/database/dao/FileInfoDao.kt

@@ -12,6 +12,9 @@ interface FileInfoDao {
     @Query("SELECT * FROM file_info WHERE folder = :folder AND path = :path")
     fun findFileInfo(folder: String, path: String): FileInfoItem?
 
+    @Query("SELECT * FROM file_info WHERE folder = :folder AND path IN (:path)")
+    fun findFileInfo(folder: String, path: List<String>): List<FileInfoItem>
+
     @Query("SELECT last_modified FROM file_info WHERE folder = :folder AND path = :path")
     fun findFileInfoLastModified(folder: String, path: String): FileInfoLastModified?
 
@@ -31,4 +34,7 @@ interface FileInfoDao {
 
     @Insert(onConflict = OnConflictStrategy.REPLACE)
     fun updateFileInfo(info: FileInfoItem)
+
+    @Insert(onConflict = OnConflictStrategy.REPLACE)
+    fun updateFileInfo(info: List<FileInfoItem>)
 }

syncthing-repository-android/src/main/java/net/syncthing/repository/android/database/dao/FolderIndexInfoDao.kt

@@ -13,4 +13,7 @@ interface FolderIndexInfoDao {
 
     @Query("SELECT * FROM folder_index_info WHERE device_id = :deviceId AND folder = :folder")
     fun findIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): FolderIndexInfoItem?
+
+    @Query("SELECT * FROM folder_index_info")
+    fun findAllIndexInfo(): List<FolderIndexInfoItem>
 }

syncthing-repository-android/src/main/java/net/syncthing/repository/android/database/item/FolderIndexInfoItem.kt

@@ -31,12 +31,12 @@ data class FolderIndexInfoItem(
 
     @delegate:Transient
     val native: IndexInfo by lazy {
-        IndexInfo.newBuilder()
-                .setFolder(folder)
-                .setDeviceId(deviceId)
-                .setIndexId(indexId)
-                .setLocalSequence(localSequence)
-                .setMaxSequence(maxSequence)
-                .build()
+        IndexInfo(
+                folderId = folder,
+                deviceId = deviceId,
+                indexId = indexId,
+                localSequence = localSequence,
+                maxSequence = maxSequence
+        )
     }
 }

syncthing-repository-android/src/main/java/net/syncthing/repository/android/database/item/FolderStatsItem.kt

@@ -25,12 +25,12 @@ data class FolderStatsItem(
 ) {
     @delegate:Transient
     val native: FolderStats by lazy {
-        FolderStats.Builder()
-                .setFolder(folder)
-                .setDirCount(dirCount)
-                .setFileCount(fileCount)
-                .setSize(size)
-                .setLastUpdate(lastUpdate)
-                .build()
+        FolderStats(
+                folderId = folder,
+                dirCount = dirCount,
+                fileCount = fileCount,
+                size = size,
+                lastUpdate = lastUpdate
+        )
     }
 }

syncthing-repository-default/src/main/kotlin/net/syncthing/java/repository/repo/SqlRepository.kt

@@ -13,37 +13,23 @@
  */
 package net.syncthing.java.repository.repo
 
-import com.google.protobuf.ByteString
-import com.google.protobuf.InvalidProtocolBufferException
 import com.zaxxer.hikari.HikariConfig
 import com.zaxxer.hikari.HikariDataSource
-import net.syncthing.java.bep.BlockExchangeExtraProtos
-import net.syncthing.java.bep.BlockExchangeProtos
-import net.syncthing.java.core.beans.*
-import net.syncthing.java.core.beans.FileInfo.FileType
-import net.syncthing.java.core.beans.FileInfo.Version
 import net.syncthing.java.core.interfaces.IndexRepository
-import net.syncthing.java.core.interfaces.Sequencer
+import net.syncthing.java.core.interfaces.IndexTransaction
 import net.syncthing.java.core.interfaces.TempRepository
-import org.apache.commons.lang3.tuple.Pair
-import org.apache.http.util.TextUtils.isBlank
-import org.bouncycastle.util.encoders.Hex
 import org.slf4j.LoggerFactory
 import java.io.Closeable
 import java.io.File
 import java.sql.Connection
-import java.sql.ResultSet
 import java.sql.SQLException
-import java.sql.Types
 import java.util.*
 
 class SqlRepository(databaseFolder: File) : Closeable, IndexRepository, TempRepository {
 
     private val logger = LoggerFactory.getLogger(javaClass)
-    private var sequencer: Sequencer = IndexRepoSequencer()
     private val dataSource: HikariDataSource
     //    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
-    private var onFolderStatsUpdatedListener: ((IndexRepository.FolderStatsUpdatedEvent) -> Unit)? = null
 
     @Throws(SQLException::class)
     private fun getConnection() = dataSource.connection
@@ -79,10 +65,6 @@ class SqlRepository(databaseFolder: File) : Closeable, IndexRepository, TempRepo
         logger.debug("database ready")
     }
 
-    override fun setOnFolderStatsUpdatedListener(listener: ((IndexRepository.FolderStatsUpdatedEvent) -> Unit)?) {
-        onFolderStatsUpdatedListener = listener
-    }
-
     private fun checkDb() {
         try {
             getConnection().use { connection ->
@@ -96,16 +78,17 @@ class SqlRepository(databaseFolder: File) : Closeable, IndexRepository, TempRepo
             }
         } catch (ex: SQLException) {
             logger.warn("Invalid database, resetting db", ex)
-            initDb()
+            getConnection().use {
+                initDb(it)
+            }
         }
     }
 
     @Throws(SQLException::class)
-    private fun initDb() {
+    private fun initDb(connection: Connection) {
         logger.info("init db")
-        getConnection().use { connection -> connection.prepareStatement("DROP ALL OBJECTS").use { prepareStatement -> prepareStatement.execute() } }
+        connection.prepareStatement("DROP ALL OBJECTS").use { prepareStatement -> prepareStatement.execute() }
 
-        getConnection().use { connection ->
             connection.prepareStatement("CREATE TABLE index_sequence (index_id BIGINT NOT NULL PRIMARY KEY, current_sequence BIGINT NOT NULL)").use { prepareStatement -> prepareStatement.execute() }
             connection.prepareStatement("CREATE TABLE folder_index_info (folder VARCHAR NOT NULL,"
                     + "device_id VARCHAR NOT NULL,"
@@ -151,7 +134,6 @@ class SqlRepository(databaseFolder: File) : Closeable, IndexRepository, TempRepo
                 prepareStatement.setInt(1, VERSION)
                 assert(prepareStatement.executeUpdate() == 1)
             }
-        }
 
         logger.info("database initialized")
     }
@@ -165,433 +147,28 @@ class SqlRepository(databaseFolder: File) : Closeable, IndexRepository, TempRepo
         }
     }
 
-    override fun getSequencer(): Sequencer = sequencer
+    override fun <T> runInTransaction(action: (IndexTransaction) -> T): T {
+        return getConnection().use {  connection ->
+            val transaction = SqlTransaction(connection, ::initDb)
 
-    @Throws(SQLException::class)
-    private fun readFolderIndexInfo(resultSet: ResultSet): IndexInfo {
-        return IndexInfo.newBuilder()
-                .setFolder(resultSet.getString("folder"))
-                .setDeviceId(resultSet.getString("device_id"))
-                .setIndexId(resultSet.getLong("index_id"))
-                .setLocalSequence(resultSet.getLong("local_sequence"))
-                .setMaxSequence(resultSet.getLong("max_sequence"))
-                .build()
-    }
+            try {
+                connection.autoCommit = false
+                connection.transactionIsolation = Connection.TRANSACTION_SERIALIZABLE
 
-    @Throws(SQLException::class)
-    override fun updateIndexInfo(indexInfo: IndexInfo) {
-        getConnection().use { connection ->
-            connection.prepareStatement("MERGE INTO folder_index_info"
-                    + " (folder,device_id,index_id,local_sequence,max_sequence)"
-                    + " VALUES (?,?,?,?,?)").use { prepareStatement ->
-                prepareStatement.setString(1, indexInfo.folderId)
-                prepareStatement.setString(2, indexInfo.deviceId)
-                prepareStatement.setLong(3, indexInfo.indexId)
-                prepareStatement.setLong(4, indexInfo.localSequence)
-                prepareStatement.setLong(5, indexInfo.maxSequence)
-                prepareStatement.executeUpdate()
+                action(transaction)
+            } catch (ex: Exception) {
+                connection.rollback()
+
+                throw ex
+            } finally {
+                transaction.close()
+
+                connection.commit()
+                connection.autoCommit = true
             }
         }
     }
 
-    override fun findIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo? {
-        val key = Pair.of(deviceId, folder)
-        return doFindIndexInfoByDeviceAndFolder(key.left, key.right)
-    }
-
-    @Throws(SQLException::class)
-    private fun doFindIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo? {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT * FROM folder_index_info WHERE device_id=? AND folder=?").use { prepareStatement ->
-                prepareStatement.setString(1, deviceId.deviceId)
-                prepareStatement.setString(2, folder)
-                val resultSet = prepareStatement.executeQuery()
-                return if (resultSet.first()) {
-                    readFolderIndexInfo(resultSet)
-                } else {
-                    null
-                }
-            }
-        }
-    }
-
-    @Throws(SQLException::class)
-    override fun findFileInfo(folder: String, path: String): FileInfo? {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT * FROM file_info WHERE folder=? AND path=?").use { prepareStatement ->
-                prepareStatement.setString(1, folder)
-                prepareStatement.setString(2, path)
-                val resultSet = prepareStatement.executeQuery()
-                return if (resultSet.first()) {
-                    readFileInfo(resultSet)
-                } else {
-                    null
-                }
-            }
-        }
-    }
-
-    @Throws(SQLException::class)
-    override fun findFileInfoLastModified(folder: String, path: String): Date? {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT last_modified FROM file_info WHERE folder=? AND path=?").use { prepareStatement ->
-                prepareStatement.setString(1, folder)
-                prepareStatement.setString(2, path)
-                val resultSet = prepareStatement.executeQuery()
-                return if (resultSet.first()) {
-                    Date(resultSet.getLong("last_modified"))
-                } else {
-                    null
-                }
-            }
-        }
-    }
-
-    @Throws(SQLException::class)
-    override fun findNotDeletedFileInfo(folder: String, path: String): FileInfo? {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT * FROM file_info WHERE folder=? AND path=? AND is_deleted=FALSE").use { prepareStatement ->
-                prepareStatement.setString(1, folder)
-                prepareStatement.setString(2, path)
-                val resultSet = prepareStatement.executeQuery()
-                return if (resultSet.first()) {
-                    readFileInfo(resultSet)
-                } else {
-                    null
-                }
-            }
-        }
-    }
-
-    @Throws(SQLException::class)
-    private fun readFileInfo(resultSet: ResultSet): FileInfo {
-        val folder = resultSet.getString("folder")
-        val path = resultSet.getString("path")
-        val fileType = FileType.valueOf(resultSet.getString("file_type"))
-        val lastModified = Date(resultSet.getLong("last_modified"))
-        val versionList = listOf(Version(resultSet.getLong("version_id"), resultSet.getLong("version_value")))
-        val isDeleted = resultSet.getBoolean("is_deleted")
-        val builder = FileInfo.Builder()
-                .setFolder(folder)
-                .setPath(path)
-                .setLastModified(lastModified)
-                .setVersionList(versionList)
-                .setDeleted(isDeleted)
-        return if (fileType == FileType.DIRECTORY) {
-            builder.setTypeDir().build()
-        } else {
-            builder.setTypeFile().setSize(resultSet.getLong("size")).setHash(resultSet.getString("hash")).build()
-        }
-    }
-
-    @Throws(SQLException::class, InvalidProtocolBufferException::class)
-    override fun findFileBlocks(folder: String, path: String): FileBlocks? {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT * FROM file_blocks WHERE folder=? AND path=?").use { prepareStatement ->
-                prepareStatement.setString(1, folder)
-                prepareStatement.setString(2, path)
-                val resultSet = prepareStatement.executeQuery()
-                return if (resultSet.first()) {
-                    readFileBlocks(resultSet)
-                } else {
-                    null
-                }
-            }
-        }
-    }
-
-    @Throws(SQLException::class, InvalidProtocolBufferException::class)
-    private fun readFileBlocks(resultSet: ResultSet): FileBlocks {
-        val blocks = BlockExchangeExtraProtos.Blocks.parseFrom(resultSet.getBytes("blocks"))
-        val blockList = blocks.blocksList.map { record ->
-            BlockInfo(record!!.offset, record.size, Hex.toHexString(record.hash.toByteArray()))
-        }
-        return FileBlocks(resultSet.getString("folder"), resultSet.getString("path"), blockList)
-    }
-
-    @Throws(SQLException::class)
-    override fun updateFileInfo(newFileInfo: FileInfo, newFileBlocks: FileBlocks?) {
-        val version = newFileInfo.versionList.last()
-        //TODO open transsaction, rollback
-        getConnection().use { connection ->
-            if (newFileBlocks != null) {
-                FileInfo.checkBlocks(newFileInfo, newFileBlocks)
-                connection.prepareStatement("MERGE INTO file_blocks"
-                        + " (folder,path,hash,size,blocks)"
-                        + " VALUES (?,?,?,?,?)").use { prepareStatement ->
-                    prepareStatement.setString(1, newFileBlocks.folder)
-                    prepareStatement.setString(2, newFileBlocks.path)
-                    prepareStatement.setString(3, newFileBlocks.hash)
-                    prepareStatement.setLong(4, newFileBlocks.size)
-                    prepareStatement.setBytes(5, BlockExchangeExtraProtos.Blocks.newBuilder()
-                            .addAllBlocks(newFileBlocks.blocks.map { input ->
-                                BlockExchangeProtos.BlockInfo.newBuilder()
-                                        .setOffset(input.offset)
-                                        .setSize(input.size)
-                                        .setHash(ByteString.copyFrom(Hex.decode(input.hash)))
-                                        .build()
-                            }).build().toByteArray())
-                    prepareStatement.executeUpdate()
-                }
-            }
-            val oldFileInfo = findFileInfo(newFileInfo.folder, newFileInfo.path)
-            connection.prepareStatement("MERGE INTO file_info"
-                    + " (folder,path,file_name,parent,size,hash,last_modified,file_type,version_id,version_value,is_deleted)"
-                    + " VALUES (?,?,?,?,?,?,?,?,?,?,?)").use { prepareStatement ->
-                prepareStatement.setString(1, newFileInfo.folder)
-                prepareStatement.setString(2, newFileInfo.path)
-                prepareStatement.setString(3, newFileInfo.fileName)
-                prepareStatement.setString(4, newFileInfo.parent)
-                prepareStatement.setLong(7, newFileInfo.lastModified.time)
-                prepareStatement.setString(8, newFileInfo.type.name)
-                prepareStatement.setLong(9, version.id)
-                prepareStatement.setLong(10, version.value)
-                prepareStatement.setBoolean(11, newFileInfo.isDeleted)
-                if (newFileInfo.isDirectory()) {
-                    prepareStatement.setNull(5, Types.BIGINT)
-                    prepareStatement.setNull(6, Types.VARCHAR)
-                } else {
-                    prepareStatement.setLong(5, newFileInfo.size!!)
-                    prepareStatement.setString(6, newFileInfo.hash)
-                }
-                prepareStatement.executeUpdate()
-            }
-            //update stats
-            var deltaFileCount: Long = 0
-            var deltaDirCount: Long = 0
-            var deltaSize: Long = 0
-            val oldMissing = oldFileInfo == null || oldFileInfo.isDeleted
-            val newMissing = newFileInfo.isDeleted
-            val oldSizeMissing = oldMissing || !oldFileInfo!!.isFile()
-            val newSizeMissing = newMissing || !newFileInfo.isFile()
-            if (!oldSizeMissing) {
-                deltaSize -= oldFileInfo!!.size!!
-            }
-            if (!newSizeMissing) {
-                deltaSize += newFileInfo.size!!
-            }
-            if (!oldMissing) {
-                if (oldFileInfo!!.isFile()) {
-                    deltaFileCount--
-                } else if (oldFileInfo.isDirectory()) {
-                    deltaDirCount--
-                }
-            }
-            if (!newMissing) {
-                if (newFileInfo.isFile()) {
-                    deltaFileCount++
-                } else if (newFileInfo.isDirectory()) {
-                    deltaDirCount++
-                }
-            }
-            val folderStats = updateFolderStats(connection, newFileInfo.folder, deltaFileCount, deltaDirCount, deltaSize, newFileInfo.lastModified)
-
-            onFolderStatsUpdatedListener?.invoke(object : IndexRepository.FolderStatsUpdatedEvent() {
-                override fun getFolderStats(): List<FolderStats> {
-                    return listOf(folderStats)
-                }
-            })
-        }
-    }
-
-    @Throws(SQLException::class)
-    override fun findNotDeletedFilesByFolderAndParent(folder: String, parentPath: String): MutableList<FileInfo> {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT * FROM file_info WHERE folder=? AND parent=? AND is_deleted=FALSE").use { prepareStatement ->
-                prepareStatement.setString(1, folder)
-                prepareStatement.setString(2, parentPath)
-                val resultSet = prepareStatement.executeQuery()
-                val list = mutableListOf<FileInfo>()
-                while (resultSet.next()) {
-                    list.add(readFileInfo(resultSet))
-                }
-                return list
-            }
-        }
-    }
-
-    @Throws(SQLException::class)
-    override fun findFileInfoBySearchTerm(query: String): List<FileInfo> {
-        assert(!isBlank(query))
-        //        checkArgument(maxResult > 0);
-        //        try (Connection connection = getConnection(); PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM file_info WHERE LOWER(file_name) LIKE ? AND is_deleted=FALSE LIMIT ?")) {
-            getConnection().use { connection ->
-                connection.prepareStatement("SELECT * FROM file_info WHERE LOWER(file_name) REGEXP ? AND is_deleted=FALSE").use { preparedStatement ->
-                    //        try (Connection connection = getConnection(); PreparedStatement prepareStatement = connection.prepareStatement("SELECT * FROM file_info LIMIT 10")) {
-                    //            preparedStatement.setString(1, "%" + query.trim().toLowerCase() + "%");
-                    preparedStatement.setString(1, query.trim { it <= ' ' }.toLowerCase())
-                    //            preparedStatement.setInt(2, maxResult);
-                    val resultSet = preparedStatement.executeQuery()
-                    val list = mutableListOf<FileInfo>()
-                    while (resultSet.next()) {
-                        list.add(readFileInfo(resultSet))
-                    }
-                    return list
-                }
-            }
-    }
-
-    @Throws(SQLException::class)
-    override fun countFileInfoBySearchTerm(query: String): Long {
-        assert(!isBlank(query))
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT COUNT(*) FROM file_info WHERE LOWER(file_name) REGEXP ? AND is_deleted=FALSE").use { preparedStatement ->
-                //        try (Connection connection = getConnection(); PreparedStatement preparedStatement = connection.prepareStatement("SELECT COUNT(*) FROM file_info")) {
-                preparedStatement.setString(1, query.trim { it <= ' ' }.toLowerCase())
-                val resultSet = preparedStatement.executeQuery()
-                assert(resultSet.first())
-                return resultSet.getLong(1)
-            }
-        }
-    }
-
-    // FILE INFO - END
-    override fun clearIndex() {
-        initDb()
-        sequencer = IndexRepoSequencer()
-    }
-
-    // FOLDER STATS - BEGIN
-    @Throws(SQLException::class)
-    private fun readFolderStats(resultSet: ResultSet): FolderStats {
-        return FolderStats.Builder()
-                .setFolder(resultSet.getString("folder"))
-                .setDirCount(resultSet.getLong("dir_count"))
-                .setFileCount(resultSet.getLong("file_count"))
-                .setSize(resultSet.getLong("size"))
-                .setLastUpdate(Date(resultSet.getLong("last_update")))
-                .build()
-    }
-
-    override fun findFolderStats(folder: String): FolderStats? {
-        return doFindFolderStats(folder)
-    }
-
-    @Throws(SQLException::class)
-    private fun doFindFolderStats(folder: String): FolderStats? {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT * FROM folder_stats WHERE folder=?").use { prepareStatement ->
-                prepareStatement.setString(1, folder)
-                val resultSet = prepareStatement.executeQuery()
-                return if (resultSet.first()) {
-                    readFolderStats(resultSet)
-                } else {
-                    null
-                }
-            }
-        }
-    }
-
-    @Throws(SQLException::class)
-    override fun findAllFolderStats(): List<FolderStats> {
-        getConnection().use { connection ->
-            connection.prepareStatement("SELECT * FROM folder_stats").use { prepareStatement ->
-                val resultSet = prepareStatement.executeQuery()
-                val list = mutableListOf<FolderStats>()
-                while (resultSet.next()) {
-                    val folderStats = readFolderStats(resultSet)
-                    list.add(folderStats)
-                }
-                return list
-            }
-        }
-    }
-
-    @Throws(SQLException::class)
-    private fun updateFolderStats(connection: Connection, folder: String, deltaFileCount: Long, deltaDirCount: Long, deltaSize: Long, lastUpdate: Date): FolderStats {
-        val oldFolderStats = findFolderStats(folder)
-        val newFolderStats: FolderStats
-        if (oldFolderStats == null) {
-            newFolderStats = FolderStats.Builder()
-                    .setDirCount(deltaDirCount)
-                    .setFileCount(deltaFileCount)
-                    .setFolder(folder)
-                    .setLastUpdate(lastUpdate)
-                    .setSize(deltaSize)
-                    .build()
-        } else {
-            newFolderStats = oldFolderStats.copyBuilder()
-                    .setDirCount(oldFolderStats.dirCount + deltaDirCount)
-                    .setFileCount(oldFolderStats.fileCount + deltaFileCount)
-                    .setSize(oldFolderStats.size + deltaSize)
-                    .setLastUpdate(if (lastUpdate.after(oldFolderStats.lastUpdate)) lastUpdate else oldFolderStats.lastUpdate)
-                    .build()
-        }
-        updateFolderStats(connection, newFolderStats)
-        return newFolderStats
-    }
-
-    //    private void updateFolderStats() {
-    //        logger.info("updateFolderStats BEGIN");
-    //        final Map<String, FolderStats.Builder> map = Maps.newHashMap();
-    //        final Function<String, FolderStats.Builder> func = new Function<String, FolderStats.Builder>() {
-    //            @Override
-    //            public FolderStats.Builder apply(String folder) {
-    //                FolderStats.Builder res = map.get(folder);
-    //                if (res == null) {
-    //                    res = FolderStats.newBuilder().setFolder(folder);
-    //                    map.put(folder, res);
-    //                }
-    //                return res;
-    //            }
-    //        };
-    //        final List<FolderStats> list;
-    //        try (Connection connection = getConnection()) {
-    //            try (PreparedStatement prepareStatement = connection.prepareStatement("SELECT folder, COUNT(*) AS file_count, SUM(size) AS size, MAX(last_modified) AS last_update FROM file_info WHERE file_type=? AND is_deleted=FALSE GROUP BY folder")) {
-    //                prepareStatement.setString(1, FileType.FILE.name());
-    //                ResultSet resultSet = prepareStatement.executeQuery();
-    //                while (resultSet.next()) {
-    //                    FolderStats.Builder builder = func.apply(resultSet.getString("folder"));
-    //                    builder.setSize(resultSet.getLong("size"));
-    //                    builder.setFileCount(resultSet.getLong("file_count"));
-    //                    builder.setLastUpdate(new Date(resultSet.getLong("last_update")));
-    //                }
-    //            }
-    //            try (PreparedStatement prepareStatement = connection.prepareStatement("SELECT folder, COUNT(*) AS dir_count FROM file_info WHERE file_type=? AND is_deleted=FALSE GROUP BY folder")) {
-    //                prepareStatement.setString(1, FileType.DIRECTORY.name());
-    //                ResultSet resultSet = prepareStatement.executeQuery();
-    //                while (resultSet.next()) {
-    //                    FolderStats.Builder builder = func.apply(resultSet.getString("folder"));
-    //                    builder.setDirCount(resultSet.getLong("dir_count"));
-    //                }
-    //            }
-    //            list = Lists.newArrayList(Iterables.transform(map.values(), new Function<FolderStats.Builder, FolderStats>() {
-    //                @Override
-    //                public FolderStats apply(FolderStats.Builder builder) {
-    //                    return builder.build();
-    //                }
-    //            }));
-    //            for (FolderStats folderStats : list) {
-    //                updateFolderStats(connection, folderStats);
-    //            }
-    //        } catch (SQLException ex) {
-    //            throw new RuntimeException(ex);
-    //        }
-    //        logger.info("updateFolderStats END");
-    //        eventBus.post(new FolderStatsUpdatedEvent() {
-    //            @Override
-    //            public List<FolderStats> getFolderStats() {
-    //                return Collections.unmodifiableList(list);
-    //            }
-    //        });
-    //    }
-    @Throws(SQLException::class)
-    private fun updateFolderStats(connection: Connection, folderStats: FolderStats) {
-        assert(folderStats.fileCount >= 0)
-        assert(folderStats.dirCount >= 0)
-        assert(folderStats.size >= 0)
-        connection.prepareStatement("MERGE INTO folder_stats"
-                + " (folder,file_count,dir_count,size,last_update)"
-                + " VALUES (?,?,?,?,?)").use { prepareStatement ->
-            prepareStatement.setString(1, folderStats.folderId)
-            prepareStatement.setLong(2, folderStats.fileCount)
-            prepareStatement.setLong(3, folderStats.dirCount)
-            prepareStatement.setLong(4, folderStats.size)
-            prepareStatement.setLong(5, folderStats.lastUpdate.time)
-            prepareStatement.executeUpdate()
-        }
-    }
-
     override fun close() {
         logger.info("closing index repository (sql)")
         //        scheduledExecutorService.shutdown();
@@ -650,68 +227,6 @@ class SqlRepository(databaseFolder: File) : Closeable, IndexRepository, TempRepo
         }
     }
 
-    //SEQUENCER
-    private inner class IndexRepoSequencer : Sequencer {
-
-        private var indexId: Long? = null
-        private var currentSequence: Long? = null
-
-        @Throws(SQLException::class)
-        @Synchronized private fun loadFromDb() {
-            getConnection().use { connection ->
-                connection.prepareStatement("SELECT index_id,current_sequence FROM index_sequence").use { statement ->
-                    val resultSet = statement.executeQuery()
-                    assert(resultSet.first())
-                    indexId = resultSet.getLong("index_id")
-                    currentSequence = resultSet.getLong("current_sequence")
-                    logger.info("loaded index info from db, index_id = {}, current_sequence = {}", indexId, currentSequence)
-                }
-            }
-        }
-
-        @Synchronized override fun indexId(): Long {
-            if (indexId == null) {
-                loadFromDb()
-            }
-            return indexId!!
-        }
-
-        @Throws(SQLException::class)
-        @Synchronized override fun nextSequence(): Long {
-            val nextSequence = currentSequence() + 1
-            getConnection().use { connection ->
-                connection.prepareStatement("UPDATE index_sequence SET current_sequence=?").use { statement ->
-                    statement.setLong(1, nextSequence)
-                    assert(statement.executeUpdate() == 1)
-                    logger.debug("update local index sequence to {}", nextSequence)
-                }
-            }
-
-            currentSequence = nextSequence
-            return nextSequence
-        }
-
-        @Synchronized override fun currentSequence(): Long {
-            if (currentSequence == null) {
-                loadFromDb()
-            }
-            return currentSequence!!
-        }
-    }
-
-    @Throws(SQLException::class)
-    private fun readDeviceAddress(resultSet: ResultSet): DeviceAddress {
-        val instanceId = resultSet.getLong("instance_id")
-        return DeviceAddress.Builder()
-                .setAddress(resultSet.getString("address_url"))
-                .setDeviceId(resultSet.getString("device_id"))
-                .setInstanceId(if (instanceId == 0L) null else instanceId)
-                .setProducer(DeviceAddress.AddressProducer.valueOf(resultSet.getString("address_producer")))
-                .setScore(resultSet.getInt("address_score"))
-                .setLastModified(Date(resultSet.getLong("last_modified")))
-                .build()
-    }
-
     companion object {
         private const val VERSION = 13
     }

syncthing-repository-default/src/main/kotlin/net/syncthing/java/repository/repo/SqlTransaction.kt

@@ -0,0 +1,482 @@
+/*
+ * Copyright (C) 2016 Davide Imbriaco
+ * Copyright (C) 2018 Jonas Lochmann
+ *
+ * This Java file is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.syncthing.java.repository.repo
+
+import com.google.protobuf.ByteString
+import com.google.protobuf.InvalidProtocolBufferException
+import net.syncthing.java.bep.BlockExchangeExtraProtos
+import net.syncthing.java.bep.BlockExchangeProtos
+import net.syncthing.java.core.beans.*
+import net.syncthing.java.core.interfaces.IndexTransaction
+import net.syncthing.java.core.interfaces.Sequencer
+import org.bouncycastle.util.encoders.Hex
+import java.sql.Connection
+import java.sql.ResultSet
+import java.sql.SQLException
+import java.sql.Types
+import java.util.*
+
+class SqlTransaction(
+        private val connection: Connection,
+        private val initDb: (Connection) -> Unit
+): IndexTransaction, Sequencer {
+    private var closed = false
+
+    fun close() {
+        closed = true
+    }
+
+    private fun <T> runIfAllowed(block: () -> T): T {
+        if (closed) {
+            throw IllegalStateException("transaction already done")
+        }
+
+        return block()
+    }
+
+    override fun getSequencer() = this
+
+    override fun indexId(): Long = runIfAllowed {
+        connection.prepareStatement("SELECT index_id FROM index_sequence").use { statement ->
+            val resultSet = statement.executeQuery()
+            assert(resultSet.first())
+            resultSet.getLong("index_id")
+        }
+    }
+
+    override fun currentSequence(): Long = runIfAllowed {
+        connection.prepareStatement("SELECT current_sequence FROM index_sequence").use { statement ->
+            val resultSet = statement.executeQuery()
+            assert(resultSet.first())
+            resultSet.getLong("current_sequence")
+        }
+    }
+
+    override fun nextSequence(): Long = runIfAllowed {
+        connection.prepareStatement("UPDATE index_sequence SET current_sequence = current_sequence + 1").use { statement ->
+            assert(statement.executeUpdate() == 1)
+        }
+
+        currentSequence()
+    }
+
+    override fun updateIndexInfo(indexInfo: IndexInfo): Unit = runIfAllowed {
+        connection.prepareStatement("MERGE INTO folder_index_info"
+                + " (folder,device_id,index_id,local_sequence,max_sequence)"
+                + " VALUES (?,?,?,?,?)").use { prepareStatement ->
+            prepareStatement.setString(1, indexInfo.folderId)
+            prepareStatement.setString(2, indexInfo.deviceId)
+            prepareStatement.setLong(3, indexInfo.indexId)
+            prepareStatement.setLong(4, indexInfo.localSequence)
+            prepareStatement.setLong(5, indexInfo.maxSequence)
+            prepareStatement.executeUpdate()
+        }
+    }
+
+    override fun findIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo? = runIfAllowed {
+        doFindIndexInfoByDeviceAndFolder(deviceId, folder)
+    }
+
+    @Throws(SQLException::class)
+    private fun doFindIndexInfoByDeviceAndFolder(deviceId: DeviceId, folder: String): IndexInfo? = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM folder_index_info WHERE device_id=? AND folder=?").use { prepareStatement ->
+            prepareStatement.setString(1, deviceId.deviceId)
+            prepareStatement.setString(2, folder)
+            val resultSet = prepareStatement.executeQuery()
+
+            if (resultSet.first()) {
+                readFolderIndexInfo(resultSet)
+            } else {
+                null
+            }
+        }
+    }
+
+    override fun findAllIndexInfos(): List<IndexInfo> = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM folder_index_info").use { prepareStatement ->
+            val resultSet = prepareStatement.executeQuery()
+            val list = mutableListOf<IndexInfo>()
+
+            while (resultSet.next()) {
+                list.add(readFolderIndexInfo(resultSet))
+            }
+
+            list
+        }
+    }
+
+    @Throws(SQLException::class)
+    override fun findFileInfo(folder: String, path: String): FileInfo? = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM file_info WHERE folder=? AND path=?").use { prepareStatement ->
+            prepareStatement.setString(1, folder)
+            prepareStatement.setString(2, path)
+            val resultSet = prepareStatement.executeQuery()
+
+            if (resultSet.first()) {
+                readFileInfo(resultSet)
+            } else {
+                null
+            }
+        }
+    }
+
+    override fun findFileInfo(folder: String, path: List<String>): Map<String, FileInfo> = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM folder_index_info WHERE folder=? AND PATH IN ?").use { prepareStatement ->
+            prepareStatement.setString(1, folder)
+            prepareStatement.setArray(2, connection.createArrayOf("VARCHAR", path.toTypedArray()))
+
+            val resultSet = prepareStatement.executeQuery()
+            val map = mutableMapOf<String, FileInfo>()
+
+            while (resultSet.next()) {
+                val item = readFileInfo(resultSet)
+
+                map[item.path] = item
+            }
+
+            map
+        }
+    }
+
+    @Throws(SQLException::class)
+    override fun findFileInfoLastModified(folder: String, path: String): Date? = runIfAllowed {
+        connection.prepareStatement("SELECT last_modified FROM file_info WHERE folder=? AND path=?").use { prepareStatement ->
+            prepareStatement.setString(1, folder)
+            prepareStatement.setString(2, path)
+            val resultSet = prepareStatement.executeQuery()
+
+            if (resultSet.first()) {
+                Date(resultSet.getLong("last_modified"))
+            } else {
+                null
+            }
+        }
+    }
+
+    @Throws(SQLException::class)
+    override fun findNotDeletedFileInfo(folder: String, path: String): FileInfo? = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM file_info WHERE folder=? AND path=? AND is_deleted=FALSE").use { prepareStatement ->
+            prepareStatement.setString(1, folder)
+            prepareStatement.setString(2, path)
+            val resultSet = prepareStatement.executeQuery()
+
+            if (resultSet.first()) {
+                readFileInfo(resultSet)
+            } else {
+                null
+            }
+        }
+    }
+
+    @Throws(SQLException::class)
+    private fun readFileInfo(resultSet: ResultSet): FileInfo {
+        val folder = resultSet.getString("folder")
+        val path = resultSet.getString("path")
+        val fileType = FileInfo.FileType.valueOf(resultSet.getString("file_type"))
+        val lastModified = Date(resultSet.getLong("last_modified"))
+        val versionList = listOf(FileInfo.Version(resultSet.getLong("version_id"), resultSet.getLong("version_value")))
+        val isDeleted = resultSet.getBoolean("is_deleted")
+        val builder = FileInfo.Builder()
+                .setFolder(folder)
+                .setPath(path)
+                .setLastModified(lastModified)
+                .setVersionList(versionList)
+                .setDeleted(isDeleted)
+        return if (fileType == FileInfo.FileType.DIRECTORY) {
+            builder.setTypeDir().build()
+        } else {
+            builder.setTypeFile().setSize(resultSet.getLong("size")).setHash(resultSet.getString("hash")).build()
+        }
+    }
+
+    @Throws(SQLException::class, InvalidProtocolBufferException::class)
+    override fun findFileBlocks(folder: String, path: String): FileBlocks? = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM file_blocks WHERE folder=? AND path=?").use { prepareStatement ->
+            prepareStatement.setString(1, folder)
+            prepareStatement.setString(2, path)
+            val resultSet = prepareStatement.executeQuery()
+
+            if (resultSet.first()) {
+                readFileBlocks(resultSet)
+            } else {
+                null
+            }
+        }
+    }
+
+    @Throws(SQLException::class, InvalidProtocolBufferException::class)
+    private fun readFileBlocks(resultSet: ResultSet): FileBlocks {
+        val blocks = BlockExchangeExtraProtos.Blocks.parseFrom(resultSet.getBytes("blocks"))
+        val blockList = blocks.blocksList.map { record ->
+            BlockInfo(record!!.offset, record.size, Hex.toHexString(record.hash.toByteArray()))
+        }
+        return FileBlocks(resultSet.getString("folder"), resultSet.getString("path"), blockList)
+    }
+
+    @Throws(SQLException::class)
+    override fun updateFileInfo(fileInfo: FileInfo, fileBlocks: FileBlocks?): Unit = runIfAllowed {
+        val version = fileInfo.versionList.last()
+
+        if (fileBlocks != null) {
+            FileInfo.checkBlocks(fileInfo, fileBlocks)
+            connection.prepareStatement("MERGE INTO file_blocks"
+                    + " (folder,path,hash,size,blocks)"
+                    + " VALUES (?,?,?,?,?)").use { prepareStatement ->
+                prepareStatement.setString(1, fileBlocks.folder)
+                prepareStatement.setString(2, fileBlocks.path)
+                prepareStatement.setString(3, fileBlocks.hash)
+                prepareStatement.setLong(4, fileBlocks.size)
+                prepareStatement.setBytes(5, BlockExchangeExtraProtos.Blocks.newBuilder()
+                        .addAllBlocks(fileBlocks.blocks.map { input ->
+                            BlockExchangeProtos.BlockInfo.newBuilder()
+                                    .setOffset(input.offset)
+                                    .setSize(input.size)
+                                    .setHash(ByteString.copyFrom(Hex.decode(input.hash)))
+                                    .build()
+                        }).build().toByteArray())
+                prepareStatement.executeUpdate()
+            }
+        }
+
+        connection.prepareStatement("MERGE INTO file_info"
+                + " (folder,path,file_name,parent,size,hash,last_modified,file_type,version_id,version_value,is_deleted)"
+                + " VALUES (?,?,?,?,?,?,?,?,?,?,?)").use { prepareStatement ->
+            prepareStatement.setString(1, fileInfo.folder)
+            prepareStatement.setString(2, fileInfo.path)
+            prepareStatement.setString(3, fileInfo.fileName)
+            prepareStatement.setString(4, fileInfo.parent)
+            prepareStatement.setLong(7, fileInfo.lastModified.time)
+            prepareStatement.setString(8, fileInfo.type.name)
+            prepareStatement.setLong(9, version.id)
+            prepareStatement.setLong(10, version.value)
+            prepareStatement.setBoolean(11, fileInfo.isDeleted)
+            if (fileInfo.isDirectory()) {
+                prepareStatement.setNull(5, Types.BIGINT)
+                prepareStatement.setNull(6, Types.VARCHAR)
+            } else {
+                prepareStatement.setLong(5, fileInfo.size!!)
+                prepareStatement.setString(6, fileInfo.hash)
+            }
+            prepareStatement.executeUpdate()
+        }
+    }
+
+    override fun updateFileInfoAndBlocks(fileInfos: List<FileInfo>, fileBlocks: List<FileBlocks>) = runIfAllowed {
+        connection.prepareStatement("MERGE INTO file_blocks"
+                + " (folder,path,hash,size,blocks)"
+                + " VALUES (?,?,?,?,?)").use { prepareStatement ->
+
+            fileBlocks.forEach { block ->
+                prepareStatement.setString(1, block.folder)
+                prepareStatement.setString(2, block.path)
+                prepareStatement.setString(3, block.hash)
+                prepareStatement.setLong(4, block.size)
+                prepareStatement.setBytes(5, BlockExchangeExtraProtos.Blocks.newBuilder()
+                        .addAllBlocks(block.blocks.map { input ->
+                            BlockExchangeProtos.BlockInfo.newBuilder()
+                                    .setOffset(input.offset)
+                                    .setSize(input.size)
+                                    .setHash(ByteString.copyFrom(Hex.decode(input.hash)))
+                                    .build()
+                        }).build().toByteArray())
+                prepareStatement.executeUpdate()
+            }
+        }
+
+        connection.prepareStatement("MERGE INTO file_info"
+                + " (folder,path,file_name,parent,size,hash,last_modified,file_type,version_id,version_value,is_deleted)"
+                + " VALUES (?,?,?,?,?,?,?,?,?,?,?)").use { prepareStatement ->
+
+            fileInfos.forEach { fileInfo ->
+                val version = fileInfo.versionList.last()
+
+                prepareStatement.setString(1, fileInfo.folder)
+                prepareStatement.setString(2, fileInfo.path)
+                prepareStatement.setString(3, fileInfo.fileName)
+                prepareStatement.setString(4, fileInfo.parent)
+                prepareStatement.setLong(7, fileInfo.lastModified.time)
+                prepareStatement.setString(8, fileInfo.type.name)
+                prepareStatement.setLong(9, version.id)
+                prepareStatement.setLong(10, version.value)
+                prepareStatement.setBoolean(11, fileInfo.isDeleted)
+                if (fileInfo.isDirectory()) {
+                    prepareStatement.setNull(5, Types.BIGINT)
+                    prepareStatement.setNull(6, Types.VARCHAR)
+                } else {
+                    prepareStatement.setLong(5, fileInfo.size!!)
+                    prepareStatement.setString(6, fileInfo.hash)
+                }
+                prepareStatement.executeUpdate()
+            }
+        }
+    }
+
+    @Throws(SQLException::class)
+    override fun findNotDeletedFilesByFolderAndParent(folder: String, parentPath: String): MutableList<FileInfo> = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM file_info WHERE folder=? AND parent=? AND is_deleted=FALSE").use { prepareStatement ->
+            prepareStatement.setString(1, folder)
+            prepareStatement.setString(2, parentPath)
+            val resultSet = prepareStatement.executeQuery()
+            val list = mutableListOf<FileInfo>()
+            while (resultSet.next()) {
+                list.add(readFileInfo(resultSet))
+            }
+
+            list
+        }
+    }
+
+    @Throws(SQLException::class)
+    override fun findFileInfoBySearchTerm(query: String): List<FileInfo> = runIfAllowed {
+        assert(query.isNotBlank())
+        //        checkArgument(maxResult > 0);
+        //        try (Connection connection = getConnection(); PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM file_info WHERE LOWER(file_name) LIKE ? AND is_deleted=FALSE LIMIT ?")) {
+        connection.prepareStatement("SELECT * FROM file_info WHERE LOWER(file_name) REGEXP ? AND is_deleted=FALSE").use { preparedStatement ->
+            //        try (Connection connection = getConnection(); PreparedStatement prepareStatement = connection.prepareStatement("SELECT * FROM file_info LIMIT 10")) {
+            //            preparedStatement.setString(1, "%" + query.trim().toLowerCase() + "%");
+            preparedStatement.setString(1, query.trim { it <= ' ' }.toLowerCase())
+            //            preparedStatement.setInt(2, maxResult);
+            val resultSet = preparedStatement.executeQuery()
+            val list = mutableListOf<FileInfo>()
+            while (resultSet.next()) {
+                list.add(readFileInfo(resultSet))
+            }
+
+            list
+        }
+    }
+
+    @Throws(SQLException::class)
+    override fun countFileInfoBySearchTerm(query: String): Long = runIfAllowed {
+        assert(query.isNotBlank())
+        connection.prepareStatement("SELECT COUNT(*) FROM file_info WHERE LOWER(file_name) REGEXP ? AND is_deleted=FALSE").use { preparedStatement ->
+            //        try (Connection connection = getConnection(); PreparedStatement preparedStatement = connection.prepareStatement("SELECT COUNT(*) FROM file_info")) {
+            preparedStatement.setString(1, query.trim { it <= ' ' }.toLowerCase())
+            val resultSet = preparedStatement.executeQuery()
+            assert(resultSet.first())
+            resultSet.getLong(1)
+        }
+    }
+
+    // FILE INFO - END
+    override fun clearIndex() = runIfAllowed {
+        initDb(connection)
+    }
+
+    // FOLDER STATS - BEGIN
+    @Throws(SQLException::class)
+    private fun readFolderStats(resultSet: ResultSet) = FolderStats(
+            folderId = resultSet.getString("folder"),
+            dirCount = resultSet.getLong("dir_count"),
+            fileCount = resultSet.getLong("file_count"),
+            size = resultSet.getLong("size"),
+            lastUpdate = Date(resultSet.getLong("last_update"))
+    )
+
+    override fun findFolderStats(folder: String): FolderStats? {
+        return doFindFolderStats(folder)
+    }
+
+    @Throws(SQLException::class)
+    private fun doFindFolderStats(folder: String): FolderStats? = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM folder_stats WHERE folder=?").use { prepareStatement ->
+            prepareStatement.setString(1, folder)
+            val resultSet = prepareStatement.executeQuery()
+
+            if (resultSet.first()) {
+                readFolderStats(resultSet)
+            } else {
+                null
+            }
+        }
+    }
+
+    @Throws(SQLException::class)
+    override fun findAllFolderStats(): List<FolderStats> = runIfAllowed {
+        connection.prepareStatement("SELECT * FROM folder_stats").use { prepareStatement ->
+            val resultSet = prepareStatement.executeQuery()
+            val list = mutableListOf<FolderStats>()
+            while (resultSet.next()) {
+                val folderStats = readFolderStats(resultSet)
+                list.add(folderStats)
+            }
+
+            list
+        }
+    }
+
+    override fun updateOrInsertFolderStats(folder: String, deltaFileCount: Long, deltaDirCount: Long, deltaSize: Long, lastUpdate: Date) {
+        updateFolderStats(connection, folder, deltaFileCount, deltaDirCount, deltaSize, lastUpdate)
+    }
+
+    @Throws(SQLException::class)
+    private fun updateFolderStats(
+            connection: Connection,
+            folder: String,
+            deltaFileCount: Long,
+            deltaDirCount: Long,
+            deltaSize: Long,
+            lastUpdate: Date
+    ): FolderStats = runIfAllowed {
+        val oldFolderStats = findFolderStats(folder)
+        val newFolderStats: FolderStats
+
+        if (oldFolderStats == null) {
+            newFolderStats = FolderStats(
+                    dirCount = deltaDirCount,
+                    fileCount = deltaFileCount,
+                    folderId = folder,
+                    lastUpdate = lastUpdate,
+                    size = deltaSize
+            )
+        } else {
+            newFolderStats = oldFolderStats.copy(
+                    dirCount = oldFolderStats.dirCount + deltaDirCount,
+                    fileCount = oldFolderStats.fileCount + deltaFileCount,
+                    size = oldFolderStats.size + deltaSize,
+                    lastUpdate = if (lastUpdate.after(oldFolderStats.lastUpdate)) lastUpdate else oldFolderStats.lastUpdate
+            )
+        }
+        updateFolderStats(connection, newFolderStats)
+
+        newFolderStats
+    }
+
+    @Throws(SQLException::class)
+    private fun updateFolderStats(connection: Connection, folderStats: FolderStats) {
+        assert(folderStats.fileCount >= 0)
+        assert(folderStats.dirCount >= 0)
+        assert(folderStats.size >= 0)
+        connection.prepareStatement("MERGE INTO folder_stats"
+                + " (folder,file_count,dir_count,size,last_update)"
+                + " VALUES (?,?,?,?,?)").use { prepareStatement ->
+            prepareStatement.setString(1, folderStats.folderId)
+            prepareStatement.setLong(2, folderStats.fileCount)
+            prepareStatement.setLong(3, folderStats.dirCount)
+            prepareStatement.setLong(4, folderStats.size)
+            prepareStatement.setLong(5, folderStats.lastUpdate.time)
+            prepareStatement.executeUpdate()
+        }
+    }
+
+    @Throws(SQLException::class)
+    private fun readFolderIndexInfo(resultSet: ResultSet) = IndexInfo(
+            folderId = resultSet.getString("folder"),
+            deviceId = resultSet.getString("device_id"),
+            indexId = resultSet.getLong("index_id"),
+            localSequence = resultSet.getLong("local_sequence"),
+            maxSequence = resultSet.getLong("max_sequence")
+    )
+}