Sync is now essentially fully atomic on the client side

Sync now aborts on most errors instead of continuing
Sync now holds screen, CPU and WIFI locks

app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt

@@ -12,7 +12,6 @@ import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.source.online.LewdSource
 import eu.kanade.tachiyomi.util.asJsoup
 import exh.metadata.EX_DATE_FORMAT
-import exh.metadata.ignore
 import exh.metadata.models.ExGalleryMetadata
 import exh.metadata.models.Tag
 import exh.metadata.nullIfBlank
@@ -20,6 +19,7 @@ import exh.metadata.parseHumanReadableByteCount
 import exh.ui.login.LoginController
 import exh.util.UriFilter
 import exh.util.UriGroup
+import exh.util.ignore
 import exh.util.urlImportFetchSearchManga
 import okhttp3.CacheControl
 import okhttp3.Headers

app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt

@@ -571,6 +571,8 @@ class LibraryController(
         favSyncDialog?.dismiss()
         favSyncDialog = null
         oldSyncStatus = null
+        //Clear flags
+        releaseSyncLocks()
     }
 
     private fun buildDialog() = activity?.let {
@@ -586,13 +588,25 @@ class LibraryController(
                 ?.show()
     }
 
+    private fun takeSyncLocks() {
+        activity?.window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
+    }
+
+    private fun releaseSyncLocks() {
+        activity?.window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
+    }
+
     private fun updateSyncStatus(status: FavoritesSyncStatus) {
         when(status) {
             is FavoritesSyncStatus.Idle -> {
+                releaseSyncLocks()
+
                 favSyncDialog?.dismiss()
                 favSyncDialog = null
             }
             is FavoritesSyncStatus.Error -> {
+                releaseSyncLocks()
+
                 favSyncDialog?.dismiss()
                 favSyncDialog = buildDialog()
                         ?.title("Favorites sync error")
@@ -606,6 +620,8 @@ class LibraryController(
             }
             is FavoritesSyncStatus.Processing,
             is FavoritesSyncStatus.Initializing -> {
+                takeSyncLocks()
+
                 if(favSyncDialog == null || (oldSyncStatus != null
                         && oldSyncStatus !is FavoritesSyncStatus.Initializing
                         && oldSyncStatus !is FavoritesSyncStatus.Processing))
@@ -613,24 +629,6 @@ class LibraryController(
 
                 favSyncDialog?.setContent(status.message)
             }
-            is FavoritesSyncStatus.Complete -> {
-                favSyncDialog?.dismiss()
-
-                if(status.errors.isNotEmpty()) {
-                    favSyncDialog = buildDialog()
-                            ?.title("Favorites sync complete with errors")
-                            ?.content("Some errors occurred during the sync process:\n\n"
-                                    + status.errors.joinToString("\n"))
-                            ?.cancelable(false)
-                            ?.positiveText("Ok")
-                            ?.onPositive { _, _ ->
-                                presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle())
-                            }
-                            ?.show()
-                } else {
-                    presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle())
-                }
-            }
         }
         oldSyncStatus = status
     }

app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt

@@ -10,6 +10,7 @@ import eu.kanade.tachiyomi.util.toast
 import exh.favorites.FavoritesIntroDialog
 import exh.favorites.LocalFavoritesStorage
 import exh.ui.login.LoginController
+import exh.util.trans
 import rx.android.schedulers.AndroidSchedulers
 import rx.schedulers.Schedulers
 
@@ -163,7 +164,13 @@ class SettingsEhController : SettingsController() {
                                 .content("Resetting the sync state can cause your next sync to be extremely slow.")
                                 .positiveText("Yes")
                                 .onPositive { _, _ ->
-                                    LocalFavoritesStorage().clearSnapshots()
+                                    LocalFavoritesStorage().apply {
+                                        getRealm().use {
+                                            it.trans {
+                                                clearSnapshots(it)
+                                            }
+                                        }
+                                    }
                                     it.toast("Sync state reset", Toast.LENGTH_LONG)
                                 }
                                 .negativeText("No")

app/src/main/java/eu/kanade/tachiyomi/util/ContextExtensions.kt

@@ -10,6 +10,7 @@ import android.content.IntentFilter
 import android.content.pm.PackageManager
 import android.content.res.Resources
 import android.net.ConnectivityManager
+import android.net.wifi.WifiManager
 import android.os.PowerManager
 import android.support.annotation.StringRes
 import android.support.v4.app.NotificationCompat
@@ -116,6 +117,12 @@ val Context.connectivityManager: ConnectivityManager
 val Context.powerManager: PowerManager
     get() = getSystemService(Context.POWER_SERVICE) as PowerManager
 
+/**
+ * Property to get the wifi manager from the context.
+ */
+val Context.wifiManager: WifiManager
+    get() = applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager
+
 /**
  * Function used to send a local broadcast asynchronous
  *

app/src/main/java/exh/favorites/FavoritesIntroDialog.kt

@@ -24,7 +24,7 @@ class FavoritesIntroDialog {
         <br><br>
         2. The favorite categories on ExHentai correspond to the <b>first 10 categories in the app</b> (excluding the 'Default' category). <i>Galleries in other categories will <b>NOT</b> be synced!</i>
         <br><br>
-        3. <font color='red'><b>ENSURE YOU HAVE A STABLE INTERNET CONNECTION WHEN SYNC IS IN PROGRESS!</b></font> If the internet disconnects while the app is syncing, your favorites may be left in a <i>partially-synced state</i>. This could be disastrous and can cause you to lose favorites. Backup regularly and be aware that <i>I will not be responsible for any loss of data</i>...
+        3. <font color='red'><b>ENSURE YOU HAVE A STABLE INTERNET CONNECTION WHEN SYNC IS IN PROGRESS!</b></font> If the internet disconnects while the app is syncing, your favorites may be left in a <i>partially-synced state</i>.
         <br><br>
         4. Keep the app open while favorites are syncing. Android will close apps that are in the background sometimes and that could be bad if it happens while the app is syncing.
         <br><br>

app/src/main/java/exh/favorites/FavoritesSyncHelper.kt

@@ -1,6 +1,8 @@
 package exh.favorites
 
 import android.content.Context
+import android.net.wifi.WifiManager
+import android.os.PowerManager
 import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.database.models.Category
 import eu.kanade.tachiyomi.data.database.models.Manga
@@ -9,10 +11,14 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.source.online.all.EHentai
+import eu.kanade.tachiyomi.util.powerManager
+import eu.kanade.tachiyomi.util.wifiManager
 import exh.EH_METADATA_SOURCE_ID
 import exh.EXH_SOURCE_ID
 import exh.GalleryAddEvent
 import exh.GalleryAdder
+import exh.util.ignore
+import exh.util.trans
 import okhttp3.FormBody
 import okhttp3.Request
 import rx.subjects.BehaviorSubject
@@ -22,7 +28,7 @@ import uy.kohesive.injekt.api.get
 import uy.kohesive.injekt.injectLazy
 import kotlin.concurrent.thread
 
-class FavoritesSyncHelper(context: Context) {
+class FavoritesSyncHelper(val context: Context) {
     private val db: DatabaseHelper by injectLazy()
 
     private val prefs: PreferencesHelper by injectLazy()
@@ -39,6 +45,9 @@ class FavoritesSyncHelper(context: Context) {
     private var lastThrottleTime: Long = 0
     private var throttleTime: Long = 0
 
+    private var wifiLock: WifiManager.WifiLock? = null
+    private var wakeLock: PowerManager.WakeLock? = null
+
     val status = BehaviorSubject.create<FavoritesSyncStatus>(FavoritesSyncStatus.Idle())
 
     @Synchronized
@@ -69,30 +78,44 @@ class FavoritesSyncHelper(context: Context) {
             return
         }
 
-        val errors = mutableListOf<String>()
-
         try {
-            db.inTransaction {
+            //Take wake + wifi locks
-                status.onNext(FavoritesSyncStatus.Processing("Calculating remote changes"))
+            wakeLock?.release()
-                val remoteChanges = storage.getChangedRemoteEntries(favorites.first)
+            wakeLock = ignore {
-                val localChanges = if(prefs.eh_readOnlySync().getOrDefault()) {
+                context.powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK,
-                    null //Do not build local changes if they are not going to be applied
+                        "ExhFavoritesSyncWakelock")
-                } else {
+            }
-                    status.onNext(FavoritesSyncStatus.Processing("Calculating local changes"))
+            wifiLock?.release()
-                    storage.getChangedDbEntries()
+            wifiLock = ignore {
+                context.wifiManager.createWifiLock(WifiManager.WIFI_MODE_FULL,
+                        "ExhFavoritesSyncWifi")
+            }
+
+            storage.getRealm().use { realm ->
+                realm.trans {
+                    db.inTransaction {
+                        status.onNext(FavoritesSyncStatus.Processing("Calculating remote changes"))
+                        val remoteChanges = storage.getChangedRemoteEntries(realm, favorites.first)
+                        val localChanges = if(prefs.eh_readOnlySync().getOrDefault()) {
+                            null //Do not build local changes if they are not going to be applied
+                        } else {
+                            status.onNext(FavoritesSyncStatus.Processing("Calculating local changes"))
+                            storage.getChangedDbEntries(realm)
+                        }
+
+                        //Apply remote categories
+                        status.onNext(FavoritesSyncStatus.Processing("Updating category names"))
+                        applyRemoteCategories(favorites.second)
+
+                        //Apply change sets
+                        applyChangeSetToLocal(remoteChanges)
+                        if(localChanges != null)
+                            applyChangeSetToRemote(localChanges)
+
+                        status.onNext(FavoritesSyncStatus.Processing("Cleaning up"))
+                        storage.snapshotEntries(realm)
+                    }
                 }
-
-                //Apply remote categories
-                status.onNext(FavoritesSyncStatus.Processing("Updating category names"))
-                applyRemoteCategories(favorites.second)
-
-                //Apply change sets
-                applyChangeSetToLocal(remoteChanges, errors)
-                if(localChanges != null)
-                    applyChangeSetToRemote(localChanges, errors)
-
-                status.onNext(FavoritesSyncStatus.Processing("Cleaning up"))
-                storage.snapshotEntries()
             }
         } catch(e: IgnoredException) {
             //Do not display error as this error has already been reported
@@ -102,9 +125,13 @@ class FavoritesSyncHelper(context: Context) {
             status.onNext(FavoritesSyncStatus.Error("Unknown error: ${e.message}"))
             Timber.e(e, "Sync error!")
             return
+        } finally {
+            //Release wake + wifi locks
+            ignore { wakeLock?.release() }
+            ignore { wifiLock?.release() }
         }
 
-        status.onNext(FavoritesSyncStatus.Complete(errors))
+        status.onNext(FavoritesSyncStatus.Idle())
     }
 
     private fun applyRemoteCategories(categories: List<String>) {
@@ -149,7 +176,7 @@ class FavoritesSyncHelper(context: Context) {
             db.insertCategories(newLocalCategories).executeAsBlocking()
     }
 
-    private fun addGalleryRemote(gallery: FavoriteEntry, errors: MutableList<String>) {
+    private fun addGalleryRemote(gallery: FavoriteEntry) {
         val url = "${exh.baseUrl}/gallerypopups.php?gid=${gallery.gid}&t=${gallery.token}&act=addfav"
 
         val request = Request.Builder()
@@ -163,11 +190,12 @@ class FavoritesSyncHelper(context: Context) {
                 .build()
 
         if(!explicitlyRetryExhRequest(10, request)) {
-            errors += "Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!"
+            status.onNext(FavoritesSyncStatus.Error("Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!"))
+            throw IgnoredException()
         }
     }
 
-    private fun explicitlyRetryExhRequest(retryCount: Int, request: Request, minDelay: Int = 0): Boolean {
+    private fun explicitlyRetryExhRequest(retryCount: Int, request: Request): Boolean {
         var success = false
 
         for(i in 1 .. retryCount) {
@@ -186,7 +214,7 @@ class FavoritesSyncHelper(context: Context) {
         return success
     }
 
-    private fun applyChangeSetToRemote(changeSet: ChangeSet, errors: MutableList<String>) {
+    private fun applyChangeSetToRemote(changeSet: ChangeSet) {
         //Apply removals
         if(changeSet.removed.isNotEmpty()) {
             status.onNext(FavoritesSyncStatus.Processing("Removing ${changeSet.removed.size} galleries from remote server"))
@@ -221,11 +249,11 @@ class FavoritesSyncHelper(context: Context) {
 
             throttle()
 
-            addGalleryRemote(it, errors)
+            addGalleryRemote(it)
         }
     }
 
-    private fun applyChangeSetToLocal(changeSet: ChangeSet, errors: MutableList<String>) {
+    private fun applyChangeSetToLocal(changeSet: ChangeSet) {
         val removedManga = mutableListOf<Manga>()
 
         //Apply removals
@@ -266,10 +294,11 @@ class FavoritesSyncHelper(context: Context) {
                     EXH_SOURCE_ID)
 
             if(result is GalleryAddEvent.Fail) {
-                errors += "Failed to add gallery to local database: " + when (result) {
+                status.onNext(FavoritesSyncStatus.Error("Failed to add gallery to local database: " + when (result) {
                     is GalleryAddEvent.Fail.Error -> "'${it.title}' ${result.logMessage}"
                     is GalleryAddEvent.Fail.UnknownType -> "'${it.title}' (${result.galleryUrl}) is not a valid gallery!"
-                }
+                }))
+                throw IgnoredException()
             } else if(result is GalleryAddEvent.Success) {
                 insertedMangaCategories += MangaCategory.create(result.manga,
                         categories[it.category])
@@ -318,5 +347,4 @@ sealed class FavoritesSyncStatus(val message: String) {
         (message + "\n\nSync is currently throttling (to avoid being banned from ExHentai) and may take a long to complete.")
     else
         message)
-    class Complete(val errors: List<String>) : FavoritesSyncStatus("Sync complete!")
 }

app/src/main/java/exh/favorites/LocalFavoritesStorage.kt

@@ -6,7 +6,6 @@ import eu.kanade.tachiyomi.source.online.all.EHentai
 import exh.EH_SOURCE_ID
 import exh.EXH_SOURCE_ID
 import exh.metadata.models.ExGalleryMetadata
-import exh.util.trans
 import io.realm.Realm
 import io.realm.RealmConfiguration
 import uy.kohesive.injekt.injectLazy
@@ -19,11 +18,10 @@ class LocalFavoritesStorage {
             .deleteRealmIfMigrationNeeded()
             .build()
 
-    private val realm
+    fun getRealm() = Realm.getInstance(realmConfig)
-        get() = Realm.getInstance(realmConfig)
 
-    fun getChangedDbEntries()
+    fun getChangedDbEntries(realm: Realm)
-            = getChangedEntries(
+            = getChangedEntries(realm,
             parseToFavoriteEntries(
                     loadDbCategories(
                             db.getFavoriteMangas()
@@ -33,8 +31,8 @@ class LocalFavoritesStorage {
             )
     )
 
-    fun getChangedRemoteEntries(entries: List<EHentai.ParsedManga>)
+    fun getChangedRemoteEntries(realm: Realm, entries: List<EHentai.ParsedManga>)
-            = getChangedEntries(
+            = getChangedEntries(realm,
             parseToFavoriteEntries(
                     entries.asSequence().map {
                         Pair(it.fav, it.manga.apply {
@@ -44,7 +42,7 @@ class LocalFavoritesStorage {
             )
     )
 
-    fun snapshotEntries() {
+    fun snapshotEntries(realm: Realm) {
         val dbMangas = parseToFavoriteEntries(
                 loadDbCategories(
                         db.getFavoriteMangas()
@@ -53,43 +51,33 @@ class LocalFavoritesStorage {
                 )
         )
 
-        realm.use { realm ->
+        //Delete old snapshot
-            realm.trans {
+        realm.delete(FavoriteEntry::class.java)
-                //Delete old snapshot
-                realm.delete(FavoriteEntry::class.java)
 
-                //Insert new snapshots
+        //Insert new snapshots
-                realm.copyToRealm(dbMangas.toList())
+        realm.copyToRealm(dbMangas.toList())
-            }
-        }
     }
 
-    fun clearSnapshots() {
+    fun clearSnapshots(realm: Realm) {
-        realm.use {
+        realm.delete(FavoriteEntry::class.java)
-            it.trans {
-                it.delete(FavoriteEntry::class.java)
-            }
-        }
     }
 
-    private fun getChangedEntries(entries: Sequence<FavoriteEntry>): ChangeSet {
+    private fun getChangedEntries(realm: Realm, entries: Sequence<FavoriteEntry>): ChangeSet {
-        return realm.use { realm ->
+        val terminated = entries.toList()
-            val terminated = entries.toList()
 
-            val added = terminated.filter {
+        val added = terminated.filter {
-                realm.queryRealmForEntry(it) == null
+            realm.queryRealmForEntry(it) == null
-            }
-
-            val removed = realm.where(FavoriteEntry::class.java)
-                    .findAll()
-                    .filter {
-                        queryListForEntry(terminated, it) == null
-                    }.map {
-                        realm.copyFromRealm(it)
-                    }
-
-            ChangeSet(added, removed)
         }
+
+        val removed = realm.where(FavoriteEntry::class.java)
+                .findAll()
+                .filter {
+                    queryListForEntry(terminated, it) == null
+                }.map {
+                    realm.copyFromRealm(it)
+                }
+
+        return ChangeSet(added, removed)
     }
 
     private fun Realm.queryRealmForEntry(entry: FavoriteEntry)
@@ -100,7 +88,7 @@ class LocalFavoritesStorage {
             .findFirst()
 
     private fun queryListForEntry(list: List<FavoriteEntry>, entry: FavoriteEntry)
-        = list.find {
+            = list.find {
         it.gid == entry.gid
                 && it.token == entry.token
                 && it.category == entry.category

app/src/main/java/exh/metadata/MetadataUtil.kt

@@ -43,10 +43,6 @@ fun String?.nullIfBlank(): String? = if(isNullOrBlank())
 else
     this
 
-fun <T> ignore(expr: () -> T): T? {
-    return try { expr() } catch (t: Throwable) { null }
-}
-
 fun <K,V> Set<Map.Entry<K,V>>.forEach(action: (K, V) -> Unit) {
     forEach { action(it.key, it.value) }
 }

app/src/main/java/exh/util/ExceptionUtil.kt

@@ -0,0 +1,7 @@
+package exh.util
+
+inline fun <T> ignore(expr: () -> T): T? {
+    return try { expr() } catch (t: Throwable) { null }
+}
+
+