Finish favorites sync

CHANGELOG.md

@@ -4,6 +4,8 @@
 - Upstream merge
 - Fix Tsumino and HentaiCafe links not triggering a link import
 - Fix many bugs in link interceptor
+- Completely revamp favorites sync algorithm
+- Various bug fixes
 
 ### 6.6.0
 - Many performance improvements

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt

@@ -120,4 +120,8 @@ object PreferenceKeys {
     fun trackToken(syncId: Int) = "track_token_$syncId"
 
     const val eh_nh_useHighQualityThumbs = "eh_nh_hq_thumbs"
+
+    const val eh_showSyncIntro = "eh_show_sync_intro"
+
+    const val eh_readOnlySync = "eh_sync_read_only"
 }

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt

@@ -206,5 +206,9 @@ class PreferencesHelper(val context: Context) {
     fun lockUseFingerprint() = rxPrefs.getBoolean("lock_finger", false)
 
     fun eh_useHighQualityThumbs() = rxPrefs.getBoolean(Keys.eh_nh_useHighQualityThumbs, false)
+
+    fun eh_showSyncIntro() = rxPrefs.getBoolean(Keys.eh_showSyncIntro, true)
+
+    fun eh_readOnlySync() = rxPrefs.getBoolean(Keys.eh_readOnlySync, false)
     // <-- EH
 }

app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt

@@ -37,6 +37,7 @@ import eu.kanade.tachiyomi.ui.migration.MigrationController
 import eu.kanade.tachiyomi.util.inflate
 import eu.kanade.tachiyomi.util.toast
 import eu.kanade.tachiyomi.widget.DrawerSwipeCloseListener
+import exh.favorites.FavoritesIntroDialog
 import exh.favorites.FavoritesSyncStatus
 import exh.metadata.loadAllMetadata
 import exh.metadata.models.SearchableGalleryMetadata
@@ -410,9 +411,14 @@ class LibraryController(
             R.id.action_source_migration -> {
                 router.pushController(MigrationController().withFadeTransaction())
             }
-            R.id.action_download_favorites -> {
+            // --> EXH
-                presenter.favoritesSync.runSync()
+            R.id.action_sync_favorites -> {
+                if(preferences.eh_showSyncIntro().getOrDefault())
+                    activity?.let { FavoritesIntroDialog().show(it) }
+                else
+                    presenter.favoritesSync.runSync()
             }
+            // <-- EXH
             else -> return super.onOptionsItemSelected(item)
         }
 

app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt

@@ -1,8 +1,14 @@
 package eu.kanade.tachiyomi.ui.setting
 
 import android.support.v7.preference.PreferenceScreen
+import android.widget.Toast
+import com.afollestad.materialdialogs.MaterialDialog
 import com.bluelinelabs.conductor.RouterTransaction
 import com.bluelinelabs.conductor.changehandler.FadeChangeHandler
+import eu.kanade.tachiyomi.data.preference.PreferenceKeys
+import eu.kanade.tachiyomi.util.toast
+import exh.favorites.FavoritesIntroDialog
+import exh.favorites.LocalFavoritesStorage
 import exh.ui.login.LoginController
 import rx.android.schedulers.AndroidSchedulers
 import rx.schedulers.Schedulers
@@ -125,5 +131,47 @@ class SettingsEhController : SettingsController() {
                     "tr_20"
             )
         }.dependency = "enable_exhentai"
+
+        preferenceCategory {
+            title = "Favorites sync"
+
+            switchPreference {
+                title = "Disable favorites uploading"
+                summary = "Favorites are only downloaded from ExHentai. Any changes to favorites in the app will not be uploaded. Prevents accidental loss of favorites on ExHentai. Note that removals will still be downloaded (if you remove a favorites on ExHentai, it will be removed in the app as well)."
+                key = PreferenceKeys.eh_readOnlySync
+            }
+
+            preference {
+                title = "Show favorites sync notes"
+                summary = "Show some information regarding the favorites sync feature"
+
+                onClick {
+                    activity?.let {
+                        FavoritesIntroDialog().show(it)
+                    }
+                }
+            }
+
+            preference {
+                title = "Force sync state reset"
+                summary = "Performs a full resynchronization on the next sync. Removals will not be synced. All favorites in the app will be re-uploaded to ExHentai and all favorites on ExHentai will be redownloaded into the app. Useful for repairing sync after sync has been interrupted."
+
+                onClick {
+                    activity?.let {
+                        MaterialDialog.Builder(it)
+                                .title("Are you sure?")
+                                .content("Resetting the sync state can cause your next sync to be extremely slow.")
+                                .positiveText("Yes")
+                                .onPositive { _, _ ->
+                                    LocalFavoritesStorage().clearSnapshots()
+                                    it.toast("Sync state reset", Toast.LENGTH_LONG)
+                                }
+                                .negativeText("No")
+                                .cancelable(false)
+                                .show()
+                    }
+                }
+            }
+        }
     }
 }

app/src/main/java/exh/favorites/FavoritesIntroDialog.kt

@@ -0,0 +1,33 @@
+package exh.favorites
+
+import android.content.Context
+import android.text.Html
+import com.afollestad.materialdialogs.MaterialDialog
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import uy.kohesive.injekt.injectLazy
+
+class FavoritesIntroDialog {
+    private val prefs: PreferencesHelper by injectLazy()
+
+    fun show(context: Context) = MaterialDialog.Builder(context)
+            .title("IMPORTANT FAVORITES SYNC NOTES")
+            .content(Html.fromHtml(FAVORITES_INTRO_TEXT))
+            .positiveText("Ok")
+            .onPositive { _, _ ->
+                prefs.eh_showSyncIntro().set(false)
+            }
+            .cancelable(false)
+            .show()
+
+    private val FAVORITES_INTRO_TEXT = """
+        1. Changes to category names in the app are <b>NOT</b> synced! Please <i>change the category names on ExHentai instead</i>. The category names will be copied from the ExHentai servers every sync.
+        <br><br>
+        2. The favorite categories on ExHentai correspond to the <b>first 10 categories in the app</b> (excluding the 'Default' category). <i>Galleries in other categories will <b>NOT</b> be synced!</i>
+        <br><br>
+        3. <font color='red'><b>ENSURE YOU HAVE A STABLE INTERNET CONNECTION WHEN SYNC IS IN PROGRESS!</b></font> If the internet disconnects while the app is syncing, your favorites may be left in a <i>partially-synced state</i>. This could be disastrous and can cause you to lose favorites. Backup regularly and be aware that <i>I will not be responsible for any loss of data</i>...
+        <br><br>
+        4. Keep the app open while favorites are syncing. Android will close apps that are in the background sometimes and that could be bad if it happens while the app is syncing.
+        <br><br>
+        This dialog will only popup once. You can read these notes again by going to 'Settings > E-Hentai > Show favorites sync notes'.
+""".trimIndent()
+}

app/src/main/java/exh/favorites/FavoritesSyncHelper.kt

@@ -36,6 +36,9 @@ class FavoritesSyncHelper(context: Context) {
 
     private val galleryAdder = GalleryAdder()
 
+    private var lastThrottleTime: Long = 0
+    private var throttleTime: Long = 0
+
     val status = BehaviorSubject.create<FavoritesSyncStatus>(FavoritesSyncStatus.Idle())
 
     @Synchronized
@@ -70,16 +73,23 @@ class FavoritesSyncHelper(context: Context) {
 
         try {
             db.inTransaction {
+                status.onNext(FavoritesSyncStatus.Processing("Calculating remote changes"))
                 val remoteChanges = storage.getChangedRemoteEntries(favorites.first)
-                val localChanges = storage.getChangedDbEntries()
+                val localChanges = if(prefs.eh_readOnlySync().getOrDefault()) {
+                    null //Do not build local changes if they are not going to be applied
+                } else {
+                    status.onNext(FavoritesSyncStatus.Processing("Calculating local changes"))
+                    storage.getChangedDbEntries()
+                }
 
                 //Apply remote categories
                 status.onNext(FavoritesSyncStatus.Processing("Updating category names"))
                 applyRemoteCategories(favorites.second)
 
-                //Apply ChangeSets
+                //Apply change sets
                 applyChangeSetToLocal(remoteChanges, errors)
-                applyChangeSetToRemote(localChanges, errors)
+                if(localChanges != null)
+                    applyChangeSetToRemote(localChanges, errors)
 
                 status.onNext(FavoritesSyncStatus.Processing("Cleaning up"))
                 storage.snapshotEntries()
@@ -157,7 +167,7 @@ class FavoritesSyncHelper(context: Context) {
         }
     }
 
-    private fun explicitlyRetryExhRequest(retryCount: Int, request: Request): Boolean {
+    private fun explicitlyRetryExhRequest(retryCount: Int, request: Request, minDelay: Int = 0): Boolean {
         var success = false
 
         for(i in 1 .. retryCount) {
@@ -204,8 +214,12 @@ class FavoritesSyncHelper(context: Context) {
         }
 
         //Apply additions
+        resetThrottle()
         changeSet.added.forEachIndexed { index, it ->
-            status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to remote server"))
+            status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to remote server",
+                    needWarnThrottle()))
+
+            throttle()
 
             addGalleryRemote(it, errors)
         }
@@ -239,8 +253,12 @@ class FavoritesSyncHelper(context: Context) {
         val categories = db.getCategories().executeAsBlocking()
 
         //Apply additions
+        resetThrottle()
         changeSet.added.forEachIndexed { index, it ->
-            status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to local library"))
+            status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to local library",
+                    needWarnThrottle()))
+
+            throttle()
 
             //Import using gallery adder
             val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}",
@@ -262,13 +280,43 @@ class FavoritesSyncHelper(context: Context) {
         db.setMangaCategories(insertedMangaCategories, insertedMangaCategoriesMangas)
     }
 
+    fun throttle() {
+        //Throttle requests if necessary
+        val now = System.currentTimeMillis()
+        val timeDiff = now - lastThrottleTime
+        if(timeDiff < throttleTime)
+            Thread.sleep(throttleTime - timeDiff)
+
+        if(throttleTime < THROTTLE_MAX)
+            throttleTime += THROTTLE_INC
+
+        lastThrottleTime = System.currentTimeMillis()
+    }
+
+    fun resetThrottle() {
+        lastThrottleTime = 0
+        throttleTime = 0
+    }
+
+    fun needWarnThrottle()
+        = throttleTime >= THROTTLE_WARN
+
     class IgnoredException : RuntimeException()
+
+    companion object {
+        private const val THROTTLE_MAX = 5000
+        private const val THROTTLE_INC = 10
+        private const val THROTTLE_WARN = 1000
+    }
 }
 
 sealed class FavoritesSyncStatus(val message: String) {
     class Error(message: String) : FavoritesSyncStatus(message)
     class Idle : FavoritesSyncStatus("Waiting for sync to start")
     class Initializing : FavoritesSyncStatus("Initializing sync")
-    class Processing(message: String) : FavoritesSyncStatus(message)
+    class Processing(message: String, isThrottle: Boolean = false) : FavoritesSyncStatus(if(isThrottle)
+        (message + "\n\nSync is currently throttling (to avoid being banned from ExHentai) and may take a long to complete.")
+    else
+        message)
     class Complete(val errors: List<String>) : FavoritesSyncStatus("Sync complete!")
 }

app/src/main/java/exh/favorites/LocalFavoritesStorage.kt

@@ -64,6 +64,14 @@ class LocalFavoritesStorage {
         }
     }
 
+    fun clearSnapshots() {
+        realm.use {
+            it.trans {
+                it.delete(FavoriteEntry::class.java)
+            }
+        }
+    }
+
     private fun getChangedEntries(entries: Sequence<FavoriteEntry>): ChangeSet {
         return realm.use { realm ->
             val terminated = entries.toList()

app/src/main/res/drawable/ic_cloud_white_24dp.xml

@@ -0,0 +1,9 @@
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+        android:width="24dp"
+        android:height="24dp"
+        android:viewportWidth="24.0"
+        android:viewportHeight="24.0">
+    <path
+        android:fillColor="#ffffff"
+        android:pathData="M19.35,10.04C18.67,6.59 15.64,4 12,4 9.11,4 6.6,5.64 5.35,8.04 2.34,8.36 0,10.91 0,14c0,3.31 2.69,6 6,6h13c2.76,0 5,-2.24 5,-5 0,-2.64 -2.05,-4.78 -4.65,-4.96z"/>
+</vector>

app/src/main/res/menu/library.xml

@@ -23,9 +23,9 @@
         app:showAsAction="ifRoom"/>
 
     <item
-        android:id="@+id/action_download_favorites"
+        android:id="@+id/action_sync_favorites"
-        android:icon="@drawable/ic_cloud_download_white_24dp"
+        android:icon="@drawable/ic_cloud_white_24dp"
-        android:title="Download favorites"
+        android:title="Sync favorites"
         app:showAsAction="ifRoom"/>
 
     <item