From 117214c671e82e71be78a8aaa499f8542f003b18 Mon Sep 17 00:00:00 2001 From: NerdNumber9 Date: Fri, 23 Feb 2018 18:57:19 -0500 Subject: [PATCH] Implement lenient sync --- .../data/preference/PreferenceKeys.kt | 2 + .../data/preference/PreferencesHelper.kt | 2 + .../tachiyomi/ui/library/LibraryController.kt | 14 ++++ .../ui/setting/SettingsEhController.kt | 10 ++- .../java/exh/favorites/FavoritesSyncHelper.kt | 76 +++++++++++++------ 5 files changed, 79 insertions(+), 25 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt index e67d4f922..53f5f40c1 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt @@ -125,6 +125,8 @@ object PreferenceKeys { const val eh_readOnlySync = "eh_sync_read_only" + const val eh_lenientSync = "eh_lenient_sync" + const val eh_useOrigImages = "eh_useOrigImages" const val eh_ehSettingsProfile = "eh_ehSettingsProfile" diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt index fc942a4ed..b6ff3feca 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt @@ -217,6 +217,8 @@ class PreferencesHelper(val context: Context) { fun eh_readOnlySync() = rxPrefs.getBoolean(Keys.eh_readOnlySync, false) + fun eh_lenientSync() = rxPrefs.getBoolean(Keys.eh_lenientSync, false) + fun eh_showSettingsUploadWarning() = rxPrefs.getBoolean(Keys.eh_showSettingsUploadWarning, true) // <-- EH } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt index 0d0cef6e9..50df61fc2 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt @@ -619,6 +619,20 @@ class LibraryController( } ?.show() } + is FavoritesSyncStatus.CompleteWithErrors -> { + releaseSyncLocks() + + favSyncDialog?.dismiss() + favSyncDialog = buildDialog() + ?.title("Favorites sync complete with errors") + ?.content("Errors occurred during the sync process that were ignored:\n${status.message}") + ?.cancelable(false) + ?.positiveText("Ok") + ?.onPositive { _, _ -> + presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle()) + } + ?.show() + } is FavoritesSyncStatus.Processing, is FavoritesSyncStatus.Initializing -> { takeSyncLocks() diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt index c08824d50..f1abf79c4 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt @@ -135,6 +135,7 @@ class SettingsEhController : SettingsController() { title = "Disable favorites uploading" summary = "Favorites are only downloaded from ExHentai. Any changes to favorites in the app will not be uploaded. Prevents accidental loss of favorites on ExHentai. Note that removals will still be downloaded (if you remove a favorites on ExHentai, it will be removed in the app as well)." key = PreferenceKeys.eh_readOnlySync + defaultValue = false } preference { @@ -148,9 +149,16 @@ class SettingsEhController : SettingsController() { } } + switchPreference { + title = "Ignore sync errors when possible" + summary = "Do not abort immediately when encountering errors during the sync process. Errors will still be displayed when the sync is complete. Can cause loss of favorites in some cases. Useful when syncing large libraries." + key = PreferenceKeys.eh_lenientSync + defaultValue = false + } + preference { title = "Force sync state reset" - summary = "Performs a full resynchronization on the next sync. Removals will not be synced. All favorites in the app will be re-uploaded to ExHentai and all favorites on ExHentai will be redownloaded into the app. Useful for repairing sync after sync has been interrupted." + summary = "Performs a full resynchronization on the next sync. Removals will not be synced. All favorites in the app will be re-uploaded to ExHentai and all favorites on ExHentai will be re-downloaded into the app. Useful for repairing sync after sync has been interrupted." onClick { activity?.let { diff --git a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt index e3c730b04..b615129af 100644 --- a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt +++ b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt @@ -80,6 +80,8 @@ class FavoritesSyncHelper(val context: Context) { return } + val errorList = mutableListOf() + try { //Take wake + wifi locks ignore { wakeLock?.release() } @@ -107,12 +109,12 @@ class FavoritesSyncHelper(val context: Context) { //Apply remote categories status.onNext(FavoritesSyncStatus.Processing("Updating category names")) - applyRemoteCategories(favorites.second) + applyRemoteCategories(errorList, favorites.second) //Apply change sets - applyChangeSetToLocal(remoteChanges) + applyChangeSetToLocal(errorList, remoteChanges) if(localChanges != null) - applyChangeSetToRemote(localChanges) + applyChangeSetToRemote(errorList, localChanges) status.onNext(FavoritesSyncStatus.Processing("Cleaning up")) storage.snapshotEntries(realm) @@ -144,10 +146,13 @@ class FavoritesSyncHelper(val context: Context) { } } - status.onNext(FavoritesSyncStatus.Idle()) + if(errorList.isEmpty()) + status.onNext(FavoritesSyncStatus.Idle()) + else + status.onNext(FavoritesSyncStatus.CompleteWithErrors(errorList)) } - private fun applyRemoteCategories(categories: List) { + private fun applyRemoteCategories(errorList: MutableList, categories: List) { val localCategories = db.getCategories().executeAsBlocking() val newLocalCategories = localCategories.toMutableList() @@ -189,7 +194,7 @@ class FavoritesSyncHelper(val context: Context) { db.insertCategories(newLocalCategories).executeAsBlocking() } - private fun addGalleryRemote(gallery: FavoriteEntry) { + private fun addGalleryRemote(errorList: MutableList, gallery: FavoriteEntry) { val url = "${exh.baseUrl}/gallerypopups.php?gid=${gallery.gid}&t=${gallery.token}&act=addfav" val request = Request.Builder() @@ -203,8 +208,14 @@ class FavoritesSyncHelper(val context: Context) { .build() if(!explicitlyRetryExhRequest(10, request)) { - status.onNext(FavoritesSyncStatus.Error("Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!")) - throw IgnoredException() + val errorString = "Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!" + + if(prefs.eh_lenientSync().getOrDefault()) { + errorList += errorString + } else { + status.onNext(FavoritesSyncStatus.Error(errorString)) + throw IgnoredException() + } } } @@ -227,7 +238,7 @@ class FavoritesSyncHelper(val context: Context) { return success } - private fun applyChangeSetToRemote(changeSet: ChangeSet) { + private fun applyChangeSetToRemote(errorList: MutableList, changeSet: ChangeSet) { //Apply removals if(changeSet.removed.isNotEmpty()) { status.onNext(FavoritesSyncStatus.Processing("Removing ${changeSet.removed.size} galleries from remote server")) @@ -247,10 +258,14 @@ class FavoritesSyncHelper(val context: Context) { .build() if(!explicitlyRetryExhRequest(10, request)) { - status.onNext(FavoritesSyncStatus.Error("Unable to delete galleries from the remote servers!")) + val errorString = "Unable to delete galleries from the remote servers!" - //It is still safe to stop here so crash - throw IgnoredException() + if(prefs.eh_lenientSync().getOrDefault()) { + errorList += errorString + } else { + status.onNext(FavoritesSyncStatus.Error(errorString)) + throw IgnoredException() + } } } @@ -262,11 +277,11 @@ class FavoritesSyncHelper(val context: Context) { throttle() - addGalleryRemote(it) + addGalleryRemote(errorList, it) } } - private fun applyChangeSetToLocal(changeSet: ChangeSet) { + private fun applyChangeSetToLocal(errorList: MutableList, changeSet: ChangeSet) { val removedManga = mutableListOf() //Apply removals @@ -287,10 +302,12 @@ class FavoritesSyncHelper(val context: Context) { } } - db.deleteOldMangasCategories(removedManga).executeAsBlocking() + // Can't do too many DB OPs in one go + removedManga.chunked(10).forEach { + db.deleteOldMangasCategories(it).executeAsBlocking() + } - val insertedMangaCategories = mutableListOf() - val insertedMangaCategoriesMangas = mutableListOf() + val insertedMangaCategories = mutableListOf>() val categories = db.getCategories().executeAsBlocking() //Apply additions @@ -307,19 +324,29 @@ class FavoritesSyncHelper(val context: Context) { EXH_SOURCE_ID) if(result is GalleryAddEvent.Fail) { - status.onNext(FavoritesSyncStatus.Error("Failed to add gallery to local database: " + when (result) { + val errorString = "Failed to add gallery to local database: " + when (result) { is GalleryAddEvent.Fail.Error -> "'${it.title}' ${result.logMessage}" is GalleryAddEvent.Fail.UnknownType -> "'${it.title}' (${result.galleryUrl}) is not a valid gallery!" - })) - throw IgnoredException() + } + + if(prefs.eh_lenientSync().getOrDefault()) { + errorList += errorString + } else { + status.onNext(FavoritesSyncStatus.Error(errorString)) + throw IgnoredException() + } } else if(result is GalleryAddEvent.Success) { insertedMangaCategories += MangaCategory.create(result.manga, - categories[it.category]) - insertedMangaCategoriesMangas += result.manga + categories[it.category]) to result.manga } } - db.setMangaCategories(insertedMangaCategories, insertedMangaCategoriesMangas) + // Can't do too many DB OPs in one go + insertedMangaCategories.chunked(10).map { + Pair(it.map { it.first }, it.map { it.second }) + }.forEach { + db.setMangaCategories(it.first, it.second) + } } fun throttle() { @@ -341,7 +368,7 @@ class FavoritesSyncHelper(val context: Context) { } fun needWarnThrottle() - = throttleTime >= THROTTLE_WARN + = throttleTime >= THROTTLE_WARN class IgnoredException : RuntimeException() @@ -360,4 +387,5 @@ sealed class FavoritesSyncStatus(val message: String) { (message + "\n\nSync is currently throttling (to avoid being banned from ExHentai) and may take a long time to complete.") else message) + class CompleteWithErrors(messages: List) : FavoritesSyncStatus(messages.joinToString("\n")) }