From 39e0d434ad24cf82f98b7a1bc00302ed416741dd Mon Sep 17 00:00:00 2001 From: NerdNumber9 Date: Fri, 19 Apr 2019 22:59:24 -0400 Subject: [PATCH] Add throttling when restoring E-Hentai/ExHentai galleries. --- .../tachiyomi/data/backup/BackupManager.kt | 11 ++++-- .../data/backup/BackupRestoreService.kt | 26 +++++++++++--- .../ui/setting/SettingsBackupController.kt | 2 +- app/src/main/java/exh/GalleryAdder.kt | 7 +++- .../java/exh/eh/EHentaiThrottleManager.kt | 31 ++++++++++++++++ .../java/exh/favorites/FavoritesSyncHelper.kt | 36 +++++-------------- 6 files changed, 76 insertions(+), 37 deletions(-) create mode 100644 app/src/main/java/exh/eh/EHentaiThrottleManager.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupManager.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupManager.kt index c58b3e603..38af057ad 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupManager.kt @@ -30,8 +30,10 @@ import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.data.track.TrackManager import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceManager +import eu.kanade.tachiyomi.source.online.all.EHentai import eu.kanade.tachiyomi.util.sendLocalBroadcast import eu.kanade.tachiyomi.util.syncChaptersWithSource +import exh.eh.EHentaiThrottleManager import rx.Observable import timber.log.Timber import uy.kohesive.injekt.injectLazy @@ -280,9 +282,12 @@ class BackupManager(val context: Context, version: Int = CURRENT_VERSION) { * @param manga manga that needs updating * @return [Observable] that contains manga */ - fun restoreChapterFetchObservable(source: Source, manga: Manga, chapters: List): Observable, List>> { - return source.fetchChapterList(manga) - .map { syncChaptersWithSource(databaseHelper, it, manga, source) } + fun restoreChapterFetchObservable(source: Source, manga: Manga, chapters: List, throttleManager: EHentaiThrottleManager): Observable, List>> { + return (if(source is EHentai) { + source.fetchChapterList(manga, throttleManager::throttle) + } else { + source.fetchChapterList(manga) + }).map { syncChaptersWithSource(databaseHelper, it, manga, source) } .doOnNext { if (it.first.isNotEmpty()) { chapters.forEach { it.manga_id = manga.id } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestoreService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestoreService.kt index 6cf397d1a..762f2b1f2 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestoreService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestoreService.kt @@ -28,11 +28,14 @@ import eu.kanade.tachiyomi.util.chop import eu.kanade.tachiyomi.util.isServiceRunning import eu.kanade.tachiyomi.util.sendLocalBroadcast import exh.BackupEntry +import exh.EH_SOURCE_ID import exh.EXHMigrations +import exh.EXH_SOURCE_ID +import exh.eh.EHentaiThrottleManager +import exh.eh.EHentaiUpdateWorker import rx.Observable import rx.Subscription import rx.schedulers.Schedulers -import timber.log.Timber import uy.kohesive.injekt.injectLazy import java.io.File import java.text.SimpleDateFormat @@ -125,6 +128,8 @@ class BackupRestoreService : Service() { private lateinit var executor: ExecutorService + private val throttleManager = EHentaiThrottleManager() + /** * Method called when the service is created. It injects dependencies and acquire the wake lock. */ @@ -167,13 +172,23 @@ class BackupRestoreService : Service() { val uri = intent.getParcelableExtra(BackupConst.EXTRA_URI) + throttleManager.resetThrottle() + // Unsubscribe from any previous subscription if needed. subscription?.unsubscribe() subscription = Observable.using( - { db.lowLevel().beginTransaction() }, + { + // Pause auto-gallery-update during restore + EHentaiUpdateWorker.cancelBackground(this) + db.lowLevel().beginTransaction() + }, { getRestoreObservable(uri).doOnNext { db.lowLevel().setTransactionSuccessful() } }, - { executor.execute { db.lowLevel().endTransaction() } }) + { + // Resume auto-gallery-update + EHentaiUpdateWorker.scheduleBackground(this) + executor.execute { db.lowLevel().endTransaction() } + }) .doAfterTerminate { stopSelf(startId) } .subscribeOn(Schedulers.from(executor)) .subscribe() @@ -340,6 +355,9 @@ class BackupRestoreService : Service() { private fun mangaFetchObservable(source: Source, manga: Manga, chapters: List, categories: List, history: List, tracks: List): Observable { + if(source.id == EH_SOURCE_ID || source.id == EXH_SOURCE_ID) + throttleManager.throttle() + return backupManager.restoreMangaFetchObservable(source, manga) .onErrorReturn { // [EXH] @@ -419,7 +437,7 @@ class BackupRestoreService : Service() { * @return [Observable] that contains manga */ private fun chapterFetchObservable(source: Source, manga: Manga, chapters: List): Observable, List>> { - return backupManager.restoreChapterFetchObservable(source, manga, chapters) + return backupManager.restoreChapterFetchObservable(source, manga, chapters, throttleManager) // If there's any error, return empty update and continue. .onErrorReturn { // [EXH] diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsBackupController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsBackupController.kt index 597cc4eef..c3d7677b5 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsBackupController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsBackupController.kt @@ -305,7 +305,7 @@ class SettingsBackupController : SettingsController() { override fun onCreateDialog(savedViewState: Bundle?): Dialog { return MaterialDialog.Builder(activity!!) .title(R.string.pref_restore_backup) - .content(R.string.backup_restore_content) + .content(activity!!.getString(R.string.backup_restore_content) + "\n\nThe app will throttle when restoring EHentai/ExHentai backups. This may cause the app to appear frozen when it is actually still working. Report an issue if the app remains frozen for more than 30 minutes however.") .positiveText(R.string.action_restore) .onPositive { _, _ -> val context = applicationContext diff --git a/app/src/main/java/exh/GalleryAdder.kt b/app/src/main/java/exh/GalleryAdder.kt index 76cb05d80..e7fc9309a 100755 --- a/app/src/main/java/exh/GalleryAdder.kt +++ b/app/src/main/java/exh/GalleryAdder.kt @@ -176,7 +176,12 @@ class GalleryAdder { //Fetch and copy chapters try { - sourceObj.fetchChapterList(manga).map { + val chapterListObs = if(sourceObj is EHentai) { + sourceObj.fetchChapterList(manga, throttleFunc) + } else { + sourceObj.fetchChapterList(manga) + } + chapterListObs.map { syncChaptersWithSource(db, it, manga, sourceObj) }.toBlocking().first() } catch (e: Exception) { diff --git a/app/src/main/java/exh/eh/EHentaiThrottleManager.kt b/app/src/main/java/exh/eh/EHentaiThrottleManager.kt new file mode 100644 index 000000000..fadffa55d --- /dev/null +++ b/app/src/main/java/exh/eh/EHentaiThrottleManager.kt @@ -0,0 +1,31 @@ +package exh.eh + +class EHentaiThrottleManager(private val max: Int = THROTTLE_MAX, + private val inc: Int = THROTTLE_INC) { + private var lastThrottleTime: Long = 0 + var throttleTime: Long = 0 + private set + + fun throttle() { + //Throttle requests if necessary + val now = System.currentTimeMillis() + val timeDiff = now - lastThrottleTime + if(timeDiff < throttleTime) + Thread.sleep(throttleTime - timeDiff) + + if(throttleTime < max) + throttleTime += inc + + lastThrottleTime = System.currentTimeMillis() + } + + fun resetThrottle() { + lastThrottleTime = 0 + throttleTime = 0 + } + + companion object { + const val THROTTLE_MAX = 5500 + const val THROTTLE_INC = 20 + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt index df542df70..e4494430a 100644 --- a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt +++ b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt @@ -17,6 +17,7 @@ import eu.kanade.tachiyomi.util.powerManager import eu.kanade.tachiyomi.util.toast import eu.kanade.tachiyomi.util.wifiManager import exh.* +import exh.eh.EHentaiThrottleManager import exh.eh.EHentaiUpdateWorker import exh.util.ignore import exh.util.trans @@ -42,8 +43,7 @@ class FavoritesSyncHelper(val context: Context) { private val galleryAdder = GalleryAdder() - private var lastThrottleTime: Long = 0 - private var throttleTime: Long = 0 + private val throttleManager = EHentaiThrottleManager() private var wifiLock: WifiManager.WifiLock? = null private var wakeLock: PowerManager.WakeLock? = null @@ -294,12 +294,12 @@ class FavoritesSyncHelper(val context: Context) { } //Apply additions - resetThrottle() + throttleManager.resetThrottle() changeSet.added.forEachIndexed { index, it -> status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to remote server", needWarnThrottle())) - throttle() + throttleManager.throttle() addGalleryRemote(errorList, it) } @@ -335,18 +335,18 @@ class FavoritesSyncHelper(val context: Context) { val categories = db.getCategories().executeAsBlocking() //Apply additions - resetThrottle() + throttleManager.resetThrottle() changeSet.added.forEachIndexed { index, it -> status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to local library", needWarnThrottle())) - throttle() + throttleManager.throttle() //Import using gallery adder val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}", true, EXH_SOURCE_ID, - ::throttle) + throttleManager::throttle) if(result is GalleryAddEvent.Fail) { if(result is GalleryAddEvent.Fail.NotFound) { @@ -380,32 +380,12 @@ class FavoritesSyncHelper(val context: Context) { } } - fun throttle() { - //Throttle requests if necessary - val now = System.currentTimeMillis() - val timeDiff = now - lastThrottleTime - if(timeDiff < throttleTime) - Thread.sleep(throttleTime - timeDiff) - - if(throttleTime < THROTTLE_MAX) - throttleTime += THROTTLE_INC - - lastThrottleTime = System.currentTimeMillis() - } - - fun resetThrottle() { - lastThrottleTime = 0 - throttleTime = 0 - } - fun needWarnThrottle() - = throttleTime >= THROTTLE_WARN + = throttleManager.throttleTime >= THROTTLE_WARN class IgnoredException : RuntimeException() companion object { - private const val THROTTLE_MAX = 5500 - private const val THROTTLE_INC = 20 private const val THROTTLE_WARN = 1000 } }