Draff/TachiyomiSY-Plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add throttling when restoring E-Hentai/ExHentai galleries.

Browse Source
This commit is contained in:
NerdNumber9 2019-04-19 22:59:24 -04:00
parent 6ada3cbf59
commit 39e0d434ad
6 changed files with 76 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupManager.kt
View File

@ -30,8 +30,10 @@ import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.data.track.TrackManager
import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.all.EHentai
import eu.kanade.tachiyomi.util.sendLocalBroadcast
import eu.kanade.tachiyomi.util.syncChaptersWithSource
import exh.eh.EHentaiThrottleManager
import rx.Observable
import timber.log.Timber
import uy.kohesive.injekt.injectLazy
@ -280,9 +282,12 @@ class BackupManager(val context: Context, version: Int = CURRENT_VERSION) {
* @param manga manga that needs updating
* @return [Observable] that contains manga
*/
fun restoreChapterFetchObservable(source: Source, manga: Manga, chapters: List<Chapter>): Observable<Pair<List<Chapter>, List<Chapter>>> {
return source.fetchChapterList(manga)
.map { syncChaptersWithSource(databaseHelper, it, manga, source) }
fun restoreChapterFetchObservable(source: Source, manga: Manga, chapters: List<Chapter>, throttleManager: EHentaiThrottleManager): Observable<Pair<List<Chapter>, List<Chapter>>> {
return (if(source is EHentai) {
source.fetchChapterList(manga, throttleManager::throttle)
} else {
source.fetchChapterList(manga)
}).map { syncChaptersWithSource(databaseHelper, it, manga, source) }
.doOnNext {
if (it.first.isNotEmpty()) {
chapters.forEach { it.manga_id = manga.id }

26
app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestoreService.kt
View File

@ -28,11 +28,14 @@ import eu.kanade.tachiyomi.util.chop
import eu.kanade.tachiyomi.util.isServiceRunning
import eu.kanade.tachiyomi.util.sendLocalBroadcast
import exh.BackupEntry
import exh.EH_SOURCE_ID
import exh.EXHMigrations
import exh.EXH_SOURCE_ID
import exh.eh.EHentaiThrottleManager
import exh.eh.EHentaiUpdateWorker
import rx.Observable
import rx.Subscription
import rx.schedulers.Schedulers
import timber.log.Timber
import uy.kohesive.injekt.injectLazy
import java.io.File
import java.text.SimpleDateFormat
@ -125,6 +128,8 @@ class BackupRestoreService : Service() {
private lateinit var executor: ExecutorService
private val throttleManager = EHentaiThrottleManager()
/**
* Method called when the service is created. It injects dependencies and acquire the wake lock.
*/
@ -167,13 +172,23 @@ class BackupRestoreService : Service() {
val uri = intent.getParcelableExtra<Uri>(BackupConst.EXTRA_URI)
throttleManager.resetThrottle()
// Unsubscribe from any previous subscription if needed.
subscription?.unsubscribe()
subscription = Observable.using(
{ db.lowLevel().beginTransaction() },
{
// Pause auto-gallery-update during restore
EHentaiUpdateWorker.cancelBackground(this)
db.lowLevel().beginTransaction()
},
{ getRestoreObservable(uri).doOnNext { db.lowLevel().setTransactionSuccessful() } },
{ executor.execute { db.lowLevel().endTransaction() } })
{
// Resume auto-gallery-update
EHentaiUpdateWorker.scheduleBackground(this)
executor.execute { db.lowLevel().endTransaction() }
})
.doAfterTerminate { stopSelf(startId) }
.subscribeOn(Schedulers.from(executor))
.subscribe()
@ -340,6 +355,9 @@ class BackupRestoreService : Service() {
private fun mangaFetchObservable(source: Source, manga: Manga, chapters: List<Chapter>,
categories: List<String>, history: List<DHistory>,
tracks: List<Track>): Observable<Manga> {
if(source.id == EH_SOURCE_ID || source.id == EXH_SOURCE_ID)
throttleManager.throttle()
return backupManager.restoreMangaFetchObservable(source, manga)
.onErrorReturn {
// [EXH]
@ -419,7 +437,7 @@ class BackupRestoreService : Service() {
* @return [Observable] that contains manga
*/
private fun chapterFetchObservable(source: Source, manga: Manga, chapters: List<Chapter>): Observable<Pair<List<Chapter>, List<Chapter>>> {
return backupManager.restoreChapterFetchObservable(source, manga, chapters)
return backupManager.restoreChapterFetchObservable(source, manga, chapters, throttleManager)
// If there's any error, return empty update and continue.
.onErrorReturn {
// [EXH]

2
app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsBackupController.kt
View File

@ -305,7 +305,7 @@ class SettingsBackupController : SettingsController() {
override fun onCreateDialog(savedViewState: Bundle?): Dialog {
return MaterialDialog.Builder(activity!!)
.title(R.string.pref_restore_backup)
.content(R.string.backup_restore_content)
.content(activity!!.getString(R.string.backup_restore_content) + "\n\nThe app will throttle when restoring EHentai/ExHentai backups. This may cause the app to appear frozen when it is actually still working. Report an issue if the app remains frozen for more than 30 minutes however.")
.positiveText(R.string.action_restore)
.onPositive { _, _ ->
val context = applicationContext

7
app/src/main/java/exh/GalleryAdder.kt
View File

@ -176,7 +176,12 @@ class GalleryAdder {
//Fetch and copy chapters
try {
sourceObj.fetchChapterList(manga).map {
val chapterListObs = if(sourceObj is EHentai) {
sourceObj.fetchChapterList(manga, throttleFunc)
} else {
sourceObj.fetchChapterList(manga)
}
chapterListObs.map {
syncChaptersWithSource(db, it, manga, sourceObj)
}.toBlocking().first()
} catch (e: Exception) {

31
app/src/main/java/exh/eh/EHentaiThrottleManager.kt Normal file
View File

@ -0,0 +1,31 @@
package exh.eh
class EHentaiThrottleManager(private val max: Int = THROTTLE_MAX,
private val inc: Int = THROTTLE_INC) {
private var lastThrottleTime: Long = 0
var throttleTime: Long = 0
private set
fun throttle() {
//Throttle requests if necessary
val now = System.currentTimeMillis()
val timeDiff = now - lastThrottleTime
if(timeDiff < throttleTime)
Thread.sleep(throttleTime - timeDiff)
if(throttleTime < max)
throttleTime += inc
lastThrottleTime = System.currentTimeMillis()
}
fun resetThrottle() {
lastThrottleTime = 0
throttleTime = 0
}
companion object {
const val THROTTLE_MAX = 5500
const val THROTTLE_INC = 20
}
}

36
app/src/main/java/exh/favorites/FavoritesSyncHelper.kt
View File

@ -17,6 +17,7 @@ import eu.kanade.tachiyomi.util.powerManager
import eu.kanade.tachiyomi.util.toast
import eu.kanade.tachiyomi.util.wifiManager
import exh.*
import exh.eh.EHentaiThrottleManager
import exh.eh.EHentaiUpdateWorker
import exh.util.ignore
import exh.util.trans
@ -42,8 +43,7 @@ class FavoritesSyncHelper(val context: Context) {
private val galleryAdder = GalleryAdder()
private var lastThrottleTime: Long = 0
private var throttleTime: Long = 0
private val throttleManager = EHentaiThrottleManager()
private var wifiLock: WifiManager.WifiLock? = null
private var wakeLock: PowerManager.WakeLock? = null
@ -294,12 +294,12 @@ class FavoritesSyncHelper(val context: Context) {
}
//Apply additions
resetThrottle()
throttleManager.resetThrottle()
changeSet.added.forEachIndexed { index, it ->
status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to remote server",
needWarnThrottle()))
throttle()
throttleManager.throttle()
addGalleryRemote(errorList, it)
}
@ -335,18 +335,18 @@ class FavoritesSyncHelper(val context: Context) {
val categories = db.getCategories().executeAsBlocking()
//Apply additions
resetThrottle()
throttleManager.resetThrottle()
changeSet.added.forEachIndexed { index, it ->
status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to local library",
needWarnThrottle()))
throttle()
throttleManager.throttle()
//Import using gallery adder
val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}",
true,
EXH_SOURCE_ID,
::throttle)
throttleManager::throttle)
if(result is GalleryAddEvent.Fail) {
if(result is GalleryAddEvent.Fail.NotFound) {
@ -380,32 +380,12 @@ class FavoritesSyncHelper(val context: Context) {
}
}
fun throttle() {
//Throttle requests if necessary
val now = System.currentTimeMillis()
val timeDiff = now - lastThrottleTime
if(timeDiff < throttleTime)
Thread.sleep(throttleTime - timeDiff)
if(throttleTime < THROTTLE_MAX)
throttleTime += THROTTLE_INC
lastThrottleTime = System.currentTimeMillis()
}
fun resetThrottle() {
lastThrottleTime = 0
throttleTime = 0
}
fun needWarnThrottle()
= throttleTime >= THROTTLE_WARN
= throttleManager.throttleTime >= THROTTLE_WARN
class IgnoredException : RuntimeException()
companion object {
private const val THROTTLE_MAX = 5500
private const val THROTTLE_INC = 20
private const val THROTTLE_WARN = 1000
}
}