diff --git a/app/build.gradle b/app/build.gradle index 153d8b657..0db11560a 100755 --- a/app/build.gradle +++ b/app/build.gradle @@ -289,6 +289,9 @@ dependencies { releaseTestImplementation 'com.ms-square:debugoverlay:1.1.3' releaseImplementation 'com.ms-square:debugoverlay-no-op:1.1.3' testImplementation 'com.ms-square:debugoverlay-no-op:1.1.3' + + // Humanize (EH) + implementation 'com.github.mfornos:humanize-slim:1.2.2' } buildscript { diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index f139f3947..8d325e744 100755 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -122,6 +122,10 @@ android:exported="false" /> + - + \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/AppModule.kt b/app/src/main/java/eu/kanade/tachiyomi/AppModule.kt index 114f231b6..d027dde89 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/AppModule.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/AppModule.kt @@ -11,6 +11,7 @@ import eu.kanade.tachiyomi.data.track.TrackManager import eu.kanade.tachiyomi.extension.ExtensionManager import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.source.SourceManager +import exh.eh.EHentaiUpdateHelper import rx.Observable import rx.schedulers.Schedulers import uy.kohesive.injekt.api.* @@ -41,6 +42,8 @@ class AppModule(val app: Application) : InjektModule { addSingletonFactory { Gson() } + addSingletonFactory { EHentaiUpdateHelper(app) } + // Asynchronously init expensive components for a faster cold start rxAsync { get() } diff --git a/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt b/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt index 48eb2f061..a6733ab4e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt @@ -57,6 +57,9 @@ object Migrations { } } } + + // ===========[ ALL MIGRATIONS ABOVE HERE HAVE BEEN ALREADY REWRITTEN ]=========== + return true } return false diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/ChapterQueries.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/ChapterQueries.kt index b6cb58670..90ac910d8 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/ChapterQueries.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/ChapterQueries.kt @@ -15,12 +15,14 @@ import java.util.* interface ChapterQueries : DbProvider { - fun getChapters(manga: Manga) = db.get() + fun getChapters(manga: Manga) = getChaptersByMangaId(manga.id) + + fun getChaptersByMangaId(mangaId: Long?) = db.get() .listOfObjects(Chapter::class.java) .withQuery(Query.builder() .table(ChapterTable.TABLE) .where("${ChapterTable.COL_MANGA_ID} = ?") - .whereArgs(manga.id) + .whereArgs(mangaId) .build()) .prepare() @@ -52,6 +54,15 @@ interface ChapterQueries : DbProvider { .build()) .prepare() + fun getChapters(url: String) = db.get() + .listOfObjects(Chapter::class.java) + .withQuery(Query.builder() + .table(ChapterTable.TABLE) + .where("${ChapterTable.COL_URL} = ?") + .whereArgs(url) + .build()) + .prepare() + fun insertChapter(chapter: Chapter) = db.put().`object`(chapter).prepare() diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt index faf44eee4..b20a62d0c 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt @@ -29,6 +29,8 @@ import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.ui.main.MainActivity import eu.kanade.tachiyomi.util.* +import exh.EH_SOURCE_ID +import exh.EXH_SOURCE_ID import rx.Observable import rx.Subscription import rx.schedulers.Schedulers @@ -283,24 +285,29 @@ class LibraryUpdateService( .doOnNext { showProgressNotification(it, count.andIncrement, mangaToUpdate.size) } // Update the chapters of the manga. .concatMap { manga -> - updateManga(manga) - // If there's any error, return empty update and continue. - .onErrorReturn { - failedUpdates.add(manga) - Pair(emptyList(), emptyList()) - } - // Filter out mangas without new chapters (or failed). - .filter { pair -> pair.first.isNotEmpty() } - .doOnNext { - if (downloadNew && (categoriesToDownload.isEmpty() || - manga.category in categoriesToDownload)) { - - downloadChapters(manga, it.first) - hasDownloads = true + if(manga.source == EXH_SOURCE_ID || manga.source == EH_SOURCE_ID) { + // Ignore EXH manga, updating chapters for every manga will get you banned + Observable.just(manga) + } else { + updateManga(manga) + // If there's any error, return empty update and continue. + .onErrorReturn { + failedUpdates.add(manga) + Pair(emptyList(), emptyList()) } - } - // Convert to the manga that contains new chapters. - .map { manga } + // Filter out mangas without new chapters (or failed). + .filter { pair -> pair.first.isNotEmpty() } + .doOnNext { + if (downloadNew && (categoriesToDownload.isEmpty() || + manga.category in categoriesToDownload)) { + + downloadChapters(manga, it.first) + hasDownloads = true + } + } + // Convert to the manga that contains new chapters. + .map { manga } + } } // Add manga with new chapters to the list. .doOnNext { manga -> diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt index e743009f9..4ce0cca63 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt @@ -186,4 +186,10 @@ object PreferenceKeys { const val eh_logLevel = "eh_log_level" const val eh_enableSourceBlacklist = "eh_enable_source_blacklist" + + const val eh_autoUpdateFrequency = "eh_auto_update_frequency" + + const val eh_autoUpdateRestrictions = "eh_auto_update_restrictions" + + const val eh_autoUpdateStats = "eh_auto_update_stats" } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt index dc785cb24..a26b655c1 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt @@ -259,4 +259,10 @@ class PreferencesHelper(val context: Context) { fun eh_logLevel() = rxPrefs.getInteger(Keys.eh_logLevel, 0) fun eh_enableSourceBlacklist() = rxPrefs.getBoolean(Keys.eh_enableSourceBlacklist, true) + + fun eh_autoUpdateFrequency() = rxPrefs.getInteger(Keys.eh_autoUpdateFrequency, 1) + + fun eh_autoUpdateRequirements() = prefs.getStringSet(Keys.eh_autoUpdateRestrictions, emptySet()) + + fun eh_autoUpdateStats() = rxPrefs.getString(Keys.eh_autoUpdateStats, "") } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/model/SManga.kt b/app/src/main/java/eu/kanade/tachiyomi/source/model/SManga.kt index 8a1ba1af0..7a6f87a56 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/model/SManga.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/model/SManga.kt @@ -23,6 +23,11 @@ interface SManga : Serializable { var initialized: Boolean fun copyFrom(other: SManga) { + // EXH --> + url = other.url // Allow dynamically mutating one manga into another + title = other.title + // EXH <-- + if (other.author != null) author = other.author diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt index 741f408bc..10edb7a72 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt @@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.source.online.all import android.content.Context import android.net.Uri +import com.elvishew.xlog.XLog import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.getOrDefault @@ -12,6 +13,7 @@ import eu.kanade.tachiyomi.source.model.* import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.LewdSource import eu.kanade.tachiyomi.util.asJsoup +import exh.eh.EHentaiUpdateHelper import exh.metadata.EX_DATE_FORMAT import exh.metadata.metadata.EHentaiSearchMetadata import exh.metadata.metadata.EHentaiSearchMetadata.Companion.EH_GENRE_NAMESPACE @@ -33,11 +35,17 @@ import uy.kohesive.injekt.injectLazy import java.net.URLEncoder import java.util.* import exh.metadata.metadata.base.RaisedTag +import exh.eh.EHentaiUpdateWorker +import exh.eh.GalleryEntry +import kotlinx.coroutines.runBlocking +import org.jsoup.nodes.TextNode +import rx.Single import java.lang.RuntimeException +// TODO Consider gallery updating when doing tabbed browsing class EHentai(override val id: Long, val exh: Boolean, - val context: Context) : HttpSource(), LewdSource { + val context: Context) : HttpSource(), LewdSource { override val metaClass = EHentaiSearchMetadata::class val schema: String @@ -58,7 +66,8 @@ class EHentai(override val id: Long, override val lang = "all" override val supportsLatest = true - val prefs: PreferencesHelper by injectLazy() + private val prefs: PreferencesHelper by injectLazy() + private val updateHelper: EHentaiUpdateHelper by injectLazy() /** * Gallery list entry @@ -115,15 +124,83 @@ class EHentai(override val id: Long, MangasPage(it.first.map { it.manga }, it.second) } - override fun fetchChapterList(manga: SManga): Observable> - = Observable.just(listOf(SChapter.create().apply { - url = manga.url - name = "Chapter" - chapter_number = 1f - })) + override fun fetchChapterList(manga: SManga) + = fetchChapterList(manga) {} + + fun fetchChapterList(manga: SManga, throttleFunc: () -> Unit): Observable> { + return Single.fromCallable { + // Pull all the way to the root gallery + // We can't do this with RxJava or we run into stack overflows on shit like this: + // https://exhentai.org/g/1073061/f9345f1c12/ + var url: String = manga.url + var doc: Document? = null + + runBlocking { + while (true) { + val gid = EHentaiSearchMetadata.galleryId(url).toInt() + val cachedParent = updateHelper.parentLookupTable.get( + gid + ) + if(cachedParent == null) { + throttleFunc() + + val resp = client.newCall(exGet(baseUrl + url)).execute() + if (!resp.isSuccessful) error("HTTP error (${resp.code()})!") + doc = resp.asJsoup() + + val parentLink = doc!!.select("#gdd .gdt1").find { el -> + el.text().toLowerCase() == "parent:" + }!!.nextElementSibling().selectFirst("a")?.attr("href") + + if (parentLink != null) { + updateHelper.parentLookupTable.put( + gid, + GalleryEntry( + EHentaiSearchMetadata.galleryId(parentLink), + EHentaiSearchMetadata.galleryToken(parentLink) + ) + ) + url = EHentaiSearchMetadata.normalizeUrl(parentLink) + } else break + } else { + XLog.d("Parent cache hit: %s!", gid) + url = EHentaiSearchMetadata.idAndTokenToUrl( + cachedParent.gId, + cachedParent.gToken + ) + } + } + } + + doc!! + }.map { d -> + val newDisplay = d.select("#gnd a") + // Build chapter for root gallery + val self = SChapter.create().apply { + url = EHentaiSearchMetadata.normalizeUrl(d.location()) + name = "v1: " + d.selectFirst("#gn").text() + chapter_number = 1f + date_upload = EX_DATE_FORMAT.parse(d.select("#gdd .gdt1").find { el -> + el.text().toLowerCase() == "posted:" + }!!.nextElementSibling().text()).time + } + // Build and append the rest of the galleries + listOf(self) + newDisplay.mapIndexed { index, newGallery -> + val link = newGallery.attr("href") + val name = newGallery.text() + val posted = (newGallery.nextSibling() as TextNode).text().removePrefix(", added ") + SChapter.create().apply { + this.url = EHentaiSearchMetadata.normalizeUrl(link) + this.name = "v${index + 2}: $name" + this.chapter_number = index + 2f + this.date_upload = EX_DATE_FORMAT.parse(posted).time + } + } + }.toObservable() + } override fun fetchPageList(chapter: SChapter) - = fetchChapterPage(chapter, "$baseUrl/${chapter.url}").map { + = fetchChapterPage(chapter, baseUrl + chapter.url).map { it.mapIndexed { i, s -> Page(i, s) } @@ -241,9 +318,20 @@ class EHentai(override val id: Long, .asObservableWithAsyncStacktrace() .flatMap { (stacktrace, response) -> if(response.isSuccessful) { - parseToManga(manga, response).andThen(Observable.just(manga.apply { - initialized = true - })) + // Pull to most recent + val doc = response.asJsoup() + val newerGallery = doc.select("#gnd a").lastOrNull() + val pre = if(newerGallery != null) { + manga.url = EHentaiSearchMetadata.normalizeUrl(newerGallery.attr("href")) + client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess().map { it.asJsoup() } + } else Observable.just(doc) + + pre.flatMap { + parseToManga(manga, it).andThen(Observable.just(manga.apply { + initialized = true + })) + } } else { response.close() @@ -261,10 +349,10 @@ class EHentai(override val id: Long, */ override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException() - override fun parseIntoMetadata(metadata: EHentaiSearchMetadata, input: Response) { + override fun parseIntoMetadata(metadata: EHentaiSearchMetadata, input: Document) { with(metadata) { - with(input.asJsoup()) { - val url = input.request().url().encodedPath() + with(input) { + val url = input.location() gId = EHentaiSearchMetadata.galleryId(url) gToken = EHentaiSearchMetadata.galleryToken(url) @@ -296,6 +384,8 @@ class EHentai(override val id: Long, .toLowerCase()) { "posted" -> datePosted = EX_DATE_FORMAT.parse(right).time // Example gallery with parent: https://e-hentai.org/g/1390451/7f181c2426/ + // Example JP gallery: https://exhentai.org/g/1375385/03519d541b/ + // Parent is older variation of the gallery "parent" -> parent = if (!right.equals("None", true)) { rightElement.child(0).attr("href") } else null @@ -312,6 +402,12 @@ class EHentai(override val id: Long, } } + lastUpdateCheck = System.currentTimeMillis() + if(datePosted != null + && lastUpdateCheck - datePosted!! > EHentaiUpdateWorker.GALLERY_AGE_TIME) { + aged = true + } + //Parse ratings ignore { averageRating = select("#rating_label") diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt index a57a2319b..e93fd5dbc 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt @@ -26,6 +26,7 @@ import eu.kanade.tachiyomi.ui.base.controller.RxController import eu.kanade.tachiyomi.ui.base.controller.TabbedController import eu.kanade.tachiyomi.ui.base.controller.requestPermissionsSafe import eu.kanade.tachiyomi.ui.manga.chapter.ChaptersController +import eu.kanade.tachiyomi.ui.manga.chapter.ChaptersPresenter import eu.kanade.tachiyomi.ui.manga.info.MangaInfoController import eu.kanade.tachiyomi.ui.manga.track.TrackController import eu.kanade.tachiyomi.util.toast @@ -48,6 +49,18 @@ class MangaController : RxController, TabbedController { } } + // EXH --> + constructor(redirect: ChaptersPresenter.EXHRedirect) : super(Bundle().apply { + putLong(MANGA_EXTRA, redirect.manga.id!!) + putBoolean(UPDATE_EXTRA, redirect.update) + }) { + this.manga = redirect.manga + if (manga != null) { + source = Injekt.get().getOrStub(redirect.manga.source) + } + } + // EXH <-- + constructor(mangaId: Long) : this( Injekt.get().getManga(mangaId).executeAsBlocking()) @@ -64,6 +77,8 @@ class MangaController : RxController, TabbedController { val fromCatalogue = args.getBoolean(FROM_CATALOGUE_EXTRA, false) + val update = args.getBoolean(UPDATE_EXTRA, false) + val lastUpdateRelay: BehaviorRelay = BehaviorRelay.create() val chapterCountRelay: BehaviorRelay = BehaviorRelay.create() @@ -180,6 +195,9 @@ class MangaController : RxController, TabbedController { companion object { + // EXH --> + const val UPDATE_EXTRA = "update" + // EXH <-- const val FROM_CATALOGUE_EXTRA = "from_catalogue" const val MANGA_EXTRA = "manga" diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersController.kt index 9b721857d..5a767d763 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersController.kt @@ -11,6 +11,7 @@ import android.support.v7.view.ActionMode import android.support.v7.widget.DividerItemDecoration import android.support.v7.widget.LinearLayoutManager import android.view.* +import com.bluelinelabs.conductor.RouterTransaction import com.elvishew.xlog.XLog import com.jakewharton.rxbinding.support.v4.widget.refreshes import com.jakewharton.rxbinding.view.clicks @@ -28,6 +29,7 @@ import eu.kanade.tachiyomi.util.getCoordinates import eu.kanade.tachiyomi.util.snack import eu.kanade.tachiyomi.util.toast import kotlinx.android.synthetic.main.chapters_controller.* +import rx.android.schedulers.AndroidSchedulers import timber.log.Timber class ChaptersController : NucleusController(), @@ -104,6 +106,14 @@ class ChaptersController : NucleusController(), view.context.toast(R.string.no_next_chapter) } } + + presenter.redirectUserRelay + .observeOn(AndroidSchedulers.mainThread()) + .subscribeUntilDestroy { redirect -> + XLog.d("Redirecting to updated manga (manga.id: %s, manga.title: %s, update: %s)!", redirect.manga.id, redirect.manga.title, redirect.update) + // Replace self + parentController?.router?.replaceTopController(RouterTransaction.with(MangaController(redirect))) + } } override fun onDestroyView(view: View) { @@ -188,6 +198,9 @@ class ChaptersController : NucleusController(), if (presenter.chapters.isEmpty()) initialFetchChapters() + if ((parentController as MangaController).update) + fetchChaptersFromSource() + val adapter = adapter ?: return adapter.updateDataSet(chapters) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt index 6fd1c4d49..c4d432597 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt @@ -15,6 +15,9 @@ import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter import eu.kanade.tachiyomi.util.isNullOrUnsubscribed import eu.kanade.tachiyomi.util.syncChaptersWithSource +import exh.EH_SOURCE_ID +import exh.EXH_SOURCE_ID +import exh.eh.EHentaiUpdateHelper import rx.Observable import rx.Subscription import rx.android.schedulers.AndroidSchedulers @@ -22,6 +25,7 @@ import rx.schedulers.Schedulers import timber.log.Timber import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get +import uy.kohesive.injekt.injectLazy import java.util.Date /** @@ -66,6 +70,14 @@ class ChaptersPresenter( */ private var observeDownloadsSubscription: Subscription? = null + // EXH --> + private val updateHelper: EHentaiUpdateHelper by injectLazy() + + val redirectUserRelay = BehaviorRelay.create() + + data class EXHRedirect(val manga: Manga, val update: Boolean) + // EXH <-- + override fun onCreate(savedState: Bundle?) { super.onCreate(savedState) @@ -100,6 +112,25 @@ class ChaptersPresenter( lastUpdateRelay.call(Date(chapters.maxBy { it.date_upload }?.date_upload ?: 0)) + // EXH --> + if(chapters.isNotEmpty() + && (source.id == EXH_SOURCE_ID || source.id == EH_SOURCE_ID)) { + // Check for gallery in library and accept manga with lowest id + // Find chapters sharing same root + add(updateHelper.findAcceptedRootAndDiscardOthers(chapters) + .subscribeOn(Schedulers.io()) + .subscribe { (acceptedChain, _) -> + // Redirect if we are not the accepted root + if(manga.id != acceptedChain.manga.id) { + // Update if any of our chapters are not in accepted manga's chapters + val ourChapterUrls = chapters.map { it.url }.toSet() + val acceptedChapterUrls = acceptedChain.chapters.map { it.url }.toSet() + val update = (ourChapterUrls - acceptedChapterUrls).isNotEmpty() + redirectUserRelay.call(EXHRedirect(acceptedChain.manga, update)) + } + }) + } + // EXH <-- } .subscribe { chaptersRelay.call(it) }) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt index 5f00f7ac1..bfc276670 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt @@ -22,6 +22,7 @@ import com.bumptech.glide.load.engine.DiskCacheStrategy import com.bumptech.glide.load.resource.bitmap.RoundedCorners import com.bumptech.glide.request.target.SimpleTarget import com.bumptech.glide.request.transition.Transition +import com.bumptech.glide.signature.ObjectKey import com.elvishew.xlog.XLog import com.jakewharton.rxbinding.support.v4.widget.refreshes import com.jakewharton.rxbinding.view.clicks @@ -74,7 +75,9 @@ class MangaInfoController : NucleusController(), */ private val preferences: PreferencesHelper by injectLazy() - private val sourceManager: SourceManager by injectLazy() + // EXH --> + private var lastMangaThumbnail: String? = null + // EXH <-- init { setHasOptionsMenu(true) @@ -181,6 +184,7 @@ class MangaInfoController : NucleusController(), // Update view. setMangaInfo(manga, source) + if((parentController as MangaController).update) fetchMangaFromSource() } else { // Initialize manga. fetchMangaFromSource() @@ -247,10 +251,17 @@ class MangaInfoController : NucleusController(), // Set the favorite drawable to the correct one. setFavoriteDrawable(manga.favorite) - // Set cover if it wasn't already. - if (manga_cover.drawable == null && !manga.thumbnail_url.isNullOrEmpty()) { + // Set cover if it matches + val tagMatches = lastMangaThumbnail == manga.thumbnail_url + val coverLoaded = manga_cover.drawable != null + if ((!tagMatches || !coverLoaded) && !manga.thumbnail_url.isNullOrEmpty()) { + lastMangaThumbnail = manga.thumbnail_url + + val coverSig = ObjectKey(manga.thumbnail_url ?: "") + GlideApp.with(view.context) .load(manga) + .signature(coverSig) .diskCacheStrategy(DiskCacheStrategy.RESOURCE) .centerCrop() .into(manga_cover) @@ -258,6 +269,7 @@ class MangaInfoController : NucleusController(), if (backdrop != null) { GlideApp.with(view.context) .load(manga) + .signature(coverSig) .diskCacheStrategy(DiskCacheStrategy.RESOURCE) .centerCrop() .into(backdrop) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt index f1abf79c4..29b7c6948 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt @@ -1,26 +1,54 @@ package eu.kanade.tachiyomi.ui.setting +import android.os.Build +import android.os.Handler import android.support.v7.preference.PreferenceScreen import android.widget.Toast import com.afollestad.materialdialogs.MaterialDialog import com.bluelinelabs.conductor.RouterTransaction import com.bluelinelabs.conductor.changehandler.FadeChangeHandler import com.f2prateek.rx.preferences.Preference +import com.github.salomonbrys.kotson.fromJson +import com.google.gson.Gson +import com.kizitonwose.time.Interval +import com.kizitonwose.time.days +import com.kizitonwose.time.hours +import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.data.database.DatabaseHelper import eu.kanade.tachiyomi.data.preference.PreferenceKeys +import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.util.toast +import exh.EH_SOURCE_ID +import exh.EXH_SOURCE_ID +import exh.eh.EHentaiUpdateWorker +import exh.eh.EHentaiUpdaterStats import exh.favorites.FavoritesIntroDialog import exh.favorites.LocalFavoritesStorage +import exh.metadata.metadata.EHentaiSearchMetadata +import exh.metadata.metadata.base.getFlatMetadataForManga +import exh.metadata.nullIfBlank import exh.uconfig.WarnConfigureDialogController import exh.ui.login.LoginController +import exh.util.await import exh.util.trans +import humanize.Humanize +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import rx.android.schedulers.AndroidSchedulers import rx.schedulers.Schedulers +import uy.kohesive.injekt.injectLazy +import java.util.* /** * EH Settings fragment */ class SettingsEhController : SettingsController() { + private val gson: Gson by injectLazy() + private val db: DatabaseHelper by injectLazy() + private fun Preference<*>.reconfigure(): Boolean { //Listen for change commit asObservable() @@ -183,5 +211,113 @@ class SettingsEhController : SettingsController() { } } } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + preferenceCategory { + title = "Gallery update checker" + + intListPreference { + key = PreferenceKeys.eh_autoUpdateFrequency + title = "Time between update batches" + entries = arrayOf("Never update galleries", "1 hour", "2 hours", "3 hours", "6 hours", "12 hours", "24 hours", "48 hours") + entryValues = arrayOf("0", "1", "2", "3", "6", "12", "24", "48") + defaultValue = "0" + + preferences.eh_autoUpdateFrequency().asObservable().subscribeUntilDestroy { newVal -> + summary = if(newVal == 0) { + "${context.getString(R.string.app_name)} will currently never check galleries in your library for updates." + } else { + "${context.getString(R.string.app_name)} checks/updates galleries in batches. " + + "This means it will wait $newVal hour(s), check ${EHentaiUpdateWorker.UPDATES_PER_ITERATION} galleries," + + " wait $newVal hour(s), check ${EHentaiUpdateWorker.UPDATES_PER_ITERATION} and so on..." + } + } + + onChange { newValue -> + val interval = (newValue as String).toInt() + EHentaiUpdateWorker.scheduleBackground(context, interval) + true + } + } + + multiSelectListPreference { + key = PreferenceKeys.eh_autoUpdateRestrictions + title = "Auto update restrictions" + entriesRes = arrayOf(R.string.wifi, R.string.charging) + entryValues = arrayOf("wifi", "ac") + summaryRes = R.string.pref_library_update_restriction_summary + + preferences.eh_autoUpdateFrequency().asObservable() + .subscribeUntilDestroy { isVisible = it > 0 } + + onChange { + // Post to event looper to allow the preference to be updated. + Handler().post { EHentaiUpdateWorker.scheduleBackground(context) } + true + } + } + + preference { + title = "Show updater statistics" + + onClick { + val progress = MaterialDialog.Builder(context) + .progress(true, 0) + .content("Collecting statistics...") + .cancelable(false) + .show() + + GlobalScope.launch(Dispatchers.IO) { + val updateInfo = try { + val stats = preferences.eh_autoUpdateStats().getOrDefault().nullIfBlank()?.let { + gson.fromJson(it) + } + + val statsText = if (stats != null) { + "The updater last ran ${Humanize.naturalTime(Date(stats.startTime))}, and checked ${stats.updateCount} out of the ${stats.possibleUpdates} galleries that were ready for checking." + } else "The updater has not ran yet." + + val allMeta = db.getMangaWithMetadata().await().filter { + it.favorite && (it.source == EH_SOURCE_ID || it.source == EXH_SOURCE_ID) + }.mapNotNull { + db.getFlatMetadataForManga(it.id!!).await()?.raise() + }.toList() + + fun metaInRelativeDuration(duration: Interval<*>): Int { + val durationMs = duration.inMilliseconds.longValue + return allMeta.asSequence().filter { + System.currentTimeMillis() - it.lastUpdateCheck < durationMs + }.count() + } + + """ + $statsText + + Galleries that were checked in the last: + - hour: ${metaInRelativeDuration(1.hours)} + - 6 hours: ${metaInRelativeDuration(6.hours)} + - 12 hours: ${metaInRelativeDuration(12.hours)} + - day: ${metaInRelativeDuration(1.days)} + - 2 days: ${metaInRelativeDuration(2.days)} + - week: ${metaInRelativeDuration(7.days)} + - month: ${metaInRelativeDuration(30.days)} + - year: ${metaInRelativeDuration(365.days)} + """.trimIndent() + } finally { + progress.dismiss() + } + + withContext(Dispatchers.Main) { + MaterialDialog.Builder(context) + .title("Gallery updater statistics") + .content(updateInfo) + .positiveText("Ok") + .show() + } + } + } + } + } + } } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/ContextExtensions.kt b/app/src/main/java/eu/kanade/tachiyomi/util/ContextExtensions.kt index 2433a1199..1dd0dda4c 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/util/ContextExtensions.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/util/ContextExtensions.kt @@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.util import android.app.ActivityManager import android.app.Notification import android.app.NotificationManager +import android.app.job.JobScheduler import android.content.* import android.content.Context.VIBRATOR_SERVICE import android.content.pm.PackageManager @@ -128,11 +129,11 @@ val Context.wifiManager: WifiManager get() = applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager // --> EH -/** - * Property to get the wifi manager from the context. - */ val Context.clipboardManager: ClipboardManager get() = applicationContext.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager + +val Context.jobScheduler: JobScheduler + get() = applicationContext.getSystemService(Context.JOB_SCHEDULER_SERVICE) as JobScheduler // <-- EH /** diff --git a/app/src/main/java/exh/EXHMigrations.kt b/app/src/main/java/exh/EXHMigrations.kt index 6d679fff8..559c99cfb 100644 --- a/app/src/main/java/exh/EXHMigrations.kt +++ b/app/src/main/java/exh/EXHMigrations.kt @@ -1,5 +1,6 @@ package exh +import com.elvishew.xlog.XLog import com.pushtorefresh.storio.sqlite.queries.Query import com.pushtorefresh.storio.sqlite.queries.RawQuery import eu.kanade.tachiyomi.data.backup.models.DHistory @@ -20,6 +21,8 @@ import java.net.URISyntaxException object EXHMigrations { private val db: DatabaseHelper by injectLazy() + private val logger = XLog.tag("EXHMigrations") + private const val CURRENT_MIGRATION_VERSION = 1 /** @@ -31,45 +34,49 @@ object EXHMigrations { fun upgrade(preferences: PreferencesHelper): Boolean { val context = preferences.context val oldVersion = preferences.eh_lastVersionCode().getOrDefault() - if (oldVersion < CURRENT_MIGRATION_VERSION) { - preferences.eh_lastVersionCode().set(CURRENT_MIGRATION_VERSION) - - if(oldVersion < 1) { - db.inTransaction { - // Migrate HentaiCafe source IDs - db.lowLevel().executeSQL(RawQuery.builder() - .query(""" + try { + if (oldVersion < CURRENT_MIGRATION_VERSION) { + if (oldVersion < 1) { + db.inTransaction { + // Migrate HentaiCafe source IDs + db.lowLevel().executeSQL(RawQuery.builder() + .query(""" UPDATE ${MangaTable.TABLE} SET ${MangaTable.COL_SOURCE} = $HENTAI_CAFE_SOURCE_ID WHERE ${MangaTable.COL_SOURCE} = 6908 """.trimIndent()) - .affectsTables(MangaTable.TABLE) - .build()) + .affectsTables(MangaTable.TABLE) + .build()) - // Migrate nhentai URLs - val nhentaiManga = db.db.get() - .listOfObjects(Manga::class.java) - .withQuery(Query.builder() - .table(MangaTable.TABLE) - .where("${MangaTable.COL_SOURCE} = $NHENTAI_SOURCE_ID") - .build()) - .prepare() - .executeAsBlocking() + // Migrate nhentai URLs + val nhentaiManga = db.db.get() + .listOfObjects(Manga::class.java) + .withQuery(Query.builder() + .table(MangaTable.TABLE) + .where("${MangaTable.COL_SOURCE} = $NHENTAI_SOURCE_ID") + .build()) + .prepare() + .executeAsBlocking() - nhentaiManga.forEach { - it.url = getUrlWithoutDomain(it.url) + nhentaiManga.forEach { + it.url = getUrlWithoutDomain(it.url) + } + + db.db.put() + .objects(nhentaiManga) + // Extremely slow without the resolver :/ + .withPutResolver(MangaUrlPutResolver()) + .prepare() + .executeAsBlocking() } - - db.db.put() - .objects(nhentaiManga) - // Extremely slow without the resolver :/ - .withPutResolver(MangaUrlPutResolver()) - .prepare() - .executeAsBlocking() } - } - return true + preferences.eh_lastVersionCode().set(CURRENT_MIGRATION_VERSION) + + return true + } + } catch(e: Exception) { + logger.e( "Failed to migrate app from $oldVersion -> $CURRENT_MIGRATION_VERSION!", e) } return false } diff --git a/app/src/main/java/exh/GalleryAdder.kt b/app/src/main/java/exh/GalleryAdder.kt index 714552d4c..76cb05d80 100755 --- a/app/src/main/java/exh/GalleryAdder.kt +++ b/app/src/main/java/exh/GalleryAdder.kt @@ -61,7 +61,8 @@ class GalleryAdder { fun addGallery(url: String, fav: Boolean = false, - forceSource: Long? = null): GalleryAddEvent { + forceSource: Long? = null, + throttleFunc: () -> Unit = {}): GalleryAddEvent { XLog.d("Importing gallery (url: %s, fav: %s, forceSource: %s)...", url, fav, forceSource) try { val urlObj = Uri.parse(url) @@ -167,7 +168,6 @@ class GalleryAdder { // Fetch and copy details val newManga = sourceObj.fetchMangaDetails(manga).toBlocking().first() manga.copyFrom(newManga) - manga.title = newManga.title //Forcibly copy title as copyFrom does not copy title manga.initialized = true if (fav) manga.favorite = true @@ -180,13 +180,13 @@ class GalleryAdder { syncChaptersWithSource(db, it, manga, sourceObj) }.toBlocking().first() } catch (e: Exception) { - XLog.w("Failed to update chapters for gallery: %s!", manga.title) + XLog.w("Failed to update chapters for gallery: ${manga.title}!", e) return GalleryAddEvent.Fail.Error(url, "Failed to update chapters for gallery: $url") } return GalleryAddEvent.Success(url, manga) } catch(e: Exception) { - XLog.w("Could not add gallery!", e) + XLog.w("Could not add gallery (url: $url)!", e) if(e is EHentai.GalleryNotFoundException) { return GalleryAddEvent.Fail.NotFound(url) diff --git a/app/src/main/java/exh/debug/DebugFunctions.kt b/app/src/main/java/exh/debug/DebugFunctions.kt index 336535d81..5a5888605 100644 --- a/app/src/main/java/exh/debug/DebugFunctions.kt +++ b/app/src/main/java/exh/debug/DebugFunctions.kt @@ -1,5 +1,7 @@ package exh.debug +import android.app.Application +import android.os.Build import com.pushtorefresh.storio.sqlite.queries.RawQuery import eu.kanade.tachiyomi.data.database.DatabaseHelper import eu.kanade.tachiyomi.data.database.tables.MangaTable @@ -7,9 +9,11 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.source.SourceManager import exh.EH_SOURCE_ID import exh.EXH_SOURCE_ID +import exh.eh.EHentaiUpdateWorker import uy.kohesive.injekt.injectLazy object DebugFunctions { + val app: Application by injectLazy() val db: DatabaseHelper by injectLazy() val prefs: PreferencesHelper by injectLazy() val sourceManager: SourceManager by injectLazy() @@ -48,6 +52,14 @@ object DebugFunctions { fun convertAllExhentaiGalleriesToEhentai() = convertSources(EXH_SOURCE_ID, EH_SOURCE_ID) + fun testLaunchBackgroundUpdater() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + EHentaiUpdateWorker.launchBackgroundTest(app) + } else { + error("OS/SDK version too old!") + } + } + private fun convertSources(from: Long, to: Long) { db.lowLevel().executeSQL(RawQuery.builder() .query(""" diff --git a/app/src/main/java/exh/eh/EHentaiUpdateHelper.kt b/app/src/main/java/exh/eh/EHentaiUpdateHelper.kt new file mode 100644 index 000000000..40cfd48cf --- /dev/null +++ b/app/src/main/java/exh/eh/EHentaiUpdateHelper.kt @@ -0,0 +1,137 @@ +package exh.eh + +import android.content.Context +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Chapter +import eu.kanade.tachiyomi.data.database.models.ChapterImpl +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.database.models.MangaCategory +import rx.Observable +import rx.Single +import uy.kohesive.injekt.injectLazy +import java.io.File + +data class ChapterChain(val manga: Manga, val chapters: List) + +class EHentaiUpdateHelper(context: Context) { + val parentLookupTable = + MemAutoFlushingLookupTable( + File(context.filesDir, "exh-plt.maftable"), + GalleryEntry.Serializer() + ) + private val db: DatabaseHelper by injectLazy() + + /** + * @param chapters Cannot be an empty list! + * + * @return Pair + */ + fun findAcceptedRootAndDiscardOthers(chapters: List): Single>> { + // Find other chains + val chainsObservable = Observable.merge(chapters.map { chapter -> + db.getChapters(chapter.url).asRxSingle().toObservable() + }).toList().map { allChapters -> + allChapters.flatMap { innerChapters -> innerChapters.map { it.manga_id!! } }.distinct() + }.flatMap { mangaIds -> + Observable.merge( + mangaIds.map { mangaId -> + Single.zip( + db.getManga(mangaId).asRxSingle(), + db.getChaptersByMangaId(mangaId).asRxSingle() + ) { manga, chapters -> + ChapterChain(manga, chapters) + }.toObservable() + } + ) + }.toList() + + // Accept oldest chain + val chainsWithAccepted = chainsObservable.map { chains -> + val acceptedChain = chains.minBy { it.manga.id!! }!! + + acceptedChain to chains + } + + return chainsWithAccepted.map { (accepted, chains) -> + val toDiscard = chains.filter { it.manga.favorite && it.manga.id != accepted.manga.id } + + if(toDiscard.isNotEmpty()) { + // Copy chain chapters to curChapters + val newChapters = toDiscard + .flatMap { it.chapters } + .fold(accepted.chapters) { curChapters, chapter -> + val existing = curChapters.find { it.url == chapter.url } + + if (existing != null) { + existing.read = existing.read || chapter.read + existing.last_page_read = existing.last_page_read.coerceAtLeast(chapter.last_page_read) + existing.bookmark = existing.bookmark || chapter.bookmark + curChapters + } else if (chapter.date_upload > 0) { // Ignore chapters using the old system + curChapters + ChapterImpl().apply { + manga_id = accepted.manga.id + url = chapter.url + name = chapter.name + read = chapter.read + bookmark = chapter.bookmark + last_page_read = chapter.last_page_read + date_fetch = chapter.date_fetch + date_upload = chapter.date_upload + } + } else curChapters + } + .filter { it.date_upload <= 0 } // Ignore chapters using the old system (filter after to prevent dupes from insert) + .sortedBy { it.date_upload } + .apply { + withIndex().map { (index, chapter) -> + chapter.name = "v${index + 1}: " + chapter.name.substringAfter(" ") + chapter.chapter_number = index + 1f + chapter.source_order = index + } + } + + toDiscard.forEach { it.manga.favorite = false } + accepted.manga.favorite = true + + val newAccepted = ChapterChain(accepted.manga, newChapters) + val rootsToMutate = toDiscard + newAccepted + + db.inTransaction { + // Apply changes to all manga + db.insertMangas(rootsToMutate.map { it.manga }).executeAsBlocking() + // Insert new chapters for accepted manga + db.insertChapters(newAccepted.chapters) + // Copy categories from all chains to accepted manga + val newCategories = rootsToMutate.flatMap { + db.getCategoriesForManga(it.manga).executeAsBlocking() + }.distinctBy { it.id }.map { + MangaCategory.create(newAccepted.manga, it) + } + db.setMangaCategories(newCategories, rootsToMutate.map { it.manga }) + } + + newAccepted to toDiscard + } else accepted to emptyList() + }.toSingle() + } +} + +data class GalleryEntry(val gId: String, val gToken: String) { + class Serializer: MemAutoFlushingLookupTable.EntrySerializer { + /** + * Serialize an entry as a String. + */ + override fun write(entry: GalleryEntry) = with(entry) { "$gId:$gToken" } + + /** + * Read an entry from a String. + */ + override fun read(string: String): GalleryEntry { + val colonIndex = string.indexOf(':') + return GalleryEntry( + string.substring(0, colonIndex), + string.substring(colonIndex + 1, string.length) + ) + } + } +} diff --git a/app/src/main/java/exh/eh/EHentaiUpdateWorker.kt b/app/src/main/java/exh/eh/EHentaiUpdateWorker.kt new file mode 100644 index 000000000..81c7a831f --- /dev/null +++ b/app/src/main/java/exh/eh/EHentaiUpdateWorker.kt @@ -0,0 +1,351 @@ +package exh.eh + +import android.app.job.JobInfo +import android.app.job.JobParameters +import android.app.job.JobScheduler +import android.app.job.JobService +import android.content.ComponentName +import android.content.Context +import android.os.Build +import android.support.annotation.RequiresApi +import com.elvishew.xlog.XLog +import com.evernote.android.job.JobRequest +import com.google.gson.Gson +import com.kizitonwose.time.days +import com.kizitonwose.time.hours +import com.kizitonwose.time.minutes +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Chapter +import eu.kanade.tachiyomi.data.database.models.ChapterImpl +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import eu.kanade.tachiyomi.source.SourceManager +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.online.all.EHentai +import eu.kanade.tachiyomi.util.jobScheduler +import eu.kanade.tachiyomi.util.syncChaptersWithSource +import exh.EH_SOURCE_ID +import exh.EXH_SOURCE_ID +import exh.metadata.metadata.EHentaiSearchMetadata +import exh.metadata.metadata.base.* +import exh.metadata.sql.models.SearchMetadata +import exh.util.await +import exh.util.awaitSuspending +import exh.util.cancellable +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.* +import rx.schedulers.Schedulers +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import uy.kohesive.injekt.injectLazy +import java.io.IOException +import kotlin.coroutines.CoroutineContext + +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +class EHentaiUpdateWorker: JobService(), CoroutineScope { + override val coroutineContext: CoroutineContext + get() = Dispatchers.Default + Job() + + private val db: DatabaseHelper by injectLazy() + private val prefs: PreferencesHelper by injectLazy() + private val gson: Gson by injectLazy() + private val sourceManager: SourceManager by injectLazy() + private val updateHelper: EHentaiUpdateHelper by injectLazy() + private val logger = XLog.tag("EHUpdater") + + /** + * This method is called if the system has determined that you must stop execution of your job + * even before you've had a chance to call [.jobFinished]. + * + * + * This will happen if the requirements specified at schedule time are no longer met. For + * example you may have requested WiFi with + * [android.app.job.JobInfo.Builder.setRequiredNetworkType], yet while your + * job was executing the user toggled WiFi. Another example is if you had specified + * [android.app.job.JobInfo.Builder.setRequiresDeviceIdle], and the phone left its + * idle maintenance window. You are solely responsible for the behavior of your application + * upon receipt of this message; your app will likely start to misbehave if you ignore it. + * + * + * Once this method returns, the system releases the wakelock that it is holding on + * behalf of the job. + * + * @param params The parameters identifying this job, as supplied to + * the job in the [.onStartJob] callback. + * @return `true` to indicate to the JobManager whether you'd like to reschedule + * this job based on the retry criteria provided at job creation-time; or `false` + * to end the job entirely. Regardless of the value returned, your job must stop executing. + */ + override fun onStopJob(params: JobParameters?): Boolean { + runBlocking { coroutineContext[Job]?.cancelAndJoin() } + return false + } + + /** + * Called to indicate that the job has begun executing. Override this method with the + * logic for your job. Like all other component lifecycle callbacks, this method executes + * on your application's main thread. + * + * + * Return `true` from this method if your job needs to continue running. If you + * do this, the job remains active until you call + * [.jobFinished] to tell the system that it has completed + * its work, or until the job's required constraints are no longer satisfied. For + * example, if the job was scheduled using + * [setRequiresCharging(true)][JobInfo.Builder.setRequiresCharging], + * it will be immediately halted by the system if the user unplugs the device from power, + * the job's [.onStopJob] callback will be invoked, and the app + * will be expected to shut down all ongoing work connected with that job. + * + * + * The system holds a wakelock on behalf of your app as long as your job is executing. + * This wakelock is acquired before this method is invoked, and is not released until either + * you call [.jobFinished], or after the system invokes + * [.onStopJob] to notify your job that it is being shut down + * prematurely. + * + * + * Returning `false` from this method means your job is already finished. The + * system's wakelock for the job will be released, and [.onStopJob] + * will not be invoked. + * + * @param params Parameters specifying info about this job, including the optional + * extras configured with [ This object serves to identify this specific running job instance when calling][JobInfo.Builder.setExtras] + */ + override fun onStartJob(params: JobParameters): Boolean { + launch { + startUpdating() + logger.d("Update job completed!") + jobFinished(params, false) + } + + return true + } + + suspend fun startUpdating() { + logger.d("Update job started!") + val startTime = System.currentTimeMillis() + + logger.d("Finding manga with metadata...") + val metadataManga = db.getMangaWithMetadata().await() + + logger.d("Filtering manga and raising metadata...") + val curTime = System.currentTimeMillis() + val allMeta = metadataManga.asFlow().cancellable().mapNotNull { manga -> + if (!manga.favorite || (manga.source != EH_SOURCE_ID && manga.source != EXH_SOURCE_ID)) + return@mapNotNull null + + val meta = db.getFlatMetadataForManga(manga.id!!).asRxSingle().await() + ?: return@mapNotNull null + + val raisedMeta = meta.raise() + + // Don't update galleries too frequently + if (raisedMeta.aged || curTime - raisedMeta.lastUpdateCheck < MIN_BACKGROUND_UPDATE_FREQ) + return@mapNotNull null + + val chapter = db.getChaptersByMangaId(manga.id!!).asRxSingle().await().minBy { + it.date_upload + } + + UpdateEntry(manga, raisedMeta, chapter) + }.toList() + + logger.d("Found %s manga to update, starting updates!", allMeta.size) + val mangaMetaToUpdateThisIter = allMeta.take(UPDATES_PER_ITERATION) + + var failuresThisIteration = 0 + var updatedThisIteration = 0 + val modifiedThisIteration = mutableSetOf() + + try { + for ((index, entry) in mangaMetaToUpdateThisIter.withIndex()) { + val (manga, meta) = entry + if (failuresThisIteration > MAX_UPDATE_FAILURES) { + logger.w("Too many update failures, aborting...") + break + } + + logger.d("Updating gallery (index: %s, manga.id: %s, meta.gId: %s, meta.gToken: %s, failures-so-far: %s, modifiedThisIteration.size: %s)...", + index, + manga.id, + meta.gId, + meta.gToken, + failuresThisIteration, + modifiedThisIteration.size) + + if (manga.id in modifiedThisIteration) { + // We already processed this manga! + logger.w("Gallery already updated this iteration, skipping...") + updatedThisIteration++ + continue + } + + val chapters = try { + updateEntryAndGetChapters(manga) + } catch (e: GalleryNotUpdatedException) { + if (e.network) { + failuresThisIteration++ + + logger.e("> Network error while updating gallery!", e) + logger.e("> (manga.id: %s, meta.gId: %s, meta.gToken: %s, failures-so-far: %s)", + manga.id, + meta.gId, + meta.gToken, + failuresThisIteration) + } + + continue + } + + if (chapters.isEmpty()) { + logger.e("No chapters found for gallery (manga.id: %s, meta.gId: %s, meta.gToken: %s, failures-so-far: %s)!", + manga.id, + meta.gId, + meta.gToken, + failuresThisIteration) + + continue + } + + // Find accepted root and discard others + val (acceptedRoot, discardedRoots) = + updateHelper.findAcceptedRootAndDiscardOthers(chapters).await() + + modifiedThisIteration += acceptedRoot.manga.id!! + modifiedThisIteration += discardedRoots.map { it.manga.id!! } + updatedThisIteration++ + } + } finally { + prefs.eh_autoUpdateStats().set( + gson.toJson( + EHentaiUpdaterStats( + startTime, + allMeta.size, + updatedThisIteration + ) + ) + ) + } + } + + suspend fun updateEntryAndGetChapters(manga: Manga): List { + val source = sourceManager.get(manga.source) as EHentai + + try { + val updatedManga = source.fetchMangaDetails(manga).toSingle().await(Schedulers.io()) + manga.copyFrom(updatedManga) + db.insertManga(manga).asRxSingle().await() + + val newChapters = source.fetchChapterList(manga).toSingle().await(Schedulers.io()) + syncChaptersWithSource(db, newChapters, manga, source) // Not suspending, but does block, maybe fix this? + return db.getChapters(manga).await() + } catch(t: Throwable) { + if(t is EHentai.GalleryNotFoundException) { + val meta = db.getFlatMetadataForManga(manga.id!!).await()?.raise() + if(meta != null) { + // Age dead galleries + meta.aged = true + db.insertFlatMetadata(meta.flatten()).await() + } + throw GalleryNotUpdatedException(false, t) + } + throw GalleryNotUpdatedException(true, t) + } + } + + companion object { + const val UPDATES_PER_ITERATION = 50 + private const val MAX_UPDATE_FAILURES = 5 + + private val MIN_BACKGROUND_UPDATE_FREQ = 1.days.inMilliseconds.longValue + + val GALLERY_AGE_TIME = 365.days.inMilliseconds.longValue + + private const val JOB_ID_UPDATE_BACKGROUND = 0 + private const val JOB_ID_UPDATE_BACKGROUND_TEST = 1 + + private val logger by lazy { XLog.tag("EHUpdaterScheduler") } + + private fun Context.componentName(): ComponentName { + return ComponentName(this, EHentaiUpdateWorker::class.java) + } + + private fun Context.baseBackgroundJobInfo(isTest: Boolean): JobInfo.Builder { + return JobInfo.Builder( + if(isTest) JOB_ID_UPDATE_BACKGROUND_TEST + else JOB_ID_UPDATE_BACKGROUND, componentName()) + .apply { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + setEstimatedNetworkBytes(15000L * UPDATES_PER_ITERATION, + 1000L * UPDATES_PER_ITERATION) + } + } + } + + private fun Context.periodicBackgroundJobInfo(period: Long, + requireCharging: Boolean, + requireUnmetered: Boolean): JobInfo { + return baseBackgroundJobInfo(false) + .setPeriodic(period) + .setPersisted(true) + .apply { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + setRequiresBatteryNotLow(true) + } + } + .setRequiresCharging(requireCharging) + .setRequiredNetworkType( + if(requireUnmetered) JobInfo.NETWORK_TYPE_UNMETERED + else JobInfo.NETWORK_TYPE_ANY) + .setRequiresDeviceIdle(true) + .build() + } + + private fun Context.testBackgroundJobInfo(): JobInfo { + return baseBackgroundJobInfo(true) + .setOverrideDeadline(1) + .build() + } + + fun launchBackgroundTest(context: Context) { + val jobScheduler = context.jobScheduler + if(jobScheduler.schedule(context.testBackgroundJobInfo()) == JobScheduler.RESULT_FAILURE) { + logger.e("Failed to schedule background test job!") + } else { + logger.d("Successfully scheduled background test job!") + } + } + + fun scheduleBackground(context: Context, prefInterval: Int? = null) { + cancelBackground(context) + + val preferences = Injekt.get() + val interval = prefInterval ?: preferences.eh_autoUpdateFrequency().getOrDefault() + if (interval > 0) { + val restrictions = preferences.eh_autoUpdateRequirements()!! + val acRestriction = "ac" in restrictions + val wifiRestriction = "wifi" in restrictions + + val jobInfo = context.periodicBackgroundJobInfo( + interval.hours.inMilliseconds.longValue, + acRestriction, + wifiRestriction + ) + + if(context.jobScheduler.schedule(jobInfo) == JobScheduler.RESULT_FAILURE) { + logger.e("Failed to schedule background update job!") + } else { + logger.d("Successfully scheduled background update job!") + } + } + } + + fun cancelBackground(context: Context) { + context.jobScheduler.cancel(JOB_ID_UPDATE_BACKGROUND) + } + } +} + +data class UpdateEntry(val manga: Manga, val meta: EHentaiSearchMetadata, val rootChapter: Chapter?) diff --git a/app/src/main/java/exh/eh/EHentaiUpdaterStats.kt b/app/src/main/java/exh/eh/EHentaiUpdaterStats.kt new file mode 100644 index 000000000..b91854b9f --- /dev/null +++ b/app/src/main/java/exh/eh/EHentaiUpdaterStats.kt @@ -0,0 +1,7 @@ +package exh.eh + +data class EHentaiUpdaterStats( + val startTime: Long, + val possibleUpdates: Int, + val updateCount: Int +) \ No newline at end of file diff --git a/app/src/main/java/exh/eh/GalleryNotUpdatedException.kt b/app/src/main/java/exh/eh/GalleryNotUpdatedException.kt new file mode 100644 index 000000000..724c43e9f --- /dev/null +++ b/app/src/main/java/exh/eh/GalleryNotUpdatedException.kt @@ -0,0 +1,3 @@ +package exh.eh + +class GalleryNotUpdatedException(val network: Boolean, cause: Throwable): RuntimeException(cause) \ No newline at end of file diff --git a/app/src/main/java/exh/eh/MemAutoFlushingLookupTable.kt b/app/src/main/java/exh/eh/MemAutoFlushingLookupTable.kt new file mode 100644 index 000000000..46b21306d --- /dev/null +++ b/app/src/main/java/exh/eh/MemAutoFlushingLookupTable.kt @@ -0,0 +1,214 @@ +package exh.eh + +import android.support.v4.util.AtomicFile +import android.util.SparseArray +import android.util.SparseIntArray +import com.elvishew.xlog.XLog +import exh.ui.captcha.SolveCaptchaActivity.Companion.launch +import kotlinx.coroutines.* +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.io.* +import java.nio.ByteBuffer +import kotlin.concurrent.thread +import kotlin.coroutines.CoroutineContext + +/** + * In memory Int -> Obj lookup table implementation that + * automatically persists itself to disk atomically and asynchronously. + * + * Thread safe + * + * @author nulldev + */ +class MemAutoFlushingLookupTable( + file: File, + private val serializer: EntrySerializer, + private val debounceTimeMs: Long = 3000 +) : CoroutineScope, Closeable { + /** + * The context of this scope. + * Context is encapsulated by the scope and used for implementation of coroutine builders that are extensions on the scope. + * Accessing this property in general code is not recommended for any purposes except accessing [Job] instance for advanced usages. + * + * By convention, should contain an instance of a [job][Job] to enforce structured concurrency. + */ + override val coroutineContext: CoroutineContext + get() = Dispatchers.IO + SupervisorJob() + + private val table = SparseArray(INITIAL_SIZE) + private val mutex = Mutex(true) + + // Used to debounce + @Volatile + private var writeCounter = Long.MIN_VALUE + @Volatile + private var flushed = true + + private val atomicFile = AtomicFile(file) + + private val shutdownHook = thread(start = false) { + if(!flushed) writeSynchronously() + } + + init { + initialLoad() + + Runtime.getRuntime().addShutdownHook(shutdownHook) + } + + private fun InputStream.requireBytes(targetArray: ByteArray, byteCount: Int): Boolean { + var readIter = 0 + while (true) { + val readThisIter = read(targetArray, readIter, byteCount - readIter) + if(readThisIter <= 0) return false // No more data to read + readIter += readThisIter + if(readIter == byteCount) return true + } + } + + private fun initialLoad() { + launch { + try { + atomicFile.openRead().buffered().use { input -> + val bb = ByteBuffer.allocate(8) + + while(true) { + if(!input.requireBytes(bb.array(), 8)) break + val k = bb.getInt(0) + val size = bb.getInt(4) + val strBArr = ByteArray(size) + if(!input.requireBytes(strBArr, size)) break + table.put(k, serializer.read(strBArr.toString(Charsets.UTF_8))) + } + } + } catch(e: FileNotFoundException) { + XLog.d("Lookup table not found!", e) + // Ignored + } + + mutex.unlock() + } + } + + private fun tryWrite() { + val id = ++writeCounter + flushed = false + launch { + delay(debounceTimeMs) + if(id != writeCounter) return@launch + + mutex.withLock { + // Second check inside of mutex to prevent dupe writes + if(id != writeCounter) return@launch + withContext(NonCancellable) { + writeSynchronously() + + // Yes there is a race here, no it's isn't critical + if (id == writeCounter) flushed = true + } + } + } + } + + private fun writeSynchronously() { + val bb = ByteBuffer.allocate(ENTRY_SIZE_BYTES) + + val fos = atomicFile.startWrite() + try { + val out = fos.buffered() + for(i in 0 until table.size()) { + val k = table.keyAt(i) + val v = serializer.write(table.valueAt(i)).toByteArray(Charsets.UTF_8) + bb.putInt(0, k) + bb.putInt(4, v.size) + out.write(bb.array()) + out.write(v) + } + out.flush() + atomicFile.finishWrite(fos) + } catch(t: Throwable) { + atomicFile.failWrite(fos) + throw t + } + } + + suspend fun put(key: Int, value: T) { + mutex.withLock { table.put(key, value) } + tryWrite() + } + + suspend fun get(key: Int): T? { + return mutex.withLock { table.get(key) } + } + + suspend fun size(): Int { + return mutex.withLock { table.size() } + } + + /** + * Closes this resource, relinquishing any underlying resources. + * This method is invoked automatically on objects managed by the + * `try`-with-resources statement. + * + * + * While this interface method is declared to throw `Exception`, implementers are *strongly* encouraged to + * declare concrete implementations of the `close` method to + * throw more specific exceptions, or to throw no exception at all + * if the close operation cannot fail. + * + * + * Cases where the close operation may fail require careful + * attention by implementers. It is strongly advised to relinquish + * the underlying resources and to internally *mark* the + * resource as closed, prior to throwing the exception. The `close` method is unlikely to be invoked more than once and so + * this ensures that the resources are released in a timely manner. + * Furthermore it reduces problems that could arise when the resource + * wraps, or is wrapped, by another resource. + * + * + * *Implementers of this interface are also strongly advised + * to not have the `close` method throw [ ].* + * + * This exception interacts with a thread's interrupted status, + * and runtime misbehavior is likely to occur if an `InterruptedException` is [ suppressed][Throwable.addSuppressed]. + * + * More generally, if it would cause problems for an + * exception to be suppressed, the `AutoCloseable.close` + * method should not throw it. + * + * + * Note that unlike the [close][java.io.Closeable.close] + * method of [java.io.Closeable], this `close` method + * is *not* required to be idempotent. In other words, + * calling this `close` method more than once may have some + * visible side effect, unlike `Closeable.close` which is + * required to have no effect if called more than once. + * + * However, implementers of this interface are strongly encouraged + * to make their `close` methods idempotent. + * + * @throws Exception if this resource cannot be closed + */ + override fun close() { + runBlocking { coroutineContext[Job]?.cancelAndJoin() } + Runtime.getRuntime().removeShutdownHook(shutdownHook) + } + + interface EntrySerializer { + /** + * Serialize an entry as a String. + */ + fun write(entry: T): String + + /** + * Read an entry from a String. + */ + fun read(string: String): T + } + + companion object { + private const val INITIAL_SIZE = 1000 + private const val ENTRY_SIZE_BYTES = 8 + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt index 960d3a976..b26d50d22 100644 --- a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt +++ b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt @@ -17,6 +17,7 @@ import eu.kanade.tachiyomi.util.powerManager import eu.kanade.tachiyomi.util.toast import eu.kanade.tachiyomi.util.wifiManager import exh.* +import exh.eh.EHentaiUpdateWorker import exh.util.ignore import exh.util.trans import okhttp3.FormBody @@ -112,6 +113,9 @@ class FavoritesSyncHelper(val context: Context) { "teh:ExhFavoritesSyncWifi") } + // Do not update galleries while syncing favorites + EHentaiUpdateWorker.cancelBackground(context) + storage.getRealm().use { realm -> realm.trans { db.inTransaction { @@ -161,6 +165,9 @@ class FavoritesSyncHelper(val context: Context) { wifiLock?.release() wifiLock = null } + + // Update galleries again! + EHentaiUpdateWorker.scheduleBackground(context) } if(errorList.isEmpty()) @@ -338,7 +345,8 @@ class FavoritesSyncHelper(val context: Context) { //Import using gallery adder val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}", true, - EXH_SOURCE_ID) + EXH_SOURCE_ID, + ::throttle) if(result is GalleryAddEvent.Fail) { if(result is GalleryAddEvent.Fail.NotFound) { @@ -396,7 +404,7 @@ class FavoritesSyncHelper(val context: Context) { class IgnoredException : RuntimeException() companion object { - private const val THROTTLE_MAX = 4500 + private const val THROTTLE_MAX = 5500 private const val THROTTLE_INC = 10 private const val THROTTLE_WARN = 1000 } diff --git a/app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt index 71f09570b..b1bf50242 100644 --- a/app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt +++ b/app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt @@ -36,6 +36,9 @@ class EHentaiSearchMetadata : RaisedSearchMetadata() { var ratingCount: Int? = null var averageRating: Double? = null + var aged: Boolean = false + var lastUpdateCheck: Long = 0 + override fun copyTo(manga: SManga) { gId?.let { gId -> gToken?.let { gToken -> diff --git a/app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt b/app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt index c0e8f8b24..0e6bdcdc1 100644 --- a/app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt +++ b/app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt @@ -25,7 +25,7 @@ data class FlatMetadata( fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation { // We have to use fromCallable because StorIO messes up the thread scheduling if we use their rx functions - fun getSingle() = Single.fromCallable { + val single = Single.fromCallable { val meta = getSearchMetadataForManga(mangaId).executeAsBlocking() if(meta != null) { val tags = getSearchTagsForManga(mangaId).executeAsBlocking() @@ -35,7 +35,11 @@ fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation { + return preparedOperationFromSingle(single) +} + +private fun preparedOperationFromSingle(single: Single): PreparedOperation { + return object : PreparedOperation { /** * Creates [rx.Observable] that emits result of Operation. * @@ -44,7 +48,7 @@ fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation Observable.melt(): Observable { val rs = ReplaySubject.create() subscribe(rs) return rs -} \ No newline at end of file +} + +suspend fun Single.await(subscribeOn: Scheduler? = null): T { + return suspendCancellableCoroutine { continuation -> + val self = if (subscribeOn != null) subscribeOn(subscribeOn) else this + lateinit var sub: Subscription + sub = self.subscribe({ + continuation.resume(it) { + sub.unsubscribe() + } + }, { + if (!continuation.isCancelled) + continuation.resumeWithException(it) + }) + + continuation.invokeOnCancellation { + sub.unsubscribe() + } + } +} + +suspend fun PreparedOperation.await(): T = asRxSingle().await() +suspend fun PreparedGetObject.await(): T? = asRxSingle().await() + +suspend fun Completable.awaitSuspending(subscribeOn: Scheduler? = null) { + return suspendCancellableCoroutine { continuation -> + val self = if (subscribeOn != null) subscribeOn(subscribeOn) else this + lateinit var sub: Subscription + sub = self.subscribe({ + continuation.resume(Unit) { + sub.unsubscribe() + } + }, { + if (!continuation.isCancelled) + continuation.resumeWithException(it) + }) + + continuation.invokeOnCancellation { + sub.unsubscribe() + } + } +}