Draff/TachiyomiSY-Plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add automatic gallery updating

Browse Source
This commit is contained in:
NerdNumber9 2019-04-18 17:40:13 -04:00
parent a218f4a48b
commit 1d36c3269e
29 changed files with 1240 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/build.gradle
View File

@ -289,6 +289,9 @@ dependencies {
releaseTestImplementation 'com.ms-square:debugoverlay:1.1.3'
releaseImplementation 'com.ms-square:debugoverlay-no-op:1.1.3'
testImplementation 'com.ms-square:debugoverlay-no-op:1.1.3'
// Humanize (EH)
implementation 'com.github.mfornos:humanize-slim:1.2.2'
}
buildscript {

6
app/src/main/AndroidManifest.xml
View File

@ -122,6 +122,10 @@
android:exported="false" />
<!-- EH -->
<service
android:name="exh.eh.EHentaiUpdateWorker"
android:permission="android.permission.BIND_JOB_SERVICE"
android:exported="true" />
<activity
android:name="exh.ui.intercept.InterceptActivity"
android:label="TachiyomiEH"
@ -218,7 +222,7 @@
android:name="exh.ui.captcha.SolveCaptchaActivity"
android:launchMode="singleInstance"
android:theme="@style/Theme.EHActivity" />
<activity android:name="exh.ui.webview.WebViewActivity"></activity>
<activity android:name="exh.ui.webview.WebViewActivity" />
</application>
</manifest>

3
app/src/main/java/eu/kanade/tachiyomi/AppModule.kt
View File

@ -11,6 +11,7 @@ import eu.kanade.tachiyomi.data.track.TrackManager
import eu.kanade.tachiyomi.extension.ExtensionManager
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.source.SourceManager
import exh.eh.EHentaiUpdateHelper
import rx.Observable
import rx.schedulers.Schedulers
import uy.kohesive.injekt.api.*
@ -41,6 +42,8 @@ class AppModule(val app: Application) : InjektModule {
addSingletonFactory { Gson() }
addSingletonFactory { EHentaiUpdateHelper(app) }
// Asynchronously init expensive components for a faster cold start
rxAsync { get<PreferencesHelper>() }

3
app/src/main/java/eu/kanade/tachiyomi/Migrations.kt
View File

@ -57,6 +57,9 @@ object Migrations {
}
}
}
// ===========[ ALL MIGRATIONS ABOVE HERE HAVE BEEN ALREADY REWRITTEN ]===========
return true
}
return false

15
app/src/main/java/eu/kanade/tachiyomi/data/database/queries/ChapterQueries.kt
View File

@ -15,12 +15,14 @@ import java.util.*
interface ChapterQueries : DbProvider {
fun getChapters(manga: Manga) = db.get()
fun getChapters(manga: Manga) = getChaptersByMangaId(manga.id)
fun getChaptersByMangaId(mangaId: Long?) = db.get()
.listOfObjects(Chapter::class.java)
.withQuery(Query.builder()
.table(ChapterTable.TABLE)
.where("${ChapterTable.COL_MANGA_ID} = ?")
.whereArgs(manga.id)
.whereArgs(mangaId)
.build())
.prepare()
@ -52,6 +54,15 @@ interface ChapterQueries : DbProvider {
.build())
.prepare()
fun getChapters(url: String) = db.get()
.listOfObjects(Chapter::class.java)
.withQuery(Query.builder()
.table(ChapterTable.TABLE)
.where("${ChapterTable.COL_URL} = ?")
.whereArgs(url)
.build())
.prepare()
fun insertChapter(chapter: Chapter) = db.put().`object`(chapter).prepare()

41
app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateService.kt
View File

@ -29,6 +29,8 @@ import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.ui.main.MainActivity
import eu.kanade.tachiyomi.util.*
import exh.EH_SOURCE_ID
import exh.EXH_SOURCE_ID
import rx.Observable
import rx.Subscription
import rx.schedulers.Schedulers
@ -283,24 +285,29 @@ class LibraryUpdateService(
.doOnNext { showProgressNotification(it, count.andIncrement, mangaToUpdate.size) }
// Update the chapters of the manga.
.concatMap { manga ->
updateManga(manga)
// If there's any error, return empty update and continue.
.onErrorReturn {
failedUpdates.add(manga)
Pair(emptyList(), emptyList())
}
// Filter out mangas without new chapters (or failed).
.filter { pair -> pair.first.isNotEmpty() }
.doOnNext {
if (downloadNew && (categoriesToDownload.isEmpty() ||
manga.category in categoriesToDownload)) {
downloadChapters(manga, it.first)
hasDownloads = true
if(manga.source == EXH_SOURCE_ID || manga.source == EH_SOURCE_ID) {
// Ignore EXH manga, updating chapters for every manga will get you banned
Observable.just(manga)
} else {
updateManga(manga)
// If there's any error, return empty update and continue.
.onErrorReturn {
failedUpdates.add(manga)
Pair(emptyList(), emptyList())
}
}
// Convert to the manga that contains new chapters.
.map { manga }
// Filter out mangas without new chapters (or failed).
.filter { pair -> pair.first.isNotEmpty() }
.doOnNext {
if (downloadNew && (categoriesToDownload.isEmpty() ||
manga.category in categoriesToDownload)) {
downloadChapters(manga, it.first)
hasDownloads = true
}
}
// Convert to the manga that contains new chapters.
.map { manga }
}
}
// Add manga with new chapters to the list.
.doOnNext { manga ->

6
app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt
View File

@ -186,4 +186,10 @@ object PreferenceKeys {
const val eh_logLevel = "eh_log_level"
const val eh_enableSourceBlacklist = "eh_enable_source_blacklist"
const val eh_autoUpdateFrequency = "eh_auto_update_frequency"
const val eh_autoUpdateRestrictions = "eh_auto_update_restrictions"
const val eh_autoUpdateStats = "eh_auto_update_stats"
}

6
app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt
View File

@ -259,4 +259,10 @@ class PreferencesHelper(val context: Context) {
fun eh_logLevel() = rxPrefs.getInteger(Keys.eh_logLevel, 0)
fun eh_enableSourceBlacklist() = rxPrefs.getBoolean(Keys.eh_enableSourceBlacklist, true)
fun eh_autoUpdateFrequency() = rxPrefs.getInteger(Keys.eh_autoUpdateFrequency, 1)
fun eh_autoUpdateRequirements() = prefs.getStringSet(Keys.eh_autoUpdateRestrictions, emptySet())
fun eh_autoUpdateStats() = rxPrefs.getString(Keys.eh_autoUpdateStats, "")
}

5
app/src/main/java/eu/kanade/tachiyomi/source/model/SManga.kt
View File

@ -23,6 +23,11 @@ interface SManga : Serializable {
var initialized: Boolean
fun copyFrom(other: SManga) {
// EXH -->
url = other.url // Allow dynamically mutating one manga into another
title = other.title
// EXH <--
if (other.author != null)
author = other.author

126
app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt
View File

@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.source.online.all
import android.content.Context
import android.net.Uri
import com.elvishew.xlog.XLog
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
@ -12,6 +13,7 @@ import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.LewdSource
import eu.kanade.tachiyomi.util.asJsoup
import exh.eh.EHentaiUpdateHelper
import exh.metadata.EX_DATE_FORMAT
import exh.metadata.metadata.EHentaiSearchMetadata
import exh.metadata.metadata.EHentaiSearchMetadata.Companion.EH_GENRE_NAMESPACE
@ -33,11 +35,17 @@ import uy.kohesive.injekt.injectLazy
import java.net.URLEncoder
import java.util.*
import exh.metadata.metadata.base.RaisedTag
import exh.eh.EHentaiUpdateWorker
import exh.eh.GalleryEntry
import kotlinx.coroutines.runBlocking
import org.jsoup.nodes.TextNode
import rx.Single
import java.lang.RuntimeException
// TODO Consider gallery updating when doing tabbed browsing
class EHentai(override val id: Long,
val exh: Boolean,
val context: Context) : HttpSource(), LewdSource<EHentaiSearchMetadata, Response> {
val context: Context) : HttpSource(), LewdSource<EHentaiSearchMetadata, Document> {
override val metaClass = EHentaiSearchMetadata::class
val schema: String
@ -58,7 +66,8 @@ class EHentai(override val id: Long,
override val lang = "all"
override val supportsLatest = true
val prefs: PreferencesHelper by injectLazy()
private val prefs: PreferencesHelper by injectLazy()
private val updateHelper: EHentaiUpdateHelper by injectLazy()
/**
* Gallery list entry
@ -115,15 +124,83 @@ class EHentai(override val id: Long,
MangasPage(it.first.map { it.manga }, it.second)
}
override fun fetchChapterList(manga: SManga): Observable<List<SChapter>>
= Observable.just(listOf(SChapter.create().apply {
url = manga.url
name = "Chapter"
chapter_number = 1f
}))
override fun fetchChapterList(manga: SManga)
= fetchChapterList(manga) {}
fun fetchChapterList(manga: SManga, throttleFunc: () -> Unit): Observable<List<SChapter>> {
return Single.fromCallable {
// Pull all the way to the root gallery
// We can't do this with RxJava or we run into stack overflows on shit like this:
// https://exhentai.org/g/1073061/f9345f1c12/
var url: String = manga.url
var doc: Document? = null
runBlocking {
while (true) {
val gid = EHentaiSearchMetadata.galleryId(url).toInt()
val cachedParent = updateHelper.parentLookupTable.get(
gid
)
if(cachedParent == null) {
throttleFunc()
val resp = client.newCall(exGet(baseUrl + url)).execute()
if (!resp.isSuccessful) error("HTTP error (${resp.code()})!")
doc = resp.asJsoup()
val parentLink = doc!!.select("#gdd .gdt1").find { el ->
el.text().toLowerCase() == "parent:"
}!!.nextElementSibling().selectFirst("a")?.attr("href")
if (parentLink != null) {
updateHelper.parentLookupTable.put(
gid,
GalleryEntry(
EHentaiSearchMetadata.galleryId(parentLink),
EHentaiSearchMetadata.galleryToken(parentLink)
)
)
url = EHentaiSearchMetadata.normalizeUrl(parentLink)
} else break
} else {
XLog.d("Parent cache hit: %s!", gid)
url = EHentaiSearchMetadata.idAndTokenToUrl(
cachedParent.gId,
cachedParent.gToken
)
}
}
}
doc!!
}.map { d ->
val newDisplay = d.select("#gnd a")
// Build chapter for root gallery
val self = SChapter.create().apply {
url = EHentaiSearchMetadata.normalizeUrl(d.location())
name = "v1: " + d.selectFirst("#gn").text()
chapter_number = 1f
date_upload = EX_DATE_FORMAT.parse(d.select("#gdd .gdt1").find { el ->
el.text().toLowerCase() == "posted:"
}!!.nextElementSibling().text()).time
}
// Build and append the rest of the galleries
listOf(self) + newDisplay.mapIndexed { index, newGallery ->
val link = newGallery.attr("href")
val name = newGallery.text()
val posted = (newGallery.nextSibling() as TextNode).text().removePrefix(", added ")
SChapter.create().apply {
this.url = EHentaiSearchMetadata.normalizeUrl(link)
this.name = "v${index + 2}: $name"
this.chapter_number = index + 2f
this.date_upload = EX_DATE_FORMAT.parse(posted).time
}
}
}.toObservable()
}
override fun fetchPageList(chapter: SChapter)
= fetchChapterPage(chapter, "$baseUrl/${chapter.url}").map {
= fetchChapterPage(chapter, baseUrl + chapter.url).map {
it.mapIndexed { i, s ->
Page(i, s)
}
@ -241,9 +318,20 @@ class EHentai(override val id: Long,
.asObservableWithAsyncStacktrace()
.flatMap { (stacktrace, response) ->
if(response.isSuccessful) {
parseToManga(manga, response).andThen(Observable.just(manga.apply {
initialized = true
}))
// Pull to most recent
val doc = response.asJsoup()
val newerGallery = doc.select("#gnd a").lastOrNull()
val pre = if(newerGallery != null) {
manga.url = EHentaiSearchMetadata.normalizeUrl(newerGallery.attr("href"))
client.newCall(mangaDetailsRequest(manga))
.asObservableSuccess().map { it.asJsoup() }
} else Observable.just(doc)
pre.flatMap {
parseToManga(manga, it).andThen(Observable.just(manga.apply {
initialized = true
}))
}
} else {
response.close()
@ -261,10 +349,10 @@ class EHentai(override val id: Long,
*/
override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException()
override fun parseIntoMetadata(metadata: EHentaiSearchMetadata, input: Response) {
override fun parseIntoMetadata(metadata: EHentaiSearchMetadata, input: Document) {
with(metadata) {
with(input.asJsoup()) {
val url = input.request().url().encodedPath()
with(input) {
val url = input.location()
gId = EHentaiSearchMetadata.galleryId(url)
gToken = EHentaiSearchMetadata.galleryToken(url)
@ -296,6 +384,8 @@ class EHentai(override val id: Long,
.toLowerCase()) {
"posted" -> datePosted = EX_DATE_FORMAT.parse(right).time
// Example gallery with parent: https://e-hentai.org/g/1390451/7f181c2426/
// Example JP gallery: https://exhentai.org/g/1375385/03519d541b/
// Parent is older variation of the gallery
"parent" -> parent = if (!right.equals("None", true)) {
rightElement.child(0).attr("href")
} else null
@ -312,6 +402,12 @@ class EHentai(override val id: Long,
}
}
lastUpdateCheck = System.currentTimeMillis()
if(datePosted != null
&& lastUpdateCheck - datePosted!! > EHentaiUpdateWorker.GALLERY_AGE_TIME) {
aged = true
}
//Parse ratings
ignore {
averageRating = select("#rating_label")

18
app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt
View File

@ -26,6 +26,7 @@ import eu.kanade.tachiyomi.ui.base.controller.RxController
import eu.kanade.tachiyomi.ui.base.controller.TabbedController
import eu.kanade.tachiyomi.ui.base.controller.requestPermissionsSafe
import eu.kanade.tachiyomi.ui.manga.chapter.ChaptersController
import eu.kanade.tachiyomi.ui.manga.chapter.ChaptersPresenter
import eu.kanade.tachiyomi.ui.manga.info.MangaInfoController
import eu.kanade.tachiyomi.ui.manga.track.TrackController
import eu.kanade.tachiyomi.util.toast
@ -48,6 +49,18 @@ class MangaController : RxController, TabbedController {
}
}
// EXH -->
constructor(redirect: ChaptersPresenter.EXHRedirect) : super(Bundle().apply {
putLong(MANGA_EXTRA, redirect.manga.id!!)
putBoolean(UPDATE_EXTRA, redirect.update)
}) {
this.manga = redirect.manga
if (manga != null) {
source = Injekt.get<SourceManager>().getOrStub(redirect.manga.source)
}
}
// EXH <--
constructor(mangaId: Long) : this(
Injekt.get<DatabaseHelper>().getManga(mangaId).executeAsBlocking())
@ -64,6 +77,8 @@ class MangaController : RxController, TabbedController {
val fromCatalogue = args.getBoolean(FROM_CATALOGUE_EXTRA, false)
val update = args.getBoolean(UPDATE_EXTRA, false)
val lastUpdateRelay: BehaviorRelay<Date> = BehaviorRelay.create()
val chapterCountRelay: BehaviorRelay<Float> = BehaviorRelay.create()
@ -180,6 +195,9 @@ class MangaController : RxController, TabbedController {
companion object {
// EXH -->
const val UPDATE_EXTRA = "update"
// EXH <--
const val FROM_CATALOGUE_EXTRA = "from_catalogue"
const val MANGA_EXTRA = "manga"

13
app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersController.kt
View File

@ -11,6 +11,7 @@ import android.support.v7.view.ActionMode
import android.support.v7.widget.DividerItemDecoration
import android.support.v7.widget.LinearLayoutManager
import android.view.*
import com.bluelinelabs.conductor.RouterTransaction
import com.elvishew.xlog.XLog
import com.jakewharton.rxbinding.support.v4.widget.refreshes
import com.jakewharton.rxbinding.view.clicks
@ -28,6 +29,7 @@ import eu.kanade.tachiyomi.util.getCoordinates
import eu.kanade.tachiyomi.util.snack
import eu.kanade.tachiyomi.util.toast
import kotlinx.android.synthetic.main.chapters_controller.*
import rx.android.schedulers.AndroidSchedulers
import timber.log.Timber
class ChaptersController : NucleusController<ChaptersPresenter>(),
@ -104,6 +106,14 @@ class ChaptersController : NucleusController<ChaptersPresenter>(),
view.context.toast(R.string.no_next_chapter)
}
}
presenter.redirectUserRelay
.observeOn(AndroidSchedulers.mainThread())
.subscribeUntilDestroy { redirect ->
XLog.d("Redirecting to updated manga (manga.id: %s, manga.title: %s, update: %s)!", redirect.manga.id, redirect.manga.title, redirect.update)
// Replace self
parentController?.router?.replaceTopController(RouterTransaction.with(MangaController(redirect)))
}
}
override fun onDestroyView(view: View) {
@ -188,6 +198,9 @@ class ChaptersController : NucleusController<ChaptersPresenter>(),
if (presenter.chapters.isEmpty())
initialFetchChapters()
if ((parentController as MangaController).update)
fetchChaptersFromSource()
val adapter = adapter ?: return
adapter.updateDataSet(chapters)

31
app/src/main/java/eu/kanade/tachiyomi/ui/manga/chapter/ChaptersPresenter.kt
View File

@ -15,6 +15,9 @@ import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter
import eu.kanade.tachiyomi.util.isNullOrUnsubscribed
import eu.kanade.tachiyomi.util.syncChaptersWithSource
import exh.EH_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.eh.EHentaiUpdateHelper
import rx.Observable
import rx.Subscription
import rx.android.schedulers.AndroidSchedulers
@ -22,6 +25,7 @@ import rx.schedulers.Schedulers
import timber.log.Timber
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import uy.kohesive.injekt.injectLazy
import java.util.Date
/**
@ -66,6 +70,14 @@ class ChaptersPresenter(
*/
private var observeDownloadsSubscription: Subscription? = null
// EXH -->
private val updateHelper: EHentaiUpdateHelper by injectLazy()
val redirectUserRelay = BehaviorRelay.create<EXHRedirect>()
data class EXHRedirect(val manga: Manga, val update: Boolean)
// EXH <--
override fun onCreate(savedState: Bundle?) {
super.onCreate(savedState)
@ -100,6 +112,25 @@ class ChaptersPresenter(
lastUpdateRelay.call(Date(chapters.maxBy { it.date_upload }?.date_upload
?: 0))
// EXH -->
if(chapters.isNotEmpty()
&& (source.id == EXH_SOURCE_ID || source.id == EH_SOURCE_ID)) {
// Check for gallery in library and accept manga with lowest id
// Find chapters sharing same root
add(updateHelper.findAcceptedRootAndDiscardOthers(chapters)
.subscribeOn(Schedulers.io())
.subscribe { (acceptedChain, _) ->
// Redirect if we are not the accepted root
if(manga.id != acceptedChain.manga.id) {
// Update if any of our chapters are not in accepted manga's chapters
val ourChapterUrls = chapters.map { it.url }.toSet()
val acceptedChapterUrls = acceptedChain.chapters.map { it.url }.toSet()
val update = (ourChapterUrls - acceptedChapterUrls).isNotEmpty()
redirectUserRelay.call(EXHRedirect(acceptedChain.manga, update))
}
})
}
// EXH <--
}
.subscribe { chaptersRelay.call(it) })
}

18
app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt
View File

@ -22,6 +22,7 @@ import com.bumptech.glide.load.engine.DiskCacheStrategy
import com.bumptech.glide.load.resource.bitmap.RoundedCorners
import com.bumptech.glide.request.target.SimpleTarget
import com.bumptech.glide.request.transition.Transition
import com.bumptech.glide.signature.ObjectKey
import com.elvishew.xlog.XLog
import com.jakewharton.rxbinding.support.v4.widget.refreshes
import com.jakewharton.rxbinding.view.clicks
@ -74,7 +75,9 @@ class MangaInfoController : NucleusController<MangaInfoPresenter>(),
*/
private val preferences: PreferencesHelper by injectLazy()
private val sourceManager: SourceManager by injectLazy()
// EXH -->
private var lastMangaThumbnail: String? = null
// EXH <--
init {
setHasOptionsMenu(true)
@ -181,6 +184,7 @@ class MangaInfoController : NucleusController<MangaInfoPresenter>(),
// Update view.
setMangaInfo(manga, source)
if((parentController as MangaController).update) fetchMangaFromSource()
} else {
// Initialize manga.
fetchMangaFromSource()
@ -247,10 +251,17 @@ class MangaInfoController : NucleusController<MangaInfoPresenter>(),
// Set the favorite drawable to the correct one.
setFavoriteDrawable(manga.favorite)
// Set cover if it wasn't already.
if (manga_cover.drawable == null && !manga.thumbnail_url.isNullOrEmpty()) {
// Set cover if it matches
val tagMatches = lastMangaThumbnail == manga.thumbnail_url
val coverLoaded = manga_cover.drawable != null
if ((!tagMatches || !coverLoaded) && !manga.thumbnail_url.isNullOrEmpty()) {
lastMangaThumbnail = manga.thumbnail_url
val coverSig = ObjectKey(manga.thumbnail_url ?: "")
GlideApp.with(view.context)
.load(manga)
.signature(coverSig)
.diskCacheStrategy(DiskCacheStrategy.RESOURCE)
.centerCrop()
.into(manga_cover)
@ -258,6 +269,7 @@ class MangaInfoController : NucleusController<MangaInfoPresenter>(),
if (backdrop != null) {
GlideApp.with(view.context)
.load(manga)
.signature(coverSig)
.diskCacheStrategy(DiskCacheStrategy.RESOURCE)
.centerCrop()
.into(backdrop)

136
app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt
View File

@ -1,26 +1,54 @@
package eu.kanade.tachiyomi.ui.setting
import android.os.Build
import android.os.Handler
import android.support.v7.preference.PreferenceScreen
import android.widget.Toast
import com.afollestad.materialdialogs.MaterialDialog
import com.bluelinelabs.conductor.RouterTransaction
import com.bluelinelabs.conductor.changehandler.FadeChangeHandler
import com.f2prateek.rx.preferences.Preference
import com.github.salomonbrys.kotson.fromJson
import com.google.gson.Gson
import com.kizitonwose.time.Interval
import com.kizitonwose.time.days
import com.kizitonwose.time.hours
import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.preference.PreferenceKeys
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.util.toast
import exh.EH_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.eh.EHentaiUpdateWorker
import exh.eh.EHentaiUpdaterStats
import exh.favorites.FavoritesIntroDialog
import exh.favorites.LocalFavoritesStorage
import exh.metadata.metadata.EHentaiSearchMetadata
import exh.metadata.metadata.base.getFlatMetadataForManga
import exh.metadata.nullIfBlank
import exh.uconfig.WarnConfigureDialogController
import exh.ui.login.LoginController
import exh.util.await
import exh.util.trans
import humanize.Humanize
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import rx.android.schedulers.AndroidSchedulers
import rx.schedulers.Schedulers
import uy.kohesive.injekt.injectLazy
import java.util.*
/**
* EH Settings fragment
*/
class SettingsEhController : SettingsController() {
private val gson: Gson by injectLazy()
private val db: DatabaseHelper by injectLazy()
private fun Preference<*>.reconfigure(): Boolean {
//Listen for change commit
asObservable()
@ -183,5 +211,113 @@ class SettingsEhController : SettingsController() {
}
}
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
preferenceCategory {
title = "Gallery update checker"
intListPreference {
key = PreferenceKeys.eh_autoUpdateFrequency
title = "Time between update batches"
entries = arrayOf("Never update galleries", "1 hour", "2 hours", "3 hours", "6 hours", "12 hours", "24 hours", "48 hours")
entryValues = arrayOf("0", "1", "2", "3", "6", "12", "24", "48")
defaultValue = "0"
preferences.eh_autoUpdateFrequency().asObservable().subscribeUntilDestroy { newVal ->
summary = if(newVal == 0) {
"${context.getString(R.string.app_name)} will currently never check galleries in your library for updates."
} else {
"${context.getString(R.string.app_name)} checks/updates galleries in batches. " +
"This means it will wait $newVal hour(s), check ${EHentaiUpdateWorker.UPDATES_PER_ITERATION} galleries," +
" wait $newVal hour(s), check ${EHentaiUpdateWorker.UPDATES_PER_ITERATION} and so on..."
}
}
onChange { newValue ->
val interval = (newValue as String).toInt()
EHentaiUpdateWorker.scheduleBackground(context, interval)
true
}
}
multiSelectListPreference {
key = PreferenceKeys.eh_autoUpdateRestrictions
title = "Auto update restrictions"
entriesRes = arrayOf(R.string.wifi, R.string.charging)
entryValues = arrayOf("wifi", "ac")
summaryRes = R.string.pref_library_update_restriction_summary
preferences.eh_autoUpdateFrequency().asObservable()
.subscribeUntilDestroy { isVisible = it > 0 }
onChange {
// Post to event looper to allow the preference to be updated.
Handler().post { EHentaiUpdateWorker.scheduleBackground(context) }
true
}
}
preference {
title = "Show updater statistics"
onClick {
val progress = MaterialDialog.Builder(context)
.progress(true, 0)
.content("Collecting statistics...")
.cancelable(false)
.show()
GlobalScope.launch(Dispatchers.IO) {
val updateInfo = try {
val stats = preferences.eh_autoUpdateStats().getOrDefault().nullIfBlank()?.let {
gson.fromJson<EHentaiUpdaterStats>(it)
}
val statsText = if (stats != null) {
"The updater last ran ${Humanize.naturalTime(Date(stats.startTime))}, and checked ${stats.updateCount} out of the ${stats.possibleUpdates} galleries that were ready for checking."
} else "The updater has not ran yet."
val allMeta = db.getMangaWithMetadata().await().filter {
it.favorite && (it.source == EH_SOURCE_ID || it.source == EXH_SOURCE_ID)
}.mapNotNull {
db.getFlatMetadataForManga(it.id!!).await()?.raise<EHentaiSearchMetadata>()
}.toList()
fun metaInRelativeDuration(duration: Interval<*>): Int {
val durationMs = duration.inMilliseconds.longValue
return allMeta.asSequence().filter {
System.currentTimeMillis() - it.lastUpdateCheck < durationMs
}.count()
}
"""
$statsText
Galleries that were checked in the last:
- hour: ${metaInRelativeDuration(1.hours)}
- 6 hours: ${metaInRelativeDuration(6.hours)}
- 12 hours: ${metaInRelativeDuration(12.hours)}
- day: ${metaInRelativeDuration(1.days)}
- 2 days: ${metaInRelativeDuration(2.days)}
- week: ${metaInRelativeDuration(7.days)}
- month: ${metaInRelativeDuration(30.days)}
- year: ${metaInRelativeDuration(365.days)}
""".trimIndent()
} finally {
progress.dismiss()
}
withContext(Dispatchers.Main) {
MaterialDialog.Builder(context)
.title("Gallery updater statistics")
.content(updateInfo)
.positiveText("Ok")
.show()
}
}
}
}
}
}
}
}

7
app/src/main/java/eu/kanade/tachiyomi/util/ContextExtensions.kt
View File

@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.util
import android.app.ActivityManager
import android.app.Notification
import android.app.NotificationManager
import android.app.job.JobScheduler
import android.content.*
import android.content.Context.VIBRATOR_SERVICE
import android.content.pm.PackageManager
@ -128,11 +129,11 @@ val Context.wifiManager: WifiManager
get() = applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager
// --> EH
/**
* Property to get the wifi manager from the context.
*/
val Context.clipboardManager: ClipboardManager
get() = applicationContext.getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
val Context.jobScheduler: JobScheduler
get() = applicationContext.getSystemService(Context.JOB_SCHEDULER_SERVICE) as JobScheduler
// <-- EH
/**

67
app/src/main/java/exh/EXHMigrations.kt
View File

@ -1,5 +1,6 @@
package exh
import com.elvishew.xlog.XLog
import com.pushtorefresh.storio.sqlite.queries.Query
import com.pushtorefresh.storio.sqlite.queries.RawQuery
import eu.kanade.tachiyomi.data.backup.models.DHistory
@ -20,6 +21,8 @@ import java.net.URISyntaxException
object EXHMigrations {
private val db: DatabaseHelper by injectLazy()
private val logger = XLog.tag("EXHMigrations")
private const val CURRENT_MIGRATION_VERSION = 1
/**
@ -31,45 +34,49 @@ object EXHMigrations {
fun upgrade(preferences: PreferencesHelper): Boolean {
val context = preferences.context
val oldVersion = preferences.eh_lastVersionCode().getOrDefault()
if (oldVersion < CURRENT_MIGRATION_VERSION) {
preferences.eh_lastVersionCode().set(CURRENT_MIGRATION_VERSION)
if(oldVersion < 1) {
db.inTransaction {
// Migrate HentaiCafe source IDs
db.lowLevel().executeSQL(RawQuery.builder()
.query("""
try {
if (oldVersion < CURRENT_MIGRATION_VERSION) {
if (oldVersion < 1) {
db.inTransaction {
// Migrate HentaiCafe source IDs
db.lowLevel().executeSQL(RawQuery.builder()
.query("""
UPDATE ${MangaTable.TABLE}
SET ${MangaTable.COL_SOURCE} = $HENTAI_CAFE_SOURCE_ID
WHERE ${MangaTable.COL_SOURCE} = 6908
""".trimIndent())
.affectsTables(MangaTable.TABLE)
.build())
.affectsTables(MangaTable.TABLE)
.build())
// Migrate nhentai URLs
val nhentaiManga = db.db.get()
.listOfObjects(Manga::class.java)
.withQuery(Query.builder()
.table(MangaTable.TABLE)
.where("${MangaTable.COL_SOURCE} = $NHENTAI_SOURCE_ID")
.build())
.prepare()
.executeAsBlocking()
// Migrate nhentai URLs
val nhentaiManga = db.db.get()
.listOfObjects(Manga::class.java)
.withQuery(Query.builder()
.table(MangaTable.TABLE)
.where("${MangaTable.COL_SOURCE} = $NHENTAI_SOURCE_ID")
.build())
.prepare()
.executeAsBlocking()
nhentaiManga.forEach {
it.url = getUrlWithoutDomain(it.url)
nhentaiManga.forEach {
it.url = getUrlWithoutDomain(it.url)
}
db.db.put()
.objects(nhentaiManga)
// Extremely slow without the resolver :/
.withPutResolver(MangaUrlPutResolver())
.prepare()
.executeAsBlocking()
}
db.db.put()
.objects(nhentaiManga)
// Extremely slow without the resolver :/
.withPutResolver(MangaUrlPutResolver())
.prepare()
.executeAsBlocking()
}
}
return true
preferences.eh_lastVersionCode().set(CURRENT_MIGRATION_VERSION)
return true
}
} catch(e: Exception) {
logger.e( "Failed to migrate app from $oldVersion -> $CURRENT_MIGRATION_VERSION!", e)
}
return false
}

8
app/src/main/java/exh/GalleryAdder.kt
View File

@ -61,7 +61,8 @@ class GalleryAdder {
fun addGallery(url: String,
fav: Boolean = false,
forceSource: Long? = null): GalleryAddEvent {
forceSource: Long? = null,
throttleFunc: () -> Unit = {}): GalleryAddEvent {
XLog.d("Importing gallery (url: %s, fav: %s, forceSource: %s)...", url, fav, forceSource)
try {
val urlObj = Uri.parse(url)
@ -167,7 +168,6 @@ class GalleryAdder {
// Fetch and copy details
val newManga = sourceObj.fetchMangaDetails(manga).toBlocking().first()
manga.copyFrom(newManga)
manga.title = newManga.title //Forcibly copy title as copyFrom does not copy title
manga.initialized = true
if (fav) manga.favorite = true
@ -180,13 +180,13 @@ class GalleryAdder {
syncChaptersWithSource(db, it, manga, sourceObj)
}.toBlocking().first()
} catch (e: Exception) {
XLog.w("Failed to update chapters for gallery: %s!", manga.title)
XLog.w("Failed to update chapters for gallery: ${manga.title}!", e)
return GalleryAddEvent.Fail.Error(url, "Failed to update chapters for gallery: $url")
}
return GalleryAddEvent.Success(url, manga)
} catch(e: Exception) {
XLog.w("Could not add gallery!", e)
XLog.w("Could not add gallery (url: $url)!", e)
if(e is EHentai.GalleryNotFoundException) {
return GalleryAddEvent.Fail.NotFound(url)

12
app/src/main/java/exh/debug/DebugFunctions.kt
View File

@ -1,5 +1,7 @@
package exh.debug
import android.app.Application
import android.os.Build
import com.pushtorefresh.storio.sqlite.queries.RawQuery
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.tables.MangaTable
@ -7,9 +9,11 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.source.SourceManager
import exh.EH_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.eh.EHentaiUpdateWorker
import uy.kohesive.injekt.injectLazy
object DebugFunctions {
val app: Application by injectLazy()
val db: DatabaseHelper by injectLazy()
val prefs: PreferencesHelper by injectLazy()
val sourceManager: SourceManager by injectLazy()
@ -48,6 +52,14 @@ object DebugFunctions {
fun convertAllExhentaiGalleriesToEhentai() = convertSources(EXH_SOURCE_ID, EH_SOURCE_ID)
fun testLaunchBackgroundUpdater() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
EHentaiUpdateWorker.launchBackgroundTest(app)
} else {
error("OS/SDK version too old!")
}
}
private fun convertSources(from: Long, to: Long) {
db.lowLevel().executeSQL(RawQuery.builder()
.query("""

137
app/src/main/java/exh/eh/EHentaiUpdateHelper.kt Normal file
View File

@ -0,0 +1,137 @@
package exh.eh
import android.content.Context
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Chapter
import eu.kanade.tachiyomi.data.database.models.ChapterImpl
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.database.models.MangaCategory
import rx.Observable
import rx.Single
import uy.kohesive.injekt.injectLazy
import java.io.File
data class ChapterChain(val manga: Manga, val chapters: List<Chapter>)
class EHentaiUpdateHelper(context: Context) {
val parentLookupTable =
MemAutoFlushingLookupTable(
File(context.filesDir, "exh-plt.maftable"),
GalleryEntry.Serializer()
)
private val db: DatabaseHelper by injectLazy()
/**
* @param chapters Cannot be an empty list!
*
* @return Pair<Accepted, Discarded>
*/
fun findAcceptedRootAndDiscardOthers(chapters: List<Chapter>): Single<Pair<ChapterChain, List<ChapterChain>>> {
// Find other chains
val chainsObservable = Observable.merge(chapters.map { chapter ->
db.getChapters(chapter.url).asRxSingle().toObservable()
}).toList().map { allChapters ->
allChapters.flatMap { innerChapters -> innerChapters.map { it.manga_id!! } }.distinct()
}.flatMap { mangaIds ->
Observable.merge(
mangaIds.map { mangaId ->
Single.zip(
db.getManga(mangaId).asRxSingle(),
db.getChaptersByMangaId(mangaId).asRxSingle()
) { manga, chapters ->
ChapterChain(manga, chapters)
}.toObservable()
}
)
}.toList()
// Accept oldest chain
val chainsWithAccepted = chainsObservable.map { chains ->
val acceptedChain = chains.minBy { it.manga.id!! }!!
acceptedChain to chains
}
return chainsWithAccepted.map { (accepted, chains) ->
val toDiscard = chains.filter { it.manga.favorite && it.manga.id != accepted.manga.id }
if(toDiscard.isNotEmpty()) {
// Copy chain chapters to curChapters
val newChapters = toDiscard
.flatMap { it.chapters }
.fold(accepted.chapters) { curChapters, chapter ->
val existing = curChapters.find { it.url == chapter.url }
if (existing != null) {
existing.read = existing.read || chapter.read
existing.last_page_read = existing.last_page_read.coerceAtLeast(chapter.last_page_read)
existing.bookmark = existing.bookmark || chapter.bookmark
curChapters
} else if (chapter.date_upload > 0) { // Ignore chapters using the old system
curChapters + ChapterImpl().apply {
manga_id = accepted.manga.id
url = chapter.url
name = chapter.name
read = chapter.read
bookmark = chapter.bookmark
last_page_read = chapter.last_page_read
date_fetch = chapter.date_fetch
date_upload = chapter.date_upload
}
} else curChapters
}
.filter { it.date_upload <= 0 } // Ignore chapters using the old system (filter after to prevent dupes from insert)
.sortedBy { it.date_upload }
.apply {
withIndex().map { (index, chapter) ->
chapter.name = "v${index + 1}: " + chapter.name.substringAfter(" ")
chapter.chapter_number = index + 1f
chapter.source_order = index
}
}
toDiscard.forEach { it.manga.favorite = false }
accepted.manga.favorite = true
val newAccepted = ChapterChain(accepted.manga, newChapters)
val rootsToMutate = toDiscard + newAccepted
db.inTransaction {
// Apply changes to all manga
db.insertMangas(rootsToMutate.map { it.manga }).executeAsBlocking()
// Insert new chapters for accepted manga
db.insertChapters(newAccepted.chapters)
// Copy categories from all chains to accepted manga
val newCategories = rootsToMutate.flatMap {
db.getCategoriesForManga(it.manga).executeAsBlocking()
}.distinctBy { it.id }.map {
MangaCategory.create(newAccepted.manga, it)
}
db.setMangaCategories(newCategories, rootsToMutate.map { it.manga })
}
newAccepted to toDiscard
} else accepted to emptyList()
}.toSingle()
}
}
data class GalleryEntry(val gId: String, val gToken: String) {
class Serializer: MemAutoFlushingLookupTable.EntrySerializer<GalleryEntry> {
/**
* Serialize an entry as a String.
*/
override fun write(entry: GalleryEntry) = with(entry) { "$gId:$gToken" }
/**
* Read an entry from a String.
*/
override fun read(string: String): GalleryEntry {
val colonIndex = string.indexOf(':')
return GalleryEntry(
string.substring(0, colonIndex),
string.substring(colonIndex + 1, string.length)
)
}
}
}

351
app/src/main/java/exh/eh/EHentaiUpdateWorker.kt Normal file
View File

@ -0,0 +1,351 @@
package exh.eh
import android.app.job.JobInfo
import android.app.job.JobParameters
import android.app.job.JobScheduler
import android.app.job.JobService
import android.content.ComponentName
import android.content.Context
import android.os.Build
import android.support.annotation.RequiresApi
import com.elvishew.xlog.XLog
import com.evernote.android.job.JobRequest
import com.google.gson.Gson
import com.kizitonwose.time.days
import com.kizitonwose.time.hours
import com.kizitonwose.time.minutes
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Chapter
import eu.kanade.tachiyomi.data.database.models.ChapterImpl
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.online.all.EHentai
import eu.kanade.tachiyomi.util.jobScheduler
import eu.kanade.tachiyomi.util.syncChaptersWithSource
import exh.EH_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.metadata.metadata.EHentaiSearchMetadata
import exh.metadata.metadata.base.*
import exh.metadata.sql.models.SearchMetadata
import exh.util.await
import exh.util.awaitSuspending
import exh.util.cancellable
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.*
import rx.schedulers.Schedulers
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import uy.kohesive.injekt.injectLazy
import java.io.IOException
import kotlin.coroutines.CoroutineContext
@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
class EHentaiUpdateWorker: JobService(), CoroutineScope {
override val coroutineContext: CoroutineContext
get() = Dispatchers.Default + Job()
private val db: DatabaseHelper by injectLazy()
private val prefs: PreferencesHelper by injectLazy()
private val gson: Gson by injectLazy()
private val sourceManager: SourceManager by injectLazy()
private val updateHelper: EHentaiUpdateHelper by injectLazy()
private val logger = XLog.tag("EHUpdater")
/**
* This method is called if the system has determined that you must stop execution of your job
* even before you've had a chance to call [.jobFinished].
*
*
* This will happen if the requirements specified at schedule time are no longer met. For
* example you may have requested WiFi with
* [android.app.job.JobInfo.Builder.setRequiredNetworkType], yet while your
* job was executing the user toggled WiFi. Another example is if you had specified
* [android.app.job.JobInfo.Builder.setRequiresDeviceIdle], and the phone left its
* idle maintenance window. You are solely responsible for the behavior of your application
* upon receipt of this message; your app will likely start to misbehave if you ignore it.
*
*
* Once this method returns, the system releases the wakelock that it is holding on
* behalf of the job.
*
* @param params The parameters identifying this job, as supplied to
* the job in the [.onStartJob] callback.
* @return `true` to indicate to the JobManager whether you'd like to reschedule
* this job based on the retry criteria provided at job creation-time; or `false`
* to end the job entirely. Regardless of the value returned, your job must stop executing.
*/
override fun onStopJob(params: JobParameters?): Boolean {
runBlocking { coroutineContext[Job]?.cancelAndJoin() }
return false
}
/**
* Called to indicate that the job has begun executing. Override this method with the
* logic for your job. Like all other component lifecycle callbacks, this method executes
* on your application's main thread.
*
*
* Return `true` from this method if your job needs to continue running. If you
* do this, the job remains active until you call
* [.jobFinished] to tell the system that it has completed
* its work, or until the job's required constraints are no longer satisfied. For
* example, if the job was scheduled using
* [setRequiresCharging(true)][JobInfo.Builder.setRequiresCharging],
* it will be immediately halted by the system if the user unplugs the device from power,
* the job's [.onStopJob] callback will be invoked, and the app
* will be expected to shut down all ongoing work connected with that job.
*
*
* The system holds a wakelock on behalf of your app as long as your job is executing.
* This wakelock is acquired before this method is invoked, and is not released until either
* you call [.jobFinished], or after the system invokes
* [.onStopJob] to notify your job that it is being shut down
* prematurely.
*
*
* Returning `false` from this method means your job is already finished. The
* system's wakelock for the job will be released, and [.onStopJob]
* will not be invoked.
*
* @param params Parameters specifying info about this job, including the optional
* extras configured with [ This object serves to identify this specific running job instance when calling][JobInfo.Builder.setExtras]
*/
override fun onStartJob(params: JobParameters): Boolean {
launch {
startUpdating()
logger.d("Update job completed!")
jobFinished(params, false)
}
return true
}
suspend fun startUpdating() {
logger.d("Update job started!")
val startTime = System.currentTimeMillis()
logger.d("Finding manga with metadata...")
val metadataManga = db.getMangaWithMetadata().await()
logger.d("Filtering manga and raising metadata...")
val curTime = System.currentTimeMillis()
val allMeta = metadataManga.asFlow().cancellable().mapNotNull { manga ->
if (!manga.favorite || (manga.source != EH_SOURCE_ID && manga.source != EXH_SOURCE_ID))
return@mapNotNull null
val meta = db.getFlatMetadataForManga(manga.id!!).asRxSingle().await()
?: return@mapNotNull null
val raisedMeta = meta.raise<EHentaiSearchMetadata>()
// Don't update galleries too frequently
if (raisedMeta.aged || curTime - raisedMeta.lastUpdateCheck < MIN_BACKGROUND_UPDATE_FREQ)
return@mapNotNull null
val chapter = db.getChaptersByMangaId(manga.id!!).asRxSingle().await().minBy {
it.date_upload
}
UpdateEntry(manga, raisedMeta, chapter)
}.toList()
logger.d("Found %s manga to update, starting updates!", allMeta.size)
val mangaMetaToUpdateThisIter = allMeta.take(UPDATES_PER_ITERATION)
var failuresThisIteration = 0
var updatedThisIteration = 0
val modifiedThisIteration = mutableSetOf<Long>()
try {
for ((index, entry) in mangaMetaToUpdateThisIter.withIndex()) {
val (manga, meta) = entry
if (failuresThisIteration > MAX_UPDATE_FAILURES) {
logger.w("Too many update failures, aborting...")
break
}
logger.d("Updating gallery (index: %s, manga.id: %s, meta.gId: %s, meta.gToken: %s, failures-so-far: %s, modifiedThisIteration.size: %s)...",
index,
manga.id,
meta.gId,
meta.gToken,
failuresThisIteration,
modifiedThisIteration.size)
if (manga.id in modifiedThisIteration) {
// We already processed this manga!
logger.w("Gallery already updated this iteration, skipping...")
updatedThisIteration++
continue
}
val chapters = try {
updateEntryAndGetChapters(manga)
} catch (e: GalleryNotUpdatedException) {
if (e.network) {
failuresThisIteration++
logger.e("> Network error while updating gallery!", e)
logger.e("> (manga.id: %s, meta.gId: %s, meta.gToken: %s, failures-so-far: %s)",
manga.id,
meta.gId,
meta.gToken,
failuresThisIteration)
}
continue
}
if (chapters.isEmpty()) {
logger.e("No chapters found for gallery (manga.id: %s, meta.gId: %s, meta.gToken: %s, failures-so-far: %s)!",
manga.id,
meta.gId,
meta.gToken,
failuresThisIteration)
continue
}
// Find accepted root and discard others
val (acceptedRoot, discardedRoots) =
updateHelper.findAcceptedRootAndDiscardOthers(chapters).await()
modifiedThisIteration += acceptedRoot.manga.id!!
modifiedThisIteration += discardedRoots.map { it.manga.id!! }
updatedThisIteration++
}
} finally {
prefs.eh_autoUpdateStats().set(
gson.toJson(
EHentaiUpdaterStats(
startTime,
allMeta.size,
updatedThisIteration
)
)
)
}
}
suspend fun updateEntryAndGetChapters(manga: Manga): List<Chapter> {
val source = sourceManager.get(manga.source) as EHentai
try {
val updatedManga = source.fetchMangaDetails(manga).toSingle().await(Schedulers.io())
manga.copyFrom(updatedManga)
db.insertManga(manga).asRxSingle().await()
val newChapters = source.fetchChapterList(manga).toSingle().await(Schedulers.io())
syncChaptersWithSource(db, newChapters, manga, source) // Not suspending, but does block, maybe fix this?
return db.getChapters(manga).await()
} catch(t: Throwable) {
if(t is EHentai.GalleryNotFoundException) {
val meta = db.getFlatMetadataForManga(manga.id!!).await()?.raise<EHentaiSearchMetadata>()
if(meta != null) {
// Age dead galleries
meta.aged = true
db.insertFlatMetadata(meta.flatten()).await()
}
throw GalleryNotUpdatedException(false, t)
}
throw GalleryNotUpdatedException(true, t)
}
}
companion object {
const val UPDATES_PER_ITERATION = 50
private const val MAX_UPDATE_FAILURES = 5
private val MIN_BACKGROUND_UPDATE_FREQ = 1.days.inMilliseconds.longValue
val GALLERY_AGE_TIME = 365.days.inMilliseconds.longValue
private const val JOB_ID_UPDATE_BACKGROUND = 0
private const val JOB_ID_UPDATE_BACKGROUND_TEST = 1
private val logger by lazy { XLog.tag("EHUpdaterScheduler") }
private fun Context.componentName(): ComponentName {
return ComponentName(this, EHentaiUpdateWorker::class.java)
}
private fun Context.baseBackgroundJobInfo(isTest: Boolean): JobInfo.Builder {
return JobInfo.Builder(
if(isTest) JOB_ID_UPDATE_BACKGROUND_TEST
else JOB_ID_UPDATE_BACKGROUND, componentName())
.apply {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
setEstimatedNetworkBytes(15000L * UPDATES_PER_ITERATION,
1000L * UPDATES_PER_ITERATION)
}
}
}
private fun Context.periodicBackgroundJobInfo(period: Long,
requireCharging: Boolean,
requireUnmetered: Boolean): JobInfo {
return baseBackgroundJobInfo(false)
.setPeriodic(period)
.setPersisted(true)
.apply {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
setRequiresBatteryNotLow(true)
}
}
.setRequiresCharging(requireCharging)
.setRequiredNetworkType(
if(requireUnmetered) JobInfo.NETWORK_TYPE_UNMETERED
else JobInfo.NETWORK_TYPE_ANY)
.setRequiresDeviceIdle(true)
.build()
}
private fun Context.testBackgroundJobInfo(): JobInfo {
return baseBackgroundJobInfo(true)
.setOverrideDeadline(1)
.build()
}
fun launchBackgroundTest(context: Context) {
val jobScheduler = context.jobScheduler
if(jobScheduler.schedule(context.testBackgroundJobInfo()) == JobScheduler.RESULT_FAILURE) {
logger.e("Failed to schedule background test job!")
} else {
logger.d("Successfully scheduled background test job!")
}
}
fun scheduleBackground(context: Context, prefInterval: Int? = null) {
cancelBackground(context)
val preferences = Injekt.get<PreferencesHelper>()
val interval = prefInterval ?: preferences.eh_autoUpdateFrequency().getOrDefault()
if (interval > 0) {
val restrictions = preferences.eh_autoUpdateRequirements()!!
val acRestriction = "ac" in restrictions
val wifiRestriction = "wifi" in restrictions
val jobInfo = context.periodicBackgroundJobInfo(
interval.hours.inMilliseconds.longValue,
acRestriction,
wifiRestriction
)
if(context.jobScheduler.schedule(jobInfo) == JobScheduler.RESULT_FAILURE) {
logger.e("Failed to schedule background update job!")
} else {
logger.d("Successfully scheduled background update job!")
}
}
}
fun cancelBackground(context: Context) {
context.jobScheduler.cancel(JOB_ID_UPDATE_BACKGROUND)
}
}
}
data class UpdateEntry(val manga: Manga, val meta: EHentaiSearchMetadata, val rootChapter: Chapter?)

7
app/src/main/java/exh/eh/EHentaiUpdaterStats.kt Normal file
View File

@ -0,0 +1,7 @@
package exh.eh
data class EHentaiUpdaterStats(
val startTime: Long,
val possibleUpdates: Int,
val updateCount: Int
)

3
app/src/main/java/exh/eh/GalleryNotUpdatedException.kt Normal file
View File

@ -0,0 +1,3 @@
package exh.eh
class GalleryNotUpdatedException(val network: Boolean, cause: Throwable): RuntimeException(cause)

214
app/src/main/java/exh/eh/MemAutoFlushingLookupTable.kt Normal file
View File

@ -0,0 +1,214 @@
package exh.eh
import android.support.v4.util.AtomicFile
import android.util.SparseArray
import android.util.SparseIntArray
import com.elvishew.xlog.XLog
import exh.ui.captcha.SolveCaptchaActivity.Companion.launch
import kotlinx.coroutines.*
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import java.io.*
import java.nio.ByteBuffer
import kotlin.concurrent.thread
import kotlin.coroutines.CoroutineContext
/**
* In memory Int -> Obj lookup table implementation that
* automatically persists itself to disk atomically and asynchronously.
*
* Thread safe
*
* @author nulldev
*/
class MemAutoFlushingLookupTable<T>(
file: File,
private val serializer: EntrySerializer<T>,
private val debounceTimeMs: Long = 3000
) : CoroutineScope, Closeable {
/**
* The context of this scope.
* Context is encapsulated by the scope and used for implementation of coroutine builders that are extensions on the scope.
* Accessing this property in general code is not recommended for any purposes except accessing [Job] instance for advanced usages.
*
* By convention, should contain an instance of a [job][Job] to enforce structured concurrency.
*/
override val coroutineContext: CoroutineContext
get() = Dispatchers.IO + SupervisorJob()
private val table = SparseArray<T>(INITIAL_SIZE)
private val mutex = Mutex(true)
// Used to debounce
@Volatile
private var writeCounter = Long.MIN_VALUE
@Volatile
private var flushed = true
private val atomicFile = AtomicFile(file)
private val shutdownHook = thread(start = false) {
if(!flushed) writeSynchronously()
}
init {
initialLoad()
Runtime.getRuntime().addShutdownHook(shutdownHook)
}
private fun InputStream.requireBytes(targetArray: ByteArray, byteCount: Int): Boolean {
var readIter = 0
while (true) {
val readThisIter = read(targetArray, readIter, byteCount - readIter)
if(readThisIter <= 0) return false // No more data to read
readIter += readThisIter
if(readIter == byteCount) return true
}
}
private fun initialLoad() {
launch {
try {
atomicFile.openRead().buffered().use { input ->
val bb = ByteBuffer.allocate(8)
while(true) {
if(!input.requireBytes(bb.array(), 8)) break
val k = bb.getInt(0)
val size = bb.getInt(4)
val strBArr = ByteArray(size)
if(!input.requireBytes(strBArr, size)) break
table.put(k, serializer.read(strBArr.toString(Charsets.UTF_8)))
}
}
} catch(e: FileNotFoundException) {
XLog.d("Lookup table not found!", e)
// Ignored
}
mutex.unlock()
}
}
private fun tryWrite() {
val id = ++writeCounter
flushed = false
launch {
delay(debounceTimeMs)
if(id != writeCounter) return@launch
mutex.withLock {
// Second check inside of mutex to prevent dupe writes
if(id != writeCounter) return@launch
withContext(NonCancellable) {
writeSynchronously()
// Yes there is a race here, no it's isn't critical
if (id == writeCounter) flushed = true
}
}
}
}
private fun writeSynchronously() {
val bb = ByteBuffer.allocate(ENTRY_SIZE_BYTES)
val fos = atomicFile.startWrite()
try {
val out = fos.buffered()
for(i in 0 until table.size()) {
val k = table.keyAt(i)
val v = serializer.write(table.valueAt(i)).toByteArray(Charsets.UTF_8)
bb.putInt(0, k)
bb.putInt(4, v.size)
out.write(bb.array())
out.write(v)
}
out.flush()
atomicFile.finishWrite(fos)
} catch(t: Throwable) {
atomicFile.failWrite(fos)
throw t
}
}
suspend fun put(key: Int, value: T) {
mutex.withLock { table.put(key, value) }
tryWrite()
}
suspend fun get(key: Int): T? {
return mutex.withLock { table.get(key) }
}
suspend fun size(): Int {
return mutex.withLock { table.size() }
}
/**
* Closes this resource, relinquishing any underlying resources.
* This method is invoked automatically on objects managed by the
* `try`-with-resources statement.
*
*
* While this interface method is declared to throw `Exception`, implementers are *strongly* encouraged to
* declare concrete implementations of the `close` method to
* throw more specific exceptions, or to throw no exception at all
* if the close operation cannot fail.
*
*
* Cases where the close operation may fail require careful
* attention by implementers. It is strongly advised to relinquish
* the underlying resources and to internally *mark* the
* resource as closed, prior to throwing the exception. The `close` method is unlikely to be invoked more than once and so
* this ensures that the resources are released in a timely manner.
* Furthermore it reduces problems that could arise when the resource
* wraps, or is wrapped, by another resource.
*
*
* *Implementers of this interface are also strongly advised
* to not have the `close` method throw [ ].*
*
* This exception interacts with a thread's interrupted status,
* and runtime misbehavior is likely to occur if an `InterruptedException` is [ suppressed][Throwable.addSuppressed].
*
* More generally, if it would cause problems for an
* exception to be suppressed, the `AutoCloseable.close`
* method should not throw it.
*
*
* Note that unlike the [close][java.io.Closeable.close]
* method of [java.io.Closeable], this `close` method
* is *not* required to be idempotent. In other words,
* calling this `close` method more than once may have some
* visible side effect, unlike `Closeable.close` which is
* required to have no effect if called more than once.
*
* However, implementers of this interface are strongly encouraged
* to make their `close` methods idempotent.
*
* @throws Exception if this resource cannot be closed
*/
override fun close() {
runBlocking { coroutineContext[Job]?.cancelAndJoin() }
Runtime.getRuntime().removeShutdownHook(shutdownHook)
}
interface EntrySerializer<T> {
/**
* Serialize an entry as a String.
*/
fun write(entry: T): String
/**
* Read an entry from a String.
*/
fun read(string: String): T
}
companion object {
private const val INITIAL_SIZE = 1000
private const val ENTRY_SIZE_BYTES = 8
}
}

12
app/src/main/java/exh/favorites/FavoritesSyncHelper.kt
View File

@ -17,6 +17,7 @@ import eu.kanade.tachiyomi.util.powerManager
import eu.kanade.tachiyomi.util.toast
import eu.kanade.tachiyomi.util.wifiManager
import exh.*
import exh.eh.EHentaiUpdateWorker
import exh.util.ignore
import exh.util.trans
import okhttp3.FormBody
@ -112,6 +113,9 @@ class FavoritesSyncHelper(val context: Context) {
"teh:ExhFavoritesSyncWifi")
}
// Do not update galleries while syncing favorites
EHentaiUpdateWorker.cancelBackground(context)
storage.getRealm().use { realm ->
realm.trans {
db.inTransaction {
@ -161,6 +165,9 @@ class FavoritesSyncHelper(val context: Context) {
wifiLock?.release()
wifiLock = null
}
// Update galleries again!
EHentaiUpdateWorker.scheduleBackground(context)
}
if(errorList.isEmpty())
@ -338,7 +345,8 @@ class FavoritesSyncHelper(val context: Context) {
//Import using gallery adder
val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}",
true,
EXH_SOURCE_ID)
EXH_SOURCE_ID,
::throttle)
if(result is GalleryAddEvent.Fail) {
if(result is GalleryAddEvent.Fail.NotFound) {
@ -396,7 +404,7 @@ class FavoritesSyncHelper(val context: Context) {
class IgnoredException : RuntimeException()
companion object {
private const val THROTTLE_MAX = 4500
private const val THROTTLE_MAX = 5500
private const val THROTTLE_INC = 10
private const val THROTTLE_WARN = 1000
}

3
app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt
View File

@ -36,6 +36,9 @@ class EHentaiSearchMetadata : RaisedSearchMetadata() {
var ratingCount: Int? = null
var averageRating: Double? = null
var aged: Boolean = false
var lastUpdateCheck: Long = 0
override fun copyTo(manga: SManga) {
gId?.let { gId ->
gToken?.let { gToken ->

17
app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt
View File

@ -25,7 +25,7 @@ data class FlatMetadata(
fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation<FlatMetadata?> {
// We have to use fromCallable because StorIO messes up the thread scheduling if we use their rx functions
fun getSingle() = Single.fromCallable {
val single = Single.fromCallable {
val meta = getSearchMetadataForManga(mangaId).executeAsBlocking()
if(meta != null) {
val tags = getSearchTagsForManga(mangaId).executeAsBlocking()
@ -35,7 +35,11 @@ fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation<Fla
} else null
}
return object : PreparedOperation<FlatMetadata?> {
return preparedOperationFromSingle(single)
}
private fun <T> preparedOperationFromSingle(single: Single<T>): PreparedOperation<T> {
return object : PreparedOperation<T> {
/**
* Creates [rx.Observable] that emits result of Operation.
*
@ -44,7 +48,7 @@ fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation<Fla
*
* @return observable result of operation with only one [rx.Observer.onNext] call.
*/
override fun createObservable() = getSingle().toObservable()
override fun createObservable() = single.toObservable()
/**
* Executes operation synchronously in current thread.
@ -57,7 +61,7 @@ fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation<Fla
*
* @return nullable result of operation.
*/
override fun executeAsBlocking() = getSingle().toBlocking().value()
override fun executeAsBlocking() = single.toBlocking().value()
/**
* Creates [rx.Observable] that emits result of Operation.
@ -67,7 +71,7 @@ fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation<Fla
*
* @return observable result of operation with only one [rx.Observer.onNext] call.
*/
override fun asRxObservable() = getSingle().toObservable()
override fun asRxObservable() = single.toObservable()
/**
* Creates [rx.Single] that emits result of Operation lazily when somebody subscribes to it.
@ -76,8 +80,7 @@ fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation<Fla
*
* @return single result of operation.
*/
override fun asRxSingle() = getSingle()
override fun asRxSingle() = single
}
}

9
app/src/main/java/exh/metadata/sql/queries/SearchMetadataQueries.kt
View File

@ -25,6 +25,15 @@ interface SearchMetadataQueries : DbProvider {
.build())
.prepare()
fun getSearchMetadataByIndexedExtra(extra: String) = db.get()
.listOfObjects(SearchMetadata::class.java)
.withQuery(Query.builder()
.table(SearchMetadataTable.TABLE)
.where("${SearchMetadataTable.COL_INDEXED_EXTRA} = ?")
.whereArgs(extra)
.build())
.prepare()
fun insertSearchMetadata(metadata: SearchMetadata) = db.put().`object`(metadata).prepare()
fun deleteSearchMetadata(metadata: SearchMetadata) = db.delete().`object`(metadata).prepare()

48
app/src/main/java/exh/util/RxUtil.kt
View File

@ -1,8 +1,11 @@
package exh.util
import rx.Observable
import rx.Single
import com.pushtorefresh.storio.operations.PreparedOperation
import com.pushtorefresh.storio.sqlite.operations.get.PreparedGetObject
import kotlinx.coroutines.suspendCancellableCoroutine
import rx.*
import rx.subjects.ReplaySubject
import kotlin.coroutines.resumeWithException
/**
* Transform a cold single to a hot single
@ -25,3 +28,44 @@ fun <T> Observable<T>.melt(): Observable<T> {
subscribe(rs)
return rs
}
suspend fun <T> Single<T>.await(subscribeOn: Scheduler? = null): T {
return suspendCancellableCoroutine { continuation ->
val self = if (subscribeOn != null) subscribeOn(subscribeOn) else this
lateinit var sub: Subscription
sub = self.subscribe({
continuation.resume(it) {
sub.unsubscribe()
}
}, {
if (!continuation.isCancelled)
continuation.resumeWithException(it)
})
continuation.invokeOnCancellation {
sub.unsubscribe()
}
}
}
suspend fun <T> PreparedOperation<T>.await(): T = asRxSingle().await()
suspend fun <T> PreparedGetObject<T>.await(): T? = asRxSingle().await()
suspend fun Completable.awaitSuspending(subscribeOn: Scheduler? = null) {
return suspendCancellableCoroutine { continuation ->
val self = if (subscribeOn != null) subscribeOn(subscribeOn) else this
lateinit var sub: Subscription
sub = self.subscribe({
continuation.resume(Unit) {
sub.unsubscribe()
}
}, {
if (!continuation.isCancelled)
continuation.resumeWithException(it)
})
continuation.invokeOnCancellation {
sub.unsubscribe()
}
}
}