From f9834fcca0c74c3a2a4ac6b1638a32cf4603d5c7 Mon Sep 17 00:00:00 2001 From: Rolando Lecca <90949336+bapeey@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:56:44 -0500 Subject: [PATCH] HeanCMS: Add fetchStrategy (#17320) * Add fetchStrategy * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com> * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com> * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com> * Remove YugenMangas companion object * Lol * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> * Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> * Fixes * get slug directly instead convert to SManga * change new result function * remove unnecessary var * unused imports * Exact search NOT RETURN LATEST TIMESTAMP O.o * parse response directly Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> * change search query --------- Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com> Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> --- .../heancms/reaperscans/src/ReaperScans.kt | 2 +- .../heancms/yugenmangas/src/YugenMangas.kt | 47 +------ .../tachiyomi/multisrc/heancms/HeanCms.kt | 124 ++++++++++++++++-- .../tachiyomi/multisrc/heancms/HeanCmsDto.kt | 13 +- .../multisrc/heancms/HeanCmsGenerator.kt | 2 +- 5 files changed, 123 insertions(+), 65 deletions(-) diff --git a/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt b/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt index 8257e9a7c..422b98aee 100644 --- a/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt +++ b/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt @@ -21,7 +21,7 @@ class ReaperScans : HeanCms( // Site changed from Madara to HeanCms. override val versionId = 2 - override val fetchAllTitles = true + override val fetchAllTitlesStrategy = FetchAllStrategy.SEARCH_ALL override val coverPath: String = "" diff --git a/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt b/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt index c6822d9a8..9a566e728 100644 --- a/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt +++ b/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt @@ -1,16 +1,8 @@ package eu.kanade.tachiyomi.extension.es.yugenmangas -import android.app.Application -import android.content.SharedPreferences -import android.widget.Toast -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat import eu.kanade.tachiyomi.multisrc.heancms.Genre import eu.kanade.tachiyomi.multisrc.heancms.HeanCms import eu.kanade.tachiyomi.network.interceptor.rateLimit -import eu.kanade.tachiyomi.source.ConfigurableSource -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get import java.text.SimpleDateFormat import java.util.TimeZone import java.util.concurrent.TimeUnit @@ -21,17 +13,12 @@ class YugenMangas : "https://yugenmangas.net", "es", "https://api.yugenmangas.net", - ), - ConfigurableSource { - - private val preferences: SharedPreferences by lazy { - Injekt.get().getSharedPreferences("source_$id", 0x0000) - } + ) { // Site changed from Madara to HeanCms. override val versionId = 2 - override val fetchAllTitles = getFetchAllSeriesPref() + override val fetchAllTitlesStrategy = FetchAllStrategy.SEARCH_EACH override val client = super.client.newBuilder() .connectTimeout(60, TimeUnit.SECONDS) @@ -45,26 +32,6 @@ class YugenMangas : timeZone = TimeZone.getTimeZone("UTC") } - private fun getFetchAllSeriesPref(): Boolean { - return preferences.getBoolean(FETCH_ALL_SERIES_PREF, FETCH_ALL_SERIES_DEFAULT_VALUE) - } - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - val fetchAllSeriesPreference = SwitchPreferenceCompat(screen.context).apply { - key = FETCH_ALL_SERIES_PREF - title = FETCH_ALL_SERIES_TITLE - summary = FETCH_ALL_SERIES_SUMMARY - setDefaultValue(FETCH_ALL_SERIES_DEFAULT_VALUE) - - setOnPreferenceChangeListener { _, newValue -> - Toast.makeText(screen.context, "Reinicia la app para aplicar los cambios.", Toast.LENGTH_LONG).show() - true - } - } - - screen.addPreference(fetchAllSeriesPreference) - } - override fun getGenreList(): List = listOf( Genre("+18", 1), Genre("Acción", 36), @@ -115,14 +82,4 @@ class YugenMangas : Genre("Yaoi", 43), Genre("Yuri", 44), ) - - companion object { - private const val FETCH_ALL_SERIES_PREF = "fetchAllSeriesPref" - private const val FETCH_ALL_SERIES_TITLE = "Buscar todas las series" - private const val FETCH_ALL_SERIES_SUMMARY = "Busca las URLs actuales de las series. " + - "Habilitar esta opción evita la necesidad de migrar, " + - "pero puede llevar tiempo dependiendo del número total de series. \n" + - "Tendrá que migrar cada vez que cambie esta opción." - private const val FETCH_ALL_SERIES_DEFAULT_VALUE = true - } } diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt index 3ac3c1128..122a56f07 100644 --- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt +++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt @@ -1,5 +1,7 @@ package eu.kanade.tachiyomi.multisrc.heancms +import android.app.Application +import android.content.SharedPreferences import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.POST import eu.kanade.tachiyomi.source.model.Filter @@ -19,6 +21,8 @@ import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import rx.Observable +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get import java.text.SimpleDateFormat import java.util.Locale @@ -29,6 +33,10 @@ abstract class HeanCms( protected val apiUrl: String = baseUrl.replace("://", "://api."), ) : HttpSource() { + private val preferences: SharedPreferences by lazy { + Injekt.get().getSharedPreferences("source_$id", 0x0000) + } + override val supportsLatest = true override val client: OkHttpClient = network.cloudflareClient @@ -45,7 +53,7 @@ abstract class HeanCms( protected val intl by lazy { HeanCmsIntl(lang) } - protected open val fetchAllTitles: Boolean = false + protected open val fetchAllTitlesStrategy = FetchAllStrategy.NONE protected open val coverPath: String = "cover/" @@ -81,7 +89,7 @@ abstract class HeanCms( if (json.startsWith("{")) { val result = json.parseAs() - val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) } + val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) } fetchAllTitles() @@ -89,7 +97,7 @@ abstract class HeanCms( } val mangaList = json.parseAs>() - .map { it.toSManga(apiUrl, coverPath, fetchAllTitles) } + .map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) } fetchAllTitles() @@ -179,14 +187,14 @@ abstract class HeanCms( val result = json.parseAs>() val mangaList = result .filter { it.type == "Comic" } - .map { it.toSManga(apiUrl, coverPath, seriesSlugMap.orEmpty(), fetchAllTitles) } + .map { it.toSManga(apiUrl, coverPath, seriesSlugMap.orEmpty(), fetchAllTitlesStrategy) } return MangasPage(mangaList, false) } if (json.startsWith("{")) { val result = json.parseAs() - val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) } + val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) } fetchAllTitles() @@ -194,7 +202,7 @@ abstract class HeanCms( } val mangaList = json.parseAs>() - .map { it.toSManga(apiUrl, coverPath, fetchAllTitles) } + .map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) } fetchAllTitles() @@ -206,12 +214,38 @@ abstract class HeanCms( .substringAfterLast("/") .replace(TIMESTAMP_REGEX, "") - val currentSlug = seriesSlugMap?.get(seriesSlug)?.slug ?: manga.url.substringAfterLast("/") + val currentSlug = if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) { + preferences.slugMap[seriesSlug] ?: manga.url.substringAfterLast("/") + } else { + seriesSlugMap?.get(seriesSlug)?.slug ?: manga.url.substringAfterLast("/") + } return "$baseUrl/series/$currentSlug" } override fun mangaDetailsRequest(manga: SManga): Request { + if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) { + val searchQuery = manga.title.trim() + .replaceAfterLast(" ", "").trim() + .let { + if (it.length > 2) it.dropLast(1).trim() else it + } + val searchPayloadObj = HeanCmsSearchPayloadDto(searchQuery) + val searchPayload = json.encodeToString(searchPayloadObj) + .toRequestBody(JSON_MEDIA_TYPE) + + val apiHeaders = headersBuilder() + .add("Accept", ACCEPT_JSON) + .add("Content-Type", searchPayload.contentType().toString()) + .build() + + val mangaSlug = manga.url + .substringAfterLast("/") + .replace(TIMESTAMP_REGEX, "") + + return POST("$apiUrl/series/search#$mangaSlug", apiHeaders, searchPayload) + } + val seriesSlug = manga.url .substringAfterLast("/") .replace(TIMESTAMP_REGEX, "") @@ -230,30 +264,72 @@ abstract class HeanCms( } override fun mangaDetailsParse(response: Response): SManga { - val result = runCatching { response.parseAs() } - val seriesDetails = result.getOrNull()?.toSManga(apiUrl, coverPath, fetchAllTitles) + val mangaStatus = response.request.url.fragment?.toIntOrNull() ?: SManga.UNKNOWN + + val result = runCatching { + if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) { + val originalSlug = response.request.url.fragment!! + val searchResult = response.parseAs>() + searchResultToSeries(originalSlug, searchResult) + } else { + response.parseAs() + } + } + + val seriesDetails = result.getOrNull()?.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) ?: throw Exception(intl.urlChangedError(name)) return seriesDetails.apply { status = status.takeUnless { it == SManga.UNKNOWN } - ?: response.request.url.fragment?.toIntOrNull() ?: SManga.UNKNOWN + ?: mangaStatus } } override fun chapterListRequest(manga: SManga): Request = mangaDetailsRequest(manga) override fun chapterListParse(response: Response): List { - val result = response.parseAs() - val seriesSlug = response.request.url.pathSegments.last() + val result = if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) { + val originalSlug = response.request.url.fragment!! + val searchResult = response.parseAs>() + searchResultToSeries(originalSlug, searchResult) + } else { + response.parseAs() + } + val currentTimestamp = System.currentTimeMillis() return result.chapters.orEmpty() .filterNot { it.price == 1 } - .map { it.toSChapter(seriesSlug, dateFormat) } + .map { it.toSChapter(result.slug, dateFormat) } .filter { it.date_upload <= currentTimestamp } .reversed() } + private fun searchResultToSeries(originalSlug: String, searchResult: List): HeanCmsSeriesDto { + val mangaSlug = searchResult + .filter { it.type == "Comic" } + .map { it.slug } + .find { it.replace(TIMESTAMP_REGEX, "") == originalSlug } + ?: throw Exception(intl.urlChangedError(name)) + + val apiHeaders = headersBuilder() + .add("Accept", ACCEPT_JSON) + .build() + + val detailsRequest = GET("$apiUrl/series/$mangaSlug", apiHeaders) + val result = client.newCall(detailsRequest).execute() + .parseAs() + + val permSlug = result.slug + .substringAfterLast("/") + .replace(TIMESTAMP_REGEX, "") + + preferences.slugMap = preferences.slugMap.toMutableMap() + .also { it[permSlug] = result.slug.substringAfterLast("/") } + + return result + } + override fun getChapterUrl(chapter: SChapter): String = baseUrl + chapter.url override fun pageListRequest(chapter: SChapter): Request { @@ -309,7 +385,7 @@ abstract class HeanCms( protected open fun getGenreList(): List = emptyList() protected open fun fetchAllTitles() { - if (!seriesSlugMap.isNullOrEmpty() || !fetchAllTitles) { + if (!seriesSlugMap.isNullOrEmpty() || fetchAllTitlesStrategy != FetchAllStrategy.SEARCH_ALL) { return } @@ -358,6 +434,18 @@ abstract class HeanCms( return POST("$apiUrl/series/querysearch", apiHeaders, payload) } + protected var SharedPreferences.slugMap: MutableMap + get() { + val jsonMap = getString(PREF_URL_MAP, "{}")!! + val slugMap = runCatching { json.decodeFromString>(jsonMap) } + return slugMap.getOrNull()?.toMutableMap() ?: mutableMapOf() + } + set(newSlugMap) { + edit() + .putString(PREF_URL_MAP, json.encodeToString(newSlugMap)) + .commit() + } + protected open fun parseAllTitles(result: List): Map { return result .filter { it.type == "Comic" } @@ -401,6 +489,12 @@ abstract class HeanCms( */ data class HeanCmsTitle(val slug: String, val thumbnailFileName: String, val status: Int) + enum class FetchAllStrategy { + NONE, + SEARCH_EACH, + SEARCH_ALL, + } + companion object { private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" private const val ACCEPT_JSON = "application/json, text/plain, */*" @@ -410,5 +504,7 @@ abstract class HeanCms( val TIMESTAMP_REGEX = "-\\d+$".toRegex() const val SEARCH_PREFIX = "slug:" + + private const val PREF_URL_MAP = "pref_url_map" } } diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt index 42b0a49a5..8116c7dea 100644 --- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt +++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt @@ -1,5 +1,6 @@ package eu.kanade.tachiyomi.multisrc.heancms +import eu.kanade.tachiyomi.multisrc.heancms.HeanCms.FetchAllStrategy import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import kotlinx.serialization.SerialName @@ -36,9 +37,9 @@ data class HeanCmsSearchDto( apiUrl: String, coverPath: String, slugMap: Map, - usePermSlug: Boolean, + fetchStrategy: FetchAllStrategy = FetchAllStrategy.NONE, ): SManga = SManga.create().apply { - val slug = if (!usePermSlug) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "") + val slug = if (fetchStrategy == FetchAllStrategy.NONE) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "") val thumbnailFileName = slugMap[slug]?.thumbnailFileName title = this@HeanCmsSearchDto.title @@ -63,9 +64,13 @@ data class HeanCmsSeriesDto( val chapters: List? = emptyList(), ) { - fun toSManga(apiUrl: String, coverPath: String, usePermSlug: Boolean): SManga = SManga.create().apply { + fun toSManga( + apiUrl: String, + coverPath: String, + fetchStrategy: FetchAllStrategy = FetchAllStrategy.NONE, + ): SManga = SManga.create().apply { val descriptionBody = this@HeanCmsSeriesDto.description?.let(Jsoup::parseBodyFragment) - val slug = if (!usePermSlug) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "") + val slug = if (fetchStrategy == FetchAllStrategy.NONE) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "") title = this@HeanCmsSeriesDto.title author = this@HeanCmsSeriesDto.author?.trim() diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt index 3bcd33152..7d3cbaa69 100644 --- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt +++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt @@ -9,7 +9,7 @@ class HeanCmsGenerator : ThemeSourceGenerator { override val themeClass = "HeanCms" - override val baseVersionCode: Int = 14 + override val baseVersionCode: Int = 15 override val sources = listOf( SingleLang("Glorious Scan", "https://gloriousscan.com", "pt-BR", overrideVersionCode = 17),