From d1d9e03560d7fcc7a183991da2571c3802816a1c Mon Sep 17 00:00:00 2001
From: bapeey <90949336+bapeey@users.noreply.github.com>
Date: Thu, 24 Aug 2023 11:15:51 -0500
Subject: [PATCH] HeanCms: Add option to use ID instead slug (#17647)
* Use ID instead slug
* Minor changes
* Opps
* ID
* I cant explain this
* Fix for search in old API
* Unnecessary IF
* Yugen domain
* Change message
* Ah xD
---
.../heancms/reaperscans/src/ReaperScans.kt | 2 +-
.../heancms/yugenmangas/src/YugenMangas.kt | 4 +-
.../tachiyomi/multisrc/heancms/HeanCms.kt | 159 ++++++++++++++++--
.../tachiyomi/multisrc/heancms/HeanCmsDto.kt | 28 ++-
.../multisrc/heancms/HeanCmsGenerator.kt | 4 +-
.../tachiyomi/multisrc/heancms/HeanCmsIntl.kt | 6 +
6 files changed, 171 insertions(+), 32 deletions(-)
diff --git a/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt b/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt
index d382dfcb7..77122a85b 100644
--- a/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt
+++ b/multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt
@@ -21,7 +21,7 @@ class ReaperScans : HeanCms(
// Site changed from Madara to HeanCms.
override val versionId = 2
- override val fetchAllTitles = true
+ override val slugStrategy = SlugStrategy.FETCH_ALL
override val useNewQueryEndpoint = true
override val coverPath: String = ""
diff --git a/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt b/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt
index 83038a2ca..60343a753 100644
--- a/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt
+++ b/multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt
@@ -11,7 +11,7 @@ import java.util.concurrent.TimeUnit
class YugenMangas :
HeanCms(
"YugenMangas",
- "https://yugenmangas.lat",
+ "https://yugenmangas.net",
"es",
"https://api.yugenmangas.net",
) {
@@ -19,7 +19,7 @@ class YugenMangas :
// Site changed from Madara to HeanCms.
override val versionId = 2
- override val fetchAllTitles = true
+ override val slugStrategy = SlugStrategy.ID
override val useNewQueryEndpoint = true
override val client = super.client.newBuilder()
diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt
index f644777f1..d02b27857 100644
--- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt
+++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt
@@ -1,5 +1,7 @@
package eu.kanade.tachiyomi.multisrc.heancms
+import android.app.Application
+import android.content.SharedPreferences
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.source.model.Filter
@@ -21,6 +23,8 @@ import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import rx.Observable
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
import java.text.SimpleDateFormat
import java.util.Locale
@@ -31,11 +35,15 @@ abstract class HeanCms(
protected val apiUrl: String = baseUrl.replace("://", "://api."),
) : HttpSource() {
+ private val preferences: SharedPreferences by lazy {
+ Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
+ }
+
override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient
- protected open val fetchAllTitles = false
+ protected open val slugStrategy = SlugStrategy.NONE
protected open val useNewQueryEndpoint = false
@@ -103,7 +111,13 @@ abstract class HeanCms(
if (json.startsWith("{")) {
val result = json.parseAs<HeanCmsQuerySearchDto>()
- val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
+ val mangaList = result.data.map {
+ if (slugStrategy != SlugStrategy.NONE) {
+ preferences.slugMap = preferences.slugMap.toMutableMap()
+ .also { map -> map[it.slug.toPermSlugIfNeeded()] = it.slug }
+ }
+ it.toSManga(apiUrl, coverPath, slugStrategy)
+ }
fetchAllTitles()
@@ -111,7 +125,13 @@ abstract class HeanCms(
}
val mangaList = json.parseAs<List<HeanCmsSeriesDto>>()
- .map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
+ .map {
+ if (slugStrategy != SlugStrategy.NONE) {
+ preferences.slugMap = preferences.slugMap.toMutableMap()
+ .also { map -> map[it.slug.toPermSlugIfNeeded()] = it.slug }
+ }
+ it.toSManga(apiUrl, coverPath, slugStrategy)
+ }
fetchAllTitles()
@@ -163,11 +183,33 @@ abstract class HeanCms(
}
val slug = query.substringAfter(SEARCH_PREFIX)
- val manga = SManga.create().apply { url = "/series/$slug" }
+ val manga = SManga.create().apply {
+ url = if (slugStrategy != SlugStrategy.NONE) {
+ val mangaId = getIdBySlug(slug)
+ "/series/${slug.toPermSlugIfNeeded()}#$mangaId"
+ } else {
+ "/series/$slug"
+ }
+ }
return fetchMangaDetails(manga).map { MangasPage(listOf(it), false) }
}
+ private fun getIdBySlug(slug: String): Int {
+ val result = runCatching {
+ val response = client.newCall(GET("$apiUrl/series/$slug", headers)).execute()
+ val json = response.body.string()
+
+ val seriesDetail = json.parseAs<HeanCmsSeriesDto>()
+
+ preferences.slugMap = preferences.slugMap.toMutableMap()
+ .also { it[seriesDetail.slug.toPermSlugIfNeeded()] = seriesDetail.slug }
+
+ seriesDetail.id
+ }
+ return result.getOrNull() ?: throw Exception(intl.idNotFoundError + slug)
+ }
+
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
if (useNewQueryEndpoint) {
return newEndpointSearchMangaRequest(page, query, filters)
@@ -242,8 +284,8 @@ abstract class HeanCms(
val mangaList = result
.filter { it.type == "Comic" }
.map {
- it.slug = it.slug.replace(TIMESTAMP_REGEX, "")
- it.toSManga(apiUrl, coverPath, seriesSlugMap.orEmpty(), fetchAllTitles)
+ it.slug = it.slug.toPermSlugIfNeeded()
+ it.toSManga(apiUrl, coverPath, seriesSlugMap.orEmpty(), slugStrategy)
}
return MangasPage(mangaList, false)
@@ -251,7 +293,13 @@ abstract class HeanCms(
if (json.startsWith("{")) {
val result = json.parseAs<HeanCmsQuerySearchDto>()
- val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
+ val mangaList = result.data.map {
+ if (slugStrategy != SlugStrategy.NONE) {
+ preferences.slugMap = preferences.slugMap.toMutableMap()
+ .also { map -> map[it.slug.toPermSlugIfNeeded()] = it.slug }
+ }
+ it.toSManga(apiUrl, coverPath, slugStrategy)
+ }
fetchAllTitles()
@@ -259,7 +307,13 @@ abstract class HeanCms(
}
val mangaList = json.parseAs<List<HeanCmsSeriesDto>>()
- .map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
+ .map {
+ if (slugStrategy != SlugStrategy.NONE) {
+ preferences.slugMap = preferences.slugMap.toMutableMap()
+ .also { map -> map[it.slug.toPermSlugIfNeeded()] = it.slug }
+ }
+ it.toSManga(apiUrl, coverPath, slugStrategy)
+ }
fetchAllTitles()
@@ -269,22 +323,34 @@ abstract class HeanCms(
override fun getMangaUrl(manga: SManga): String {
val seriesSlug = manga.url
.substringAfterLast("/")
+ .substringBefore("#")
.toPermSlugIfNeeded()
- val currentSlug = seriesSlugMap?.get(seriesSlug)?.slug ?: seriesSlug
+ val currentSlug = if (slugStrategy != SlugStrategy.NONE) {
+ preferences.slugMap[seriesSlug] ?: seriesSlug
+ } else {
+ seriesSlug
+ }
return "$baseUrl/series/$currentSlug"
}
override fun mangaDetailsRequest(manga: SManga): Request {
- if (fetchAllTitles && manga.url.contains(TIMESTAMP_REGEX)) {
+ if (slugStrategy != SlugStrategy.NONE && (manga.url.contains(TIMESTAMP_REGEX))) {
+ throw Exception(intl.urlChangedError(name))
+ }
+
+ if (slugStrategy == SlugStrategy.ID && !manga.url.contains("#")) {
throw Exception(intl.urlChangedError(name))
}
val seriesSlug = manga.url
.substringAfterLast("/")
+ .substringBefore("#")
.toPermSlugIfNeeded()
+ val seriesId = manga.url.substringAfterLast("#")
+
fetchAllTitles()
val seriesDetails = seriesSlugMap?.get(seriesSlug)
@@ -295,7 +361,11 @@ abstract class HeanCms(
.add("Accept", ACCEPT_JSON)
.build()
- return GET("$apiUrl/series/$currentSlug#$currentStatus", apiHeaders)
+ return if (slugStrategy == SlugStrategy.ID) {
+ GET("$apiUrl/series/id/$seriesId", apiHeaders)
+ } else {
+ GET("$apiUrl/series/$currentSlug#$currentStatus", apiHeaders)
+ }
}
override fun mangaDetailsParse(response: Response): SManga {
@@ -303,8 +373,14 @@ abstract class HeanCms(
val result = runCatching { response.parseAs<HeanCmsSeriesDto>() }
- val seriesDetails = result.getOrNull()?.toSManga(apiUrl, coverPath, fetchAllTitles)
- ?: throw Exception(intl.urlChangedError(name))
+ val seriesResult = result.getOrNull() ?: throw Exception(intl.urlChangedError(name))
+
+ if (slugStrategy != SlugStrategy.NONE) {
+ preferences.slugMap = preferences.slugMap.toMutableMap()
+ .also { it[seriesResult.slug.toPermSlugIfNeeded()] = seriesResult.slug }
+ }
+
+ val seriesDetails = seriesResult.toSManga(apiUrl, coverPath, slugStrategy)
return seriesDetails.apply {
status = status.takeUnless { it == SManga.UNKNOWN }
@@ -323,21 +399,39 @@ abstract class HeanCms(
return result.seasons.orEmpty()
.flatMap { it.chapters.orEmpty() }
.filterNot { it.price == 1 }
- .map { it.toSChapter(result.slug, dateFormat) }
+ .map { it.toSChapter(result.slug, dateFormat, slugStrategy) }
.filter { it.date_upload <= currentTimestamp }
}
return result.chapters.orEmpty()
.filterNot { it.price == 1 }
- .map { it.toSChapter(result.slug, dateFormat) }
+ .map { it.toSChapter(result.slug, dateFormat, slugStrategy) }
.filter { it.date_upload <= currentTimestamp }
.reversed()
}
- override fun getChapterUrl(chapter: SChapter): String = baseUrl + chapter.url
+ override fun getChapterUrl(chapter: SChapter): String {
+ if (slugStrategy == SlugStrategy.NONE) return baseUrl + chapter.url
+
+ val seriesSlug = chapter.url
+ .substringAfter("/series/")
+ .substringBefore("/")
+ .toPermSlugIfNeeded()
+
+ val currentSlug = preferences.slugMap[seriesSlug] ?: seriesSlug
+ val chapterUrl = chapter.url.replaceFirst(seriesSlug, currentSlug)
+
+ return baseUrl + chapterUrl
+ }
override fun pageListRequest(chapter: SChapter): Request {
if (useNewQueryEndpoint) {
+ if (slugStrategy != SlugStrategy.NONE) {
+ val seriesPermSlug = chapter.url.substringAfter("/series/").substringBefore("/")
+ val seriesSlug = preferences.slugMap[seriesPermSlug] ?: seriesPermSlug
+ val chapterUrl = chapter.url.replaceFirst(seriesPermSlug, seriesSlug)
+ return GET(baseUrl + chapterUrl, headers)
+ }
return GET(baseUrl + chapter.url, headers)
}
@@ -389,7 +483,7 @@ abstract class HeanCms(
}
protected open fun fetchAllTitles() {
- if (!seriesSlugMap.isNullOrEmpty() || !fetchAllTitles) {
+ if (!seriesSlugMap.isNullOrEmpty() || slugStrategy != SlugStrategy.FETCH_ALL) {
return
}
@@ -418,6 +512,8 @@ abstract class HeanCms(
}
seriesSlugMap = result.getOrNull()
+ preferences.slugMap = preferences.slugMap.toMutableMap()
+ .also { it.putAll(seriesSlugMap.orEmpty().mapValues { (_, v) -> v.slug }) }
}
protected open fun allTitlesRequest(page: Int): Request {
@@ -468,8 +564,21 @@ abstract class HeanCms(
*/
data class HeanCmsTitle(val slug: String, val thumbnailFileName: String, val status: Int)
+ /**
+ * Used to specify the strategy to use when fetching the slug for a manga.
+ * This is needed because some sources change the slug periodically.
+ * [NONE]: Use series_slug without changes.
+ * [ID]: Use series_id to fetch the slug from the API.
+ * IMPORTANT: [ID] is only available in the new query endpoint.
+ * [FETCH_ALL]: Convert the slug to a permanent slug by removing the timestamp.
+ * At extension start, all the slugs are fetched and stored in a map.
+ */
+ enum class SlugStrategy {
+ NONE, ID, FETCH_ALL
+ }
+
private fun String.toPermSlugIfNeeded(): String {
- return if (fetchAllTitles) {
+ return if (slugStrategy != SlugStrategy.NONE) {
this.replace(TIMESTAMP_REGEX, "")
} else {
this
@@ -514,6 +623,18 @@ abstract class HeanCms(
protected inline fun <reified R> List<*>.firstInstanceOrNull(): R? =
filterIsInstance<R>().firstOrNull()
+ protected var SharedPreferences.slugMap: MutableMap<String, String>
+ get() {
+ val jsonMap = getString(PREF_URL_MAP_SLUG, "{}")!!
+ val slugMap = runCatching { json.decodeFromString<Map<String, String>>(jsonMap) }
+ return slugMap.getOrNull()?.toMutableMap() ?: mutableMapOf()
+ }
+ set(newSlugMap) {
+ edit()
+ .putString(PREF_URL_MAP_SLUG, json.encodeToString(newSlugMap))
+ .commit()
+ }
+
companion object {
private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
private const val ACCEPT_JSON = "application/json, text/plain, */*"
@@ -525,5 +646,7 @@ abstract class HeanCms(
private const val PER_PAGE_MANGA_TITLES = 10000
const val SEARCH_PREFIX = "slug:"
+
+ private const val PREF_URL_MAP_SLUG = "pref_url_map"
}
}
diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt
index 6e765ad44..f9c2578c4 100644
--- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt
+++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt
@@ -1,5 +1,6 @@
package eu.kanade.tachiyomi.multisrc.heancms
+import eu.kanade.tachiyomi.multisrc.heancms.HeanCms.SlugStrategy
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import kotlinx.serialization.SerialName
@@ -36,9 +37,9 @@ data class HeanCmsSearchDto(
apiUrl: String,
coverPath: String,
slugMap: Map<String, HeanCms.HeanCmsTitle>,
- fetchAllTiles: Boolean,
+ slugStrategy: SlugStrategy,
): SManga = SManga.create().apply {
- val slugOnly = slug.toPermSlugIfNeeded(fetchAllTiles)
+ val slugOnly = slug.toPermSlugIfNeeded(slugStrategy)
val thumbnailFileName = slugMap[slugOnly]?.thumbnailFileName
title = this@HeanCmsSearchDto.title
thumbnail_url = thumbnail?.toAbsoluteThumbnailUrl(apiUrl, coverPath)
@@ -66,10 +67,10 @@ data class HeanCmsSeriesDto(
fun toSManga(
apiUrl: String,
coverPath: String,
- fetchAllTiles: Boolean,
+ slugStrategy: SlugStrategy,
): SManga = SManga.create().apply {
val descriptionBody = this@HeanCmsSeriesDto.description?.let(Jsoup::parseBodyFragment)
- val slugOnly = slug.toPermSlugIfNeeded(fetchAllTiles)
+ val slugOnly = slug.toPermSlugIfNeeded(slugStrategy)
title = this@HeanCmsSeriesDto.title
author = this@HeanCmsSeriesDto.author?.trim()
@@ -83,7 +84,11 @@ data class HeanCmsSeriesDto(
thumbnail_url = thumbnail.ifEmpty { null }
?.toAbsoluteThumbnailUrl(apiUrl, coverPath)
status = this@HeanCmsSeriesDto.status?.toStatus() ?: SManga.UNKNOWN
- url = "/series/$slugOnly"
+ url = if (slugStrategy != SlugStrategy.NONE) {
+ "/series/$slugOnly#$id"
+ } else {
+ "/series/$slug"
+ }
}
}
@@ -105,11 +110,16 @@ data class HeanCmsChapterDto(
@SerialName("created_at") val createdAt: String,
val price: Int? = null,
) {
- fun toSChapter(seriesSlug: String, dateFormat: SimpleDateFormat): SChapter = SChapter.create().apply {
+ fun toSChapter(
+ seriesSlug: String,
+ dateFormat: SimpleDateFormat,
+ slugStrategy: SlugStrategy,
+ ): SChapter = SChapter.create().apply {
+ val seriesSlugOnly = seriesSlug.toPermSlugIfNeeded(slugStrategy)
name = this@HeanCmsChapterDto.name.trim()
date_upload = runCatching { dateFormat.parse(createdAt)?.time }
.getOrNull() ?: 0L
- url = "/series/$seriesSlug/$slug#$id"
+ url = "/series/$seriesSlugOnly/$slug#$id"
}
}
@@ -140,8 +150,8 @@ private fun String.toAbsoluteThumbnailUrl(apiUrl: String, coverPath: String): St
return if (startsWith("https://")) this else "$apiUrl/$coverPath$this"
}
-private fun String.toPermSlugIfNeeded(fetchAllTitles: Boolean): String {
- return if (fetchAllTitles) {
+private fun String.toPermSlugIfNeeded(slugStrategy: SlugStrategy): String {
+ return if (slugStrategy != SlugStrategy.NONE) {
this.replace(HeanCms.TIMESTAMP_REGEX, "")
} else {
this
diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt
index 57c75cd24..5477173e3 100644
--- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt
+++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt
@@ -9,13 +9,13 @@ class HeanCmsGenerator : ThemeSourceGenerator {
override val themeClass = "HeanCms"
- override val baseVersionCode: Int = 17
+ override val baseVersionCode: Int = 18
override val sources = listOf(
SingleLang("Glorious Scan", "https://gloriousscan.com", "pt-BR", overrideVersionCode = 17),
SingleLang("Omega Scans", "https://omegascans.org", "en", isNsfw = true, overrideVersionCode = 17),
SingleLang("Reaper Scans", "https://reaperscans.net", "pt-BR", overrideVersionCode = 36),
- SingleLang("YugenMangas", "https://yugenmangas.lat", "es", isNsfw = true, overrideVersionCode = 7),
+ SingleLang("YugenMangas", "https://yugenmangas.net", "es", isNsfw = true, overrideVersionCode = 7),
)
companion object {
diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsIntl.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsIntl.kt
index 60bb20f22..5b0074145 100644
--- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsIntl.kt
+++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsIntl.kt
@@ -88,6 +88,12 @@ class HeanCmsIntl(lang: String) {
"to $sourceName to update the URL."
}
+ val idNotFoundError: String = when (availableLang) {
+ BRAZILIAN_PORTUGUESE -> "Falha ao obter o ID do slug: "
+ SPANISH -> "No se pudo encontrar el ID para: "
+ else -> "Failed to get the ID for slug: "
+ }
+
companion object {
const val BRAZILIAN_PORTUGUESE = "pt-BR"
const val ENGLISH = "en"