MadTheme: Fetch chapters like the site (#7978)

fetch chapters like the site

lib-multisrc/madtheme/build.gradle.kts

@@ -2,4 +2,4 @@ plugins {
     id("lib-multisrc")
 }
 
-baseVersionCode = 17
+baseVersionCode = 18

lib-multisrc/madtheme/src/eu/kanade/tachiyomi/multisrc/madtheme/MadTheme.kt

@@ -41,6 +41,10 @@ abstract class MadTheme(
         .rateLimit(1, 1, TimeUnit.SECONDS)
         .build()
 
+    protected open val useLegacyApi = false
+
+    protected open val useSlugSearch = false
+
     // TODO: better cookie sharing
     // TODO: don't count cached responses against rate limit
     private val chapterClient: OkHttpClient = network.cloudflareClient.newBuilder()
@@ -177,59 +181,57 @@ abstract class MadTheme(
 
         val document = response.asJsoup()
 
-        // Need the total chapters to check against the request
-        val totalChapters = document.selectFirst(".title span:containsOwn(CHAPTERS \\()")?.text()
-            ?.substringAfter("(")
-            ?.substringBefore(")")
-            ?.toIntOrNull()
-
         val script = document.selectFirst("script:containsData(bookId)")
             ?: throw Exception("Cannot find script")
         val bookId = script.data().substringAfter("bookId = ").substringBefore(";")
         val bookSlug = script.data().substringAfter("bookSlug = \"").substringBefore("\";")
 
-        // Use slug search by default
+        var chaptersList = document.select(chapterListSelector()).map { chapterFromElement(it) }
-        val slugRequest = chapterClient.newCall(GET(buildChapterUrl(bookSlug), headers)).execute()
+
-        if (!slugRequest.isSuccessful) {
+        val fetchApi = document.selectFirst("div#show-more-chapters > span")
-            throw Exception("HTTP error ${slugRequest.code}")
+            ?.attr("onclick")?.equals("getChapters()")
+            ?: false
+
+        if (fetchApi) {
+            val apiChapters = client.newCall(GET(buildChapterUrl(bookId, bookSlug), headers)).execute()
+                .asJsoup().select(chapterListSelector()).map { chapterFromElement(it) }
+
+            val cutIndex = chaptersList.indexOfFirst { chapter ->
+                apiChapters.any { it.url == chapter.url }
+            }.takeIf { it != -1 } ?: chaptersList.size
+
+            chaptersList = (chaptersList.subList(0, cutIndex) + apiChapters)
         }
 
-        var finalDocument = slugRequest.asJsoup().select(chapterListSelector())
+        return chaptersList
-
-        if (totalChapters != null && finalDocument.size < totalChapters) {
-            val idRequest = chapterClient.newCall(GET(buildChapterUrl(bookId), headers)).execute()
-            finalDocument = idRequest.asJsoup().select(chapterListSelector())
     }
 
-        return finalDocument.map {
+    private fun buildChapterUrl(mangaId: String, mangaSlug: String): HttpUrl {
-            SChapter.create().apply {
-                url = it.selectFirst("a")!!.absUrl("href").removePrefix(baseUrl)
-                name = it.selectFirst(".chapter-title")!!.text()
-                date_upload = parseChapterDate(it.selectFirst(".chapter-update")?.text())
-            }
-        }
-    }
-
-    private fun buildChapterUrl(fetchByParam: String): HttpUrl {
         return baseUrl.toHttpUrl().newBuilder().apply {
             addPathSegment("api")
             addPathSegment("manga")
-            addPathSegment(fetchByParam)
+            addPathSegment(if (useSlugSearch) mangaSlug else mangaId)
             addPathSegment("chapters")
             addQueryParameter("source", "detail")
         }.build()
     }
 
-    override fun chapterListRequest(manga: SManga): Request =
+    override fun chapterListRequest(manga: SManga): Request {
-        MANGA_ID_REGEX.find(manga.url)?.groupValues?.get(1)?.let { mangaId ->
+        if (useLegacyApi) {
-            val url = "$baseUrl/service/backend/chaplist/".toHttpUrl().newBuilder()
+            val mangaId = MANGA_ID_REGEX.find(manga.url)?.groupValues?.get(1)
-                .addQueryParameter("manga_id", mangaId)
+            val url = mangaId?.let {
+                "$baseUrl/service/backend/chaplist/".toHttpUrl().newBuilder()
+                    .addQueryParameter("manga_id", it)
                     .addQueryParameter("manga_name", manga.title)
                     .fragment("idFound")
                     .build()
+                    .toString()
+            } ?: (baseUrl + manga.url)
 
-            GET(url, headers)
+            return GET(url, headers)
-        } ?: GET("$baseUrl${manga.url}", headers)
+        }
+        return GET(baseUrl + manga.url, headers)
+    }
 
     override fun searchMangaParse(response: Response): MangasPage {
         if (genresList == null) {

src/en/beehentai/src/eu/kanade/tachiyomi/extension/en/beehentai/BeeHentai.kt

@@ -2,4 +2,6 @@ package eu.kanade.tachiyomi.extension.en.beehentai
 
 import eu.kanade.tachiyomi.multisrc.madtheme.MadTheme
 
-class BeeHentai : MadTheme("BeeHentai", "https://beehentai.com", "en")
+class BeeHentai : MadTheme("BeeHentai", "https://beehentai.com", "en") {
+    override val useSlugSearch = true
+}

src/en/kaliscancom/src/eu/kanade/tachiyomi/extension/en/kaliscancom/KaliScanCom.kt

@@ -2,4 +2,6 @@ package eu.kanade.tachiyomi.extension.en.kaliscancom
 
 import eu.kanade.tachiyomi.multisrc.madtheme.MadTheme
 
-class KaliScanCom : MadTheme("KaliScan.com", "https://kaliscan.com", "en")
+class KaliScanCom : MadTheme("KaliScan.com", "https://kaliscan.com", "en") {
+    override val useLegacyApi = true
+}

src/en/kaliscanio/src/eu/kanade/tachiyomi/extension/en/kaliscanio/KaliScanIo.kt

@@ -2,4 +2,6 @@ package eu.kanade.tachiyomi.extension.en.kaliscanio
 
 import eu.kanade.tachiyomi.multisrc.madtheme.MadTheme
 
-class KaliScanIo : MadTheme("KaliScan.io", "https://kaliscan.io", "en")
+class KaliScanIo : MadTheme("KaliScan.io", "https://kaliscan.io", "en") {
+    override val useLegacyApi = true
+}

src/en/kaliscanme/src/eu/kanade/tachiyomi/extension/en/kaliscanme/KaliScanMe.kt

@@ -2,4 +2,6 @@ package eu.kanade.tachiyomi.extension.en.kaliscanme
 
 import eu.kanade.tachiyomi.multisrc.madtheme.MadTheme
 
-class KaliScanMe : MadTheme("KaliScan.me", "https://kaliscan.me", "en")
+class KaliScanMe : MadTheme("KaliScan.me", "https://kaliscan.me", "en") {
+    override val useLegacyApi = true
+}

src/en/mgjinx/src/eu/kanade/tachiyomi/extension/en/mgjinx/MGJinx.kt

@@ -2,4 +2,6 @@ package eu.kanade.tachiyomi.extension.en.mgjinx
 
 import eu.kanade.tachiyomi.multisrc.madtheme.MadTheme
 
-class MGJinx : MadTheme("MGJinx", "https://mgjinx.com", "en")
+class MGJinx : MadTheme("MGJinx", "https://mgjinx.com", "en") {
+    override val useLegacyApi = true
+}

src/en/toonilyme/src/eu/kanade/tachiyomi/extension/en/toonilyme/ToonilyMe.kt

@@ -2,4 +2,6 @@ package eu.kanade.tachiyomi.extension.en.toonilyme
 
 import eu.kanade.tachiyomi.multisrc.madtheme.MadTheme
 
-class ToonilyMe : MadTheme("Toonily.me", "https://toonily.me", "en")
+class ToonilyMe : MadTheme("Toonily.me", "https://toonily.me", "en") {
+    override val useSlugSearch = true
+}

src/en/toonitube/src/eu/kanade/tachiyomi/extension/en/toonitube/TooniTube.kt

@@ -2,4 +2,6 @@ package eu.kanade.tachiyomi.extension.en.toonitube
 
 import eu.kanade.tachiyomi.multisrc.madtheme.MadTheme
 
-class TooniTube : MadTheme("TooniTube", "https://toonitube.com", "en")
+class TooniTube : MadTheme("TooniTube", "https://toonitube.com", "en") {
+    override val useSlugSearch = true
+}