HeavenManga: Fix chapter list and add latest tab (#1206)

Fixes

src/es/heavenmanga/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'HeavenManga'
     extClass = '.HeavenManga'
-    extVersionCode = 6
+    extVersionCode = 7
     isNsfw = true
 }
 

src/es/heavenmanga/src/eu/kanade/tachiyomi/extension/es/heavenmanga/HeavenManga.kt

@@ -9,12 +9,17 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.Headers
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
+import uy.kohesive.injekt.injectLazy
+import java.text.SimpleDateFormat
+import java.util.Locale
 
 class HeavenManga : ParsedHttpSource() {
 
@@ -24,81 +29,20 @@ class HeavenManga : ParsedHttpSource() {
 
     override val lang = "es"
 
-    // latest is broken on the site, it's the same as popular so turning it off
+    override val supportsLatest = true
-    override val supportsLatest = false
+
+    private val json: Json by injectLazy()
 
     override val client: OkHttpClient = network.cloudflareClient
 
-    override fun headersBuilder(): Headers.Builder {
+    override fun headersBuilder() = super.headersBuilder()
-        return Headers.Builder()
+        .add("Referer", "$baseUrl/")
-            .add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) Gecko/20100101 Firefox/75")
-    }
-
-    override fun popularMangaSelector() = "div.page-item-detail"
-
-    override fun latestUpdatesSelector() = "#container .ultimos_epis .not"
-
-    override fun searchMangaSelector() = "div.c-tabs-item__content, ${popularMangaSelector()}"
-
-    override fun chapterListSelector() = "div.listing-chapters_wrap tr"
-
-    override fun popularMangaNextPageSelector() = "ul.pagination a[rel=next]"
-
-    override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
-
-    override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
 
     override fun popularMangaRequest(page: Int) = GET("$baseUrl/top?orderby=views&page=$page", headers)
 
-    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/", headers)
+    override fun popularMangaSelector() = "div.page-item-detail"
 
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+    override fun popularMangaNextPageSelector() = "ul.pagination a[rel=next]"
-        val searchUrl = "$baseUrl/buscar?query=$query"
-
-        // Filter
-        val pageParameter = if (page > 1) "?page=$page" else ""
-
-        if (query.isBlank()) {
-            val ext = ".html"
-            var name: String
-            filters.forEach { filter ->
-                when (filter) {
-                    is GenreFilter -> {
-                        if (filter.toUriPart().isNotBlank() && filter.state != 0) {
-                            name = filter.toUriPart()
-                            return GET("$baseUrl/genero/$name$ext$pageParameter", headers)
-                        }
-                    }
-                    is AlphabeticoFilter -> {
-                        if (filter.toUriPart().isNotBlank() && filter.state != 0) {
-                            name = filter.toUriPart()
-                            return GET("$baseUrl/letra/$name$ext$pageParameter", headers)
-                        }
-                    }
-                    is ListaCompletasFilter -> {
-                        if (filter.toUriPart().isNotBlank() && filter.state != 0) {
-                            name = filter.toUriPart()
-                            return GET("$baseUrl/$name$pageParameter", headers)
-                        }
-                    }
-                    else -> {}
-                }
-            }
-        }
-
-        return GET(searchUrl, headers)
-    }
-
-    override fun searchMangaParse(response: Response): MangasPage {
-        return if (response.request.url.toString().contains("query=")) {
-            super.searchMangaParse(response)
-        } else {
-            popularMangaParse(response)
-        }
-    }
-
-    // get contents of a url
-    private fun getUrlContents(url: String): Document = client.newCall(GET(url, headers)).execute().asJsoup()
 
     override fun popularMangaFromElement(element: Element): SManga {
         return SManga.create().apply {
@@ -108,16 +52,93 @@ class HeavenManga : ParsedHttpSource() {
         }
     }
 
+    override fun latestUpdatesRequest(page: Int) = GET(baseUrl, headers)
+
+    override fun latestUpdatesParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+
+        val mangas = document.selectFirst(latestUpdatesWrapperSelector())!!
+            .select(latestUpdatesSelector())
+            .map { element ->
+                latestUpdatesFromElement(element)
+            }.distinctBy { it.url }
+
+        val hasNextPage = latestUpdatesNextPageSelector()?.let { selector ->
+            document.select(selector).first()
+        } != null
+
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    private fun latestUpdatesWrapperSelector() = ".container #loop-content "
+
+    override fun latestUpdatesSelector() = "span.list-group-item:not(:has(> div.row:containsOwn(Novela)))"
+
+    override fun latestUpdatesNextPageSelector(): String? = null
+
     override fun latestUpdatesFromElement(element: Element) = SManga.create().apply {
-        element.select("a").let {
+        with(element.selectFirst("a")!!) {
-            val latestChapter = getUrlContents(it.attr("href"))
+            val mangaUrl = attr("href").substringBeforeLast("/")
-            val url = latestChapter.select(".rpwe-clearfix:last-child a")
+            setUrlWithoutDomain(mangaUrl)
-            setUrlWithoutDomain(url.attr("href"))
+            title = select(".captitle").text()
-            title = it.select("span span").text()
+            thumbnail_url = mangaUrl.replace("/manga/", "/uploads/manga/") + "/cover/cover_250x350.jpg"
-            thumbnail_url = it.select("img").attr("src")
         }
     }
 
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        val url = baseUrl.toHttpUrl().newBuilder()
+        if (query.isNotBlank()) {
+            if (query.length < 3) throw Exception("La búsqueda debe tener al menos 3 caracteres")
+            url.addPathSegment("buscar")
+                .addQueryParameter("query", query)
+        } else {
+            val ext = ".html"
+            var name: String
+            filters.forEach { filter ->
+                when (filter) {
+                    is GenreFilter -> {
+                        if (filter.toUriPart().isNotBlank() && filter.state != 0) {
+                            name = filter.toUriPart()
+                            url.addPathSegment("genero")
+                                .addPathSegment(name + ext)
+                        }
+                    }
+                    is AlphabeticoFilter -> {
+                        if (filter.toUriPart().isNotBlank() && filter.state != 0) {
+                            name = filter.toUriPart()
+                            url.addPathSegment("letra")
+                                .addPathSegment("manga$ext")
+                                .addQueryParameter("alpha", name)
+                        }
+                    }
+                    is ListaCompletasFilter -> {
+                        if (filter.toUriPart().isNotBlank() && filter.state != 0) {
+                            name = filter.toUriPart()
+                            url.addPathSegment(name)
+                        }
+                    }
+                    else -> {}
+                }
+            }
+        }
+
+        if (page > 1) url.addQueryParameter("page", page.toString())
+
+        return GET(url.build(), headers)
+    }
+
+    override fun searchMangaParse(response: Response): MangasPage {
+        return if (response.request.url.pathSegments.contains("buscar")) {
+            super.searchMangaParse(response)
+        } else {
+            popularMangaParse(response)
+        }
+    }
+
+    override fun searchMangaSelector() = "div.c-tabs-item__content"
+
+    override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
+
     override fun searchMangaFromElement(element: Element) = SManga.create().apply {
         element.select("h4 a").let {
             title = it.text()
@@ -126,16 +147,6 @@ class HeavenManga : ParsedHttpSource() {
         thumbnail_url = element.select("img").attr("abs:data-src")
     }
 
-    override fun chapterFromElement(element: Element): SChapter {
-        return SChapter.create().apply {
-            element.select("a").let {
-                name = it.text()
-                setUrlWithoutDomain(it.attr("href"))
-            }
-            scanlator = element.select("span.pull-right").text()
-        }
-    }
-
     override fun mangaDetailsParse(document: Document) = SManga.create().apply {
         document.select("div.tab-summary").let { info ->
             genre = info.select("div.genres-content a").joinToString { it.text() }
@@ -144,27 +155,60 @@ class HeavenManga : ParsedHttpSource() {
         description = document.select("div.description-summary p").text()
     }
 
-    override fun imageUrlParse(document: Document) = throw UnsupportedOperationException()
+    override fun chapterListRequest(manga: SManga): Request {
+        val mangaUrl = (baseUrl + manga.url).toHttpUrl().newBuilder()
+            .addQueryParameter("columns[0][data]", "number")
+            .addQueryParameter("columns[0][orderable]", "true")
+            .addQueryParameter("columns[1][data]", "created_at")
+            .addQueryParameter("columns[1][searchable]", "true")
+            .addQueryParameter("order[0][column]", "1")
+            .addQueryParameter("order[0][dir]", "desc")
+            .addQueryParameter("start", "0")
+            .addQueryParameter("length", CHAPTER_LIST_LIMIT.toString())
+
+        val headers = headersBuilder()
+            .add("X-Requested-With", "XMLHttpRequest")
+            .build()
+
+        return GET(mangaUrl.build(), headers)
+    }
+
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val mangaUrl = response.request.url.toString().substringBefore("?").removeSuffix("/")
+        val result = json.decodeFromString<PayloadChaptersDto>(response.body.string())
+        return result.data.map {
+            SChapter.create().apply {
+                name = "Capítulo: ${it.slug}"
+                setUrlWithoutDomain("$mangaUrl/${it.slug}#${it.id}")
+                date_upload = runCatching { dateFormat.parse(it.createdAt)?.time ?: 0 }.getOrDefault(0)
+            }
+        }
+    }
+
+    override fun chapterListSelector() = throw UnsupportedOperationException()
+
+    override fun chapterFromElement(element: Element) = throw UnsupportedOperationException()
 
     override fun pageListRequest(chapter: SChapter): Request {
-        return getUrlContents(baseUrl + chapter.url).select("a[id=leer]").attr("abs:href")
+        val chapterId = chapter.url.substringAfterLast("#")
-            .let { GET(it, headers) }
+        if (chapterId.isBlank()) throw Exception("Error al obtener el id del capítulo. Actualice la lista")
+        val url = "$baseUrl/manga/leer/$chapterId"
+        return GET(url, headers)
     }
 
     override fun pageListParse(document: Document): List<Page> {
-        return document.select("script:containsData(pUrl)").first()!!.data()
+        val data = document.select("script:containsData(pUrl)").first()!!.data()
-            .substringAfter("pUrl=[").substringBefore("\"},];").split("\"},")
+        val jsonString = PAGES_REGEX.find(data)!!.groupValues[1].removeTrailingComma()
-            .mapIndexed { i, string -> Page(i, "", string.substringAfterLast("\"")) }
+        val pages = json.decodeFromString<List<PageDto>>(jsonString)
+        return pages.mapIndexed { i, dto -> Page(i, "", dto.imgURL) }
     }
 
-    /**
+    override fun imageUrlParse(document: Document) = throw UnsupportedOperationException()
-     * Array.from(document.querySelectorAll('.categorias a')).map(a => `Pair("${a.textContent}", "${a.getAttribute('href')}")`).join(',\n')
+
-     * on https://heavenmanga.com/top/
-     * */
     private class GenreFilter : UriPartFilter(
         "Géneros",
         arrayOf(
-            Pair("Todo", ""),
+            Pair("<Seleccionar>", ""),
             Pair("Accion", "accion"),
             Pair("Adulto", "adulto"),
             Pair("Aventura", "aventura"),
@@ -258,7 +302,8 @@ class HeavenManga : ParsedHttpSource() {
     private class AlphabeticoFilter : UriPartFilter(
         "Alfabético",
         arrayOf(
-            Pair("Todo", ""),
+            Pair("<Seleccionar>", ""),
+            Pair("Other", "Other"),
             Pair("A", "a"),
             Pair("B", "b"),
             Pair("C", "c"),
@@ -296,7 +341,7 @@ class HeavenManga : ParsedHttpSource() {
     private class ListaCompletasFilter : UriPartFilter(
         "Lista Completa",
         arrayOf(
-            Pair("Todo", ""),
+            Pair("<Seleccionar>", ""),
             Pair("Lista Comis", "comic"),
             Pair("Lista Novelas", "novela"),
             Pair("Lista Adulto", "adulto"),
@@ -317,4 +362,13 @@ class HeavenManga : ParsedHttpSource() {
         Filter.Select<String>(displayName, vals.map { it.first }.toTypedArray()) {
         fun toUriPart() = vals[state].second
     }
+
+    private fun String.removeTrailingComma() = replace(TRAILING_COMMA_REGEX, "$1")
+
+    companion object {
+        val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
+        val PAGES_REGEX = """pUrl\s*=\s*(\[.*\])\s*;""".toRegex()
+        val TRAILING_COMMA_REGEX = """,\s*(\}|\])""".toRegex()
+        private const val CHAPTER_LIST_LIMIT = 10000
+    }
 }

src/es/heavenmanga/src/eu/kanade/tachiyomi/extension/es/heavenmanga/HeavenMangaDto.kt

@@ -0,0 +1,21 @@
+package eu.kanade.tachiyomi.extension.es.heavenmanga
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class PayloadChaptersDto(
+    val data: List<ChapterDto>,
+)
+
+@Serializable
+data class ChapterDto(
+    val id: Int,
+    val slug: String,
+    @SerialName("created_at") val createdAt: String,
+)
+
+@Serializable
+data class PageDto(
+    val imgURL: String,
+)