ReadMangas: Fix loading content (#8239)

* Fix details, chapter and page

* Fix popular and latest

* Fix search

* Add change suggestion

src/pt/readmangas/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'Read Mangas'
     extClass = '.ReadMangas'
-    extVersionCode = 38
+    extVersionCode = 39
 }
 
 apply from: "$rootDir/common.gradle"

src/pt/readmangas/src/eu/kanade/tachiyomi/extension/pt/readmangas/ReadMangas.kt

@@ -1,9 +1,13 @@
 package eu.kanade.tachiyomi.extension.pt.readmangas
 
 import android.annotation.SuppressLint
+import android.app.Application
+import android.os.Handler
+import android.os.Looper
+import android.widget.Toast
+import app.cash.quickjs.QuickJs
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.network.interceptor.rateLimit
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
@@ -11,79 +15,70 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
-import kotlinx.serialization.decodeFromString
+import keiyoushi.utils.parseAs
-import kotlinx.serialization.json.Json
+import keiyoushi.utils.tryParse
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.buildJsonArray
 import kotlinx.serialization.json.buildJsonObject
 import kotlinx.serialization.json.put
-import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.Request
 import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
-import org.json.JSONObject
+import org.jsoup.nodes.Element
 import rx.Observable
-import uy.kohesive.injekt.injectLazy
+import uy.kohesive.injekt.Injekt
-import java.net.URLEncoder
+import uy.kohesive.injekt.api.get
 import java.text.SimpleDateFormat
-import java.util.Date
-import java.util.Locale
-import java.util.TimeZone
-import java.util.concurrent.TimeUnit
 
 class ReadMangas() : HttpSource() {
 
     override val name = "Read Mangas"
 
-    override val baseUrl = "https://app.loobyt.com"
+    override val baseUrl = "https://mangalivre.one"
 
     override val lang = "pt-BR"
 
     override val supportsLatest = true
 
-    private val json: Json by injectLazy()
-
     override val client = network.cloudflareClient.newBuilder()
-        .readTimeout(2, TimeUnit.MINUTES)
-        .rateLimit(1, 2)
         .build()
 
     override val versionId = 2
 
+    private val application: Application = Injekt.get<Application>()
+
     // =========================== Popular ================================
 
     private var popularNextCursorPage = ""
 
+    private val popularMangaToken: String by lazy {
+        getToken(popularMangaRequest(1), "script[src*='projects/page-']")
+    }
+
     override fun popularMangaRequest(page: Int): Request {
+        val url = "$baseUrl/projects"
         if (page == 1) {
-            popularNextCursorPage = ""
+            return GET(url, headers)
         }
 
-        val input = buildJsonObject {
+        val payload = buildJsonArray {
-            put(
+            add(
-                "0",
                 buildJsonObject {
-                    put(
+                    put("cursor", popularNextCursorPage)
-                        "json",
-                        buildJsonObject {
-                            put("direction", "forward")
-                            if (popularNextCursorPage.isNotBlank()) {
-                                put("cursor", popularNextCursorPage)
-                            }
-                        },
-                    )
                 },
             )
         }
 
-        val url = "$baseUrl/api/deprecated/manga.getAllManga?batch=1".toHttpUrl().newBuilder()
+        val newHeaders = headers.newBuilder()
-            .addQueryParameter("batch", "1")
+            .set("Next-Action", popularMangaToken)
-            .addQueryParameter("input", input.toString())
             .build()
-        return GET(url, headers)
+
+        return POST(url, newHeaders, payload.toRequestBody())
     }
 
     override fun popularMangaParse(response: Response): MangasPage {
-        val (mangaPage, nextCursor) = mangasPageParse(response)
+        val (mangaPage, nextCursor) = response.mangasPageParse<PopularResultDto>()
         popularNextCursorPage = nextCursor
         return mangaPage
     }
@@ -92,59 +87,57 @@ class ReadMangas() : HttpSource() {
 
     private var latestNextCursorPage = ""
 
-    override fun latestUpdatesRequest(page: Int): Request {
+    private val latestMangaToken: String by lazy {
-        if (page == 1) {
+        getToken(latestUpdatesRequest(1), "script[src*='updates/page-']")
-            latestNextCursorPage = Date().let { latestUpdateDateFormat.format(it) }
+    }
-        }
 
-        val input = buildJsonObject {
+    override fun latestUpdatesRequest(page: Int): Request {
-            put(
+        val url = "$baseUrl/updates"
-                "0",
+        if (page == 1) {
+            return GET(url, headers)
+        }
+        val payload = buildJsonArray {
+            add(
                 buildJsonObject {
-                    put(
+                    put("limit", 20)
-                        "json",
+                    put("cursor", latestNextCursorPage)
-                        buildJsonObject {
-                            put("direction", "forward")
-                            put("limit", 20)
-                            put("cursor", latestNextCursorPage)
-                        },
-                    )
                 },
             )
         }
 
-        val url = "$baseUrl/api/deprecated/discover.updated".toHttpUrl().newBuilder()
+        val newHeaders = headers.newBuilder()
-            .addQueryParameter("batch", "1")
+            .set("Next-Action", latestMangaToken)
-            .addQueryParameter("input", input.toString())
             .build()
-        return GET(url, headers)
+
+        return POST(url, newHeaders, payload.toRequestBody())
     }
 
     override fun latestUpdatesParse(response: Response): MangasPage {
-        val (mangaPage, nextCursor) = mangasPageParse(response)
+        val (mangaPage, nextCursor) = response.mangasPageParse<LatestResultDto>()
         latestNextCursorPage = nextCursor
         return mangaPage
     }
 
     // =========================== Search =================================
 
+    private val searchMangaToken: String by lazy {
+        getToken(latestUpdatesRequest(1), "script[src*='app/layout-']")
+    }
+
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val url = "$baseUrl/api/deprecated/discover.search?batch=1"
+        val payload = buildJsonArray {
-        val payload = buildJsonObject {
+            add(
-            put(
-                "0",
                 buildJsonObject {
-                    put(
+                    put("name", query)
-                        "json",
-                        buildJsonObject {
-                            put("name", query)
-                        },
-                    )
                 },
             )
-        }.toString().toRequestBody("application/json".toMediaType())
+        }
 
-        return POST(url, headers, payload)
+        val newHeaders = headers.newBuilder()
+            .set("Next-Action", searchMangaToken)
+            .build()
+
+        return POST(baseUrl, newHeaders, payload.toRequestBody())
     }
 
     override fun searchMangaParse(response: Response) = latestUpdatesParse(response)
@@ -152,23 +145,15 @@ class ReadMangas() : HttpSource() {
     // =========================== Details =================================
 
     override fun mangaDetailsParse(response: Response): SManga {
-        val document = response.asJsoup()
+        val json = response.parseScriptToJson()!!
-        return SManga.create().apply {
+        return with(json.parseAs<MangaDetailsDto>()) {
-            title = document.selectFirst("h1")!!.text()
+            SManga.create().apply {
-
+                title = details.title
-            thumbnail_url = document.selectFirst("img.w-full")?.absUrl("src")
+                thumbnail_url = details.thumbnailUrl
-
+                description = details.description
-            genre = document.select("div > label + div > div").joinToString { it.text() }
+                genre = details.genres.joinToString { it.name }
-
+                status = details.status.toStatus()
-            description = document.select("script").map { it.data() }
+                url = "/title/$slug#${details.id}"
-                .firstOrNull { it.contains("description", ignoreCase = true) }
-                ?.let {
-                    val jsonObject = JSONObject(it)
-                    jsonObject.optString("description", "")
-                }
-
-            document.selectFirst("div.flex > div.inline-flex.items-center:last-child")?.text()?.let {
-                status = it.toStatus()
             }
         }
     }
@@ -179,62 +164,63 @@ class ReadMangas() : HttpSource() {
 
     override fun chapterListParse(response: Response) = throw UnsupportedOperationException()
 
-    private fun chapterListRequest(manga: SManga, page: Int): Request {
+    private fun chapterListRequest(manga: SManga, page: Int, chapterToken: String): Request {
         val id = manga.url.substringAfterLast("#")
-        val input = buildJsonObject {
+        val payload = buildJsonArray {
-            put(
+            add(
-                "0",
                 buildJsonObject {
-                    put(
+                    put("id", id)
-                        "json",
+                    put("page", page)
-                        buildJsonObject {
+                    put("limit", 50)
-                            put("id", id)
+                    put("sort", "desc")
-                            put("page", page)
+                    put("search", "\$undefined")
-                            put("limit", 50)
-                            put("sort", "desc")
-                            put("search", "")
-                        },
-                    )
                 },
             )
         }
 
-        val url = "$baseUrl/api/deprecated/chapter.publicAllChapters".toHttpUrl().newBuilder()
+        val newHeaders = headers.newBuilder()
-            .addQueryParameter("batch", "1")
+            .set("Next-Action", chapterToken)
-            .addQueryParameter("input", input.toString())
             .build()
 
-        val encodedUrl = URLEncoder.encode(manga.url.substringBeforeLast("#"), "UTF-8")
+        return POST("$baseUrl/title/$id", newHeaders, payload.toRequestBody())
-
-        val apiHeaders = headers.newBuilder()
-            .set("Referer", "$baseUrl$encodedUrl")
-            .set("Content-Type", "application/json")
-            .set("Cache-Control", "no-cache")
-            .build()
-
-        return GET(url, apiHeaders)
     }
 
+    private fun findChapterToken(manga: SManga): String {
+        var response = client.newCall(super.chapterListRequest(manga)).execute()
+        val document = response.asJsoup()
+
+        val scriptUlr = document.selectFirst("""script[src*="%5Boid%5D/page"]""")
+            ?.absUrl("src")
+            ?: throw Exception("Token não encontrado")
+
+        response = client.newCall(GET(scriptUlr, headers)).execute()
+
+        return TOKEN_REGEX.find(response.body.string())?.groups?.get(1)?.value
+            ?: throw Exception("Não foi possivel obter token")
+    }
     override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        val chapterToken = findChapterToken(manga)
         val chapters = mutableListOf<SChapter>()
         var page = 1
-        do {
+        try {
-            val response = tryFetchChapterPage(manga, page++)
+            do {
-            val dto = response
+                val response = tryFetchChapterPage(manga, page++, chapterToken)
-                .parseAs<List<WrapperResult<ChapterListDto>>>()
+                val json = CHAPTERS_REGEX.find(response.body.string())?.groups?.get(0)?.value!!
-                .firstNotNullOf { it.result }
+                val dto = json.parseAs<ChapterListDto>()
-                .data.json
+                chapters += chapterListParse(dto.chapters)
-            chapters += chapterListParse(dto.chapters)
+            } while (dto.hasNext())
-        } while (dto.hasNext())
+        } catch (e: Exception) {
+            showToast(e.message!!)
+        }
 
         return Observable.just(chapters)
     }
 
-    private val attempts = 3
+    private val attempts = 2
 
-    private fun tryFetchChapterPage(manga: SManga, page: Int): Response {
+    private fun tryFetchChapterPage(manga: SManga, page: Int, chapterToken: String): Response {
         repeat(attempts) { index ->
-            try { return client.newCall(this.chapterListRequest(manga, page)).execute() } catch (e: Exception) { /* do nothing */ }
+            try { return client.newCall(this.chapterListRequest(manga, page, chapterToken)).execute() } catch (e: Exception) { /* do nothing */ }
         }
         throw Exception("Não foi possivel obter os capitulos da página: $page")
     }
@@ -244,7 +230,7 @@ class ReadMangas() : HttpSource() {
             SChapter.create().apply {
                 name = it.title
                 chapter_number = it.number.toFloat()
-                date_upload = it.createdAt.toDate()
+                date_upload = dateFormat.tryParse(it.createdAt)
                 url = "/readme/${it.id}"
             }
         }
@@ -266,11 +252,19 @@ class ReadMangas() : HttpSource() {
 
     // =========================== Utilities ===============================
 
-    private fun mangasPageParse(response: Response): Pair<MangasPage, String> {
+    private inline fun <reified T : ResultDto> Response.mangasPageParse(): Pair<MangasPage, String> {
-        val dto = response.parseAs<List<WrapperResult<MangaListDto>>>().first()
+        val json = when (request.method) {
-        val data = dto.result?.data?.json ?: return MangasPage(emptyList(), false) to ""
+            "GET" -> parseScriptToJson()
+            else -> JSON_REGEX.find(body.string())?.groups?.get(0)?.value
+        }
 
-        val mangas = data.mangas.map {
+        if (json.isNullOrBlank()) {
+            return MangasPage(emptyList(), false) to ""
+        }
+
+        val dto = json.parseAs<T>()
+
+        val mangas = dto.mangas.map {
             SManga.create().apply {
                 title = it.title
                 thumbnail_url = it.thumbnailUrl
@@ -279,33 +273,74 @@ class ReadMangas() : HttpSource() {
                 url = "/title/${it.slug}#${it.id}"
             }
         }
-        return MangasPage(mangas, data.nextCursor != null) to (data.nextCursor ?: "")
+        return MangasPage(mangas, dto.hasNextPage) to dto.nextCursor
     }
 
-    private inline fun <reified T> Response.parseAs(): T {
-        return json.decodeFromString(body.string())
-    }
-
-    private fun String.toDate() =
-        try { dateFormat.parse(this)!!.time } catch (_: Exception) { 0L }
-
     private fun String.toStatus() = when (lowercase()) {
         "ongoing" -> SManga.ONGOING
         "hiatus" -> SManga.ON_HIATUS
         else -> SManga.UNKNOWN
     }
 
+    private fun Response.parseScriptToJson(): String? {
+        val document = asJsoup()
+        val script = document.select("script")
+            .map(Element::data)
+            .filter {
+                it.contains("self.__next_f")
+            }
+            .joinToString("\n")
+
+        val content = QuickJs.create().use {
+            it.evaluate(
+                """
+                globalThis.self = globalThis;
+                $script
+                self.__next_f.map(it => it[it.length - 1]).join('')
+                """.trimIndent(),
+            ) as String
+        }
+
+        return JSON_REGEX.find(content)?.groups?.get(0)?.value
+    }
+
+    private fun JsonElement.toRequestBody() = toString().toRequestBody(APPLICATION_JSON)
+
+    private fun getToken(request: Request, selector: String): String {
+        var document = client.newCall(request).execute().asJsoup()
+        val url = document.selectFirst(selector)?.absUrl("src")
+            ?: return ""
+
+        val script = client.newCall(GET(url, headers))
+            .execute().body.string()
+
+        return TOKEN_REGEX.find(script)?.groups?.get(1)?.value ?: ""
+    }
+
+    private val handler = Handler(Looper.getMainLooper())
+
+    private fun showToast(message: String) {
+        handler.post {
+            Toast.makeText(application, message, Toast.LENGTH_LONG).show()
+        }
+    }
+
     @SuppressLint("SimpleDateFormat")
     companion object {
         val IMAGE_URL_REGEX = """url\\":\\"([^(\\")]+)""".toRegex()
+        val POPULAR_REGEX = """\{"(?:cursor|mangas)".+?\}{2}""".toRegex()
+        val LATEST_REGEX = """\{"(?:items|mangas)".+?(?:hasNextPage[^,]+|query.+\})""".toRegex()
+        val DETAILS_REGEX = """\{"oId".+\}{3}""".toRegex()
+        val CHAPTERS_REGEX = """\{"count".+totalPages.+\}""".toRegex()
+        val TOKEN_REGEX = """\("([^\)]+)",[^"]+"(?:getChapters|getProjects|getUpdatedProjects|searchProjects)""".toRegex()
+        val JSON_REGEX = listOf(
+            POPULAR_REGEX,
+            LATEST_REGEX,
+            DETAILS_REGEX,
+        ).joinToString("|").toRegex()
 
-        val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
+        val dateFormat = SimpleDateFormat("'\$D'yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
 
-        val latestUpdateDateFormat = SimpleDateFormat(
+        val APPLICATION_JSON = "application/json".toMediaType()
-            "EEE MMM dd yyyy HH:mm:ss 'GMT'Z '(Coordinated Universal Time)'",
-            Locale.ENGLISH,
-        ).apply {
-            timeZone = TimeZone.getTimeZone("UTC")
-        }
     }
 }

src/pt/readmangas/src/eu/kanade/tachiyomi/extension/pt/readmangas/ReadMangasDto.kt

@@ -4,22 +4,66 @@ import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.JsonNames
 
+interface ResultDto {
+    val mangas: List<MangaDto>
+    val nextCursor: String
+    val hasNextPage: Boolean
+}
+
 @Serializable
-class WrapperResult<T>(
+class PopularResultDto(
-    val result: Result<T>? = null,
+    @JsonNames("initialData")
+    val result: MangaListDto?,
+    @SerialName("mangas")
+    val list: List<MangaDto> = emptyList(),
+    override val nextCursor: String = result?.nextCursor ?: "",
+) : ResultDto {
+    override val mangas: List<MangaDto> get() = result?.mangas ?: list
+    override val hasNextPage: Boolean = nextCursor.isNotBlank()
+}
+
+@Serializable
+class LatestResultDto(
+    @SerialName("items")
+    override val mangas: List<MangaDto>,
+    override val nextCursor: String = "",
+    override val hasNextPage: Boolean = false,
+) : ResultDto
+
+@Serializable
+class MangaDetailsDto(
+    @SerialName("oId")
+    val slug: String,
+    @SerialName("data")
+    val details: MangaDto,
 ) {
-    @Serializable
-    class Result<T>(val `data`: Data<T>)
 
     @Serializable
-    class Data<T>(val json: T)
+    class MangaDto(
+        val id: String,
+        @SerialName("title")
+        val titles: List<Map<String, String>>,
+        val description: String,
+        @SerialName("coverImage")
+        val thumbnailUrl: String,
+        val status: String,
+        val genres: List<Genre>,
+    ) {
+        val title: String get() = titles.first().values.first()
+    }
+
+    @Serializable
+    class Genre(
+        val name: String,
+    )
 }
 
 @Serializable
 class MangaListDto(
-    @JsonNames("items")
+    @JsonNames("pages")
     val mangas: List<MangaDto>,
-    val nextCursor: String?,
+    @JsonNames("pageParams")
+    val nextCursor: String = "",
 )
 
 @Serializable