Fix VoyceMe JSON parsing and rename source (#17461)

Fix VoyceMe JSON parsing and rename source.

src/en/voyceme/build.gradle

@@ -6,7 +6,7 @@ ext {
     extName = 'Voyce.Me'
     pkgNameSuffix = 'en.voyceme'
     extClass = '.VoyceMe'
-    extVersionCode = 2
+    extVersionCode = 3
 }
 
 apply from: "$rootDir/common.gradle"

src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMe.kt

@@ -2,41 +2,32 @@ package eu.kanade.tachiyomi.extension.en.voyceme
 
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.interceptor.rateLimitHost
-import eu.kanade.tachiyomi.network.interceptor.rateLimit
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
-import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.buildJsonObject
-import kotlinx.serialization.json.decodeFromJsonElement
-import kotlinx.serialization.json.jsonArray
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
-import kotlinx.serialization.json.put
-import kotlinx.serialization.json.putJsonObject
 import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
-import org.jsoup.Jsoup
-import org.jsoup.parser.Parser
-import rx.Observable
 import uy.kohesive.injekt.injectLazy
-import java.text.ParseException
-import java.text.SimpleDateFormat
-import java.util.Locale
 import java.util.concurrent.TimeUnit
 
 class VoyceMe : HttpSource() {
 
-    override val name = "Voyce.Me"
+    // Renamed from "Voyce.Me" to "VoyceMe" as the site uses.
+    override val id = 4815322300278778429
+
+    override val name = "VoyceMe"
 
     override val baseUrl = "http://voyce.me"
 
@@ -45,7 +36,8 @@ class VoyceMe : HttpSource() {
     override val supportsLatest = true
 
     override val client: OkHttpClient = network.cloudflareClient.newBuilder()
-        .rateLimit(2, 1, TimeUnit.SECONDS)
+        .rateLimitHost(GRAPHQL_URL.toHttpUrl(), 1, 1, TimeUnit.SECONDS)
+        .rateLimitHost(STATIC_URL.toHttpUrl(), 2, 1, TimeUnit.SECONDS)
         .build()
 
     private val json: Json by injectLazy()
@@ -55,23 +47,16 @@ class VoyceMe : HttpSource() {
         .add("Origin", baseUrl)
         .add("Referer", "$baseUrl/")
 
-    private fun genericComicBookFromObject(comic: VoyceMeComic): SManga =
-        SManga.create().apply {
-            title = comic.title
-            url = "/series/${comic.slug}"
-            thumbnail_url = STATIC_URL + comic.thumbnail
-        }
-
     override fun popularMangaRequest(page: Int): Request {
-        val payload = buildJsonObject {
+        val payload = GraphQlQuery(
-            put("query", POPULAR_QUERY)
+            query = POPULAR_QUERY,
-            putJsonObject("variables") {
+            variables = PopularQueryVariables(
-                put("offset", (page - 1) * POPULAR_PER_PAGE)
+                offset = (page - 1) * POPULAR_PER_PAGE,
-                put("limit", POPULAR_PER_PAGE)
+                limit = POPULAR_PER_PAGE,
-            }
+            ),
-        }
+        )
 
-        val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
+        val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
 
         val newHeaders = headersBuilder()
             .add("Content-Length", body.contentLength().toString())
@@ -82,26 +67,24 @@ class VoyceMe : HttpSource() {
     }
 
     override fun popularMangaParse(response: Response): MangasPage {
-        val result = json.parseToJsonElement(response.body.string()).jsonObject
+        val comicList = response.parseAs<VoyceMeSeriesResponse>()
+            .data.series.map(VoyceMeComic::toSManga)
 
-        val comicList = result["data"]!!.jsonObject["voyce_series"]!!
-            .let { json.decodeFromJsonElement<List<VoyceMeComic>>(it) }
-            .map(::genericComicBookFromObject)
         val hasNextPage = comicList.size == POPULAR_PER_PAGE
 
         return MangasPage(comicList, hasNextPage)
     }
 
     override fun latestUpdatesRequest(page: Int): Request {
-        val payload = buildJsonObject {
+        val payload = GraphQlQuery(
-            put("query", LATEST_QUERY)
+            query = LATEST_QUERY,
-            putJsonObject("variables") {
+            variables = LatestQueryVariables(
-                put("offset", (page - 1) * POPULAR_PER_PAGE)
+                offset = (page - 1) * POPULAR_PER_PAGE,
-                put("limit", POPULAR_PER_PAGE)
+                limit = POPULAR_PER_PAGE,
-            }
+            ),
-        }
+        )
 
-        val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
+        val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
 
         val newHeaders = headersBuilder()
             .add("Content-Length", body.contentLength().toString())
@@ -111,28 +94,19 @@ class VoyceMe : HttpSource() {
         return POST(GRAPHQL_URL, newHeaders, body)
     }
 
-    override fun latestUpdatesParse(response: Response): MangasPage {
+    override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response)
-        val result = json.parseToJsonElement(response.body.string()).jsonObject
-
-        val comicList = result["data"]!!.jsonObject["voyce_series"]!!
-            .let { json.decodeFromJsonElement<List<VoyceMeComic>>(it) }
-            .map(::genericComicBookFromObject)
-        val hasNextPage = comicList.size == POPULAR_PER_PAGE
-
-        return MangasPage(comicList, hasNextPage)
-    }
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val payload = buildJsonObject {
+        val payload = GraphQlQuery(
-            put("query", SEARCH_QUERY)
+            query = SEARCH_QUERY,
-            putJsonObject("variables") {
+            variables = SearchQueryVariables(
-                put("searchTerm", "%$query%")
+                searchTerm = "%$query%",
-                put("offset", (page - 1) * POPULAR_PER_PAGE)
+                offset = (page - 1) * POPULAR_PER_PAGE,
-                put("limit", POPULAR_PER_PAGE)
+                limit = POPULAR_PER_PAGE,
-            }
+            ),
-        }
+        )
 
-        val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
+        val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
 
         val newHeaders = headersBuilder()
             .add("Content-Length", body.contentLength().toString())
@@ -142,39 +116,19 @@ class VoyceMe : HttpSource() {
         return POST(GRAPHQL_URL, newHeaders, body)
     }
 
-    override fun searchMangaParse(response: Response): MangasPage {
+    override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response)
-        val result = json.parseToJsonElement(response.body.string()).jsonObject
 
-        val comicList = result["data"]!!.jsonObject["voyce_series"]!!
+    override fun mangaDetailsRequest(manga: SManga): Request {
-            .let { json.decodeFromJsonElement<List<VoyceMeComic>>(it) }
-            .map(::genericComicBookFromObject)
-        val hasNextPage = comicList.size == POPULAR_PER_PAGE
-
-        return MangasPage(comicList, hasNextPage)
-    }
-
-    // Workaround to allow "Open in browser" use the real URL.
-    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
-        return client.newCall(mangaDetailsApiRequest(manga))
-            .asObservableSuccess()
-            .map { response ->
-                mangaDetailsParse(response).apply { initialized = true }
-            }
-    }
-
-    private fun mangaDetailsApiRequest(manga: SManga): Request {
         val comicSlug = manga.url
             .substringAfter("/series/")
             .substringBefore("/")
 
-        val payload = buildJsonObject {
+        val payload = GraphQlQuery(
-            put("query", DETAILS_QUERY)
+            query = DETAILS_QUERY,
-            putJsonObject("variables") {
+            variables = DetailsQueryVariables(slug = comicSlug),
-                put("slug", comicSlug)
+        )
-            }
-        }
 
-        val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
+        val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
 
         val newHeaders = headersBuilder()
             .add("Content-Length", body.contentLength().toString())
@@ -185,18 +139,11 @@ class VoyceMe : HttpSource() {
         return POST(GRAPHQL_URL, newHeaders, body)
     }
 
-    override fun mangaDetailsParse(response: Response): SManga = SManga.create().apply {
+    override fun getMangaUrl(manga: SManga) = baseUrl + manga.url
-        val result = json.parseToJsonElement(response.body.string()).jsonObject
-        val comic = result["data"]!!.jsonObject["voyce_series"]!!.jsonArray[0].jsonObject
-            .let { json.decodeFromJsonElement<VoyceMeComic>(it) }
 
-        title = comic.title
+    override fun mangaDetailsParse(response: Response): SManga {
-        author = comic.author?.username.orEmpty()
+        return response.parseAs<VoyceMeSeriesResponse>()
-        description = Parser.unescapeEntities(comic.description.orEmpty(), true)
+            .data.series.first().toSManga()
-            .let { Jsoup.parse(it).text() }
-        status = comic.status.orEmpty().toStatus()
-        genre = comic.genres.mapNotNull { it.genre?.title }.joinToString(", ")
-        thumbnail_url = STATIC_URL + comic.thumbnail
     }
 
     override fun chapterListRequest(manga: SManga): Request {
@@ -204,14 +151,12 @@ class VoyceMe : HttpSource() {
             .substringAfter("/series/")
             .substringBefore("/")
 
-        val payload = buildJsonObject {
+        val payload = GraphQlQuery(
-            put("query", CHAPTERS_QUERY)
+            query = CHAPTERS_QUERY,
-            putJsonObject("variables") {
+            variables = ChaptersQueryVariables(slug = comicSlug),
-                put("slug", comicSlug)
+        )
-            }
-        }
 
-        val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
+        val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
 
         val newHeaders = headersBuilder()
             .add("Content-Length", body.contentLength().toString())
@@ -223,70 +168,39 @@ class VoyceMe : HttpSource() {
     }
 
     override fun chapterListParse(response: Response): List<SChapter> {
-        val result = json.parseToJsonElement(response.body.string()).jsonObject
+        val comic = response.parseAs<VoyceMeSeriesResponse>().data.series.first()
-        val comicBook = result["data"]!!.jsonObject["voyce_series"]!!.jsonArray[0].jsonObject
-            .let { json.decodeFromJsonElement<VoyceMeComic>(it) }
 
-        return comicBook.chapters
+        return comic.chapters
-            .map { chapter -> chapterFromObject(chapter, comicBook) }
+            .map { it.toSChapter(comic.slug) }
-            .distinctBy { chapter -> chapter.name }
+            .distinctBy(SChapter::name)
-    }
-
-    private fun chapterFromObject(chapter: VoyceMeChapter, comic: VoyceMeComic): SChapter =
-        SChapter.create().apply {
-            name = chapter.title
-            date_upload = chapter.createdAt.toDate()
-            url = "/series/${comic.slug}/${chapter.id}#comic"
     }
 
     override fun pageListRequest(chapter: SChapter): Request {
-        val newHeaders = headersBuilder()
+        val chapterId = chapter.url
-            .set("Referer", baseUrl + chapter.url.substringBeforeLast("/"))
-            .build()
-
-        return GET(baseUrl + chapter.url, newHeaders)
-    }
-
-    private fun pageListApiRequest(buildId: String, chapterUrl: String): Request {
-        val newHeaders = headersBuilder()
-            .set("Referer", baseUrl + chapterUrl)
-            .build()
-
-        val comicSlug = chapterUrl
-            .substringAfter("/series/")
-            .substringBefore("/")
-        val chapterId = chapterUrl
-            .substringAfterLast("/")
-            .substringBefore("#")
-
-        return GET("$baseUrl/_next/data/$buildId/series/$comicSlug/$chapterId.json", newHeaders)
-    }
-
-    override fun pageListParse(response: Response): List<Page> {
-        // GraphQL endpoints do not have the chapter images, so we need
-        // to get the buildId to fetch the chapter from NextJS static data.
-        val document = response.asJsoup()
-        val nextData = document.selectFirst("script#__NEXT_DATA__")!!.data()
-        val nextJson = json.parseToJsonElement(nextData).jsonObject
-
-        val buildId = nextJson["buildId"]!!.jsonPrimitive.content
-        val chapterUrl = response.request.url.toString().substringAfter(baseUrl)
-
-        val dataRequest = pageListApiRequest(buildId, chapterUrl)
-        val dataResponse = client.newCall(dataRequest).execute()
-        val dataJson = json.parseToJsonElement(dataResponse.body.string()).jsonObject
-
-        val comic = dataJson["pageProps"]!!.jsonObject["series"]!!
-            .let { json.decodeFromJsonElement<VoyceMeComic>(it) }
-
-        val chapterId = response.request.url.toString()
             .substringAfterLast("/")
             .substringBefore("#")
             .toInt()
-        val chapter = comic.chapters.firstOrNull { it.id == chapterId }
-            ?: throw Exception(CHAPTER_DATA_NOT_FOUND)
 
-        return chapter.images.mapIndexed { i, page ->
+        val payload = GraphQlQuery(
+            query = PAGES_QUERY,
+            variables = PagesQueryVariables(chapterId = chapterId),
+        )
+
+        val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
+
+        val newHeaders = headersBuilder()
+            .add("Content-Length", body.contentLength().toString())
+            .add("Content-Type", body.contentType().toString())
+            .build()
+
+        return POST(GRAPHQL_URL, newHeaders, body)
+    }
+
+    override fun getChapterUrl(chapter: SChapter) = baseUrl + chapter.url
+
+    override fun pageListParse(response: Response): List<Page> {
+        return response.parseAs<VoyceMeChapterImagesResponse>().data.images
+            .mapIndexed { i, page ->
                 Page(i, baseUrl, STATIC_URL + page.image)
             }
     }
@@ -302,33 +216,19 @@ class VoyceMe : HttpSource() {
         return GET(page.imageUrl!!, newHeaders)
     }
 
-    private fun String.toDate(): Long {
+    private inline fun <reified T> Response.parseAs(): T = use {
-        return try {
+        json.decodeFromString(it.body.string())
-            DATE_FORMATTER.parse(this)?.time ?: 0L
-        } catch (e: ParseException) {
-            0L
-        }
-    }
-
-    private fun String.toStatus(): Int = when (this) {
-        "completed" -> SManga.COMPLETED
-        "ongoing" -> SManga.ONGOING
-        else -> SManga.UNKNOWN
     }
 
     companion object {
         private const val ACCEPT_ALL = "*/*"
         private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
 
-        private const val STATIC_URL = "https://dlkfxmdtxtzpb.cloudfront.net/"
+        const val STATIC_URL = "https://dlkfxmdtxtzpb.cloudfront.net/"
         private const val GRAPHQL_URL = "https://graphql.voyce.me/v1/graphql"
 
         private const val POPULAR_PER_PAGE = 10
 
         private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaTypeOrNull()
-
-        private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) }
-
-        private const val CHAPTER_DATA_NOT_FOUND = "Chapter data not found in website."
     }
 }

src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeDto.kt

@@ -1,7 +1,13 @@
 package eu.kanade.tachiyomi.extension.en.voyceme
 
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import org.jsoup.Jsoup
+import org.jsoup.parser.Parser
+import java.text.SimpleDateFormat
+import java.util.Locale
 
 @Serializable
 data class VoyceMeComic(
@@ -14,7 +20,24 @@ data class VoyceMeComic(
     val status: String? = "",
     val thumbnail: String = "",
     val title: String = "",
-)
+) {
+
+    fun toSManga(): SManga = SManga.create().apply {
+        title = this@VoyceMeComic.title
+        author = this@VoyceMeComic.author?.username.orEmpty()
+        description = Parser
+            .unescapeEntities(this@VoyceMeComic.description.orEmpty(), true)
+            .let { Jsoup.parseBodyFragment(it).text() }
+        status = when (this@VoyceMeComic.status.orEmpty()) {
+            "completed" -> SManga.COMPLETED
+            "ongoing" -> SManga.ONGOING
+            else -> SManga.UNKNOWN
+        }
+        genre = genres.mapNotNull { it.genre?.title }.joinToString(", ")
+        url = "/series/$slug"
+        thumbnail_url = VoyceMe.STATIC_URL + thumbnail
+    }
+}
 
 @Serializable
 data class VoyceMeAuthor(
@@ -37,9 +60,67 @@ data class VoyceMeChapter(
     val id: Int = -1,
     val images: List<VoyceMePage> = emptyList(),
     val title: String = "",
-)
+) {
+
+    fun toSChapter(comicSlug: String): SChapter = SChapter.create().apply {
+        name = title
+        date_upload = runCatching { DATE_FORMATTER.parse(createdAt)?.time }
+            .getOrNull() ?: 0L
+        url = "/series/$comicSlug/$id#comic"
+    }
+
+    companion object {
+        private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) }
+    }
+}
 
 @Serializable
 data class VoyceMePage(
     val image: String = "",
 )
+
+@Serializable
+data class GraphQlQuery<T>(
+    val variables: T,
+    val query: String,
+)
+
+@Serializable
+data class GraphQlResponse<T>(val data: T)
+
+typealias VoyceMeSeriesResponse = GraphQlResponse<VoyceMeSeriesCollection>
+typealias VoyceMeChapterImagesResponse = GraphQlResponse<VoyceChapterImagesCollection>
+
+@Serializable
+data class VoyceMeSeriesCollection(
+    @SerialName("voyce_series")
+    val series: List<VoyceMeComic> = emptyList(),
+)
+
+@Serializable
+data class VoyceChapterImagesCollection(
+    @SerialName("voyce_chapter_images")
+    val images: List<VoyceMePage> = emptyList(),
+)
+
+@Serializable
+data class PopularQueryVariables(
+    val offset: Int,
+    val limit: Int,
+)
+
+@Serializable
+data class SearchQueryVariables(
+    val offset: Int,
+    val limit: Int,
+    val searchTerm: String,
+)
+
+@Serializable
+data class DetailsQueryVariables(val slug: String)
+
+@Serializable
+data class PagesQueryVariables(val chapterId: Int)
+
+typealias LatestQueryVariables = PopularQueryVariables
+typealias ChaptersQueryVariables = DetailsQueryVariables

src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeQueries.kt

@@ -113,3 +113,16 @@ val CHAPTERS_QUERY: String = buildQuery {
         }
     """.trimIndent()
 }
+
+val PAGES_QUERY: String = buildQuery {
+    """
+        query(%chapterId: Int!) {
+            voyce_chapter_images(
+                where: { chapter_id: { _eq: %chapterId } },
+                order_by: { sort_order: asc }
+            ) {
+                image
+            }
+        }
+    """.trimIndent()
+}