From 77bb7872e393b5d066e39f5c5ed6c68aaf905dff Mon Sep 17 00:00:00 2001 From: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> Date: Wed, 9 Aug 2023 19:25:53 -0300 Subject: [PATCH] Fix VoyceMe JSON parsing and rename source (#17461) Fix VoyceMe JSON parsing and rename source. --- src/en/voyceme/build.gradle | 2 +- .../tachiyomi/extension/en/voyceme/VoyceMe.kt | 268 ++++++------------ .../extension/en/voyceme/VoyceMeDto.kt | 85 +++++- .../extension/en/voyceme/VoyceMeQueries.kt | 13 + 4 files changed, 181 insertions(+), 187 deletions(-) diff --git a/src/en/voyceme/build.gradle b/src/en/voyceme/build.gradle index b75a7a480..9743e42cb 100644 --- a/src/en/voyceme/build.gradle +++ b/src/en/voyceme/build.gradle @@ -6,7 +6,7 @@ ext { extName = 'Voyce.Me' pkgNameSuffix = 'en.voyceme' extClass = '.VoyceMe' - extVersionCode = 2 + extVersionCode = 3 } apply from: "$rootDir/common.gradle" diff --git a/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMe.kt b/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMe.kt index 46b705a9b..0e5a30e2b 100644 --- a/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMe.kt +++ b/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMe.kt @@ -2,41 +2,32 @@ package eu.kanade.tachiyomi.extension.en.voyceme import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.POST -import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.network.interceptor.rateLimit +import eu.kanade.tachiyomi.network.interceptor.rateLimitHost import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json -import kotlinx.serialization.json.buildJsonObject -import kotlinx.serialization.json.decodeFromJsonElement -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive -import kotlinx.serialization.json.put -import kotlinx.serialization.json.putJsonObject import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response -import org.jsoup.Jsoup -import org.jsoup.parser.Parser -import rx.Observable import uy.kohesive.injekt.injectLazy -import java.text.ParseException -import java.text.SimpleDateFormat -import java.util.Locale import java.util.concurrent.TimeUnit class VoyceMe : HttpSource() { - override val name = "Voyce.Me" + // Renamed from "Voyce.Me" to "VoyceMe" as the site uses. + override val id = 4815322300278778429 + + override val name = "VoyceMe" override val baseUrl = "http://voyce.me" @@ -45,7 +36,8 @@ class VoyceMe : HttpSource() { override val supportsLatest = true override val client: OkHttpClient = network.cloudflareClient.newBuilder() - .rateLimit(2, 1, TimeUnit.SECONDS) + .rateLimitHost(GRAPHQL_URL.toHttpUrl(), 1, 1, TimeUnit.SECONDS) + .rateLimitHost(STATIC_URL.toHttpUrl(), 2, 1, TimeUnit.SECONDS) .build() private val json: Json by injectLazy() @@ -55,23 +47,16 @@ class VoyceMe : HttpSource() { .add("Origin", baseUrl) .add("Referer", "$baseUrl/") - private fun genericComicBookFromObject(comic: VoyceMeComic): SManga = - SManga.create().apply { - title = comic.title - url = "/series/${comic.slug}" - thumbnail_url = STATIC_URL + comic.thumbnail - } - override fun popularMangaRequest(page: Int): Request { - val payload = buildJsonObject { - put("query", POPULAR_QUERY) - putJsonObject("variables") { - put("offset", (page - 1) * POPULAR_PER_PAGE) - put("limit", POPULAR_PER_PAGE) - } - } + val payload = GraphQlQuery( + query = POPULAR_QUERY, + variables = PopularQueryVariables( + offset = (page - 1) * POPULAR_PER_PAGE, + limit = POPULAR_PER_PAGE, + ), + ) - val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE) + val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE) val newHeaders = headersBuilder() .add("Content-Length", body.contentLength().toString()) @@ -82,26 +67,24 @@ class VoyceMe : HttpSource() { } override fun popularMangaParse(response: Response): MangasPage { - val result = json.parseToJsonElement(response.body.string()).jsonObject + val comicList = response.parseAs() + .data.series.map(VoyceMeComic::toSManga) - val comicList = result["data"]!!.jsonObject["voyce_series"]!! - .let { json.decodeFromJsonElement>(it) } - .map(::genericComicBookFromObject) val hasNextPage = comicList.size == POPULAR_PER_PAGE return MangasPage(comicList, hasNextPage) } override fun latestUpdatesRequest(page: Int): Request { - val payload = buildJsonObject { - put("query", LATEST_QUERY) - putJsonObject("variables") { - put("offset", (page - 1) * POPULAR_PER_PAGE) - put("limit", POPULAR_PER_PAGE) - } - } + val payload = GraphQlQuery( + query = LATEST_QUERY, + variables = LatestQueryVariables( + offset = (page - 1) * POPULAR_PER_PAGE, + limit = POPULAR_PER_PAGE, + ), + ) - val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE) + val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE) val newHeaders = headersBuilder() .add("Content-Length", body.contentLength().toString()) @@ -111,28 +94,19 @@ class VoyceMe : HttpSource() { return POST(GRAPHQL_URL, newHeaders, body) } - override fun latestUpdatesParse(response: Response): MangasPage { - val result = json.parseToJsonElement(response.body.string()).jsonObject - - val comicList = result["data"]!!.jsonObject["voyce_series"]!! - .let { json.decodeFromJsonElement>(it) } - .map(::genericComicBookFromObject) - val hasNextPage = comicList.size == POPULAR_PER_PAGE - - return MangasPage(comicList, hasNextPage) - } + override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response) override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val payload = buildJsonObject { - put("query", SEARCH_QUERY) - putJsonObject("variables") { - put("searchTerm", "%$query%") - put("offset", (page - 1) * POPULAR_PER_PAGE) - put("limit", POPULAR_PER_PAGE) - } - } + val payload = GraphQlQuery( + query = SEARCH_QUERY, + variables = SearchQueryVariables( + searchTerm = "%$query%", + offset = (page - 1) * POPULAR_PER_PAGE, + limit = POPULAR_PER_PAGE, + ), + ) - val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE) + val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE) val newHeaders = headersBuilder() .add("Content-Length", body.contentLength().toString()) @@ -142,39 +116,19 @@ class VoyceMe : HttpSource() { return POST(GRAPHQL_URL, newHeaders, body) } - override fun searchMangaParse(response: Response): MangasPage { - val result = json.parseToJsonElement(response.body.string()).jsonObject + override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response) - val comicList = result["data"]!!.jsonObject["voyce_series"]!! - .let { json.decodeFromJsonElement>(it) } - .map(::genericComicBookFromObject) - val hasNextPage = comicList.size == POPULAR_PER_PAGE - - return MangasPage(comicList, hasNextPage) - } - - // Workaround to allow "Open in browser" use the real URL. - override fun fetchMangaDetails(manga: SManga): Observable { - return client.newCall(mangaDetailsApiRequest(manga)) - .asObservableSuccess() - .map { response -> - mangaDetailsParse(response).apply { initialized = true } - } - } - - private fun mangaDetailsApiRequest(manga: SManga): Request { + override fun mangaDetailsRequest(manga: SManga): Request { val comicSlug = manga.url .substringAfter("/series/") .substringBefore("/") - val payload = buildJsonObject { - put("query", DETAILS_QUERY) - putJsonObject("variables") { - put("slug", comicSlug) - } - } + val payload = GraphQlQuery( + query = DETAILS_QUERY, + variables = DetailsQueryVariables(slug = comicSlug), + ) - val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE) + val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE) val newHeaders = headersBuilder() .add("Content-Length", body.contentLength().toString()) @@ -185,18 +139,11 @@ class VoyceMe : HttpSource() { return POST(GRAPHQL_URL, newHeaders, body) } - override fun mangaDetailsParse(response: Response): SManga = SManga.create().apply { - val result = json.parseToJsonElement(response.body.string()).jsonObject - val comic = result["data"]!!.jsonObject["voyce_series"]!!.jsonArray[0].jsonObject - .let { json.decodeFromJsonElement(it) } + override fun getMangaUrl(manga: SManga) = baseUrl + manga.url - title = comic.title - author = comic.author?.username.orEmpty() - description = Parser.unescapeEntities(comic.description.orEmpty(), true) - .let { Jsoup.parse(it).text() } - status = comic.status.orEmpty().toStatus() - genre = comic.genres.mapNotNull { it.genre?.title }.joinToString(", ") - thumbnail_url = STATIC_URL + comic.thumbnail + override fun mangaDetailsParse(response: Response): SManga { + return response.parseAs() + .data.series.first().toSManga() } override fun chapterListRequest(manga: SManga): Request { @@ -204,14 +151,12 @@ class VoyceMe : HttpSource() { .substringAfter("/series/") .substringBefore("/") - val payload = buildJsonObject { - put("query", CHAPTERS_QUERY) - putJsonObject("variables") { - put("slug", comicSlug) - } - } + val payload = GraphQlQuery( + query = CHAPTERS_QUERY, + variables = ChaptersQueryVariables(slug = comicSlug), + ) - val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE) + val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE) val newHeaders = headersBuilder() .add("Content-Length", body.contentLength().toString()) @@ -223,72 +168,41 @@ class VoyceMe : HttpSource() { } override fun chapterListParse(response: Response): List { - val result = json.parseToJsonElement(response.body.string()).jsonObject - val comicBook = result["data"]!!.jsonObject["voyce_series"]!!.jsonArray[0].jsonObject - .let { json.decodeFromJsonElement(it) } + val comic = response.parseAs().data.series.first() - return comicBook.chapters - .map { chapter -> chapterFromObject(chapter, comicBook) } - .distinctBy { chapter -> chapter.name } + return comic.chapters + .map { it.toSChapter(comic.slug) } + .distinctBy(SChapter::name) } - private fun chapterFromObject(chapter: VoyceMeChapter, comic: VoyceMeComic): SChapter = - SChapter.create().apply { - name = chapter.title - date_upload = chapter.createdAt.toDate() - url = "/series/${comic.slug}/${chapter.id}#comic" - } - override fun pageListRequest(chapter: SChapter): Request { - val newHeaders = headersBuilder() - .set("Referer", baseUrl + chapter.url.substringBeforeLast("/")) - .build() - - return GET(baseUrl + chapter.url, newHeaders) - } - - private fun pageListApiRequest(buildId: String, chapterUrl: String): Request { - val newHeaders = headersBuilder() - .set("Referer", baseUrl + chapterUrl) - .build() - - val comicSlug = chapterUrl - .substringAfter("/series/") - .substringBefore("/") - val chapterId = chapterUrl - .substringAfterLast("/") - .substringBefore("#") - - return GET("$baseUrl/_next/data/$buildId/series/$comicSlug/$chapterId.json", newHeaders) - } - - override fun pageListParse(response: Response): List { - // GraphQL endpoints do not have the chapter images, so we need - // to get the buildId to fetch the chapter from NextJS static data. - val document = response.asJsoup() - val nextData = document.selectFirst("script#__NEXT_DATA__")!!.data() - val nextJson = json.parseToJsonElement(nextData).jsonObject - - val buildId = nextJson["buildId"]!!.jsonPrimitive.content - val chapterUrl = response.request.url.toString().substringAfter(baseUrl) - - val dataRequest = pageListApiRequest(buildId, chapterUrl) - val dataResponse = client.newCall(dataRequest).execute() - val dataJson = json.parseToJsonElement(dataResponse.body.string()).jsonObject - - val comic = dataJson["pageProps"]!!.jsonObject["series"]!! - .let { json.decodeFromJsonElement(it) } - - val chapterId = response.request.url.toString() + val chapterId = chapter.url .substringAfterLast("/") .substringBefore("#") .toInt() - val chapter = comic.chapters.firstOrNull { it.id == chapterId } - ?: throw Exception(CHAPTER_DATA_NOT_FOUND) - return chapter.images.mapIndexed { i, page -> - Page(i, baseUrl, STATIC_URL + page.image) - } + val payload = GraphQlQuery( + query = PAGES_QUERY, + variables = PagesQueryVariables(chapterId = chapterId), + ) + + val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE) + + val newHeaders = headersBuilder() + .add("Content-Length", body.contentLength().toString()) + .add("Content-Type", body.contentType().toString()) + .build() + + return POST(GRAPHQL_URL, newHeaders, body) + } + + override fun getChapterUrl(chapter: SChapter) = baseUrl + chapter.url + + override fun pageListParse(response: Response): List { + return response.parseAs().data.images + .mapIndexed { i, page -> + Page(i, baseUrl, STATIC_URL + page.image) + } } override fun imageUrlParse(response: Response): String = "" @@ -302,33 +216,19 @@ class VoyceMe : HttpSource() { return GET(page.imageUrl!!, newHeaders) } - private fun String.toDate(): Long { - return try { - DATE_FORMATTER.parse(this)?.time ?: 0L - } catch (e: ParseException) { - 0L - } - } - - private fun String.toStatus(): Int = when (this) { - "completed" -> SManga.COMPLETED - "ongoing" -> SManga.ONGOING - else -> SManga.UNKNOWN + private inline fun Response.parseAs(): T = use { + json.decodeFromString(it.body.string()) } companion object { private const val ACCEPT_ALL = "*/*" private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" - private const val STATIC_URL = "https://dlkfxmdtxtzpb.cloudfront.net/" + const val STATIC_URL = "https://dlkfxmdtxtzpb.cloudfront.net/" private const val GRAPHQL_URL = "https://graphql.voyce.me/v1/graphql" private const val POPULAR_PER_PAGE = 10 private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaTypeOrNull() - - private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) } - - private const val CHAPTER_DATA_NOT_FOUND = "Chapter data not found in website." } } diff --git a/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeDto.kt b/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeDto.kt index 45308fc4f..0a04f3e7f 100644 --- a/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeDto.kt +++ b/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeDto.kt @@ -1,7 +1,13 @@ package eu.kanade.tachiyomi.extension.en.voyceme +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import org.jsoup.Jsoup +import org.jsoup.parser.Parser +import java.text.SimpleDateFormat +import java.util.Locale @Serializable data class VoyceMeComic( @@ -14,7 +20,24 @@ data class VoyceMeComic( val status: String? = "", val thumbnail: String = "", val title: String = "", -) +) { + + fun toSManga(): SManga = SManga.create().apply { + title = this@VoyceMeComic.title + author = this@VoyceMeComic.author?.username.orEmpty() + description = Parser + .unescapeEntities(this@VoyceMeComic.description.orEmpty(), true) + .let { Jsoup.parseBodyFragment(it).text() } + status = when (this@VoyceMeComic.status.orEmpty()) { + "completed" -> SManga.COMPLETED + "ongoing" -> SManga.ONGOING + else -> SManga.UNKNOWN + } + genre = genres.mapNotNull { it.genre?.title }.joinToString(", ") + url = "/series/$slug" + thumbnail_url = VoyceMe.STATIC_URL + thumbnail + } +} @Serializable data class VoyceMeAuthor( @@ -37,9 +60,67 @@ data class VoyceMeChapter( val id: Int = -1, val images: List = emptyList(), val title: String = "", -) +) { + + fun toSChapter(comicSlug: String): SChapter = SChapter.create().apply { + name = title + date_upload = runCatching { DATE_FORMATTER.parse(createdAt)?.time } + .getOrNull() ?: 0L + url = "/series/$comicSlug/$id#comic" + } + + companion object { + private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) } + } +} @Serializable data class VoyceMePage( val image: String = "", ) + +@Serializable +data class GraphQlQuery( + val variables: T, + val query: String, +) + +@Serializable +data class GraphQlResponse(val data: T) + +typealias VoyceMeSeriesResponse = GraphQlResponse +typealias VoyceMeChapterImagesResponse = GraphQlResponse + +@Serializable +data class VoyceMeSeriesCollection( + @SerialName("voyce_series") + val series: List = emptyList(), +) + +@Serializable +data class VoyceChapterImagesCollection( + @SerialName("voyce_chapter_images") + val images: List = emptyList(), +) + +@Serializable +data class PopularQueryVariables( + val offset: Int, + val limit: Int, +) + +@Serializable +data class SearchQueryVariables( + val offset: Int, + val limit: Int, + val searchTerm: String, +) + +@Serializable +data class DetailsQueryVariables(val slug: String) + +@Serializable +data class PagesQueryVariables(val chapterId: Int) + +typealias LatestQueryVariables = PopularQueryVariables +typealias ChaptersQueryVariables = DetailsQueryVariables diff --git a/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeQueries.kt b/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeQueries.kt index 2a0326cfd..18d62a7a4 100644 --- a/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeQueries.kt +++ b/src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeQueries.kt @@ -113,3 +113,16 @@ val CHAPTERS_QUERY: String = buildQuery { } """.trimIndent() } + +val PAGES_QUERY: String = buildQuery { + """ + query(%chapterId: Int!) { + voyce_chapter_images( + where: { chapter_id: { _eq: %chapterId } }, + order_by: { sort_order: asc } + ) { + image + } + } + """.trimIndent() +}