Draff
/
Tachiyomi-Extensions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix VoyceMe JSON parsing and rename source (#17461) 

Fix VoyceMe JSON parsing and rename source.
This commit is contained in:
Alessandro Jean 2023-08-09 19:25:53 -03:00 committed by GitHub
parent 770f84fe1c
commit 77bb7872e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 181 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/en/voyceme/build.gradle
View File

@ -6,7 +6,7 @@ ext {
extName = 'Voyce.Me'
pkgNameSuffix = 'en.voyceme'
extClass = '.VoyceMe'
extVersionCode = 2
extVersionCode = 3
}
apply from: "$rootDir/common.gradle"

268
src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMe.kt
View File

@ -2,41 +2,32 @@ package eu.kanade.tachiyomi.extension.en.voyceme
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.network.interceptor.rateLimit
import eu.kanade.tachiyomi.network.interceptor.rateLimitHost
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.decodeFromJsonElement
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonObject
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import org.jsoup.Jsoup
import org.jsoup.parser.Parser
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.text.ParseException
import java.text.SimpleDateFormat
import java.util.Locale
import java.util.concurrent.TimeUnit
class VoyceMe : HttpSource() {
override val name = "Voyce.Me"
// Renamed from "Voyce.Me" to "VoyceMe" as the site uses.
override val id = 4815322300278778429
override val name = "VoyceMe"
override val baseUrl = "http://voyce.me"
@ -45,7 +36,8 @@ class VoyceMe : HttpSource() {
override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient.newBuilder()
.rateLimit(2, 1, TimeUnit.SECONDS)
.rateLimitHost(GRAPHQL_URL.toHttpUrl(), 1, 1, TimeUnit.SECONDS)
.rateLimitHost(STATIC_URL.toHttpUrl(), 2, 1, TimeUnit.SECONDS)
.build()
private val json: Json by injectLazy()
@ -55,23 +47,16 @@ class VoyceMe : HttpSource() {
.add("Origin", baseUrl)
.add("Referer", "$baseUrl/")
private fun genericComicBookFromObject(comic: VoyceMeComic): SManga =
SManga.create().apply {
title = comic.title
url = "/series/${comic.slug}"
thumbnail_url = STATIC_URL + comic.thumbnail
}
override fun popularMangaRequest(page: Int): Request {
val payload = buildJsonObject {
put("query", POPULAR_QUERY)
putJsonObject("variables") {
put("offset", (page - 1) * POPULAR_PER_PAGE)
put("limit", POPULAR_PER_PAGE)
}
}
val payload = GraphQlQuery(
query = POPULAR_QUERY,
variables = PopularQueryVariables(
offset = (page - 1) * POPULAR_PER_PAGE,
limit = POPULAR_PER_PAGE,
),
)
val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
val newHeaders = headersBuilder()
.add("Content-Length", body.contentLength().toString())
@ -82,26 +67,24 @@ class VoyceMe : HttpSource() {
}
override fun popularMangaParse(response: Response): MangasPage {
val result = json.parseToJsonElement(response.body.string()).jsonObject
val comicList = response.parseAs<VoyceMeSeriesResponse>()
.data.series.map(VoyceMeComic::toSManga)
val comicList = result["data"]!!.jsonObject["voyce_series"]!!
.let { json.decodeFromJsonElement<List<VoyceMeComic>>(it) }
.map(::genericComicBookFromObject)
val hasNextPage = comicList.size == POPULAR_PER_PAGE
return MangasPage(comicList, hasNextPage)
}
override fun latestUpdatesRequest(page: Int): Request {
val payload = buildJsonObject {
put("query", LATEST_QUERY)
putJsonObject("variables") {
put("offset", (page - 1) * POPULAR_PER_PAGE)
put("limit", POPULAR_PER_PAGE)
}
}
val payload = GraphQlQuery(
query = LATEST_QUERY,
variables = LatestQueryVariables(
offset = (page - 1) * POPULAR_PER_PAGE,
limit = POPULAR_PER_PAGE,
),
)
val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
val newHeaders = headersBuilder()
.add("Content-Length", body.contentLength().toString())
@ -111,28 +94,19 @@ class VoyceMe : HttpSource() {
return POST(GRAPHQL_URL, newHeaders, body)
}
override fun latestUpdatesParse(response: Response): MangasPage {
val result = json.parseToJsonElement(response.body.string()).jsonObject
val comicList = result["data"]!!.jsonObject["voyce_series"]!!
.let { json.decodeFromJsonElement<List<VoyceMeComic>>(it) }
.map(::genericComicBookFromObject)
val hasNextPage = comicList.size == POPULAR_PER_PAGE
return MangasPage(comicList, hasNextPage)
}
override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response)
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val payload = buildJsonObject {
put("query", SEARCH_QUERY)
putJsonObject("variables") {
put("searchTerm", "%$query%")
put("offset", (page - 1) * POPULAR_PER_PAGE)
put("limit", POPULAR_PER_PAGE)
}
}
val payload = GraphQlQuery(
query = SEARCH_QUERY,
variables = SearchQueryVariables(
searchTerm = "%$query%",
offset = (page - 1) * POPULAR_PER_PAGE,
limit = POPULAR_PER_PAGE,
),
)
val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
val newHeaders = headersBuilder()
.add("Content-Length", body.contentLength().toString())
@ -142,39 +116,19 @@ class VoyceMe : HttpSource() {
return POST(GRAPHQL_URL, newHeaders, body)
}
override fun searchMangaParse(response: Response): MangasPage {
val result = json.parseToJsonElement(response.body.string()).jsonObject
override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response)
val comicList = result["data"]!!.jsonObject["voyce_series"]!!
.let { json.decodeFromJsonElement<List<VoyceMeComic>>(it) }
.map(::genericComicBookFromObject)
val hasNextPage = comicList.size == POPULAR_PER_PAGE
return MangasPage(comicList, hasNextPage)
}
// Workaround to allow "Open in browser" use the real URL.
override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
return client.newCall(mangaDetailsApiRequest(manga))
.asObservableSuccess()
.map { response ->
mangaDetailsParse(response).apply { initialized = true }
}
}
private fun mangaDetailsApiRequest(manga: SManga): Request {
override fun mangaDetailsRequest(manga: SManga): Request {
val comicSlug = manga.url
.substringAfter("/series/")
.substringBefore("/")
val payload = buildJsonObject {
put("query", DETAILS_QUERY)
putJsonObject("variables") {
put("slug", comicSlug)
}
}
val payload = GraphQlQuery(
query = DETAILS_QUERY,
variables = DetailsQueryVariables(slug = comicSlug),
)
val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
val newHeaders = headersBuilder()
.add("Content-Length", body.contentLength().toString())
@ -185,18 +139,11 @@ class VoyceMe : HttpSource() {
return POST(GRAPHQL_URL, newHeaders, body)
}
override fun mangaDetailsParse(response: Response): SManga = SManga.create().apply {
val result = json.parseToJsonElement(response.body.string()).jsonObject
val comic = result["data"]!!.jsonObject["voyce_series"]!!.jsonArray[0].jsonObject
.let { json.decodeFromJsonElement<VoyceMeComic>(it) }
override fun getMangaUrl(manga: SManga) = baseUrl + manga.url
title = comic.title
author = comic.author?.username.orEmpty()
description = Parser.unescapeEntities(comic.description.orEmpty(), true)
.let { Jsoup.parse(it).text() }
status = comic.status.orEmpty().toStatus()
genre = comic.genres.mapNotNull { it.genre?.title }.joinToString(", ")
thumbnail_url = STATIC_URL + comic.thumbnail
override fun mangaDetailsParse(response: Response): SManga {
return response.parseAs<VoyceMeSeriesResponse>()
.data.series.first().toSManga()
}
override fun chapterListRequest(manga: SManga): Request {
@ -204,14 +151,12 @@ class VoyceMe : HttpSource() {
.substringAfter("/series/")
.substringBefore("/")
val payload = buildJsonObject {
put("query", CHAPTERS_QUERY)
putJsonObject("variables") {
put("slug", comicSlug)
}
}
val payload = GraphQlQuery(
query = CHAPTERS_QUERY,
variables = ChaptersQueryVariables(slug = comicSlug),
)
val body = payload.toString().toRequestBody(JSON_MEDIA_TYPE)
val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
val newHeaders = headersBuilder()
.add("Content-Length", body.contentLength().toString())
@ -223,72 +168,41 @@ class VoyceMe : HttpSource() {
}
override fun chapterListParse(response: Response): List<SChapter> {
val result = json.parseToJsonElement(response.body.string()).jsonObject
val comicBook = result["data"]!!.jsonObject["voyce_series"]!!.jsonArray[0].jsonObject
.let { json.decodeFromJsonElement<VoyceMeComic>(it) }
val comic = response.parseAs<VoyceMeSeriesResponse>().data.series.first()
return comicBook.chapters
.map { chapter -> chapterFromObject(chapter, comicBook) }
.distinctBy { chapter -> chapter.name }
return comic.chapters
.map { it.toSChapter(comic.slug) }
.distinctBy(SChapter::name)
}
private fun chapterFromObject(chapter: VoyceMeChapter, comic: VoyceMeComic): SChapter =
SChapter.create().apply {
name = chapter.title
date_upload = chapter.createdAt.toDate()
url = "/series/${comic.slug}/${chapter.id}#comic"
}
override fun pageListRequest(chapter: SChapter): Request {
val newHeaders = headersBuilder()
.set("Referer", baseUrl + chapter.url.substringBeforeLast("/"))
.build()
return GET(baseUrl + chapter.url, newHeaders)
}
private fun pageListApiRequest(buildId: String, chapterUrl: String): Request {
val newHeaders = headersBuilder()
.set("Referer", baseUrl + chapterUrl)
.build()
val comicSlug = chapterUrl
.substringAfter("/series/")
.substringBefore("/")
val chapterId = chapterUrl
.substringAfterLast("/")
.substringBefore("#")
return GET("$baseUrl/_next/data/$buildId/series/$comicSlug/$chapterId.json", newHeaders)
}
override fun pageListParse(response: Response): List<Page> {
// GraphQL endpoints do not have the chapter images, so we need
// to get the buildId to fetch the chapter from NextJS static data.
val document = response.asJsoup()
val nextData = document.selectFirst("script#__NEXT_DATA__")!!.data()
val nextJson = json.parseToJsonElement(nextData).jsonObject
val buildId = nextJson["buildId"]!!.jsonPrimitive.content
val chapterUrl = response.request.url.toString().substringAfter(baseUrl)
val dataRequest = pageListApiRequest(buildId, chapterUrl)
val dataResponse = client.newCall(dataRequest).execute()
val dataJson = json.parseToJsonElement(dataResponse.body.string()).jsonObject
val comic = dataJson["pageProps"]!!.jsonObject["series"]!!
.let { json.decodeFromJsonElement<VoyceMeComic>(it) }
val chapterId = response.request.url.toString()
val chapterId = chapter.url
.substringAfterLast("/")
.substringBefore("#")
.toInt()
val chapter = comic.chapters.firstOrNull { it.id == chapterId }
?: throw Exception(CHAPTER_DATA_NOT_FOUND)
return chapter.images.mapIndexed { i, page ->
Page(i, baseUrl, STATIC_URL + page.image)
}
val payload = GraphQlQuery(
query = PAGES_QUERY,
variables = PagesQueryVariables(chapterId = chapterId),
)
val body = json.encodeToString(payload).toRequestBody(JSON_MEDIA_TYPE)
val newHeaders = headersBuilder()
.add("Content-Length", body.contentLength().toString())
.add("Content-Type", body.contentType().toString())
.build()
return POST(GRAPHQL_URL, newHeaders, body)
}
override fun getChapterUrl(chapter: SChapter) = baseUrl + chapter.url
override fun pageListParse(response: Response): List<Page> {
return response.parseAs<VoyceMeChapterImagesResponse>().data.images
.mapIndexed { i, page ->
Page(i, baseUrl, STATIC_URL + page.image)
}
}
override fun imageUrlParse(response: Response): String = ""
@ -302,33 +216,19 @@ class VoyceMe : HttpSource() {
return GET(page.imageUrl!!, newHeaders)
}
private fun String.toDate(): Long {
return try {
DATE_FORMATTER.parse(this)?.time ?: 0L
} catch (e: ParseException) {
0L
}
}
private fun String.toStatus(): Int = when (this) {
"completed" -> SManga.COMPLETED
"ongoing" -> SManga.ONGOING
else -> SManga.UNKNOWN
private inline fun <reified T> Response.parseAs(): T = use {
json.decodeFromString(it.body.string())
}
companion object {
private const val ACCEPT_ALL = "*/*"
private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
private const val STATIC_URL = "https://dlkfxmdtxtzpb.cloudfront.net/"
const val STATIC_URL = "https://dlkfxmdtxtzpb.cloudfront.net/"
private const val GRAPHQL_URL = "https://graphql.voyce.me/v1/graphql"
private const val POPULAR_PER_PAGE = 10
private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaTypeOrNull()
private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) }
private const val CHAPTER_DATA_NOT_FOUND = "Chapter data not found in website."
}
}

85
src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeDto.kt
View File

@ -1,7 +1,13 @@
package eu.kanade.tachiyomi.extension.en.voyceme
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import org.jsoup.Jsoup
import org.jsoup.parser.Parser
import java.text.SimpleDateFormat
import java.util.Locale
@Serializable
data class VoyceMeComic(
@ -14,7 +20,24 @@ data class VoyceMeComic(
val status: String? = "",
val thumbnail: String = "",
val title: String = "",
)
) {
fun toSManga(): SManga = SManga.create().apply {
title = this@VoyceMeComic.title
author = this@VoyceMeComic.author?.username.orEmpty()
description = Parser
.unescapeEntities(this@VoyceMeComic.description.orEmpty(), true)
.let { Jsoup.parseBodyFragment(it).text() }
status = when (this@VoyceMeComic.status.orEmpty()) {
"completed" -> SManga.COMPLETED
"ongoing" -> SManga.ONGOING
else -> SManga.UNKNOWN
}
genre = genres.mapNotNull { it.genre?.title }.joinToString(", ")
url = "/series/$slug"
thumbnail_url = VoyceMe.STATIC_URL + thumbnail
}
}
@Serializable
data class VoyceMeAuthor(
@ -37,9 +60,67 @@ data class VoyceMeChapter(
val id: Int = -1,
val images: List<VoyceMePage> = emptyList(),
val title: String = "",
)
) {
fun toSChapter(comicSlug: String): SChapter = SChapter.create().apply {
name = title
date_upload = runCatching { DATE_FORMATTER.parse(createdAt)?.time }
.getOrNull() ?: 0L
url = "/series/$comicSlug/$id#comic"
}
companion object {
private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) }
}
}
@Serializable
data class VoyceMePage(
val image: String = "",
)
@Serializable
data class GraphQlQuery<T>(
val variables: T,
val query: String,
)
@Serializable
data class GraphQlResponse<T>(val data: T)
typealias VoyceMeSeriesResponse = GraphQlResponse<VoyceMeSeriesCollection>
typealias VoyceMeChapterImagesResponse = GraphQlResponse<VoyceChapterImagesCollection>
@Serializable
data class VoyceMeSeriesCollection(
@SerialName("voyce_series")
val series: List<VoyceMeComic> = emptyList(),
)
@Serializable
data class VoyceChapterImagesCollection(
@SerialName("voyce_chapter_images")
val images: List<VoyceMePage> = emptyList(),
)
@Serializable
data class PopularQueryVariables(
val offset: Int,
val limit: Int,
)
@Serializable
data class SearchQueryVariables(
val offset: Int,
val limit: Int,
val searchTerm: String,
)
@Serializable
data class DetailsQueryVariables(val slug: String)
@Serializable
data class PagesQueryVariables(val chapterId: Int)
typealias LatestQueryVariables = PopularQueryVariables
typealias ChaptersQueryVariables = DetailsQueryVariables

13
src/en/voyceme/src/eu/kanade/tachiyomi/extension/en/voyceme/VoyceMeQueries.kt
View File

@ -113,3 +113,16 @@ val CHAPTERS_QUERY: String = buildQuery {
}
""".trimIndent()
}
val PAGES_QUERY: String = buildQuery {
"""
query(%chapterId: Int!) {
voyce_chapter_images(
where: { chapter_id: { _eq: %chapterId } },
order_by: { sort_order: asc }
) {
image
}
}
""".trimIndent()
}