From 1175b0d1c70b3fdf5d5742b10d21e0c8e69261ff Mon Sep 17 00:00:00 2001 From: h-hyuuga <83582211+h-hyuuga@users.noreply.github.com> Date: Tue, 15 Jun 2021 09:02:46 -0400 Subject: [PATCH] Replace json library with kotlinx.serialization in multiple sources (#7407) * Catmanga: Replace org.json with kotlinx.serialization + Light Refactor of #7451 * Genkan IO: Replace gson + Make livewire interceptor * Genkan IO: Tail Call Optimization to avoid blowing stack * Comick.fun: kotlinx.serialization migration * Remanga: kotlinx.serialzation migration --- src/all/comickfun/build.gradle | 3 +- .../extension/all/comickfun/ComickFun.kt | 172 +++------- .../all/comickfun/ComickFunSerialization.kt | 261 +++++++++++++++ src/all/genkanio/build.gradle | 3 +- .../extension/all/genkanio/GenkanIO.kt | 314 ++++++++++-------- src/en/catmanga/build.gradle | 1 + .../extension/en/catmanga/CatManga.kt | 286 +++++++--------- src/ru/remanga/build.gradle | 3 +- .../tachiyomi/extension/ru/remanga/Remanga.kt | 52 +-- .../tachiyomi/extension/ru/remanga/dto/Dto.kt | 25 +- 10 files changed, 653 insertions(+), 467 deletions(-) create mode 100644 src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunSerialization.kt diff --git a/src/all/comickfun/build.gradle b/src/all/comickfun/build.gradle index cd4b91cf1..f753a22fb 100644 --- a/src/all/comickfun/build.gradle +++ b/src/all/comickfun/build.gradle @@ -1,11 +1,12 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' +apply plugin: 'kotlinx-serialization' ext { extName = 'Comick.fun' pkgNameSuffix = 'all.comickfun' extClass = '.ComickFunFactory' - extVersionCode = 1 + extVersionCode = 3 libVersion = '1.2' containsNsfw = true } diff --git a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt index 0f9ca0503..c79b22235 100644 --- a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt +++ b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt @@ -1,13 +1,5 @@ package eu.kanade.tachiyomi.extension.all.comickfun -import android.os.Build -import android.text.Html -import com.github.salomonbrys.kotson.array -import com.github.salomonbrys.kotson.get -import com.github.salomonbrys.kotson.nullString -import com.github.salomonbrys.kotson.obj -import com.google.gson.JsonElement -import com.google.gson.JsonParser import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess @@ -18,6 +10,13 @@ import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.HttpSource +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromJsonElement +import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.modules.polymorphic import okhttp3.CacheControl import okhttp3.Headers import okhttp3.HttpUrl @@ -27,10 +26,9 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response import rx.Observable +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get import java.lang.UnsupportedOperationException -import java.text.SimpleDateFormat -import kotlin.math.pow -import kotlin.math.truncate const val SEARCH_PAGE_LIMIT = 100 @@ -40,7 +38,18 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S private val apiBase = "$baseUrl/api" override val supportsLatest = true - private val mangaIdCache = mutableMapOf() + @ExperimentalSerializationApi + private val json: Json by lazy { + Json(from = Injekt.get()) { + serializersModule = SerializersModule { + polymorphic(SManga::class) { default { SMangaDeserializer() } } + polymorphic(SChapter::class) { default { SChapterDeserializer() } } + } + } + } + + @ExperimentalSerializationApi + private val mangaIdCache = SMangaDeserializer.mangaIdCache final override fun headersBuilder() = Headers.Builder().apply { add("User-Agent", "Tachiyomi " + System.getProperty("http.agent")) @@ -80,92 +89,37 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S /** Utils **/ - /** - * Parses a json object with information suitable for showing an entry of a manga within a - * catalogue - * - * Attempts to cache the manga's numerical Id - * - * @return SManga - with url, thumbnail_url and title set - */ - private fun parseMangaObj(it: JsonElement) = it.asJsonObject.let { info -> - info["id"]?.asInt?.let { mangaIdCache.getOrPut(info["slug"].asString, { it }) } - val thumbnail = info["coverURL"]?.nullString - ?: info["md_covers"]?.asJsonArray?.get(0)?.asJsonObject?.let { cover -> - cover["gpurl"]?.nullString ?: "$baseUrl${cover["url"].asString}" - } - - SManga.create().apply { - url = "/comic/${info["slug"].asString}" - thumbnail_url = thumbnail - title = info["title"].asString - } - } - /** Returns an observable which emits a single value -> the manga's id **/ + @ExperimentalSerializationApi private fun chapterId(manga: SManga): Observable { val mangaSlug = slug(manga) return mangaIdCache[mangaSlug]?.let { Observable.just(it) } ?: fetchMangaDetails(manga).map { mangaIdCache[mangaSlug] } } - private fun parseStatus(status: Int) = when (status) { - 1 -> SManga.ONGOING - 2 -> SManga.COMPLETED - else -> SManga.UNKNOWN - } - - /** Attempts to parse an ISO-8601 compliant Date Time string with offset to epoch. - * @returns epochtime on success, 0 on failure - **/ - private fun parseISO8601(s: String): Long { - var fractionalPart_ms: Long = 0 - val sNoFraction = Regex("""\.\d+""").replace(s) { match -> - fractionalPart_ms = truncate( - match.value.substringAfter(".").toFloat() * 10.0f.pow(-(match.value.length - 1)) * // seconds - 1000 // milliseconds - ).toLong() - "" - } - - val ret = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZZZZZ").parse(sNoFraction)?.let { - fractionalPart_ms + it.time - } ?: 0 - return ret - } - /** Returns an identifier referred to as `hid` for chapter **/ private fun hid(chapter: SChapter) = "$baseUrl${chapter.url}".toHttpUrl().pathSegments[2].substringBefore("-") /** Returns an identifier referred to as a `slug` for manga **/ private fun slug(manga: SManga) = "$baseUrl${manga.url}".toHttpUrl().pathSegments[1] - private fun formatChapterTitle(title: String?, chap: String?, vol: String?): String { - val numNonNull = listOfNotNull(title.takeIf { !it.isNullOrBlank() }, chap, vol).size - if (numNonNull == 0) throw RuntimeException("formatChapterTitle requires at least one non-null argument") - - var formattedTitle = StringBuilder() - if (vol != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Vol." } ?: "Volume"} $vol") - if (vol != null && chap != null) formattedTitle.append(", ") - if (chap != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Ch." } ?: "Chapter"} $chap") - if (!title.isNullOrBlank()) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { ": " } ?: ""} $title") - return formattedTitle.toString() - } - /** Popular Manga **/ + @ExperimentalSerializationApi override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", FilterList(emptyList())) override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException("Not used") override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Not used") /** Latest Manga **/ + @ExperimentalSerializationApi override fun latestUpdatesParse(response: Response): MangasPage { val noResults = MangasPage(emptyList(), false) if (response.code == 204) return noResults - return JsonParser.parseString(response.body!!.string()).obj["data"]?.array?.let { manga -> - MangasPage(manga.map { parseMangaObj(it["md_comics"]) }, true) - } ?: noResults + return json.decodeFromString( + deserializer = deepSelectDeserializer>("data"), + response.body!!.string() + ).let { MangasPage(it, true) } } override fun latestUpdatesRequest(page: Int): Request { @@ -175,6 +129,7 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S return GET("$url", headers) } + @ExperimentalSerializationApi override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { if (!query.startsWith(SLUG_SEARCH_PREFIX)) return super.fetchSearchManga(page, query, filters) @@ -205,11 +160,14 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S return GET("$url", headers) } - override fun searchMangaParse(response: Response): MangasPage = JsonParser.parseString(response.body!!.string()).let { - if (it.isJsonObject) - MangasPage(it["comics"].array.map(::parseMangaObj), it["comics"].array.size() == SEARCH_PAGE_LIMIT) - else // search_title isn't paginated - MangasPage(it.array.map(::parseMangaObj), false) + @ExperimentalSerializationApi + override fun searchMangaParse(response: Response): MangasPage = json.parseToJsonElement(response.body!!.string()).let { parsed -> + when (parsed) { + is JsonObject -> json.decodeFromJsonElement>(parsed["comics"]!!) + .let { MangasPage(it, it.size == SEARCH_PAGE_LIMIT) } + is JsonArray -> MangasPage(json.decodeFromJsonElement(parsed), false) + else -> MangasPage(emptyList(), false) + } } /** Manga Details **/ @@ -219,6 +177,7 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S } // Shenanigans to allow "open in webview" to show a webpage instead of JSON + @ExperimentalSerializationApi override fun fetchMangaDetails(manga: SManga): Observable { return client.newCall(apiMangaDetailsRequest(manga)) .asObservableSuccess() @@ -227,36 +186,18 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S } } - override fun mangaDetailsParse(response: Response) = JsonParser.parseString(response.body!!.string())["data"].let { data -> - fun cleanDesc(s: String) = ( - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) - Html.fromHtml(s, Html.FROM_HTML_MODE_LEGACY) else Html.fromHtml(s) - ).toString() - - fun nameList(e: JsonElement?) = e?.array?.asSequence()?.map { it["name"].asString } - data["comic"]["id"].asInt.let { mangaIdCache.getOrPut(response.request.url.queryParameter("slug")!!, { it }) } - SManga.create().apply { - title = data["comic"]["title"].asString - thumbnail_url = data["coverURL"].asString - description = cleanDesc(data["comic"]["desc"].asString) - status = parseStatus(data["comic"]["status"].asInt) - artist = nameList(data["artists"])?.joinToString(", ") - author = nameList(data["authors"])?.joinToString(", ") - genre = ( - (nameList(data["genres"]) ?: sequenceOf()) + sequence { - data["demographic"].nullString?.let { yield(it) } - mapOf("kr" to "Manhwa", "jp" to "Manga", "cn" to "Manhua")[data["comic"]["country"].nullString] - ?.let { yield(it) } - } - ).joinToString(", ") - } - } + @ExperimentalSerializationApi + override fun mangaDetailsParse(response: Response) = json.decodeFromString( + deserializer = deepSelectDeserializer("data", tDeserializer = jsonFlatten(objKey = "comic", "id", "title", "desc", "status", "country", "slug")), + response.body!!.string() + ) /** Chapter List **/ private fun chapterListRequest(page: Int, mangaId: Int) = GET("$apiBase/get_chapters?comicid=$mangaId&page=$page&limit=$SEARCH_PAGE_LIMIT", headers) + @ExperimentalSerializationApi override fun fetchChapterList(manga: SManga): Observable> { return if (manga.status != SManga.LICENSED) { chapterId(manga).concatMap { id -> @@ -281,25 +222,22 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S } } - override fun chapterListParse(response: Response) = JsonParser.parseString(response.body!!.string()).obj["data"]["chapters"].array.map { elem -> - val chapter = elem.asJsonObject - val num = chapter["chap"].nullString ?: "-1" - SChapter.create().apply { - date_upload = parseISO8601(chapter["created_at"].asString) - name = formatChapterTitle(chapter["title"].nullString, chapter["chap"].nullString, chapter["vol"].nullString) - chapter_number = num.toFloat() - url = "/${chapter["hid"].asString}-chapter-${chapter["chap"].nullString}-${chapter["iso639_1"].asString}" // incomplete, is finished in fetchChapterList - scanlator = chapter.get("md_groups")?.array?.get(0)?.obj?.get("title")?.asString - } - } + @ExperimentalSerializationApi + override fun chapterListParse(response: Response) = json.decodeFromString( + deserializer = deepSelectDeserializer>("data", "chapters"), + response.body!!.string() + ) /** Page List **/ override fun pageListRequest(chapter: SChapter) = GET("$apiBase/get_chapter?hid=${hid(chapter)}", headers, CacheControl.FORCE_NETWORK) - override fun pageListParse(response: Response) = JsonParser.parseString(response.body!!.string())["data"]["chapter"]["images"].array.mapIndexed { i, url -> - Page(i, imageUrl = url.asString) - } + @ExperimentalSerializationApi + override fun pageListParse(response: Response) = + json.decodeFromString( + deserializer = deepSelectDeserializer>("data", "chapter", "images"), + response.body!!.string() + ).mapIndexed { i, url -> Page(i, imageUrl = url) } override fun imageUrlParse(response: Response) = "" // idk what this does, leave me alone kotlin diff --git a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunSerialization.kt b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunSerialization.kt new file mode 100644 index 000000000..3d4c9f82e --- /dev/null +++ b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunSerialization.kt @@ -0,0 +1,261 @@ +package eu.kanade.tachiyomi.extension.all.comickfun + +import android.os.Build +import android.text.Html +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.descriptors.buildClassSerialDescriptor +import kotlinx.serialization.descriptors.element +import kotlinx.serialization.encoding.CompositeDecoder +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.encoding.decodeStructure +import kotlinx.serialization.json.JsonDecoder +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonTransformingSerializer +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.serializer +import java.text.SimpleDateFormat +import kotlin.math.pow +import kotlin.math.truncate + +/** + * A serializer of type T which selects the value of type T by traversing down a chain of json objects + * + * e.g + * { + * "user": { + * "name": { + * "first": "John", + * "last": "Smith" + * } + * } + * } + * + * deepSelectDeserializer<String>("user", "name", "first") deserializes the above into "John" + */ +@ExperimentalSerializationApi +inline fun deepSelectDeserializer(vararg keys: String, tDeserializer: KSerializer = serializer()): KSerializer { + val descriptors = keys.foldRight(listOf(tDeserializer.descriptor)) { x, acc -> + acc + acc.last().let { + buildClassSerialDescriptor("$x\$${it.serialName}") { element(x, it) } + } + }.asReversed() + var a: ((Int) -> KSerializer)? = null + val b = { depth: Int -> + object : KSerializer { + override val descriptor = descriptors[depth] + + override fun deserialize(decoder: Decoder): T { + return if (depth == keys.size) decoder.decodeSerializableValue(tDeserializer) + else decoder.decodeStructureByKnownName(descriptor) { names -> + names.filter { (name, _) -> name == keys[depth] } + .map { (_, index) -> decodeSerializableElement(descriptor, index, a!!(depth + 1)) } + .single() + } + } + + override fun serialize(encoder: Encoder, value: T) = throw UnsupportedOperationException("Not supported") + } + } + a = b // this is the hackiest of hacky hacks to get around not being able to define recursive inline functions + return a(0) +} + +/** + * Transforms given json element by lifting specified keys in `element[objKey]` up into `element` + * Existing conflicts are overwritten + * + * @param objKey: String - A key identifying an object in JsonElement + * @param keys: vararg String - Keys identifying values to lift from objKey + */ +inline fun jsonFlatten(objKey: String, vararg keys: String, tDeserializer: KSerializer = serializer()): JsonTransformingSerializer { + return object : JsonTransformingSerializer(tDeserializer) { + override fun transformDeserialize(element: JsonElement) = buildJsonObject { + require(element is JsonObject) + element.entries.forEach { (key, value) -> put(key, value) } + val fromObj = element[objKey] + require(fromObj is JsonObject) + keys.forEach { put(it, fromObj[it]!!) } + } + } +} + +@ExperimentalSerializationApi +inline fun Decoder.decodeStructureByKnownName(descriptor: SerialDescriptor, decodeFn: CompositeDecoder.(Sequence>) -> T): T { + return decodeStructure(descriptor) { + decodeFn( + generateSequence { decodeElementIndex(descriptor) } + .takeWhile { it != CompositeDecoder.DECODE_DONE } + .filter { it != CompositeDecoder.UNKNOWN_NAME } + .map { descriptor.getElementName(it) to it } + ) + } +} + +@ExperimentalSerializationApi +class SChapterDeserializer : KSerializer { + override val descriptor = buildClassSerialDescriptor(SChapter::class.qualifiedName!!) { + element("chap") + element("hid") + element("title") + element("vol", isOptional = true) + element("created_at") + element("iso639_1") + element>("images", isOptional = true) + element>("md_groups", isOptional = true) + } + + /** Attempts to parse an ISO-8601 compliant Date Time string with offset to epoch. + * @returns epochtime on success, 0 on failure + **/ + private fun parseISO8601(s: String): Long { + var fractionalPart_ms: Long = 0 + val sNoFraction = Regex("""\.\d+""").replace(s) { match -> + fractionalPart_ms = truncate( + match.value.substringAfter(".").toFloat() * 10.0f.pow(-(match.value.length - 1)) * // seconds + 1000 // milliseconds + ).toLong() + "" + } + + return SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZZZZZ").parse(sNoFraction)?.let { + fractionalPart_ms + it.time + } ?: 0 + } + + private fun formatChapterTitle(title: String?, chap: String?, vol: String?): String { + val numNonNull = listOfNotNull(title.takeIf { !it.isNullOrBlank() }, chap, vol).size + if (numNonNull == 0) throw RuntimeException("formatChapterTitle requires at least one non-null argument") + + val formattedTitle = StringBuilder() + if (vol != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Vol." } ?: "Volume"} $vol") + if (vol != null && chap != null) formattedTitle.append(", ") + if (chap != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Ch." } ?: "Chapter"} $chap") + if (!title.isNullOrBlank()) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { ": " } ?: ""} $title") + return formattedTitle.toString() + } + + @ExperimentalSerializationApi + override fun deserialize(decoder: Decoder): SChapter { + return SChapter.create().apply { + var chap: String? = null + var vol: String? = null + var title: String? = null + var hid = "" + var iso639_1 = "" + require(decoder is JsonDecoder) + decoder.decodeStructureByKnownName(descriptor) { names -> + for ((name, index) in names) { + when (name) { + "created_at" -> date_upload = parseISO8601(decodeStringElement(descriptor, index)) + "title" -> title = decodeNullableSerializableElement(descriptor, index, serializer()) + "vol" -> vol = decodeNullableSerializableElement(descriptor, index, serializer()) + "chap" -> { + chap = decodeStringElement(descriptor, index) + chapter_number = chap!!.toFloat() + } + "hid" -> hid = decodeStringElement(descriptor, index) + "iso639_1" -> iso639_1 = decodeStringElement(descriptor, index) + "md_groups" -> scanlator = decodeSerializableElement(descriptor, index, ListSerializer(deepSelectDeserializer("title"))).joinToString(", ") + } + } + } + name = formatChapterTitle(title, chap, vol) + url = "/$hid-chapter-$chap-$iso639_1" // incomplete, is finished in fetchChapterList + } + } + + override fun serialize(encoder: Encoder, value: SChapter) = throw UnsupportedOperationException("Unsupported") +} + +@ExperimentalSerializationApi +class SMangaDeserializer : KSerializer { + private fun cleanDesc(s: String) = ( + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) + Html.fromHtml(s, Html.FROM_HTML_MODE_LEGACY) else Html.fromHtml(s) + ).toString() + + private fun parseStatus(status: Int) = when (status) { + 1 -> SManga.ONGOING + 2 -> SManga.COMPLETED + else -> SManga.UNKNOWN + } + + override val descriptor = buildClassSerialDescriptor(SManga::class.qualifiedName!!) { + element("slug") + element("title") + element("coverURL") + element("id", isOptional = true) + element>("artists", isOptional = true) + element>("authors", isOptional = true) + element("desc", isOptional = true) + element("demographic", isOptional = true) + element>("genres", isOptional = true) + element("status", isOptional = true) + element("country", isOptional = true) + } + + @ExperimentalSerializationApi + override fun deserialize(decoder: Decoder): SManga { + return SManga.create().apply { + var id: Int? = null + var slug: String? = null + val tryTo = ( + { + var hasThrown = false; + { fn: () -> Unit -> + if (!hasThrown) { + try { + fn() + } catch (_: java.lang.Exception) { + hasThrown = true + } + } + } + } + )() + decoder.decodeStructureByKnownName(descriptor) { names -> + for ((name, index) in names) { + val sluggedNameSerializer = ListSerializer(deepSelectDeserializer("name")) + fun nameList() = decodeSerializableElement(descriptor, index, sluggedNameSerializer).joinToString(", ") + when (name) { + "slug" -> { + slug = decodeStringElement(descriptor, index) + url = "/comic/$slug" + } + "title" -> title = decodeStringElement(descriptor, index) + "coverURL" -> thumbnail_url = decodeStringElement(descriptor, index) + "id" -> id = decodeIntElement(descriptor, index) + "artists" -> artist = nameList() + "authors" -> author = nameList() + "desc" -> description = cleanDesc(decodeStringElement(descriptor, index)) + // Isn't always a string in every api call + "demographic" -> tryTo { genre = listOfNotNull(genre, decodeStringElement(descriptor, index)).joinToString(", ") } + // Isn't always a list of objects in every api call + "genres" -> tryTo { genre = listOfNotNull(genre, nameList()).joinToString(", ") } + "status" -> status = parseStatus(decodeIntElement(descriptor, index)) + "country" -> genre = listOfNotNull( + genre, + mapOf("kr" to "Manhwa", "jp" to "Manga", "cn" to "Manhua")[decodeStringElement(descriptor, index)] + ).joinToString(", ") + } + } + } + if (id != null && slug != null) { + mangaIdCache[slug!!] = id!! + } + } + } + + override fun serialize(encoder: Encoder, value: SManga) = throw UnsupportedOperationException("Not supported") + + companion object { + val mangaIdCache = mutableMapOf() + } +} diff --git a/src/all/genkanio/build.gradle b/src/all/genkanio/build.gradle index 9ed57456c..9dc24ff36 100644 --- a/src/all/genkanio/build.gradle +++ b/src/all/genkanio/build.gradle @@ -1,11 +1,12 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' +apply plugin: 'kotlinx-serialization' ext { extName = 'Genkan.io' pkgNameSuffix = "all.genkanio" extClass = '.GenkanIO' - extVersionCode = 2 + extVersionCode = 3 libVersion = '1.2' } diff --git a/src/all/genkanio/src/eu/kanade/tachiyomi/extension/all/genkanio/GenkanIO.kt b/src/all/genkanio/src/eu/kanade/tachiyomi/extension/all/genkanio/GenkanIO.kt index e0c6d7a71..0ef7e826b 100644 --- a/src/all/genkanio/src/eu/kanade/tachiyomi/extension/all/genkanio/GenkanIO.kt +++ b/src/all/genkanio/src/eu/kanade/tachiyomi/extension/all/genkanio/GenkanIO.kt @@ -1,32 +1,38 @@ package eu.kanade.tachiyomi.extension.all.genkanio import android.util.Log -import com.github.salomonbrys.kotson.keys -import com.github.salomonbrys.kotson.put -import com.google.gson.JsonArray -import com.google.gson.JsonElement -import com.google.gson.JsonObject -import com.google.gson.JsonParser import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.POST import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.ParsedHttpSource import eu.kanade.tachiyomi.util.asJsoup -import okhttp3.Headers +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.boolean +import kotlinx.serialization.json.buildJsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody +import okio.Buffer import org.jsoup.nodes.Document import org.jsoup.nodes.Element import rx.Observable +import uy.kohesive.injekt.injectLazy import java.util.Calendar open class GenkanIO : ParsedHttpSource() { @@ -35,37 +41,141 @@ open class GenkanIO : ParsedHttpSource() { final override val baseUrl = "https://genkan.io" final override val supportsLatest = false - data class LiveWireRPC(val csrf: String, val state: JsonObject) - private var livewire: LiveWireRPC? = null + private val json: Json by injectLazy() - /** - * Given a string encoded with html entities and escape sequences, makes an attempt to decode - * and returns decoded string - * - * Warning: This is not all all exhaustive, and probably misses edge cases - * - * @Returns decoded string + /** An interceptor which encapsulates the logic needed to interoperate with Genkan.io's + * livewire server, which uses a form a Remote Procedure call */ - private fun htmlDecode(html: String): String { - return html.replace(Regex("&([A-Za-z]+);")) { match -> - mapOf( - "raquo" to "»", - "laquo" to "«", - "amp" to "&", - "lt" to "<", - "gt" to ">", - "quot" to "\"" - )[match.groups[1]!!.value] ?: match.groups[0]!!.value - }.replace(Regex("\\\\(.)")) { match -> - mapOf( - "t" to "\t", - "n" to "\n", - "r" to "\r", - "b" to "\b" - )[match.groups[1]!!.value] ?: match.groups[1]!!.value + private val livewireInterceptor = object : Interceptor { + private lateinit var fingerprint: JsonElement + lateinit var serverMemo: JsonObject + private lateinit var csrf: String + var initialized = false + val serverUrl = "$baseUrl/livewire/message/manga.list-all-manga" + + /** + * Given a string encoded with html entities and escape sequences, makes an attempt to decode + * and returns decoded string + * + * Warning: This is not all all exhaustive, and probably misses edge cases + * + * @Returns decoded string + */ + private fun htmlDecode(html: String): String { + return html.replace(Regex("&([A-Za-z]+);")) { match -> + mapOf( + "raquo" to "»", + "laquo" to "«", + "amp" to "&", + "lt" to "<", + "gt" to ">", + "quot" to "\"" + )[match.groups[1]!!.value] ?: match.groups[0]!!.value + }.replace(Regex("\\\\(.)")) { match -> + mapOf( + "t" to "\t", + "n" to "\n", + "r" to "\r", + "b" to "\b" + )[match.groups[1]!!.value] ?: match.groups[1]!!.value + } + } + + /** + * Recursively merges j2 onto j1 in place + * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's + * + */ + private fun mergeLeft(j1: JsonObject, j2: JsonObject): JsonObject = buildJsonObject { + j1.keys.forEach { put(it, j1[it]!!) } + j2.keys.forEach { k -> + when { + j1[k] !is JsonObject -> put(k, j2[k]!!) + j1[k] is JsonObject && j2[k] is JsonObject -> put(k, mergeLeft(j1[k]!!.jsonObject, j2[k]!!.jsonObject)) + } + } + } + + /** + * Initializes lateinit member vars + */ + private fun initLivewire(chain: Interceptor.Chain) { + val response = chain.proceed(GET("$baseUrl/manga", headers)) + val soup = response.asJsoup() + response.body?.close() + val csrfToken = soup.selectFirst("meta[name=csrf-token]")?.attr("content") + + val initialProps = soup.selectFirst("div[wire:initial-data]")?.attr("wire:initial-data")?.let { + json.parseToJsonElement(htmlDecode(it)) + } + + if (csrfToken != null && initialProps is JsonObject) { + csrf = csrfToken + serverMemo = initialProps["serverMemo"]!!.jsonObject + fingerprint = initialProps["fingerprint"]!! + initialized = true + } else { + Log.e("GenkanIo", soup.selectFirst("div[wire:initial-data]")?.toString() ?: "null") + } + } + + /** + * Builds a request for livewire, augmenting the request with required body fields and headers + * + * @param req: Request - A request with a json encoded body, which represent the updates sent to server + * + */ + private fun livewireRequest(req: Request): Request { + val payload = buildJsonObject { + put("fingerprint", fingerprint) + put("serverMemo", serverMemo) + put("updates", json.parseToJsonElement(Buffer().apply { req.body!!.writeTo(this) }.readUtf8())) + }.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) + + return req.newBuilder() + .method(req.method, payload) + .addHeader("x-csrf-token", csrf) + .addHeader("x-livewire", "true") + .build() + } + + /** + * Transforms json response from livewire server into a response which returns html + * + * @param response: Response - The response of sending a message to genkan's livewire server + * + * @return HTML Response - The html embedded within the provided response + */ + private fun livewireResponse(response: Response): Response { + if (!response.isSuccessful) return response + val body = response.body!!.string() + val responseJson = json.parseToJsonElement(body).jsonObject + + // response contains state that we need to preserve + serverMemo = mergeLeft(serverMemo, responseJson["serverMemo"]!!.jsonObject) + + // this seems to be an error state, so reset everything + if (responseJson["effects"]?.jsonObject?.get("html") is JsonNull) { + initialized = false + } + + // Build html response + return response.newBuilder() + .body(htmlDecode("${responseJson["effects"]?.jsonObject?.get("html")}").toResponseBody("Content-Type: text/html; charset=UTF-8".toMediaTypeOrNull())) + .build() + } + + override fun intercept(chain: Interceptor.Chain): Response { + if (chain.request().url.toString() != serverUrl) + return chain.proceed(chain.request()) + + if (!initialized) initLivewire(chain) + return livewireResponse(chain.proceed(livewireRequest(chain.request()))) } } + override val client = super.client.newBuilder().addInterceptor(livewireInterceptor).build() + // popular manga override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", FilterList(emptyList())) @@ -83,119 +193,33 @@ open class GenkanIO : ParsedHttpSource() { // search - /** - * initializes `livewire` local variable using data from https://genkan.io/manga - */ - private fun initLiveWire(response: Response) { - val soup = response.asJsoup() - val csrf = soup.selectFirst("meta[name=csrf-token]")?.attr("content") - - val initialProps = soup.selectFirst("div[wire:initial-data]")?.attr("wire:initial-data")?.let { - JsonParser.parseString(htmlDecode(it)) - } - - if (csrf != null && initialProps?.asJsonObject != null) { - livewire = LiveWireRPC(csrf, initialProps.asJsonObject) - } else { - Log.e("GenkanIo", soup.selectFirst("div[wire:initial-data]")?.toString() ?: "null") - } - } - - /** - * Prepares a request which'll send a message to livewire server - * - * @param url: String - Message endpoint - * @param updates: JsonElement - JsonElement which describes the actions taken by server - * - * @return Request - */ - private fun livewireRequest(url: String, updates: JsonElement): Request { - // assert(livewire != null) - val payload = JsonObject() - payload.put("fingerprint" to livewire!!.state.get("fingerprint")) - payload.put("serverMemo" to livewire!!.state.get("serverMemo")) - payload.put("updates" to updates) - - // not sure why this isn't getting added automatically - val cookie = client.cookieJar.loadForRequest(url.toHttpUrlOrNull()!!).joinToString("; ") { "${it.name}=${it.value}" } - return POST( - url, - Headers.headersOf("x-csrf-token", livewire!!.csrf, "x-livewire", "true", "cookie", cookie, "cache-control", "no-cache, private"), - payload.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) - ) - } - - /** - * Transforms json response from livewire server into a response which returns html - * Also updates `livewire` variable with state returned by livewire server - * - * @param response: Response - The response of sending a message to genkan's livewire server - * - * @return HTML Response - The html embedded within the provided response - */ - private fun livewireResponse(response: Response): Response { - val body = response.body?.string() - val responseJson = JsonParser.parseString(body).asJsonObject - - // response contains state that we need to preserve - mergeLeft(livewire!!.state.get("serverMemo").asJsonObject, responseJson.get("serverMemo").asJsonObject) - - // this seems to be an error state, so reset everything - if (responseJson.get("effects")?.asJsonObject?.get("html")?.isJsonNull == true) { - livewire = null - } - - // Build html response - return response.newBuilder() - .body(htmlDecode("${responseJson.get("effects")?.asJsonObject?.get("html")}").toResponseBody("Content-Type: text/html; charset=UTF-8".toMediaTypeOrNull())) - .build() - } - - /** - * Recursively merges j2 onto j1 in place - * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's - * - */ - private fun mergeLeft(j1: JsonObject, j2: JsonObject) { - j2.keys().forEach { k -> - if (j1.get(k)?.isJsonObject != true) - j1.put(k to j2.get(k)) - else if (j1.get(k).isJsonObject && j2.get(k).isJsonObject) - mergeLeft(j1.get(k).asJsonObject, j2.get(k).asJsonObject) - } - } - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - fun searchRequest() = client.newCall(searchMangaRequest(page, query, filters)).asObservableSuccess().map(::livewireResponse) - return if (livewire == null) { - client.newCall(GET("$baseUrl/manga", headers)) - .asObservableSuccess() - .doOnNext(::initLiveWire) - .concatWith(Observable.defer(::searchRequest)) - .reduce { _, x -> x } - } else { - searchRequest() - }.map(::searchMangaParse) - } - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - // assert(livewire != null) - val updates = JsonArray() - val data = livewire!!.state.get("serverMemo")?.asJsonObject?.get("data")?.asJsonObject!! - if (data["readyToLoad"]?.asBoolean == false) { - updates.add(JsonParser.parseString("""{"type":"callMethod","payload":{"method":"loadManga","params":[]}}""")) - } - val isNewQuery = query != data["search"]?.asString - if (isNewQuery) { - updates.add(JsonParser.parseString("""{"type": "syncInput", "payload": {"name": "search", "value": "$query"}}""")) + val data = if (livewireInterceptor.initialized) livewireInterceptor.serverMemo["data"]!!.jsonObject else buildJsonObject { + put("readyToLoad", JsonPrimitive(false)) + put("page", JsonPrimitive(1)) + put("search", JsonPrimitive("")) } - val currPage = if (isNewQuery) 1 else data["page"]?.asInt + val updates = buildJsonArray { + if (data["readyToLoad"]?.jsonPrimitive?.boolean == false) { + add(json.parseToJsonElement("""{"type":"callMethod","payload":{"method":"loadManga","params":[]}}""")) + } + val isNewQuery = query != data["search"]?.jsonPrimitive?.content + if (isNewQuery) { + add(json.parseToJsonElement("""{"type": "syncInput", "payload": {"name": "search", "value": "$query"}}""")) + } - for (i in (currPage!! + 1)..page) - updates.add(JsonParser.parseString("""{"type":"callMethod","payload":{"method":"nextPage","params":[]}}""")) + val currPage = if (isNewQuery) 1 else data["page"]!!.jsonPrimitive.int - return livewireRequest("$baseUrl/livewire/message/manga.list-all-manga", updates) + for (i in (currPage + 1)..page) + add(json.parseToJsonElement("""{"type":"callMethod","payload":{"method":"nextPage","params":[]}}""")) + } + + return POST( + livewireInterceptor.serverUrl, + headers, + updates.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) + ) } override fun searchMangaFromElement(element: Element): SManga { @@ -220,15 +244,15 @@ open class GenkanIO : ParsedHttpSource() { return if (manga.status != SManga.LICENSED) { // Returns an observable which emits the list of chapters found on a page, // for every page starting from specified page - fun getAllPagesFrom(page: Int): Observable> = + fun getAllPagesFrom(page: Int, pred: Observable> = Observable.just(emptyList())): Observable> = client.newCall(chapterListRequest(manga, page)) .asObservableSuccess() .concatMap { response -> val cp = chapterPageParse(response) if (cp.hasnext) - Observable.just(cp.chapters).concatWith(getAllPagesFrom(page + 1)) + getAllPagesFrom(page + 1, pred = pred.concatWith(Observable.just(cp.chapters))) // tail call to avoid blowing the stack else - Observable.just(cp.chapters) + pred.concatWith(Observable.just(cp.chapters)) } getAllPagesFrom(1).reduce(List::plus) } else { diff --git a/src/en/catmanga/build.gradle b/src/en/catmanga/build.gradle index 2ecc2bdb5..6b98b6bac 100644 --- a/src/en/catmanga/build.gradle +++ b/src/en/catmanga/build.gradle @@ -1,5 +1,6 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' +apply plugin: 'kotlinx-serialization' ext { extName = 'CatManga' diff --git a/src/en/catmanga/src/eu/kanade/tachiyomi/extension/en/catmanga/CatManga.kt b/src/en/catmanga/src/eu/kanade/tachiyomi/extension/en/catmanga/CatManga.kt index 7654730dd..9735e852f 100644 --- a/src/en/catmanga/src/eu/kanade/tachiyomi/extension/en/catmanga/CatManga.kt +++ b/src/en/catmanga/src/eu/kanade/tachiyomi/extension/en/catmanga/CatManga.kt @@ -10,12 +10,20 @@ import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.util.asJsoup +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.decodeFromJsonElement +import kotlinx.serialization.json.jsonObject +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Protocol import okhttp3.Response -import org.json.JSONArray -import org.json.JSONObject +import okhttp3.ResponseBody.Companion.toResponseBody import org.jsoup.nodes.Document import rx.Observable import uy.kohesive.injekt.injectLazy +import java.net.HttpURLConnection class CatManga : HttpSource() { @@ -25,206 +33,111 @@ class CatManga : HttpSource() { override val baseUrl = "https://catmanga.org" override val supportsLatest = true override val lang = "en" + private val json: Json by injectLazy() - override fun popularMangaRequest(page: Int) = GET(baseUrl) + private lateinit var seriesCache: LinkedHashMap // LinkedHashMap to preserve insertion order + private lateinit var latestSeries: List + override val client = super.client.newBuilder().addInterceptor { chain -> + // An interceptor which facilitates caching the data retrieved from the homepage + when (chain.request().url) { + doNothingRequest.url -> Response.Builder().body( + "".toResponseBody("text/plain; charset=utf-8".toMediaType()) + ).code(HttpURLConnection.HTTP_NO_CONTENT).message("").protocol(Protocol.HTTP_1_0).request(chain.request()).build() + homepageRequest.url -> { + /* Homepage embeds a Json Object with information about every single series in the service */ + val response = chain.proceed(chain.request()) + val responseBody = response.peekBody(Long.MAX_VALUE).string() + val seriesList = response.asJsoup(responseBody).getDataJsonObject()["props"]!!.jsonObject["pageProps"]!!.jsonObject["series"]!! + val latests = response.asJsoup(responseBody).getDataJsonObject()["props"]!!.jsonObject["pageProps"]!!.jsonObject["latests"]!! + seriesCache = linkedMapOf( + *json.decodeFromJsonElement>(seriesList).map { it.series_id to it }.toTypedArray() + ) + latestSeries = json.decodeFromJsonElement>>(latests).map { json.decodeFromJsonElement(it[0]).series_id } + response + } + else -> chain.proceed(chain.request()) + } + }.build() - override fun latestUpdatesRequest(page: Int) = popularMangaRequest(page) + private val homepageRequest = GET(baseUrl) + private val doNothingRequest = GET("https://dev.null") - override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = popularMangaRequest(page) + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = if (this::seriesCache.isInitialized) doNothingRequest else homepageRequest + override fun popularMangaRequest(page: Int) = if (this::seriesCache.isInitialized) doNothingRequest else homepageRequest + override fun latestUpdatesRequest(page: Int) = if (this::seriesCache.isInitialized) doNothingRequest else homepageRequest + override fun chapterListRequest(manga: SManga) = homepageRequest + private fun idOf(manga: SManga) = manga.url.substringAfterLast("/") override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - return client.newCall(popularMangaRequest(page)) + return client.newCall(searchMangaRequest(page, query, filters)) .asObservableSuccess() - .map { response -> - val mangas = if (query.startsWith(SERIES_ID_SEARCH_PREFIX)) { - getFilteredSeriesList( - response.asJsoup().getDataJsonObject(), - idFilter = query.removePrefix(SERIES_ID_SEARCH_PREFIX) - ) - } else { - getFilteredSeriesList( - response.asJsoup().getDataJsonObject(), - titleFilter = query - ) - } - MangasPage(mangas, false) + .map { + val manga = seriesCache.asSequence().map { it.value }.filter { + if (query.startsWith(SERIES_ID_SEARCH_PREFIX)) { + return@filter it.series_id.contains(query.removePrefix(SERIES_ID_SEARCH_PREFIX), true) + } + sequence { yieldAll(it.alt_titles); yield(it.title) } + .any { title -> title.contains(query, true) } + }.map { it.toSManga() }.toList() + + MangasPage(manga, false) } } - override fun fetchMangaDetails(manga: SManga): Observable { - return client.newCall(popularMangaRequest(0)) - .asObservableSuccess() - .map { response -> - manga.also { - getSeriesObject(response.asJsoup().getDataJsonObject(), it)?.let { series -> - it.title = series.getString("title") - it.author = series.getJSONArray("authors").joinToString(", ") - it.description = series.getString("description") - it.genre = series.getJSONArray("genres").joinToString(", ") - it.status = when (series.getString("status")) { - "ongoing" -> SManga.ONGOING - "completed" -> SManga.COMPLETED - else -> SManga.UNKNOWN - } - it.thumbnail_url = series.getJSONObject("cover_art").getString("source") - } - } - } - } + override fun fetchMangaDetails(manga: SManga): Observable = client.newCall(homepageRequest) + .asObservableSuccess() + .map { seriesCache[idOf(manga)]?.toSManga() ?: manga } override fun fetchChapterList(manga: SManga): Observable> { val seriesId = manga.url.substringAfter("/series/") - return client.newCall(popularMangaRequest(0)) + return client.newCall(chapterListRequest(manga)) .asObservableSuccess() - .map { response -> - var returnChapter = emptyList() + .map { + val seriesPrefs = application.getSharedPreferences("source_${id}_time_found:$seriesId", 0) + val seriesPrefsEditor = seriesPrefs.edit() + val cl = seriesCache[idOf(manga)]!!.chapters.asReversed().map { + val title = it.title ?: "" + val groups = it.groups.joinToString(", ") + val number = it.number.content + val displayNumber = it.display_number ?: number + SChapter.create().apply { + url = "${manga.url}/$number" + chapter_number = number.toFloat() + name = "Chapter $displayNumber" + if (title.isNotBlank()) " - $title" else "" + scanlator = groups - val series = getSeriesObject(response.asJsoup().getDataJsonObject(), manga) - if (series != null) { - val seriesPrefs = application.getSharedPreferences("source_${id}_time_found:$seriesId", 0) - val seriesPrefsEditor = seriesPrefs.edit() - - val chapters = series.getJSONArray("chapters") - returnChapter = (0 until chapters.length()).reversed().map { i -> - val chapter = chapters.getJSONObject(i) - val title = chapter.optString("title") - val groups = chapter.getJSONArray("groups").joinToString() - val number = chapter.getString("number") - val displayNumber = chapter.optString("display_number", number) - SChapter.create().apply { - url = "${manga.url}/$number" - chapter_number = number.toFloat() - name = "Chapter $displayNumber" + if (title.isNotBlank()) " - $title" else "" - scanlator = groups - - // Save current time when a chapter is found for the first time, and reuse it on future - // checks to prevent manga entry without any new chapter bumped to the top of - // "Latest chapter" list when the library is updated. - val currentTimeMillis = System.currentTimeMillis() - if (!seriesPrefs.contains(number)) { - seriesPrefsEditor.putLong(number, currentTimeMillis) - } - date_upload = seriesPrefs.getLong(number, currentTimeMillis) + // Save current time when a chapter is found for the first time, and reuse it on future checks to + // prevent manga entry without any new chapter bumped to the top of "Latest chapter" list + // when the library is updated. + val currentTimeMillis = System.currentTimeMillis() + if (!seriesPrefs.contains(number)) { + seriesPrefsEditor.putLong(number, currentTimeMillis) } + date_upload = seriesPrefs.getLong(number, currentTimeMillis) } - seriesPrefsEditor.apply() } - - returnChapter + seriesPrefsEditor.apply() + cl } } - override fun popularMangaParse(response: Response): MangasPage { - val mangas = getFilteredSeriesList(response.asJsoup().getDataJsonObject()) - return MangasPage(mangas, false) - } + override fun popularMangaParse(response: Response) = MangasPage(seriesCache.map { it.value.toSManga() }, false) - override fun latestUpdatesParse(response: Response): MangasPage { - val latests = response.asJsoup().getDataJsonObject() - .getJSONObject("props") - .getJSONObject("pageProps") - .getJSONArray("latests") - val mangas = (0 until latests.length()).map { i -> - val manga = latests.getJSONArray(i).getJSONObject(0) - SManga.create().apply { - url = "/series/${manga.getString("series_id")}" - title = manga.getString("title") - thumbnail_url = manga.getJSONObject("cover_art").getString("source") - } - } - return MangasPage(mangas, false) - } + override fun latestUpdatesParse(response: Response) = MangasPage( + latestSeries.map { seriesCache[it]!!.toSManga() }, + false + ) override fun pageListParse(response: Response): List { - val pages = response.asJsoup().getDataJsonObject() - .getJSONObject("props") - .getJSONObject("pageProps") - .getJSONArray("pages") - return (0 until pages.length()).map { i -> Page(i, "", pages.getString(i)) } + return json.decodeFromJsonElement>(response.asJsoup().getDataJsonObject()["props"]!!.jsonObject["pageProps"]!!.jsonObject["pages"]!!).mapIndexed { index, s -> + Page(index, "", s) + } } /** * Returns json object of site data */ - private fun Document.getDataJsonObject(): JSONObject { - return JSONObject(getElementById("__NEXT_DATA__").html()) - } - - /** - * Returns JSONObject for [manga] from site data - */ - private fun getSeriesObject(jsonObject: JSONObject, manga: SManga): JSONObject? { - val seriesId = manga.url.substringAfter("/series/") - val seriesArray = jsonObject - .getJSONObject("props") - .getJSONObject("pageProps") - .getJSONArray("series") - val seriesIndex = (0 until seriesArray.length()).firstOrNull { i -> - seriesArray.getJSONObject(i).optString("series_id").takeIf { it.isNotBlank() } == seriesId - } - return if (seriesIndex != null) seriesArray.getJSONObject(seriesIndex) else null - } - - /** - * @return filtered series from home page - * @param data json data from [getDataJsonObject] - * @param titleFilter will be used to check against title and alt_titles, null to disable filter - * @param idFilter will be used to check against id, null to disable filter, only used when [titleFilter] is unset - */ - private fun getFilteredSeriesList( - data: JSONObject, - titleFilter: String? = null, - idFilter: String? = null - ): List { - val series = data.getJSONObject("props").getJSONObject("pageProps").getJSONArray("series") - val mangas = mutableListOf() - for (i in 0 until series.length()) { - val manga = series.getJSONObject(i) - val mangaId = manga.getString("series_id") - val mangaTitle = manga.getString("title") - val mangaAltTitles = manga.getJSONArray("alt_titles") - - // Filtering - if (titleFilter != null) { - if (!(mangaTitle.contains(titleFilter, true) || mangaAltTitles.contains(titleFilter))) { - continue - } - } else if (idFilter != null) { - if (!mangaId.contains(idFilter, true)) { - continue - } - } - - mangas += SManga.create().apply { - url = "/series/$mangaId" - title = mangaTitle - thumbnail_url = manga.getJSONObject("cover_art").getString("source") - } - } - return mangas.toList() - } - - private fun JSONArray.joinToString(separator: String = ", "): String { - val stringBuilder = StringBuilder() - for (i in 0 until length()) { - if (i > 0) stringBuilder.append(separator) - val item = getString(i) - stringBuilder.append(item) - } - return stringBuilder.toString() - } - - /** - * For string objects - */ - private operator fun JSONArray.contains(other: CharSequence): Boolean { - for (i in 0 until length()) { - if (optString(i, "").contains(other, true)) { - return true - } - } - return false - } + private fun Document.getDataJsonObject() = json.parseToJsonElement(getElementById("__NEXT_DATA__").html()).jsonObject override fun mangaDetailsParse(response: Response): SManga { throw UnsupportedOperationException("Not used.") @@ -246,3 +159,28 @@ class CatManga : HttpSource() { const val SERIES_ID_SEARCH_PREFIX = "series_id:" } } + +@Serializable +private data class JsonImage(val source: String, val width: Int, val height: Int) + +@Serializable +private data class JsonChapter(val title: String? = null, val groups: List, val number: JsonPrimitive, val display_number: String? = null, val volume: Int? = null) + +@Serializable +private data class JsonSeries(val alt_titles: List, val authors: List, val genres: List, val chapters: List, val title: String, val series_id: String, val description: String, val status: String, val cover_art: JsonImage, val all_covers: List) { + fun toSManga() = this.let { jsonSeries -> + SManga.create().apply { + url = "/series/${jsonSeries.series_id}" + title = jsonSeries.title + thumbnail_url = jsonSeries.cover_art.source + author = jsonSeries.authors.joinToString(", ") + description = jsonSeries.description + genre = jsonSeries.genres.joinToString(", ") + status = when (jsonSeries.status) { + "ongoing" -> SManga.ONGOING + "completed" -> SManga.COMPLETED + else -> SManga.UNKNOWN + } + } + } +} diff --git a/src/ru/remanga/build.gradle b/src/ru/remanga/build.gradle index 3a54deef1..b0bc6695a 100644 --- a/src/ru/remanga/build.gradle +++ b/src/ru/remanga/build.gradle @@ -1,11 +1,12 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' +apply plugin: 'kotlinx-serialization' ext { extName = 'Remanga' pkgNameSuffix = 'ru.remanga' extClass = '.Remanga' - extVersionCode = 28 + extVersionCode = 29 libVersion = '1.2' } diff --git a/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/Remanga.kt b/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/Remanga.kt index c33e1dc21..a6fec3295 100644 --- a/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/Remanga.kt +++ b/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/Remanga.kt @@ -17,10 +17,6 @@ import android.content.SharedPreferences import android.os.Build import android.text.InputType import android.widget.Toast -import com.github.salomonbrys.kotson.fromJson -import com.google.gson.Gson -import com.google.gson.JsonObject -import com.google.gson.JsonSyntaxException import eu.kanade.tachiyomi.lib.dataimage.DataImageInterceptor import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.POST @@ -34,6 +30,13 @@ import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.HttpSource +import kotlinx.serialization.SerializationException +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.decodeFromJsonElement +import kotlinx.serialization.json.put import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.Interceptor @@ -42,17 +45,16 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response -import org.json.JSONObject import org.jsoup.Jsoup import rx.Observable import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get +import uy.kohesive.injekt.injectLazy import java.text.SimpleDateFormat import java.util.Date import java.util.Locale import kotlin.math.absoluteValue import kotlin.random.Random - class Remanga : ConfigurableSource, HttpSource() { override val name = "Remanga" @@ -100,15 +102,16 @@ class Remanga : ConfigurableSource, HttpSource() { private var branches = mutableMapOf>() private fun login(chain: Interceptor.Chain, username: String, password: String): String { - val jsonObject = JSONObject() - jsonObject.put("user", username) - jsonObject.put("password", password) + val jsonObject = buildJsonObject { + put("user", username) + put("password", password) + } val body = jsonObject.toString().toRequestBody(MEDIA_TYPE) val response = chain.proceed(POST("$baseUrl/api/users/login/", headers, body)) if (response.code >= 400) { throw Exception("Failed to login") } - val user = gson.fromJson>(response.body?.charStream()!!) + val user = json.decodeFromString>(response.body!!.string()) return user.content.access_token } @@ -121,7 +124,7 @@ class Remanga : ConfigurableSource, HttpSource() { override fun latestUpdatesParse(response: Response): MangasPage = searchMangaParse(response) override fun searchMangaParse(response: Response): MangasPage { - val page = gson.fromJson>(response.body?.charStream()!!) + val page = json.decodeFromString>(response.body!!.string()) val mangas = page.content.map { it.toSManga() } @@ -159,7 +162,7 @@ class Remanga : ConfigurableSource, HttpSource() { when (filter) { is OrderBy -> { val ord = arrayOf("id", "chapter_date", "rating", "votes", "views", "count_chapters", "random")[filter.state!!.index] - url.addQueryParameter("ordering", if (filter.state!!.ascending) "$ord" else "-$ord") + url.addQueryParameter("ordering", if (filter.state!!.ascending) ord else "-$ord") } is CategoryList -> filter.state.forEach { category -> if (category.state != Filter.TriState.STATE_IGNORE) { @@ -273,7 +276,7 @@ class Remanga : ConfigurableSource, HttpSource() { return GET(baseUrl.replace("api.", "") + "/manga/" + manga.url.substringAfter("/api/titles/", "/"), headers) } override fun mangaDetailsParse(response: Response): SManga { - val series = gson.fromJson>(response.body?.charStream()!!) + val series = json.decodeFromString>(response.body!!.string()) branches[series.content.en_name] = series.content.branches return series.content.toSManga() } @@ -281,10 +284,11 @@ class Remanga : ConfigurableSource, HttpSource() { private fun mangaBranches(manga: SManga): List { val responseString = client.newCall(GET("$baseUrl/${manga.url}")).execute().body?.string() ?: return emptyList() // manga requiring login return "content" as a JsonArray instead of the JsonObject we expect - return if (gson.fromJson(responseString)["content"].isJsonObject) { - val series = gson.fromJson>(responseString) - branches[series.content.en_name] = series.content.branches - series.content.branches + val content = json.decodeFromString(responseString)["content"] + return if (content is JsonObject) { + val series = json.decodeFromJsonElement(content) + branches[series.en_name] = series.branches + series.branches } else { emptyList() } @@ -325,8 +329,8 @@ class Remanga : ConfigurableSource, HttpSource() { } override fun chapterListParse(response: Response): List { - val chapters = gson.fromJson>(response.body?.charStream()!!) - return chapters.content.filter { !it.is_paid or it.is_bought }.map { chapter -> + val chapters = json.decodeFromString>>(response.body!!.string()) + return chapters.content.filter { !it.is_paid or (it.is_bought == true) }.map { chapter -> SChapter.create().apply { chapter_number = chapter.chapter.split(".").take(2).joinToString(".").toFloat() name = chapterName(chapter) @@ -343,12 +347,12 @@ class Remanga : ConfigurableSource, HttpSource() { override fun pageListParse(response: Response): List { val body = response.body?.string()!! return try { - val page = gson.fromJson>(body) + val page = json.decodeFromString>(body) page.content.pages.filter { it.height > 1 }.map { Page(it.page, "", it.link) } - } catch (e: JsonSyntaxException) { - val page = gson.fromJson>(body) + } catch (e: SerializationException) { + val page = json.decodeFromString>(body) val result = mutableListOf() page.content.pages.forEach { it.filter { page -> page.height > 10 }.forEach { page -> @@ -596,7 +600,7 @@ class Remanga : ConfigurableSource, HttpSource() { dialogTitle = title if (isPassword) { - if (!value.isNullOrBlank()) { summary = "*****" } + if (value.isNotBlank()) { summary = "*****" } setOnBindEditTextListener { it.inputType = InputType.TYPE_CLASS_TEXT or InputType.TYPE_TEXT_VARIATION_PASSWORD } @@ -617,7 +621,7 @@ class Remanga : ConfigurableSource, HttpSource() { private fun getPrefUsername(): String = preferences.getString(USERNAME_TITLE, USERNAME_DEFAULT)!! private fun getPrefPassword(): String = preferences.getString(PASSWORD_TITLE, PASSWORD_DEFAULT)!! - private val gson by lazy { Gson() } + private val json: Json by injectLazy() private val username by lazy { getPrefUsername() } private val password by lazy { getPrefPassword() } diff --git a/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/dto/Dto.kt b/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/dto/Dto.kt index 08d59147b..c173c5396 100644 --- a/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/dto/Dto.kt +++ b/src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/dto/Dto.kt @@ -1,19 +1,25 @@ +import kotlinx.serialization.Serializable + +@Serializable data class GenresDto( val id: Int, val name: String ) +@Serializable data class BranchesDto( val id: Long, val count_chapters: Int ) +@Serializable data class ImgDto( val high: String, val mid: String, val low: String ) +@Serializable data class LibraryDto( val id: Long, val en_name: String, @@ -24,11 +30,13 @@ data class LibraryDto( val img: ImgDto ) +@Serializable data class StatusDto( val id: Int, val name: String ) +@Serializable data class MangaDetDto( val id: Long, val en_name: String, @@ -47,30 +55,34 @@ data class MangaDetDto( val age_limit: Int ) +@Serializable data class PropsDto( val total_items: Int, val total_pages: Int, val page: Int ) +@Serializable data class PageWrapperDto( val msg: String, val content: List, val props: PropsDto, - val last: Boolean +// val last: Boolean ) +@Serializable data class SeriesWrapperDto( val msg: String, val content: T, - val props: PropsDto +// val props: PropsDto ) +@Serializable data class PublisherDto( val name: String, - val dir: String ) +@Serializable data class BookDto( val id: Long, val tome: Int, @@ -78,10 +90,11 @@ data class BookDto( val name: String, val upload_date: String, val is_paid: Boolean, - val is_bought: Boolean, + val is_bought: Boolean?, val publishers: List ) +@Serializable data class PagesDto( val id: Int, val height: Int, @@ -90,14 +103,17 @@ data class PagesDto( val count_comments: Int ) +@Serializable data class PageDto( val pages: List ) +@Serializable data class UserDto( val access_token: String ) +@Serializable data class PaidPagesDto( val id: Long, val link: String, @@ -105,6 +121,7 @@ data class PaidPagesDto( val page: Int ) +@Serializable data class PaidPageDto( val pages: List> )