Replace json library with kotlinx.serialization in multiple sources (#7407)

* Catmanga: Replace org.json with kotlinx.serialization + Light Refactor of #7451

* Genkan IO: Replace gson + Make livewire interceptor

* Genkan IO: Tail Call Optimization to avoid blowing stack

* Comick.fun: kotlinx.serialization migration

* Remanga: kotlinx.serialzation migration

src/all/comickfun/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'Comick.fun'
     pkgNameSuffix = 'all.comickfun'
     extClass = '.ComickFunFactory'
-    extVersionCode = 1
+    extVersionCode = 3
     libVersion = '1.2'
     containsNsfw = true
 }

src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt

@@ -1,13 +1,5 @@
 package eu.kanade.tachiyomi.extension.all.comickfun
 
-import android.os.Build
-import android.text.Html
-import com.github.salomonbrys.kotson.array
-import com.github.salomonbrys.kotson.get
-import com.github.salomonbrys.kotson.nullString
-import com.github.salomonbrys.kotson.obj
-import com.google.gson.JsonElement
-import com.google.gson.JsonParser
 import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.asObservableSuccess
@@ -18,6 +10,13 @@ import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.decodeFromJsonElement
+import kotlinx.serialization.modules.SerializersModule
+import kotlinx.serialization.modules.polymorphic
 import okhttp3.CacheControl
 import okhttp3.Headers
 import okhttp3.HttpUrl
@@ -27,10 +26,9 @@ import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
 import rx.Observable
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
 import java.lang.UnsupportedOperationException
-import java.text.SimpleDateFormat
-import kotlin.math.pow
-import kotlin.math.truncate
 
 const val SEARCH_PAGE_LIMIT = 100
 
@@ -40,7 +38,18 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S
     private val apiBase = "$baseUrl/api"
     override val supportsLatest = true
 
-    private val mangaIdCache = mutableMapOf<String, Int>()
+    @ExperimentalSerializationApi
+    private val json: Json by lazy {
+        Json(from = Injekt.get()) {
+            serializersModule = SerializersModule {
+                polymorphic(SManga::class) { default { SMangaDeserializer() } }
+                polymorphic(SChapter::class) { default { SChapterDeserializer() } }
+            }
+        }
+    }
+
+    @ExperimentalSerializationApi
+    private val mangaIdCache = SMangaDeserializer.mangaIdCache
 
     final override fun headersBuilder() = Headers.Builder().apply {
         add("User-Agent", "Tachiyomi " + System.getProperty("http.agent"))
@@ -80,92 +89,37 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S
 
     /**  Utils **/
 
-    /**
-     * Parses a json object with information suitable for showing an entry of a manga within a
-     * catalogue
-     *
-     * Attempts to cache the manga's numerical Id
-     *
-     * @return SManga - with url, thumbnail_url and title set
-     */
-    private fun parseMangaObj(it: JsonElement) = it.asJsonObject.let { info ->
-        info["id"]?.asInt?.let { mangaIdCache.getOrPut(info["slug"].asString, { it }) }
-        val thumbnail = info["coverURL"]?.nullString
-            ?: info["md_covers"]?.asJsonArray?.get(0)?.asJsonObject?.let { cover ->
-                cover["gpurl"]?.nullString ?: "$baseUrl${cover["url"].asString}"
-            }
-
-        SManga.create().apply {
-            url = "/comic/${info["slug"].asString}"
-            thumbnail_url = thumbnail
-            title = info["title"].asString
-        }
-    }
-
     /** Returns an observable which emits a single value -> the manga's id **/
+    @ExperimentalSerializationApi
     private fun chapterId(manga: SManga): Observable<Int> {
         val mangaSlug = slug(manga)
         return mangaIdCache[mangaSlug]?.let { Observable.just(it) }
             ?: fetchMangaDetails(manga).map { mangaIdCache[mangaSlug] }
     }
 
-    private fun parseStatus(status: Int) = when (status) {
-        1 -> SManga.ONGOING
-        2 -> SManga.COMPLETED
-        else -> SManga.UNKNOWN
-    }
-
-    /** Attempts to parse an ISO-8601 compliant Date Time string with offset to epoch.
-     * @returns epochtime on success, 0 on failure
-     **/
-    private fun parseISO8601(s: String): Long {
-        var fractionalPart_ms: Long = 0
-        val sNoFraction = Regex("""\.\d+""").replace(s) { match ->
-            fractionalPart_ms = truncate(
-                match.value.substringAfter(".").toFloat() * 10.0f.pow(-(match.value.length - 1)) * // seconds
-                    1000 // milliseconds
-            ).toLong()
-            ""
-        }
-
-        val ret = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZZZZZ").parse(sNoFraction)?.let {
-            fractionalPart_ms + it.time
-        } ?: 0
-        return ret
-    }
-
     /** Returns an identifier referred to as `hid` for chapter **/
     private fun hid(chapter: SChapter) = "$baseUrl${chapter.url}".toHttpUrl().pathSegments[2].substringBefore("-")
 
     /** Returns an identifier referred to as a  `slug` for manga **/
     private fun slug(manga: SManga) = "$baseUrl${manga.url}".toHttpUrl().pathSegments[1]
 
-    private fun formatChapterTitle(title: String?, chap: String?, vol: String?): String {
-        val numNonNull = listOfNotNull(title.takeIf { !it.isNullOrBlank() }, chap, vol).size
-        if (numNonNull == 0) throw RuntimeException("formatChapterTitle requires at least one non-null argument")
-
-        var formattedTitle = StringBuilder()
-        if (vol != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Vol." } ?: "Volume"} $vol")
-        if (vol != null && chap != null) formattedTitle.append(", ")
-        if (chap != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Ch." } ?: "Chapter"} $chap")
-        if (!title.isNullOrBlank()) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { ": " } ?: ""} $title")
-        return formattedTitle.toString()
-    }
-
     /** Popular Manga **/
 
+    @ExperimentalSerializationApi
     override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", FilterList(emptyList()))
     override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException("Not used")
     override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Not used")
 
     /** Latest Manga **/
+    @ExperimentalSerializationApi
     override fun latestUpdatesParse(response: Response): MangasPage {
         val noResults = MangasPage(emptyList(), false)
         if (response.code == 204)
             return noResults
-        return JsonParser.parseString(response.body!!.string()).obj["data"]?.array?.let { manga ->
+        return json.decodeFromString(
-            MangasPage(manga.map { parseMangaObj(it["md_comics"]) }, true)
+            deserializer = deepSelectDeserializer<List<SManga>>("data"),
-        } ?: noResults
+            response.body!!.string()
+        ).let { MangasPage(it, true) }
     }
 
     override fun latestUpdatesRequest(page: Int): Request {
@@ -175,6 +129,7 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S
         return GET("$url", headers)
     }
 
+    @ExperimentalSerializationApi
     override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
         if (!query.startsWith(SLUG_SEARCH_PREFIX))
             return super.fetchSearchManga(page, query, filters)
@@ -205,11 +160,14 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S
         return GET("$url", headers)
     }
 
-    override fun searchMangaParse(response: Response): MangasPage = JsonParser.parseString(response.body!!.string()).let {
+    @ExperimentalSerializationApi
-        if (it.isJsonObject)
+    override fun searchMangaParse(response: Response): MangasPage = json.parseToJsonElement(response.body!!.string()).let { parsed ->
-            MangasPage(it["comics"].array.map(::parseMangaObj), it["comics"].array.size() == SEARCH_PAGE_LIMIT)
+        when (parsed) {
-        else // search_title isn't paginated
+            is JsonObject -> json.decodeFromJsonElement<List<SManga>>(parsed["comics"]!!)
-            MangasPage(it.array.map(::parseMangaObj), false)
+                .let { MangasPage(it, it.size == SEARCH_PAGE_LIMIT) }
+            is JsonArray -> MangasPage(json.decodeFromJsonElement(parsed), false)
+            else -> MangasPage(emptyList(), false)
+        }
     }
 
     /** Manga Details **/
@@ -219,6 +177,7 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S
     }
 
     // Shenanigans to allow "open in webview" to show a webpage instead of JSON
+    @ExperimentalSerializationApi
     override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
         return client.newCall(apiMangaDetailsRequest(manga))
             .asObservableSuccess()
@@ -227,36 +186,18 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S
             }
     }
 
-    override fun mangaDetailsParse(response: Response) = JsonParser.parseString(response.body!!.string())["data"].let { data ->
+    @ExperimentalSerializationApi
-        fun cleanDesc(s: String) = (
+    override fun mangaDetailsParse(response: Response) = json.decodeFromString(
-            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N)
+        deserializer = deepSelectDeserializer<SManga>("data", tDeserializer = jsonFlatten(objKey = "comic", "id", "title", "desc", "status", "country", "slug")),
-                Html.fromHtml(s, Html.FROM_HTML_MODE_LEGACY) else Html.fromHtml(s)
+        response.body!!.string()
-            ).toString()
+    )
-
-        fun nameList(e: JsonElement?) = e?.array?.asSequence()?.map { it["name"].asString }
-        data["comic"]["id"].asInt.let { mangaIdCache.getOrPut(response.request.url.queryParameter("slug")!!, { it }) }
-        SManga.create().apply {
-            title = data["comic"]["title"].asString
-            thumbnail_url = data["coverURL"].asString
-            description = cleanDesc(data["comic"]["desc"].asString)
-            status = parseStatus(data["comic"]["status"].asInt)
-            artist = nameList(data["artists"])?.joinToString(", ")
-            author = nameList(data["authors"])?.joinToString(", ")
-            genre = (
-                (nameList(data["genres"]) ?: sequenceOf()) + sequence {
-                    data["demographic"].nullString?.let { yield(it) }
-                    mapOf("kr" to "Manhwa", "jp" to "Manga", "cn" to "Manhua")[data["comic"]["country"].nullString]
-                        ?.let { yield(it) }
-                }
-                ).joinToString(", ")
-        }
-    }
 
     /** Chapter List **/
 
     private fun chapterListRequest(page: Int, mangaId: Int) =
         GET("$apiBase/get_chapters?comicid=$mangaId&page=$page&limit=$SEARCH_PAGE_LIMIT", headers)
 
+    @ExperimentalSerializationApi
     override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
         return if (manga.status != SManga.LICENSED) {
             chapterId(manga).concatMap { id ->
@@ -281,25 +222,22 @@ abstract class ComickFun(override val lang: String, private val comickFunLang: S
         }
     }
 
-    override fun chapterListParse(response: Response) = JsonParser.parseString(response.body!!.string()).obj["data"]["chapters"].array.map { elem ->
+    @ExperimentalSerializationApi
-        val chapter = elem.asJsonObject
+    override fun chapterListParse(response: Response) = json.decodeFromString(
-        val num = chapter["chap"].nullString ?: "-1"
+        deserializer = deepSelectDeserializer<List<SChapter>>("data", "chapters"),
-        SChapter.create().apply {
+        response.body!!.string()
-            date_upload = parseISO8601(chapter["created_at"].asString)
+    )
-            name = formatChapterTitle(chapter["title"].nullString, chapter["chap"].nullString, chapter["vol"].nullString)
-            chapter_number = num.toFloat()
-            url = "/${chapter["hid"].asString}-chapter-${chapter["chap"].nullString}-${chapter["iso639_1"].asString}" // incomplete, is finished in fetchChapterList
-            scanlator = chapter.get("md_groups")?.array?.get(0)?.obj?.get("title")?.asString
-        }
-    }
 
     /** Page List **/
 
     override fun pageListRequest(chapter: SChapter) = GET("$apiBase/get_chapter?hid=${hid(chapter)}", headers, CacheControl.FORCE_NETWORK)
 
-    override fun pageListParse(response: Response) = JsonParser.parseString(response.body!!.string())["data"]["chapter"]["images"].array.mapIndexed { i, url ->
+    @ExperimentalSerializationApi
-        Page(i, imageUrl = url.asString)
+    override fun pageListParse(response: Response) =
-    }
+        json.decodeFromString(
+            deserializer = deepSelectDeserializer<List<String>>("data", "chapter", "images"),
+            response.body!!.string()
+        ).mapIndexed { i, url -> Page(i, imageUrl = url) }
 
     override fun imageUrlParse(response: Response) = "" // idk what this does, leave me alone kotlin
 

src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunSerialization.kt

@@ -0,0 +1,261 @@
+package eu.kanade.tachiyomi.extension.all.comickfun
+
+import android.os.Build
+import android.text.Html
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.builtins.ListSerializer
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.descriptors.buildClassSerialDescriptor
+import kotlinx.serialization.descriptors.element
+import kotlinx.serialization.encoding.CompositeDecoder
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.encoding.decodeStructure
+import kotlinx.serialization.json.JsonDecoder
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonTransformingSerializer
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.serializer
+import java.text.SimpleDateFormat
+import kotlin.math.pow
+import kotlin.math.truncate
+
+/**
+ * A serializer of type T which selects the value of type T by traversing down a chain of json objects
+ *
+ * e.g
+ *  {
+ *     "user": {
+ *          "name": {
+ *              "first": "John",
+ *              "last": "Smith"
+ *          }
+ *     }
+ *  }
+ *
+ * deepSelectDeserializer<String>("user", "name", "first") deserializes the above into "John"
+ */
+@ExperimentalSerializationApi
+inline fun <reified T : Any> deepSelectDeserializer(vararg keys: String, tDeserializer: KSerializer<T> = serializer()): KSerializer<T> {
+    val descriptors = keys.foldRight(listOf(tDeserializer.descriptor)) { x, acc ->
+        acc + acc.last().let {
+            buildClassSerialDescriptor("$x\$${it.serialName}") { element(x, it) }
+        }
+    }.asReversed()
+    var a: ((Int) -> KSerializer<T>)? = null
+    val b = { depth: Int ->
+        object : KSerializer<T> {
+            override val descriptor = descriptors[depth]
+
+            override fun deserialize(decoder: Decoder): T {
+                return if (depth == keys.size) decoder.decodeSerializableValue(tDeserializer)
+                else decoder.decodeStructureByKnownName(descriptor) { names ->
+                    names.filter { (name, _) -> name == keys[depth] }
+                        .map { (_, index) -> decodeSerializableElement(descriptor, index, a!!(depth + 1)) }
+                        .single()
+                }
+            }
+
+            override fun serialize(encoder: Encoder, value: T) = throw UnsupportedOperationException("Not supported")
+        }
+    }
+    a = b // this is the hackiest of hacky hacks to get around not being able to define recursive inline functions
+    return a(0)
+}
+
+/**
+ * Transforms given json element by lifting specified keys in `element[objKey]` up into `element`
+ * Existing conflicts are overwritten
+ *
+ * @param objKey: String - A key identifying an object in JsonElement
+ * @param keys: vararg String - Keys identifying values to lift from objKey
+ */
+inline fun <reified T : Any> jsonFlatten(objKey: String, vararg keys: String, tDeserializer: KSerializer<T> = serializer()): JsonTransformingSerializer<T> {
+    return object : JsonTransformingSerializer<T>(tDeserializer) {
+        override fun transformDeserialize(element: JsonElement) = buildJsonObject {
+            require(element is JsonObject)
+            element.entries.forEach { (key, value) -> put(key, value) }
+            val fromObj = element[objKey]
+            require(fromObj is JsonObject)
+            keys.forEach { put(it, fromObj[it]!!) }
+        }
+    }
+}
+
+@ExperimentalSerializationApi
+inline fun <T> Decoder.decodeStructureByKnownName(descriptor: SerialDescriptor, decodeFn: CompositeDecoder.(Sequence<Pair<String, Int>>) -> T): T {
+    return decodeStructure(descriptor) {
+        decodeFn(
+            generateSequence { decodeElementIndex(descriptor) }
+                .takeWhile { it != CompositeDecoder.DECODE_DONE }
+                .filter { it != CompositeDecoder.UNKNOWN_NAME }
+                .map { descriptor.getElementName(it) to it }
+        )
+    }
+}
+
+@ExperimentalSerializationApi
+class SChapterDeserializer : KSerializer<SChapter> {
+    override val descriptor = buildClassSerialDescriptor(SChapter::class.qualifiedName!!) {
+        element<String>("chap")
+        element<String>("hid")
+        element<String?>("title")
+        element<String?>("vol", isOptional = true)
+        element<String>("created_at")
+        element<String>("iso639_1")
+        element<List<String>>("images", isOptional = true)
+        element<List<JsonObject>>("md_groups", isOptional = true)
+    }
+
+    /** Attempts to parse an ISO-8601 compliant Date Time string with offset to epoch.
+     * @returns epochtime on success, 0 on failure
+     **/
+    private fun parseISO8601(s: String): Long {
+        var fractionalPart_ms: Long = 0
+        val sNoFraction = Regex("""\.\d+""").replace(s) { match ->
+            fractionalPart_ms = truncate(
+                match.value.substringAfter(".").toFloat() * 10.0f.pow(-(match.value.length - 1)) * // seconds
+                    1000 // milliseconds
+            ).toLong()
+            ""
+        }
+
+        return SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZZZZZ").parse(sNoFraction)?.let {
+            fractionalPart_ms + it.time
+        } ?: 0
+    }
+
+    private fun formatChapterTitle(title: String?, chap: String?, vol: String?): String {
+        val numNonNull = listOfNotNull(title.takeIf { !it.isNullOrBlank() }, chap, vol).size
+        if (numNonNull == 0) throw RuntimeException("formatChapterTitle requires at least one non-null argument")
+
+        val formattedTitle = StringBuilder()
+        if (vol != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Vol." } ?: "Volume"} $vol")
+        if (vol != null && chap != null) formattedTitle.append(", ")
+        if (chap != null) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { "Ch." } ?: "Chapter"} $chap")
+        if (!title.isNullOrBlank()) formattedTitle.append("${numNonNull.takeIf { it > 1 }?.let { ": " } ?: ""} $title")
+        return formattedTitle.toString()
+    }
+
+    @ExperimentalSerializationApi
+    override fun deserialize(decoder: Decoder): SChapter {
+        return SChapter.create().apply {
+            var chap: String? = null
+            var vol: String? = null
+            var title: String? = null
+            var hid = ""
+            var iso639_1 = ""
+            require(decoder is JsonDecoder)
+            decoder.decodeStructureByKnownName(descriptor) { names ->
+                for ((name, index) in names) {
+                    when (name) {
+                        "created_at" -> date_upload = parseISO8601(decodeStringElement(descriptor, index))
+                        "title" -> title = decodeNullableSerializableElement(descriptor, index, serializer())
+                        "vol" -> vol = decodeNullableSerializableElement(descriptor, index, serializer())
+                        "chap" -> {
+                            chap = decodeStringElement(descriptor, index)
+                            chapter_number = chap!!.toFloat()
+                        }
+                        "hid" -> hid = decodeStringElement(descriptor, index)
+                        "iso639_1" -> iso639_1 = decodeStringElement(descriptor, index)
+                        "md_groups" -> scanlator = decodeSerializableElement(descriptor, index, ListSerializer(deepSelectDeserializer<String>("title"))).joinToString(", ")
+                    }
+                }
+            }
+            name = formatChapterTitle(title, chap, vol)
+            url = "/$hid-chapter-$chap-$iso639_1" // incomplete, is finished in fetchChapterList
+        }
+    }
+
+    override fun serialize(encoder: Encoder, value: SChapter) = throw UnsupportedOperationException("Unsupported")
+}
+
+@ExperimentalSerializationApi
+class SMangaDeserializer : KSerializer<SManga> {
+    private fun cleanDesc(s: String) = (
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N)
+            Html.fromHtml(s, Html.FROM_HTML_MODE_LEGACY) else Html.fromHtml(s)
+        ).toString()
+
+    private fun parseStatus(status: Int) = when (status) {
+        1 -> SManga.ONGOING
+        2 -> SManga.COMPLETED
+        else -> SManga.UNKNOWN
+    }
+
+    override val descriptor = buildClassSerialDescriptor(SManga::class.qualifiedName!!) {
+        element<String>("slug")
+        element<String>("title")
+        element<String>("coverURL")
+        element<String>("id", isOptional = true)
+        element<List<JsonObject>>("artists", isOptional = true)
+        element<List<JsonObject>>("authors", isOptional = true)
+        element<String>("desc", isOptional = true)
+        element<String>("demographic", isOptional = true)
+        element<List<JsonObject>>("genres", isOptional = true)
+        element<Int>("status", isOptional = true)
+        element<String>("country", isOptional = true)
+    }
+
+    @ExperimentalSerializationApi
+    override fun deserialize(decoder: Decoder): SManga {
+        return SManga.create().apply {
+            var id: Int? = null
+            var slug: String? = null
+            val tryTo = (
+                {
+                    var hasThrown = false;
+                    { fn: () -> Unit ->
+                        if (!hasThrown) {
+                            try {
+                                fn()
+                            } catch (_: java.lang.Exception) {
+                                hasThrown = true
+                            }
+                        }
+                    }
+                }
+                )()
+            decoder.decodeStructureByKnownName(descriptor) { names ->
+                for ((name, index) in names) {
+                    val sluggedNameSerializer = ListSerializer(deepSelectDeserializer<String>("name"))
+                    fun nameList() = decodeSerializableElement(descriptor, index, sluggedNameSerializer).joinToString(", ")
+                    when (name) {
+                        "slug" -> {
+                            slug = decodeStringElement(descriptor, index)
+                            url = "/comic/$slug"
+                        }
+                        "title" -> title = decodeStringElement(descriptor, index)
+                        "coverURL" -> thumbnail_url = decodeStringElement(descriptor, index)
+                        "id" -> id = decodeIntElement(descriptor, index)
+                        "artists" -> artist = nameList()
+                        "authors" -> author = nameList()
+                        "desc" -> description = cleanDesc(decodeStringElement(descriptor, index))
+                        // Isn't always a string in every api call
+                        "demographic" -> tryTo { genre = listOfNotNull(genre, decodeStringElement(descriptor, index)).joinToString(", ") }
+                        // Isn't always a list of objects in every api call
+                        "genres" -> tryTo { genre = listOfNotNull(genre, nameList()).joinToString(", ") }
+                        "status" -> status = parseStatus(decodeIntElement(descriptor, index))
+                        "country" -> genre = listOfNotNull(
+                            genre,
+                            mapOf("kr" to "Manhwa", "jp" to "Manga", "cn" to "Manhua")[decodeStringElement(descriptor, index)]
+                        ).joinToString(", ")
+                    }
+                }
+            }
+            if (id != null && slug != null) {
+                mangaIdCache[slug!!] = id!!
+            }
+        }
+    }
+
+    override fun serialize(encoder: Encoder, value: SManga) = throw UnsupportedOperationException("Not supported")
+
+    companion object {
+        val mangaIdCache = mutableMapOf<String, Int>()
+    }
+}

src/all/genkanio/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'Genkan.io'
     pkgNameSuffix = "all.genkanio"
     extClass = '.GenkanIO'
-    extVersionCode = 2
+    extVersionCode = 3
     libVersion = '1.2'
 }
 

src/all/genkanio/src/eu/kanade/tachiyomi/extension/all/genkanio/GenkanIO.kt

@@ -1,32 +1,38 @@
 package eu.kanade.tachiyomi.extension.all.genkanio
 
 import android.util.Log
-import com.github.salomonbrys.kotson.keys
-import com.github.salomonbrys.kotson.put
-import com.google.gson.JsonArray
-import com.google.gson.JsonElement
-import com.google.gson.JsonObject
-import com.google.gson.JsonParser
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.Headers
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonNull
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.boolean
+import kotlinx.serialization.json.buildJsonArray
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.int
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
 import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import okhttp3.Interceptor
 import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.Request
 import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
 import okhttp3.ResponseBody.Companion.toResponseBody
+import okio.Buffer
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import rx.Observable
+import uy.kohesive.injekt.injectLazy
 import java.util.Calendar
 
 open class GenkanIO : ParsedHttpSource() {
@@ -35,37 +41,141 @@ open class GenkanIO : ParsedHttpSource() {
     final override val baseUrl = "https://genkan.io"
     final override val supportsLatest = false
 
-    data class LiveWireRPC(val csrf: String, val state: JsonObject)
+    private val json: Json by injectLazy()
-    private var livewire: LiveWireRPC? = null
 
-    /**
+    /** An interceptor which encapsulates the logic needed to interoperate with Genkan.io's
-     * Given a string encoded with html entities and escape sequences, makes an attempt to decode
+     *  livewire server, which uses a form a Remote Procedure call
-     * and returns decoded string
-     *
-     * Warning: This is not all all exhaustive, and probably misses edge cases
-     *
-     * @Returns decoded string
      */
-    private fun htmlDecode(html: String): String {
+    private val livewireInterceptor = object : Interceptor {
-        return html.replace(Regex("&([A-Za-z]+);")) { match ->
+        private lateinit var fingerprint: JsonElement
-            mapOf(
+        lateinit var serverMemo: JsonObject
-                "raquo" to "»",
+        private lateinit var csrf: String
-                "laquo" to "«",
+        var initialized = false
-                "amp" to "&",
+        val serverUrl = "$baseUrl/livewire/message/manga.list-all-manga"
-                "lt" to "<",
+
-                "gt" to ">",
+        /**
-                "quot" to "\""
+         * Given a string encoded with html entities and escape sequences, makes an attempt to decode
-            )[match.groups[1]!!.value] ?: match.groups[0]!!.value
+         * and returns decoded string
-        }.replace(Regex("\\\\(.)")) { match ->
+         *
-            mapOf(
+         * Warning: This is not all all exhaustive, and probably misses edge cases
-                "t" to "\t",
+         *
-                "n" to "\n",
+         * @Returns decoded string
-                "r" to "\r",
+         */
-                "b" to "\b"
+        private fun htmlDecode(html: String): String {
-            )[match.groups[1]!!.value] ?: match.groups[1]!!.value
+            return html.replace(Regex("&([A-Za-z]+);")) { match ->
+                mapOf(
+                    "raquo" to "»",
+                    "laquo" to "«",
+                    "amp" to "&",
+                    "lt" to "<",
+                    "gt" to ">",
+                    "quot" to "\""
+                )[match.groups[1]!!.value] ?: match.groups[0]!!.value
+            }.replace(Regex("\\\\(.)")) { match ->
+                mapOf(
+                    "t" to "\t",
+                    "n" to "\n",
+                    "r" to "\r",
+                    "b" to "\b"
+                )[match.groups[1]!!.value] ?: match.groups[1]!!.value
+            }
+        }
+
+        /**
+         * Recursively merges j2 onto j1 in place
+         * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's
+         *
+         */
+        private fun mergeLeft(j1: JsonObject, j2: JsonObject): JsonObject = buildJsonObject {
+            j1.keys.forEach { put(it, j1[it]!!) }
+            j2.keys.forEach { k ->
+                when {
+                    j1[k] !is JsonObject -> put(k, j2[k]!!)
+                    j1[k] is JsonObject && j2[k] is JsonObject -> put(k, mergeLeft(j1[k]!!.jsonObject, j2[k]!!.jsonObject))
+                }
+            }
+        }
+
+        /**
+         * Initializes lateinit member vars
+         */
+        private fun initLivewire(chain: Interceptor.Chain) {
+            val response = chain.proceed(GET("$baseUrl/manga", headers))
+            val soup = response.asJsoup()
+            response.body?.close()
+            val csrfToken = soup.selectFirst("meta[name=csrf-token]")?.attr("content")
+
+            val initialProps = soup.selectFirst("div[wire:initial-data]")?.attr("wire:initial-data")?.let {
+                json.parseToJsonElement(htmlDecode(it))
+            }
+
+            if (csrfToken != null && initialProps is JsonObject) {
+                csrf = csrfToken
+                serverMemo = initialProps["serverMemo"]!!.jsonObject
+                fingerprint = initialProps["fingerprint"]!!
+                initialized = true
+            } else {
+                Log.e("GenkanIo", soup.selectFirst("div[wire:initial-data]")?.toString() ?: "null")
+            }
+        }
+
+        /**
+         * Builds a request for livewire, augmenting the request with required body fields and headers
+         *
+         * @param req: Request - A request with a json encoded body, which represent the updates sent to server
+         *
+         */
+        private fun livewireRequest(req: Request): Request {
+            val payload = buildJsonObject {
+                put("fingerprint", fingerprint)
+                put("serverMemo", serverMemo)
+                put("updates", json.parseToJsonElement(Buffer().apply { req.body!!.writeTo(this) }.readUtf8()))
+            }.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())
+
+            return req.newBuilder()
+                .method(req.method, payload)
+                .addHeader("x-csrf-token", csrf)
+                .addHeader("x-livewire", "true")
+                .build()
+        }
+
+        /**
+         * Transforms  json response from livewire server into a response which returns html
+         *
+         * @param response: Response - The response of sending a message to genkan's livewire server
+         *
+         * @return HTML Response - The html embedded within the provided response
+         */
+        private fun livewireResponse(response: Response): Response {
+            if (!response.isSuccessful) return response
+            val body = response.body!!.string()
+            val responseJson = json.parseToJsonElement(body).jsonObject
+
+            // response contains state that we need to preserve
+            serverMemo = mergeLeft(serverMemo, responseJson["serverMemo"]!!.jsonObject)
+
+            // this seems to be an error  state, so reset everything
+            if (responseJson["effects"]?.jsonObject?.get("html") is JsonNull) {
+                initialized = false
+            }
+
+            // Build html response
+            return response.newBuilder()
+                .body(htmlDecode("${responseJson["effects"]?.jsonObject?.get("html")}").toResponseBody("Content-Type: text/html; charset=UTF-8".toMediaTypeOrNull()))
+                .build()
+        }
+
+        override fun intercept(chain: Interceptor.Chain): Response {
+            if (chain.request().url.toString() != serverUrl)
+                return chain.proceed(chain.request())
+
+            if (!initialized) initLivewire(chain)
+            return livewireResponse(chain.proceed(livewireRequest(chain.request())))
         }
     }
 
+    override val client = super.client.newBuilder().addInterceptor(livewireInterceptor).build()
+
     // popular manga
 
     override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", FilterList(emptyList()))
@@ -83,119 +193,33 @@ open class GenkanIO : ParsedHttpSource() {
 
     // search
 
-    /**
-     * initializes `livewire` local variable using data from https://genkan.io/manga
-     */
-    private fun initLiveWire(response: Response) {
-        val soup = response.asJsoup()
-        val csrf = soup.selectFirst("meta[name=csrf-token]")?.attr("content")
-
-        val initialProps = soup.selectFirst("div[wire:initial-data]")?.attr("wire:initial-data")?.let {
-            JsonParser.parseString(htmlDecode(it))
-        }
-
-        if (csrf != null && initialProps?.asJsonObject != null) {
-            livewire = LiveWireRPC(csrf, initialProps.asJsonObject)
-        } else {
-            Log.e("GenkanIo", soup.selectFirst("div[wire:initial-data]")?.toString() ?: "null")
-        }
-    }
-
-    /**
-     * Prepares  a request which'll send a message to livewire server
-     *
-     * @param url: String - Message endpoint
-     * @param updates: JsonElement - JsonElement which describes the actions taken by server
-     *
-     * @return Request
-     */
-    private fun livewireRequest(url: String, updates: JsonElement): Request {
-        // assert(livewire != null)
-        val payload = JsonObject()
-        payload.put("fingerprint" to livewire!!.state.get("fingerprint"))
-        payload.put("serverMemo" to livewire!!.state.get("serverMemo"))
-        payload.put("updates" to updates)
-
-        // not sure why this isn't getting added automatically
-        val cookie = client.cookieJar.loadForRequest(url.toHttpUrlOrNull()!!).joinToString("; ") { "${it.name}=${it.value}" }
-        return POST(
-            url,
-            Headers.headersOf("x-csrf-token", livewire!!.csrf, "x-livewire", "true", "cookie", cookie, "cache-control", "no-cache, private"),
-            payload.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())
-        )
-    }
-
-    /**
-     * Transforms  json response from livewire server into a response which returns html
-     * Also updates `livewire` variable with state returned by livewire server
-     *
-     * @param response: Response - The response of sending a message to genkan's livewire server
-     *
-     * @return HTML Response - The html embedded within the provided response
-     */
-    private fun livewireResponse(response: Response): Response {
-        val body = response.body?.string()
-        val responseJson = JsonParser.parseString(body).asJsonObject
-
-        // response contains state that we need to preserve
-        mergeLeft(livewire!!.state.get("serverMemo").asJsonObject, responseJson.get("serverMemo").asJsonObject)
-
-        // this seems to be an error  state, so reset everything
-        if (responseJson.get("effects")?.asJsonObject?.get("html")?.isJsonNull == true) {
-            livewire = null
-        }
-
-        // Build html response
-        return response.newBuilder()
-            .body(htmlDecode("${responseJson.get("effects")?.asJsonObject?.get("html")}").toResponseBody("Content-Type: text/html; charset=UTF-8".toMediaTypeOrNull()))
-            .build()
-    }
-
-    /**
-     * Recursively merges j2 onto j1 in place
-     * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's
-     *
-     */
-    private fun mergeLeft(j1: JsonObject, j2: JsonObject) {
-        j2.keys().forEach { k ->
-            if (j1.get(k)?.isJsonObject != true)
-                j1.put(k to j2.get(k))
-            else if (j1.get(k).isJsonObject && j2.get(k).isJsonObject)
-                mergeLeft(j1.get(k).asJsonObject, j2.get(k).asJsonObject)
-        }
-    }
-
-    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
-        fun searchRequest() = client.newCall(searchMangaRequest(page, query, filters)).asObservableSuccess().map(::livewireResponse)
-        return if (livewire == null) {
-            client.newCall(GET("$baseUrl/manga", headers))
-                .asObservableSuccess()
-                .doOnNext(::initLiveWire)
-                .concatWith(Observable.defer(::searchRequest))
-                .reduce { _, x -> x }
-        } else {
-            searchRequest()
-        }.map(::searchMangaParse)
-    }
-
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        //        assert(livewire != null)
+        val data = if (livewireInterceptor.initialized) livewireInterceptor.serverMemo["data"]!!.jsonObject else buildJsonObject {
-        val updates = JsonArray()
+            put("readyToLoad", JsonPrimitive(false))
-        val data = livewire!!.state.get("serverMemo")?.asJsonObject?.get("data")?.asJsonObject!!
+            put("page", JsonPrimitive(1))
-        if (data["readyToLoad"]?.asBoolean == false) {
+            put("search", JsonPrimitive(""))
-            updates.add(JsonParser.parseString("""{"type":"callMethod","payload":{"method":"loadManga","params":[]}}"""))
-        }
-        val isNewQuery = query != data["search"]?.asString
-        if (isNewQuery) {
-            updates.add(JsonParser.parseString("""{"type": "syncInput", "payload": {"name": "search", "value": "$query"}}"""))
         }
 
-        val currPage = if (isNewQuery) 1 else data["page"]?.asInt
+        val updates = buildJsonArray {
+            if (data["readyToLoad"]?.jsonPrimitive?.boolean == false) {
+                add(json.parseToJsonElement("""{"type":"callMethod","payload":{"method":"loadManga","params":[]}}"""))
+            }
+            val isNewQuery = query != data["search"]?.jsonPrimitive?.content
+            if (isNewQuery) {
+                add(json.parseToJsonElement("""{"type": "syncInput", "payload": {"name": "search", "value": "$query"}}"""))
+            }
 
-        for (i in (currPage!! + 1)..page)
+            val currPage = if (isNewQuery) 1 else data["page"]!!.jsonPrimitive.int
-            updates.add(JsonParser.parseString("""{"type":"callMethod","payload":{"method":"nextPage","params":[]}}"""))
 
-        return livewireRequest("$baseUrl/livewire/message/manga.list-all-manga", updates)
+            for (i in (currPage + 1)..page)
+                add(json.parseToJsonElement("""{"type":"callMethod","payload":{"method":"nextPage","params":[]}}"""))
+        }
+
+        return POST(
+            livewireInterceptor.serverUrl,
+            headers,
+            updates.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())
+        )
     }
 
     override fun searchMangaFromElement(element: Element): SManga {
@@ -220,15 +244,15 @@ open class GenkanIO : ParsedHttpSource() {
         return if (manga.status != SManga.LICENSED) {
             // Returns an observable which emits the list of chapters found on a page,
             // for every page starting from specified page
-            fun getAllPagesFrom(page: Int): Observable<List<SChapter>> =
+            fun getAllPagesFrom(page: Int, pred: Observable<List<SChapter>> = Observable.just(emptyList())): Observable<List<SChapter>> =
                 client.newCall(chapterListRequest(manga, page))
                     .asObservableSuccess()
                     .concatMap { response ->
                         val cp = chapterPageParse(response)
                         if (cp.hasnext)
-                            Observable.just(cp.chapters).concatWith(getAllPagesFrom(page + 1))
+                            getAllPagesFrom(page + 1, pred = pred.concatWith(Observable.just(cp.chapters))) // tail call to avoid blowing the stack
                         else
-                            Observable.just(cp.chapters)
+                            pred.concatWith(Observable.just(cp.chapters))
                     }
             getAllPagesFrom(1).reduce(List<SChapter>::plus)
         } else {

src/en/catmanga/build.gradle

@@ -1,5 +1,6 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'CatManga'

src/en/catmanga/src/eu/kanade/tachiyomi/extension/en/catmanga/CatManga.kt

@@ -10,12 +10,20 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.decodeFromJsonElement
+import kotlinx.serialization.json.jsonObject
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.Protocol
 import okhttp3.Response
-import org.json.JSONArray
+import okhttp3.ResponseBody.Companion.toResponseBody
-import org.json.JSONObject
 import org.jsoup.nodes.Document
 import rx.Observable
 import uy.kohesive.injekt.injectLazy
+import java.net.HttpURLConnection
 
 class CatManga : HttpSource() {
 
@@ -25,206 +33,111 @@ class CatManga : HttpSource() {
     override val baseUrl = "https://catmanga.org"
     override val supportsLatest = true
     override val lang = "en"
+    private val json: Json by injectLazy()
 
-    override fun popularMangaRequest(page: Int) = GET(baseUrl)
+    private lateinit var seriesCache: LinkedHashMap<String, JsonSeries> // LinkedHashMap to preserve insertion order
+    private lateinit var latestSeries: List<String>
+    override val client = super.client.newBuilder().addInterceptor { chain ->
+        // An interceptor which facilitates caching the data retrieved from the homepage
+        when (chain.request().url) {
+            doNothingRequest.url -> Response.Builder().body(
+                "".toResponseBody("text/plain; charset=utf-8".toMediaType())
+            ).code(HttpURLConnection.HTTP_NO_CONTENT).message("").protocol(Protocol.HTTP_1_0).request(chain.request()).build()
+            homepageRequest.url -> {
+                /* Homepage embeds a Json Object with information about every single series in the service */
+                val response = chain.proceed(chain.request())
+                val responseBody = response.peekBody(Long.MAX_VALUE).string()
+                val seriesList = response.asJsoup(responseBody).getDataJsonObject()["props"]!!.jsonObject["pageProps"]!!.jsonObject["series"]!!
+                val latests = response.asJsoup(responseBody).getDataJsonObject()["props"]!!.jsonObject["pageProps"]!!.jsonObject["latests"]!!
+                seriesCache = linkedMapOf(
+                    *json.decodeFromJsonElement<List<JsonSeries>>(seriesList).map { it.series_id to it }.toTypedArray()
+                )
+                latestSeries = json.decodeFromJsonElement<List<List<JsonElement>>>(latests).map { json.decodeFromJsonElement<JsonSeries>(it[0]).series_id }
+                response
+            }
+            else -> chain.proceed(chain.request())
+        }
+    }.build()
 
-    override fun latestUpdatesRequest(page: Int) = popularMangaRequest(page)
+    private val homepageRequest = GET(baseUrl)
+    private val doNothingRequest = GET("https://dev.null")
 
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = popularMangaRequest(page)
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = if (this::seriesCache.isInitialized) doNothingRequest else homepageRequest
+    override fun popularMangaRequest(page: Int) = if (this::seriesCache.isInitialized) doNothingRequest else homepageRequest
+    override fun latestUpdatesRequest(page: Int) = if (this::seriesCache.isInitialized) doNothingRequest else homepageRequest
+    override fun chapterListRequest(manga: SManga) = homepageRequest
 
+    private fun idOf(manga: SManga) = manga.url.substringAfterLast("/")
     override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
-        return client.newCall(popularMangaRequest(page))
+        return client.newCall(searchMangaRequest(page, query, filters))
             .asObservableSuccess()
-            .map { response ->
+            .map {
-                val mangas = if (query.startsWith(SERIES_ID_SEARCH_PREFIX)) {
+                val manga = seriesCache.asSequence().map { it.value }.filter {
-                    getFilteredSeriesList(
+                    if (query.startsWith(SERIES_ID_SEARCH_PREFIX)) {
-                        response.asJsoup().getDataJsonObject(),
+                        return@filter it.series_id.contains(query.removePrefix(SERIES_ID_SEARCH_PREFIX), true)
-                        idFilter = query.removePrefix(SERIES_ID_SEARCH_PREFIX)
+                    }
-                    )
+                    sequence { yieldAll(it.alt_titles); yield(it.title) }
-                } else {
+                        .any { title -> title.contains(query, true) }
-                    getFilteredSeriesList(
+                }.map { it.toSManga() }.toList()
-                        response.asJsoup().getDataJsonObject(),
+
-                        titleFilter = query
+                MangasPage(manga, false)
-                    )
-                }
-                MangasPage(mangas, false)
             }
     }
 
-    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> = client.newCall(homepageRequest)
-        return client.newCall(popularMangaRequest(0))
+        .asObservableSuccess()
-            .asObservableSuccess()
+        .map { seriesCache[idOf(manga)]?.toSManga() ?: manga }
-            .map { response ->
-                manga.also {
-                    getSeriesObject(response.asJsoup().getDataJsonObject(), it)?.let { series ->
-                        it.title = series.getString("title")
-                        it.author = series.getJSONArray("authors").joinToString(", ")
-                        it.description = series.getString("description")
-                        it.genre = series.getJSONArray("genres").joinToString(", ")
-                        it.status = when (series.getString("status")) {
-                            "ongoing" -> SManga.ONGOING
-                            "completed" -> SManga.COMPLETED
-                            else -> SManga.UNKNOWN
-                        }
-                        it.thumbnail_url = series.getJSONObject("cover_art").getString("source")
-                    }
-                }
-            }
-    }
 
     override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
         val seriesId = manga.url.substringAfter("/series/")
-        return client.newCall(popularMangaRequest(0))
+        return client.newCall(chapterListRequest(manga))
             .asObservableSuccess()
-            .map { response ->
+            .map {
-                var returnChapter = emptyList<SChapter>()
+                val seriesPrefs = application.getSharedPreferences("source_${id}_time_found:$seriesId", 0)
+                val seriesPrefsEditor = seriesPrefs.edit()
+                val cl = seriesCache[idOf(manga)]!!.chapters.asReversed().map {
+                    val title = it.title ?: ""
+                    val groups = it.groups.joinToString(", ")
+                    val number = it.number.content
+                    val displayNumber = it.display_number ?: number
+                    SChapter.create().apply {
+                        url = "${manga.url}/$number"
+                        chapter_number = number.toFloat()
+                        name = "Chapter $displayNumber" + if (title.isNotBlank()) " - $title" else ""
+                        scanlator = groups
 
-                val series = getSeriesObject(response.asJsoup().getDataJsonObject(), manga)
+                        // Save current time when a chapter is found for the first time, and reuse it on future checks to
-                if (series != null) {
+                        // prevent manga entry without any new chapter bumped to the top of "Latest chapter" list
-                    val seriesPrefs = application.getSharedPreferences("source_${id}_time_found:$seriesId", 0)
+                        // when the library is updated.
-                    val seriesPrefsEditor = seriesPrefs.edit()
+                        val currentTimeMillis = System.currentTimeMillis()
-
+                        if (!seriesPrefs.contains(number)) {
-                    val chapters = series.getJSONArray("chapters")
+                            seriesPrefsEditor.putLong(number, currentTimeMillis)
-                    returnChapter = (0 until chapters.length()).reversed().map { i ->
-                        val chapter = chapters.getJSONObject(i)
-                        val title = chapter.optString("title")
-                        val groups = chapter.getJSONArray("groups").joinToString()
-                        val number = chapter.getString("number")
-                        val displayNumber = chapter.optString("display_number", number)
-                        SChapter.create().apply {
-                            url = "${manga.url}/$number"
-                            chapter_number = number.toFloat()
-                            name = "Chapter $displayNumber" + if (title.isNotBlank()) " - $title" else ""
-                            scanlator = groups
-
-                            // Save current time when a chapter is found for the first time, and reuse it on future
-                            // checks to prevent manga entry without any new chapter bumped to the top of
-                            // "Latest chapter" list when the library is updated.
-                            val currentTimeMillis = System.currentTimeMillis()
-                            if (!seriesPrefs.contains(number)) {
-                                seriesPrefsEditor.putLong(number, currentTimeMillis)
-                            }
-                            date_upload = seriesPrefs.getLong(number, currentTimeMillis)
                         }
+                        date_upload = seriesPrefs.getLong(number, currentTimeMillis)
                     }
-                    seriesPrefsEditor.apply()
                 }
-
+                seriesPrefsEditor.apply()
-                returnChapter
+                cl
             }
     }
 
-    override fun popularMangaParse(response: Response): MangasPage {
+    override fun popularMangaParse(response: Response) = MangasPage(seriesCache.map { it.value.toSManga() }, false)
-        val mangas = getFilteredSeriesList(response.asJsoup().getDataJsonObject())
-        return MangasPage(mangas, false)
-    }
 
-    override fun latestUpdatesParse(response: Response): MangasPage {
+    override fun latestUpdatesParse(response: Response) = MangasPage(
-        val latests = response.asJsoup().getDataJsonObject()
+        latestSeries.map { seriesCache[it]!!.toSManga() },
-            .getJSONObject("props")
+        false
-            .getJSONObject("pageProps")
+    )
-            .getJSONArray("latests")
-        val mangas = (0 until latests.length()).map { i ->
-            val manga = latests.getJSONArray(i).getJSONObject(0)
-            SManga.create().apply {
-                url = "/series/${manga.getString("series_id")}"
-                title = manga.getString("title")
-                thumbnail_url = manga.getJSONObject("cover_art").getString("source")
-            }
-        }
-        return MangasPage(mangas, false)
-    }
 
     override fun pageListParse(response: Response): List<Page> {
-        val pages = response.asJsoup().getDataJsonObject()
+        return json.decodeFromJsonElement<List<String>>(response.asJsoup().getDataJsonObject()["props"]!!.jsonObject["pageProps"]!!.jsonObject["pages"]!!).mapIndexed { index, s ->
-            .getJSONObject("props")
+            Page(index, "", s)
-            .getJSONObject("pageProps")
+        }
-            .getJSONArray("pages")
-        return (0 until pages.length()).map { i -> Page(i, "", pages.getString(i)) }
     }
 
     /**
      * Returns json object of site data
      */
-    private fun Document.getDataJsonObject(): JSONObject {
+    private fun Document.getDataJsonObject() = json.parseToJsonElement(getElementById("__NEXT_DATA__").html()).jsonObject
-        return JSONObject(getElementById("__NEXT_DATA__").html())
-    }
-
-    /**
-     * Returns JSONObject for [manga] from site data
-     */
-    private fun getSeriesObject(jsonObject: JSONObject, manga: SManga): JSONObject? {
-        val seriesId = manga.url.substringAfter("/series/")
-        val seriesArray = jsonObject
-            .getJSONObject("props")
-            .getJSONObject("pageProps")
-            .getJSONArray("series")
-        val seriesIndex = (0 until seriesArray.length()).firstOrNull { i ->
-            seriesArray.getJSONObject(i).optString("series_id").takeIf { it.isNotBlank() } == seriesId
-        }
-        return if (seriesIndex != null) seriesArray.getJSONObject(seriesIndex) else null
-    }
-
-    /**
-     * @return filtered series from home page
-     * @param data json data from [getDataJsonObject]
-     * @param titleFilter will be used to check against title and alt_titles, null to disable filter
-     * @param idFilter will be used to check against id, null to disable filter, only used when [titleFilter] is unset
-     */
-    private fun getFilteredSeriesList(
-        data: JSONObject,
-        titleFilter: String? = null,
-        idFilter: String? = null
-    ): List<SManga> {
-        val series = data.getJSONObject("props").getJSONObject("pageProps").getJSONArray("series")
-        val mangas = mutableListOf<SManga>()
-        for (i in 0 until series.length()) {
-            val manga = series.getJSONObject(i)
-            val mangaId = manga.getString("series_id")
-            val mangaTitle = manga.getString("title")
-            val mangaAltTitles = manga.getJSONArray("alt_titles")
-
-            // Filtering
-            if (titleFilter != null) {
-                if (!(mangaTitle.contains(titleFilter, true) || mangaAltTitles.contains(titleFilter))) {
-                    continue
-                }
-            } else if (idFilter != null) {
-                if (!mangaId.contains(idFilter, true)) {
-                    continue
-                }
-            }
-
-            mangas += SManga.create().apply {
-                url = "/series/$mangaId"
-                title = mangaTitle
-                thumbnail_url = manga.getJSONObject("cover_art").getString("source")
-            }
-        }
-        return mangas.toList()
-    }
-
-    private fun JSONArray.joinToString(separator: String = ", "): String {
-        val stringBuilder = StringBuilder()
-        for (i in 0 until length()) {
-            if (i > 0) stringBuilder.append(separator)
-            val item = getString(i)
-            stringBuilder.append(item)
-        }
-        return stringBuilder.toString()
-    }
-
-    /**
-     * For string objects
-     */
-    private operator fun JSONArray.contains(other: CharSequence): Boolean {
-        for (i in 0 until length()) {
-            if (optString(i, "").contains(other, true)) {
-                return true
-            }
-        }
-        return false
-    }
 
     override fun mangaDetailsParse(response: Response): SManga {
         throw UnsupportedOperationException("Not used.")
@@ -246,3 +159,28 @@ class CatManga : HttpSource() {
         const val SERIES_ID_SEARCH_PREFIX = "series_id:"
     }
 }
+
+@Serializable
+private data class JsonImage(val source: String, val width: Int, val height: Int)
+
+@Serializable
+private data class JsonChapter(val title: String? = null, val groups: List<String>, val number: JsonPrimitive, val display_number: String? = null, val volume: Int? = null)
+
+@Serializable
+private data class JsonSeries(val alt_titles: List<String>, val authors: List<String>, val genres: List<String>, val chapters: List<JsonChapter>, val title: String, val series_id: String, val description: String, val status: String, val cover_art: JsonImage, val all_covers: List<JsonImage>) {
+    fun toSManga() = this.let { jsonSeries ->
+        SManga.create().apply {
+            url = "/series/${jsonSeries.series_id}"
+            title = jsonSeries.title
+            thumbnail_url = jsonSeries.cover_art.source
+            author = jsonSeries.authors.joinToString(", ")
+            description = jsonSeries.description
+            genre = jsonSeries.genres.joinToString(", ")
+            status = when (jsonSeries.status) {
+                "ongoing" -> SManga.ONGOING
+                "completed" -> SManga.COMPLETED
+                else -> SManga.UNKNOWN
+            }
+        }
+    }
+}

src/ru/remanga/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'Remanga'
     pkgNameSuffix = 'ru.remanga'
     extClass = '.Remanga'
-    extVersionCode = 28
+    extVersionCode = 29
     libVersion = '1.2'
 }
 

src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/Remanga.kt

@@ -17,10 +17,6 @@ import android.content.SharedPreferences
 import android.os.Build
 import android.text.InputType
 import android.widget.Toast
-import com.github.salomonbrys.kotson.fromJson
-import com.google.gson.Gson
-import com.google.gson.JsonObject
-import com.google.gson.JsonSyntaxException
 import eu.kanade.tachiyomi.lib.dataimage.DataImageInterceptor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
@@ -34,6 +30,13 @@ import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.decodeFromJsonElement
+import kotlinx.serialization.json.put
 import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
 import okhttp3.Interceptor
@@ -42,17 +45,16 @@ import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
-import org.json.JSONObject
 import org.jsoup.Jsoup
 import rx.Observable
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
+import uy.kohesive.injekt.injectLazy
 import java.text.SimpleDateFormat
 import java.util.Date
 import java.util.Locale
 import kotlin.math.absoluteValue
 import kotlin.random.Random
-
 class Remanga : ConfigurableSource, HttpSource() {
     override val name = "Remanga"
 
@@ -100,15 +102,16 @@ class Remanga : ConfigurableSource, HttpSource() {
     private var branches = mutableMapOf<String, List<BranchesDto>>()
 
     private fun login(chain: Interceptor.Chain, username: String, password: String): String {
-        val jsonObject = JSONObject()
+        val jsonObject = buildJsonObject {
-        jsonObject.put("user", username)
+            put("user", username)
-        jsonObject.put("password", password)
+            put("password", password)
+        }
         val body = jsonObject.toString().toRequestBody(MEDIA_TYPE)
         val response = chain.proceed(POST("$baseUrl/api/users/login/", headers, body))
         if (response.code >= 400) {
             throw Exception("Failed to login")
         }
-        val user = gson.fromJson<SeriesWrapperDto<UserDto>>(response.body?.charStream()!!)
+        val user = json.decodeFromString<SeriesWrapperDto<UserDto>>(response.body!!.string())
         return user.content.access_token
     }
 
@@ -121,7 +124,7 @@ class Remanga : ConfigurableSource, HttpSource() {
     override fun latestUpdatesParse(response: Response): MangasPage = searchMangaParse(response)
 
     override fun searchMangaParse(response: Response): MangasPage {
-        val page = gson.fromJson<PageWrapperDto<LibraryDto>>(response.body?.charStream()!!)
+        val page = json.decodeFromString<PageWrapperDto<LibraryDto>>(response.body!!.string())
         val mangas = page.content.map {
             it.toSManga()
         }
@@ -159,7 +162,7 @@ class Remanga : ConfigurableSource, HttpSource() {
             when (filter) {
                 is OrderBy -> {
                     val ord = arrayOf("id", "chapter_date", "rating", "votes", "views", "count_chapters", "random")[filter.state!!.index]
-                    url.addQueryParameter("ordering", if (filter.state!!.ascending) "$ord" else "-$ord")
+                    url.addQueryParameter("ordering", if (filter.state!!.ascending) ord else "-$ord")
                 }
                 is CategoryList -> filter.state.forEach { category ->
                     if (category.state != Filter.TriState.STATE_IGNORE) {
@@ -273,7 +276,7 @@ class Remanga : ConfigurableSource, HttpSource() {
         return GET(baseUrl.replace("api.", "") + "/manga/" + manga.url.substringAfter("/api/titles/", "/"), headers)
     }
     override fun mangaDetailsParse(response: Response): SManga {
-        val series = gson.fromJson<SeriesWrapperDto<MangaDetDto>>(response.body?.charStream()!!)
+        val series = json.decodeFromString<SeriesWrapperDto<MangaDetDto>>(response.body!!.string())
         branches[series.content.en_name] = series.content.branches
         return series.content.toSManga()
     }
@@ -281,10 +284,11 @@ class Remanga : ConfigurableSource, HttpSource() {
     private fun mangaBranches(manga: SManga): List<BranchesDto> {
         val responseString = client.newCall(GET("$baseUrl/${manga.url}")).execute().body?.string() ?: return emptyList()
         // manga requiring login return "content" as a JsonArray instead of the JsonObject we expect
-        return if (gson.fromJson<JsonObject>(responseString)["content"].isJsonObject) {
+        val content = json.decodeFromString<JsonObject>(responseString)["content"]
-            val series = gson.fromJson<SeriesWrapperDto<MangaDetDto>>(responseString)
+        return if (content is JsonObject) {
-            branches[series.content.en_name] = series.content.branches
+            val series = json.decodeFromJsonElement<MangaDetDto>(content)
-            series.content.branches
+            branches[series.en_name] = series.branches
+            series.branches
         } else {
             emptyList()
         }
@@ -325,8 +329,8 @@ class Remanga : ConfigurableSource, HttpSource() {
     }
 
     override fun chapterListParse(response: Response): List<SChapter> {
-        val chapters = gson.fromJson<PageWrapperDto<BookDto>>(response.body?.charStream()!!)
+        val chapters = json.decodeFromString<SeriesWrapperDto<List<BookDto>>>(response.body!!.string())
-        return chapters.content.filter { !it.is_paid or it.is_bought }.map { chapter ->
+        return chapters.content.filter { !it.is_paid or (it.is_bought == true) }.map { chapter ->
             SChapter.create().apply {
                 chapter_number = chapter.chapter.split(".").take(2).joinToString(".").toFloat()
                 name = chapterName(chapter)
@@ -343,12 +347,12 @@ class Remanga : ConfigurableSource, HttpSource() {
     override fun pageListParse(response: Response): List<Page> {
         val body = response.body?.string()!!
         return try {
-            val page = gson.fromJson<SeriesWrapperDto<PageDto>>(body)
+            val page = json.decodeFromString<SeriesWrapperDto<PageDto>>(body)
             page.content.pages.filter { it.height > 1 }.map {
                 Page(it.page, "", it.link)
             }
-        } catch (e: JsonSyntaxException) {
+        } catch (e: SerializationException) {
-            val page = gson.fromJson<SeriesWrapperDto<PaidPageDto>>(body)
+            val page = json.decodeFromString<SeriesWrapperDto<PaidPageDto>>(body)
             val result = mutableListOf<Page>()
             page.content.pages.forEach {
                 it.filter { page -> page.height > 10 }.forEach { page ->
@@ -596,7 +600,7 @@ class Remanga : ConfigurableSource, HttpSource() {
             dialogTitle = title
 
             if (isPassword) {
-                if (!value.isNullOrBlank()) { summary = "*****" }
+                if (value.isNotBlank()) { summary = "*****" }
                 setOnBindEditTextListener {
                     it.inputType = InputType.TYPE_CLASS_TEXT or InputType.TYPE_TEXT_VARIATION_PASSWORD
                 }
@@ -617,7 +621,7 @@ class Remanga : ConfigurableSource, HttpSource() {
     private fun getPrefUsername(): String = preferences.getString(USERNAME_TITLE, USERNAME_DEFAULT)!!
     private fun getPrefPassword(): String = preferences.getString(PASSWORD_TITLE, PASSWORD_DEFAULT)!!
 
-    private val gson by lazy { Gson() }
+    private val json: Json by injectLazy()
     private val username by lazy { getPrefUsername() }
     private val password by lazy { getPrefPassword() }
 

src/ru/remanga/src/eu/kanade/tachiyomi/extension/ru/remanga/dto/Dto.kt

@@ -1,19 +1,25 @@
+import kotlinx.serialization.Serializable
+
+@Serializable
 data class GenresDto(
     val id: Int,
     val name: String
 )
 
+@Serializable
 data class BranchesDto(
     val id: Long,
     val count_chapters: Int
 )
 
+@Serializable
 data class ImgDto(
     val high: String,
     val mid: String,
     val low: String
 )
 
+@Serializable
 data class LibraryDto(
     val id: Long,
     val en_name: String,
@@ -24,11 +30,13 @@ data class LibraryDto(
     val img: ImgDto
 )
 
+@Serializable
 data class StatusDto(
     val id: Int,
     val name: String
 )
 
+@Serializable
 data class MangaDetDto(
     val id: Long,
     val en_name: String,
@@ -47,30 +55,34 @@ data class MangaDetDto(
     val age_limit: Int
 )
 
+@Serializable
 data class PropsDto(
     val total_items: Int,
     val total_pages: Int,
     val page: Int
 )
 
+@Serializable
 data class PageWrapperDto<T>(
     val msg: String,
     val content: List<T>,
     val props: PropsDto,
-    val last: Boolean
+//    val last: Boolean
 )
 
+@Serializable
 data class SeriesWrapperDto<T>(
     val msg: String,
     val content: T,
-    val props: PropsDto
+//    val props: PropsDto
 )
 
+@Serializable
 data class PublisherDto(
     val name: String,
-    val dir: String
 )
 
+@Serializable
 data class BookDto(
     val id: Long,
     val tome: Int,
@@ -78,10 +90,11 @@ data class BookDto(
     val name: String,
     val upload_date: String,
     val is_paid: Boolean,
-    val is_bought: Boolean,
+    val is_bought: Boolean?,
     val publishers: List<PublisherDto>
 )
 
+@Serializable
 data class PagesDto(
     val id: Int,
     val height: Int,
@@ -90,14 +103,17 @@ data class PagesDto(
     val count_comments: Int
 )
 
+@Serializable
 data class PageDto(
     val pages: List<PagesDto>
 )
 
+@Serializable
 data class UserDto(
     val access_token: String
 )
 
+@Serializable
 data class PaidPagesDto(
     val id: Long,
     val link: String,
@@ -105,6 +121,7 @@ data class PaidPagesDto(
     val page: Int
 )
 
+@Serializable
 data class PaidPageDto(
     val pages: List<List<PaidPagesDto>>
 )