Fix search for genkan io (#6919)

src/all/genkanio/build.gradle

@@ -5,7 +5,7 @@ ext {
     extName = 'Genkan.io'
     pkgNameSuffix = "all.genkanio"
     extClass = '.GenkanIO'
-    extVersionCode = 1
+    extVersionCode = 2
     libVersion = '1.2'
 }
 

src/all/genkanio/src/eu/kanade/tachiyomi/extension/all/genkanio/GenkanIO.kt

@@ -1,6 +1,14 @@
 package eu.kanade.tachiyomi.extension.all.genkanio
 
+import android.util.Log
+import com.github.salomonbrys.kotson.keys
+import com.github.salomonbrys.kotson.put
+import com.google.gson.JsonArray
+import com.google.gson.JsonElement
+import com.google.gson.JsonObject
+import com.google.gson.JsonParser
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
@@ -9,42 +17,62 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
+import okhttp3.ResponseBody.Companion.toResponseBody
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import rx.Observable
 import java.util.Calendar
 
-open class GenkanIO() : ParsedHttpSource() {
+open class GenkanIO : ParsedHttpSource() {
     override val lang = "all"
     final override val name = "Genkan.io"
     final override val baseUrl = "https://genkan.io"
     final override val supportsLatest = false
 
-    // genkan.io defaults to listing manga alphabetically, and provides no configuration
+    data class LiveWireRPC(val csrf: String, val state: JsonObject)
-    fun alphabeticalMangaRequest(page: Int): Request = GET("$baseUrl/manga?page=$page", headers)
+    private var livewire: LiveWireRPC? = null
 
-    fun alphabeticalMangaFromElement(element: Element): SManga {
+    /**
-        val manga = SManga.create()
+     * Given a string encoded with html entities and escape sequences, makes an attempt to decode
-
+     * and returns decoded string
-        element.select("a").let {
+     *
-            manga.url = it.attr("href").substringAfter(baseUrl)
+     * Warning: This is not all all exhaustive, and probably misses edge cases
-            manga.title = it.text()
+     *
+     * @Returns decoded string
+     */
+    private fun htmlDecode(html: String): String {
+        return html.replace(Regex("&([A-Za-z]+);")) { match ->
+            mapOf(
+                "raquo" to "»",
+                "laquo" to "«",
+                "amp" to "&",
+                "lt" to "<",
+                "gt" to ">",
+                "quot" to "\""
+            )[match.groups[1]!!.value] ?: match.groups[0]!!.value
+        }.replace(Regex("\\\\(.)")) { match ->
+            mapOf(
+                "t" to "\t",
+                "n" to "\n",
+                "r" to "\r",
+                "b" to "\b"
+            )[match.groups[1]!!.value] ?: match.groups[1]!!.value
         }
-        manga.thumbnail_url = element.select("img").attr("src")
-        return manga
     }
 
-    fun alphabeticalMangaSelector() = "ul[role=list] > li"
+    // popular manga
-    fun alphabeticalMangaNextPageSelector() = "a[rel=next]"
 
-    // popular
+    override fun fetchPopularManga(page: Int) = fetchSearchManga(page, "", FilterList(emptyList()))
-    override fun popularMangaRequest(page: Int) = alphabeticalMangaRequest(page)
+    override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException("Not used")
-    override fun popularMangaFromElement(element: Element) = alphabeticalMangaFromElement(element)
+    override fun popularMangaFromElement(element: Element) = throw UnsupportedOperationException("Not used")
-    override fun popularMangaSelector() = alphabeticalMangaSelector()
+    override fun popularMangaSelector() = throw UnsupportedOperationException("Not used")
-    override fun popularMangaNextPageSelector() = alphabeticalMangaNextPageSelector()
+    override fun popularMangaNextPageSelector() = throw UnsupportedOperationException("Not used")
 
     // latest
 
@@ -54,29 +82,134 @@ open class GenkanIO() : ParsedHttpSource() {
     override fun latestUpdatesNextPageSelector() = throw UnsupportedOperationException("Not used")
 
     // search
+
+    /**
+     * initializes `livewire` local variable using data from https://genkan.io/manga
+     */
+    private fun initLiveWire(response: Response) {
+        val soup = response.asJsoup()
+        val csrf = soup.selectFirst("meta[name=csrf-token]")?.attr("content")
+
+        val initialProps = soup.selectFirst("div[wire:initial-data]")?.attr("wire:initial-data")?.let {
+            JsonParser.parseString(htmlDecode(it))
+        }
+
+        if (csrf != null && initialProps?.asJsonObject != null) {
+            livewire = LiveWireRPC(csrf, initialProps.asJsonObject)
+        } else {
+            Log.e("GenkanIo", soup.selectFirst("div[wire:initial-data]")?.toString() ?: "null")
+        }
+    }
+
+    /**
+     * Prepares  a request which'll send a message to livewire server
+     *
+     * @param url: String - Message endpoint
+     * @param updates: JsonElement - JsonElement which describes the actions taken by server
+     *
+     * @return Request
+     */
+    private fun livewireRequest(url: String, updates: JsonElement): Request {
+        // assert(livewire != null)
+        val payload = JsonObject()
+        payload.put("fingerprint" to livewire!!.state.get("fingerprint"))
+        payload.put("serverMemo" to livewire!!.state.get("serverMemo"))
+        payload.put("updates" to updates)
+
+        // not sure why this isn't getting added automatically
+        val cookie = client.cookieJar.loadForRequest(url.toHttpUrlOrNull()!!).joinToString("; ") { "${it.name}=${it.value}" }
+        return POST(
+            url,
+            Headers.headersOf("x-csrf-token", livewire!!.csrf, "x-livewire", "true", "cookie", cookie, "cache-control", "no-cache, private"),
+            payload.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())
+        )
+    }
+
+    /**
+     * Transforms  json response from livewire server into a response which returns html
+     * Also updates `livewire` variable with state returned by livewire server
+     *
+     * @param response: Response - The response of sending a message to genkan's livewire server
+     *
+     * @return HTML Response - The html embedded within the provided response
+     */
+    private fun livewireResponse(response: Response): Response {
+        val body = response.body?.string()
+        val responseJson = JsonParser.parseString(body).asJsonObject
+
+        // response contains state that we need to preserve
+        mergeLeft(livewire!!.state.get("serverMemo").asJsonObject, responseJson.get("serverMemo").asJsonObject)
+
+        // this seems to be an error  state, so reset everything
+        if (responseJson.get("effects")?.asJsonObject?.get("html")?.isJsonNull == true) {
+            livewire = null
+        }
+
+        // Build html response
+        return response.newBuilder()
+            .body(htmlDecode("${responseJson.get("effects")?.asJsonObject?.get("html")}").toResponseBody("Content-Type: text/html; charset=UTF-8".toMediaTypeOrNull()))
+            .build()
+    }
+
+    /**
+     * Recursively merges j2 onto j1 in place
+     * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's
+     *
+     */
+    private fun mergeLeft(j1: JsonObject, j2: JsonObject) {
+        j2.keys().forEach { k ->
+            if (j1.get(k)?.isJsonObject != true)
+                j1.put(k to j2.get(k))
+            else if (j1.get(k).isJsonObject && j2.get(k).isJsonObject)
+                mergeLeft(j1.get(k).asJsonObject, j2.get(k).asJsonObject)
+        }
+    }
+
     override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
-        return client.newCall(searchMangaRequest(page, query, filters))
+        fun searchRequest() = client.newCall(searchMangaRequest(page, query, filters)).asObservableSuccess().map(::livewireResponse)
-            .asObservableSuccess()
+        return if (livewire == null) {
-            .map { response ->
+            client.newCall(GET("$baseUrl/manga", headers))
-                // Genkan.io redirects if the search query contains characters it deems "illegal" (e.g: '-')
+                .asObservableSuccess()
-                // Return no responses if any redirects occurred
+                .doOnNext(::initLiveWire)
-                if (response.priorResponse != null)
+                .concatWith(Observable.defer(::searchRequest))
-                    MangasPage(emptyList(), false)
+                .reduce { _, x -> x }
-                else
+        } else {
-                    searchMangaParse(response)
+            searchRequest()
-            }
+        }.map(::searchMangaParse)
     }
 
-    override fun searchMangaRequest(page: Int, query: String, @Suppress("UNUSED_PARAMETER") filters: FilterList): Request {
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val url = "$baseUrl/manga".toHttpUrlOrNull()!!.newBuilder()
+        //        assert(livewire != null)
-            .addQueryParameter("page", "$page")
+        val updates = JsonArray()
-            .addQueryParameter("search", query)
+        val data = livewire!!.state.get("serverMemo")?.asJsonObject?.get("data")?.asJsonObject!!
-        return GET("$url")
+        if (data["readyToLoad"]?.asBoolean == false) {
+            updates.add(JsonParser.parseString("""{"type":"callMethod","payload":{"method":"loadManga","params":[]}}"""))
+        }
+        val isNewQuery = query != data["search"]?.asString
+        if (isNewQuery) {
+            updates.add(JsonParser.parseString("""{"type": "syncInput", "payload": {"name": "search", "value": "$query"}}"""))
+        }
+
+        val currPage = if (isNewQuery) 1 else data["page"]?.asInt
+
+        for (i in (currPage!! + 1)..page)
+            updates.add(JsonParser.parseString("""{"type":"callMethod","payload":{"method":"nextPage","params":[]}}"""))
+
+        return livewireRequest("$baseUrl/livewire/message/manga.list-all-manga", updates)
     }
 
-    override fun searchMangaFromElement(element: Element) = alphabeticalMangaFromElement(element)
+    override fun searchMangaFromElement(element: Element): SManga {
-    override fun searchMangaSelector() = alphabeticalMangaSelector()
+        val manga = SManga.create()
-    override fun searchMangaNextPageSelector() = alphabeticalMangaNextPageSelector()
+        element.select("a").let {
+            manga.url = it.attr("href").substringAfter(baseUrl)
+            manga.title = it.text()
+        }
+        manga.thumbnail_url = element.select("img").attr("src")
+        return manga
+    }
+
+    override fun searchMangaSelector() = "ul[role=list]:has(a)> li"
+    override fun searchMangaNextPageSelector() = "button[rel=next]"
 
     // chapter list (is paginated),
     override fun chapterListParse(response: Response): List<SChapter> = throw UnsupportedOperationException("Not used")
@@ -106,7 +239,7 @@ open class GenkanIO() : ParsedHttpSource() {
     private fun chapterPageParse(response: Response): ChapterPage {
         val document = response.asJsoup()
 
-        val mangas = document.select(chapterListSelector()).map { element ->
+        val manga = document.select(chapterListSelector()).map { element ->
             chapterFromElement(element)
         }
 
@@ -114,7 +247,7 @@ open class GenkanIO() : ParsedHttpSource() {
             document.select(selector).first()
         } != null
 
-        return ChapterPage(mangas, hasNextPage)
+        return ChapterPage(manga, hasNextPage)
     }
 
     private fun chapterListRequest(manga: SManga, page: Int): Request {
@@ -123,21 +256,21 @@ open class GenkanIO() : ParsedHttpSource() {
         return GET("$url", headers)
     }
 
-    override fun chapterFromElement(element: Element): SChapter = element.children().let { tablerow ->
+    override fun chapterFromElement(element: Element): SChapter = element.children().let { tableRow ->
         val isTitleBlank: (String) -> Boolean = { s: String -> s == "-" || s.isBlank() }
-        val (numElem, nameElem, languageElem, groupElem, viewsElem) = tablerow
+        val (numElem, nameElem, languageElem, groupElem, viewsElem) = tableRow
-        val (releasedElem, urlElem) = Pair(tablerow[5], tablerow[6])
+        val (releasedElem, urlElem) = Pair(tableRow[5], tableRow[6])
         SChapter.create().apply {
             name = if (isTitleBlank(nameElem.text())) "Chapter ${numElem.text()}" else "Ch. ${numElem.text()}: ${nameElem.text()}"
             url = urlElem.select("a").attr("href").substringAfter(baseUrl)
-            date_upload = parseRelativeDate(releasedElem.text()) ?: 0
+            date_upload = parseRelativeDate(releasedElem.text())
             scanlator = "${groupElem.text()} - ${languageElem.text()}"
             chapter_number = numElem.text().toFloat()
         }
     }
 
     override fun chapterListSelector() = "tbody > tr"
-    fun chapterListNextPageSelector() = "a[rel=next]"
+    private fun chapterListNextPageSelector() = "a[rel=next]"
 
     // manga
 
@@ -156,7 +289,7 @@ open class GenkanIO() : ParsedHttpSource() {
             )
     }
 
-    private fun parseRelativeDate(date: String): Long? {
+    private fun parseRelativeDate(date: String): Long {
         val trimmedDate = date.substringBefore(" ago").removeSuffix("s").split(" ")
 
         val calendar = Calendar.getInstance()