ReadComicOnline - Update Parsing to Use QuickJS and Make it Configurable (#8672)

* unscuff code, update regexes as configurable

* Update default values, improved pref

* Update Readcomiconline.kt

Add placeholders for future implementation

* Updated page list parsing to use quickjs

* add json config, remove decryption class

* review changes, updated default config path

* review changes, lint

* lint

* lint...

src/en/readcomiconline/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'ReadComicOnline'
     extClass = '.Readcomiconline'
-    extVersionCode = 35
+    extVersionCode = 36
 }
 
 apply from: "$rootDir/common.gradle"

src/en/readcomiconline/config.json

@@ -0,0 +1,5 @@
+{
+  "imageDecryptEval": "const matches=[..._encryptedString.matchAll(/(cdk|pth)\\s*=\\s*['\"](.*?)['\"]\\s*;?/gs)];const pageLinks=new Array;matches.forEach((t=>{if(t[2])pageLinks.push(decryptLink(t[2]))}));function atob(t){const e=\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\";let s=String(t).replace(/=+$/,\"\");if(s.length%4===1)throw new Error(\"'atob' failed: The string to be decoded is not correctly encoded.\");let n=\"\";for(let t=0,r,c,i=0;c=s.charAt(i++);~c&&(r=t%4?r*64+c:c,t++%4)?n+=String.fromCharCode(255&r>>(-2*t&6)):0)c=e.indexOf(c);return n}function decryptLink(t){let e=t.replace(/\\w{5}__\\w{3}__/g,\"g\").replace(/\\w{2}__\\w{6}_/g,\"a\").replace(/b/g,\"pw_.g28x\").replace(/h/g,\"d2pr.x_27\").replace(/pw_.g28x/g,\"b\").replace(/d2pr.x_27/g,\"h\");if(!e.startsWith(\"https\")){const t=e.indexOf(\"?\");const s=e.substring(t);const n=e.includes(\"=s0?\");const r=n?e.indexOf(\"=s0?\"):e.indexOf(\"=s1600?\");let c=e.substring(0,r);c=c.substring(15,33)+c.substring(50);const i=c.length;c=c.substring(0,i-11)+c[i-2]+c[i-1];const o=atob(c);let g=decodeURIComponent(o);g=g.substring(0,13)+g.substring(17);g=g.substring(0,g.length-2)+(n?\"=s0\":\"=s1600\");e=`https://2.bp.blogspot.com/${g}${s}`}return e}JSON.stringify(pageLinks);",
+  "postDecryptEval": null,
+  "shouldVerifyLinks": false
+}

src/en/readcomiconline/src/eu/kanade/tachiyomi/extension/en/readcomiconline/Readcomiconline.kt

@@ -1,6 +1,7 @@
 package eu.kanade.tachiyomi.extension.en.readcomiconline
 
 import android.content.SharedPreferences
+import app.cash.quickjs.QuickJs
 import eu.kanade.tachiyomi.lib.randomua.UserAgentType
 import eu.kanade.tachiyomi.lib.randomua.setRandomUserAgent
 import eu.kanade.tachiyomi.network.GET
@@ -13,7 +14,11 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import keiyoushi.utils.getPreferencesLazy
+import keiyoushi.utils.parseAs
 import keiyoushi.utils.tryParse
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.Interceptor
 import okhttp3.OkHttpClient
@@ -22,7 +27,9 @@ import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import rx.Observable
+import java.io.IOException
 import java.text.SimpleDateFormat
+import java.util.Calendar
 import java.util.Locale
 
 class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
@@ -35,18 +42,12 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
 
     override val supportsLatest = true
 
-    override fun headersBuilder() = super.headersBuilder()
+    override fun headersBuilder() = super.headersBuilder().set("Referer", "$baseUrl/")
-        .set("Referer", "$baseUrl/")
-    private val scriptPageRegex = """(?s)pth\s*=\s*['"](.*?)['"]\s*;?""".toRegex()
-    private val urlDecryptionRegex = """l\s*\.replace\(\s*/(.*?)/([gimuy]*)\s*,\s*(['"`])(.*?)\3\s*\)""".toRegex()
 
-    override val client: OkHttpClient = network.cloudflareClient.newBuilder()
+    override val client: OkHttpClient = network.cloudflareClient.newBuilder().setRandomUserAgent(
-        .setRandomUserAgent(
+        userAgentType = UserAgentType.DESKTOP,
-            userAgentType = UserAgentType.DESKTOP,
+        filterInclude = listOf("chrome"),
-            filterInclude = listOf("chrome"),
+    ).addNetworkInterceptor(::captchaInterceptor).build()
-        )
-        .addNetworkInterceptor(::captchaInterceptor)
-        .build()
 
     private fun captchaInterceptor(chain: Interceptor.Chain): Response {
         val request = chain.request()
@@ -93,8 +94,14 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
 
     override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
 
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { // publisher > writer > artist + sorting for both if else
+    override fun searchMangaRequest(
-        if (query.isEmpty() && (if (filters.isEmpty()) getFilterList() else filters).filterIsInstance<GenreList>().all { it.included.isEmpty() && it.excluded.isEmpty() }) {
+        page: Int,
+        query: String,
+        filters: FilterList,
+    ): Request { // publisher > writer > artist + sorting for both if else
+        if (query.isEmpty() && (if (filters.isEmpty()) getFilterList() else filters).filterIsInstance<GenreList>()
+            .all { it.included.isEmpty() && it.excluded.isEmpty() }
+        ) {
             val url = baseUrl.toHttpUrl().newBuilder().apply {
                 var pathSegmentAdded = false
 
@@ -106,18 +113,21 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
                                 pathSegmentAdded = true
                             }
                         }
+
                         is WriterFilter -> {
                             if (filter.state.isNotEmpty()) {
                                 addPathSegments("Writer/${filter.state.replace(" ", "-")}")
                                 pathSegmentAdded = true
                             }
                         }
+
                         is ArtistFilter -> {
                             if (filter.state.isNotEmpty()) {
                                 addPathSegments("Artist/${filter.state.replace(" ", "-")}")
                                 pathSegmentAdded = true
                             }
                         }
+
                         else -> {}
                     }
 
@@ -125,7 +135,10 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
                         break
                     }
                 }
-                addPathSegment((if (filters.isEmpty()) getFilterList() else filters).filterIsInstance<SortFilter>().first().selected.toString())
+                addPathSegment(
+                    (if (filters.isEmpty()) getFilterList() else filters).filterIsInstance<SortFilter>()
+                        .first().selected.toString(),
+                )
                 addQueryParameter("page", page.toString())
             }.build()
             return GET(url, headers)
@@ -135,11 +148,16 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
                 addQueryParameter("page", page.toString())
                 for (filter in if (filters.isEmpty()) getFilterList() else filters) {
                     when (filter) {
-                        is Status -> addQueryParameter("status", arrayOf("", "Completed", "Ongoing")[filter.state])
+                        is Status -> addQueryParameter(
+                            "status",
+                            arrayOf("", "Completed", "Ongoing")[filter.state],
+                        )
+
                         is GenreList -> {
                             addQueryParameter("ig", filter.included.joinToString(","))
                             addQueryParameter("eg", filter.excluded.joinToString(","))
                         }
+
                         else -> {}
                     }
                 }
@@ -165,21 +183,20 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
         manga.author = infoElement.select("p:has(span:contains(Writer:)) > a").first()?.text()
         manga.genre = infoElement.select("p:has(span:contains(Genres:)) > *:gt(0)").text()
         manga.description = infoElement.select("p:has(span:contains(Summary:)) ~ p").text()
-        manga.status = infoElement.select("p:has(span:contains(Status:))").first()?.text().orEmpty().let { parseStatus(it) }
+        manga.status = infoElement.select("p:has(span:contains(Status:))").first()?.text().orEmpty()
+            .let { parseStatus(it) }
         manga.thumbnail_url = document.select(".rightBox:eq(0) img").first()?.absUrl("src")
         return manga
     }
 
     override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
-        return client.newCall(realMangaDetailsRequest(manga))
+        return client.newCall(realMangaDetailsRequest(manga)).asObservableSuccess()
-            .asObservableSuccess()
             .map { response ->
                 mangaDetailsParse(response).apply { initialized = true }
             }
     }
 
-    private fun realMangaDetailsRequest(manga: SManga): Request =
+    private fun realMangaDetailsRequest(manga: SManga): Request = super.mangaDetailsRequest(manga)
-        super.mangaDetailsRequest(manga)
 
     override fun mangaDetailsRequest(manga: SManga): Request =
         captchaUrl?.let { GET(it, headers) }.also { captchaUrl = null }
@@ -206,54 +223,64 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
     private val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale.getDefault())
 
     override fun pageListRequest(chapter: SChapter): Request {
-        val qualitySuffix = if ((qualitypref() != "lq" && serverpref() != "s2") || (qualitypref() == "lq" && serverpref() == "s2")) {
+        val qualitySuffix =
-            "&s=${serverpref()}&quality=${qualitypref()}&readType=1"
+            if ((qualityPref() != "lq" && serverPref() != "s2") || (qualityPref() == "lq" && serverPref() == "s2")) {
-        } else {
+                "&s=${serverPref()}&quality=${qualityPref()}&readType=1"
-            "&s=${serverpref()}&readType=1"
+            } else {
-        }
+                "&s=${serverPref()}&readType=1"
+            }
 
         return GET(baseUrl + chapter.url + qualitySuffix, headers)
     }
 
     override fun pageListParse(document: Document): List<Page> {
         // Declare some important values first
-        val encryptedLinks = mutableListOf<String>()
+        var encryptedLinks = mutableListOf<String>()
-        val decryptionRegexKeys = mutableListOf<Pair<String, String>>()
 
         // Get script elements
-        val scripts = document.select("script[type=text/javascript]")
+        val scripts = document.select("script")
+
+        // We'll evaluate every script that exists in the HTML
+        if (remoteConfigItem == null) {
+            throw IOException("Failed to retrieve configuration")
+        }
 
-        // We'll get a bunch of results on the selector but we only need 2: The script that contains the encrypted links and the script
-        // that contains the partial decryption key.
         for (script in scripts) {
-            val scriptContent = script.data()
+            QuickJs.create().use {
-            if (scriptContent.isNotEmpty()) {
+                val eval =
-                val encryptedValues = scriptPageRegex.findAll(scriptContent)
+                    "let _encryptedString = `${script.data()}`;${remoteConfigItem!!.imageDecryptEval}"
-                val decryptionKeys = urlDecryptionRegex.findAll(scriptContent)
+                val evalResult = (it.evaluate(eval) as String).parseAs<List<String>>()
 
-                // We found the encrypted links
+                // Add results to 'encryptedLinks'
-                if (encryptedValues.count() > 0) {
+                encryptedLinks.addAll(evalResult)
-                    encryptedValues.forEach {
-                        val url = it.groupValues[1]
-
-                        if (url.isNotBlank()) {
-                            encryptedLinks.add(url)
-                        }
-                    }
-                }
-
-                // We found the keys
-                if (decryptionKeys.count() > 0) {
-                    decryptionKeys.forEach {
-                        // Corresponds to Pair<RegexPattern, ReplacementValue>
-                        decryptionRegexKeys.add(Pair(it.groupValues[1], it.groupValues[4]))
-                    }
-                }
             }
         }
 
-        return encryptedLinks.mapIndexed { idx, rawUrl ->
+        encryptedLinks = encryptedLinks.let { links ->
-            Page(idx, imageUrl = decryptLink(rawUrl, decryptionRegexKeys, ""))
+            if (remoteConfigItem!!.postDecryptEval != null) {
+                QuickJs.create().use {
+                    val eval = "let _decryptedLinks = ${Json.encodeToString(links)}"
+                    (it.evaluate(eval) as String).parseAs<MutableList<String>>()
+                }
+            } else {
+                links
+            }
+        }
+
+        return encryptedLinks.mapIndexedNotNull { idx, url ->
+            if (!remoteConfigItem!!.shouldVerifyLinks) {
+                Page(idx, imageUrl = url)
+            } else {
+                val request = Request.Builder().url(url).head().build()
+
+                client.newCall(request).execute().use {
+                    if (it.isSuccessful) {
+                        Page(idx, imageUrl = url)
+                    } else {
+                        null // Remove from list
+                    }
+                }
+            }
         }
     }
 
@@ -268,10 +295,12 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
         val excluded: List<String>
             get() = state.filter { it.isExcluded() }.map { it.gid }
     }
-    open class SelectFilter(displayName: String, private val options: Array<Pair<String, String>>) : Filter.Select<String>(
+
-        displayName,
+    open class SelectFilter(displayName: String, private val options: Array<Pair<String, String>>) :
-        options.map { it.first }.toTypedArray(),
+        Filter.Select<String>(
-    ) {
+            displayName,
+            options.map { it.first }.toTypedArray(),
+        ) {
         open val selected get() = options[state].second.takeUnless { it.isEmpty() }
     }
 
@@ -355,8 +384,28 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
     // Preferences Code
 
     override fun setupPreferenceScreen(screen: androidx.preference.PreferenceScreen) {
-        val qualitypref = androidx.preference.ListPreference(screen.context).apply {
+        val remoteConfigPref = androidx.preference.EditTextPreference(screen.context).apply {
-            key = QUALITY_PREF_TITLE
+            key = IMAGE_REMOTE_CONFIG_PREF
+            title = IMAGE_REMOTE_CONFIG_TITLE
+            summary = IMAGE_REMOTE_CONFIG_SUMMARY
+            setDefaultValue(IMAGE_REMOTE_CONFIG_DEFAULT)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val commitResult =
+                    preferences.edit().putString(IMAGE_REMOTE_CONFIG_PREF, newValue as String)
+                        .commit()
+
+                if (commitResult) {
+                    // Make it null so remoteConfigItem would request for a link again
+                    remoteConfigItem = null
+                }
+
+                commitResult
+            }
+        }
+
+        val qualityPref = androidx.preference.ListPreference(screen.context).apply {
+            key = QUALITY_PREF
             title = QUALITY_PREF_TITLE
             entries = arrayOf("High Quality", "Low Quality")
             entryValues = arrayOf("hq", "lq")
@@ -369,9 +418,9 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
                 preferences.edit().putString(QUALITY_PREF, entry).commit()
             }
         }
-        screen.addPreference(qualitypref)
+
-        val serverpref = androidx.preference.ListPreference(screen.context).apply {
+        val serverPref = androidx.preference.ListPreference(screen.context).apply {
-            key = SERVER_PREF_TITLE
+            key = SERVER_PREF
             title = SERVER_PREF_TITLE
             entries = arrayOf("Server 1", "Server 2")
             entryValues = arrayOf("", "s2")
@@ -384,17 +433,58 @@ class Readcomiconline : ConfigurableSource, ParsedHttpSource() {
                 preferences.edit().putString(SERVER_PREF, entry).commit()
             }
         }
-        screen.addPreference(serverpref)
+
+        screen.addPreference(remoteConfigPref)
+        screen.addPreference(qualityPref)
+        screen.addPreference(serverPref)
     }
 
-    private fun qualitypref() = preferences.getString(QUALITY_PREF, "hq")
+    private fun qualityPref() = preferences.getString(QUALITY_PREF, "hq")
 
-    private fun serverpref() = preferences.getString(SERVER_PREF, "")
+    private fun serverPref() = preferences.getString(SERVER_PREF, "")
+
+    private var remoteConfigItem: RemoteConfigDTO? = null
+        get() {
+            if (field != null) {
+                return field
+            }
+
+            val configLink = preferences.getString(
+                IMAGE_REMOTE_CONFIG_PREF.addBustQuery(),
+                IMAGE_REMOTE_CONFIG_DEFAULT.addBustQuery(),
+            ) ?: return null
+
+            try {
+                val configResponse = client.newCall(GET(configLink)).execute()
+
+                field = configResponse.parseAs<RemoteConfigDTO>()
+                configResponse.close()
+                return field
+            } catch (_: IOException) {
+                return null
+            }
+        }
+
+    private fun String.addBustQuery(): String {
+        return "$this?bust=${Calendar.getInstance().time.time}"
+    }
+
+    @Serializable
+    private class RemoteConfigDTO(
+        val imageDecryptEval: String,
+        val postDecryptEval: String?,
+        val shouldVerifyLinks: Boolean,
+    )
 
     companion object {
         private const val QUALITY_PREF_TITLE = "Image Quality Selector"
         private const val QUALITY_PREF = "qualitypref"
         private const val SERVER_PREF_TITLE = "Server Preference"
         private const val SERVER_PREF = "serverpref"
+        private const val IMAGE_REMOTE_CONFIG_TITLE = "Remote Config"
+        private const val IMAGE_REMOTE_CONFIG_SUMMARY = "Remote Config Link"
+        private const val IMAGE_REMOTE_CONFIG_PREF = "imageuseremotelinkpref"
+        private const val IMAGE_REMOTE_CONFIG_DEFAULT =
+            "https://raw.githubusercontent.com/keiyoushi/extensions-source/refs/heads/main/src/en/readcomiconline/config.json"
     }
 }

src/en/readcomiconline/src/eu/kanade/tachiyomi/extension/en/readcomiconline/ReadcomiconlinePageListDecrypt.kt

@@ -1,71 +0,0 @@
-package eu.kanade.tachiyomi.extension.en.readcomiconline
-
-import android.util.Base64
-import java.net.URLDecoder
-
-private fun step1(param: String): String {
-    return param.substring(15, 15 + 18) + param.substring(15 + 18 + 17)
-}
-
-private fun step2(param: String): String {
-    return param.substring(0, param.length - (9 + 2)) +
-        param[param.length - 2] +
-        param[param.length - 1]
-}
-
-fun decryptLink(
-    firstStringFormat: String,
-    partialDecryptKeys: List<Pair<String, String>>,
-    formatter: String = "",
-): String {
-    var processedString = firstStringFormat
-
-    partialDecryptKeys.forEach {
-        processedString = processedString.replace(it.first.toRegex(), it.second)
-    }
-
-    processedString = processedString
-        .replace("pw_.g28x", "b")
-        .replace("d2pr.x_27", "h")
-
-    if (!processedString.startsWith("https")) {
-        val firstStringFormatLocalVar = processedString
-        val firstStringSubS = firstStringFormatLocalVar.substring(
-            firstStringFormatLocalVar.indexOf("?"),
-        )
-
-        processedString = if (firstStringFormatLocalVar.contains("=s0?")) {
-            firstStringFormatLocalVar.substring(0, firstStringFormatLocalVar.indexOf("=s0?"))
-        } else {
-            firstStringFormatLocalVar.substring(0, firstStringFormatLocalVar.indexOf("=s1600?"))
-        }
-
-        processedString = step1(processedString)
-        processedString = step2(processedString)
-
-        // Base64 decode and URL decode
-        val decodedBytes = Base64.decode(processedString, Base64.DEFAULT)
-        processedString = URLDecoder.decode(String(decodedBytes), "UTF-8")
-
-        processedString = processedString.substring(0, 13) +
-            processedString.substring(17)
-
-        processedString = if (firstStringFormat.contains("=s0")) {
-            processedString.substring(0, processedString.length - 2) + "=s0"
-        } else {
-            processedString.substring(0, processedString.length - 2) + "=s1600"
-        }
-
-        processedString += firstStringSubS
-        processedString = "https://2.bp.blogspot.com/$processedString"
-    }
-
-    if (formatter.isNotEmpty()) {
-        processedString = processedString.replace(
-            "https://2.bp.blogspot.com",
-            formatter,
-        )
-    }
-
-    return processedString
-}