MangaFire: get/generate vrf for ajax calls (#10988)

* fix page list
- remove ajax call from chapter list
- use webview to get vrf

* fix query search

- get vrf from webview

* bump

* trigger search on script load, reduce wait time

* vrf script by user podimium on Discord

Co-authored-by: Trung0246 <11626920+Trung0246@users.noreply.github.com>

* use vrf script for search as webview isn't reliable for that

* remove unused

---------

Co-authored-by: Trung0246 <11626920+Trung0246@users.noreply.github.com>

src/all/mangafire/assets/vrf.js

@@ -0,0 +1,294 @@
+/**
+  Copyright © 2019 W3C and Jeff Carpenter <jeffcarp@chromium.org>
+
+  atob and btoa source code is released under the 3-Clause BSD license.
+*/
+ function atob(data) {
+     if (arguments.length === 0) {
+         throw new TypeError("1 argument required, but only 0 present.");
+     }
+
+     const keystr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+     function atobLookup(chr) {
+         const index = keystr.indexOf(chr);
+         // Throw exception if character is not in the lookup string; should not be hit in tests
+         return index < 0 ? undefined : index;
+     }
+
+     data = `${data}`;
+     data = data.replace(/[ \t\n\f\r]/g, "");
+     if (data.length % 4 === 0) {
+         data = data.replace(/==?$/, "");
+     }
+     if (data.length % 4 === 1 || /[^+/0-9A-Za-z]/.test(data)) {
+         return null;
+     }
+     let output = "";
+     let buffer = 0;
+     let accumulatedBits = 0;
+     for (let i = 0; i < data.length; i++) {
+         buffer <<= 6;
+         buffer |= atobLookup(data[i]);
+         accumulatedBits += 6;
+         if (accumulatedBits === 24) {
+         output += String.fromCharCode((buffer & 0xff0000) >> 16);
+         output += String.fromCharCode((buffer & 0xff00) >> 8);
+         output += String.fromCharCode(buffer & 0xff);
+         buffer = accumulatedBits = 0;
+         }
+     }
+     if (accumulatedBits === 12) {
+         buffer >>= 4;
+         output += String.fromCharCode(buffer);
+     } else if (accumulatedBits === 18) {
+         buffer >>= 2;
+         output += String.fromCharCode((buffer & 0xff00) >> 8);
+         output += String.fromCharCode(buffer & 0xff);
+     }
+     return output;
+ }
+
+ function btoa(s) {
+     if (arguments.length === 0) {
+       throw new TypeError("1 argument required, but only 0 present.");
+     }
+
+     const keystr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+     function btoaLookup(index) {
+         if (index >= 0 && index < 64) {
+         return keystr[index];
+         }
+
+         return undefined;
+     }
+
+     let i;
+     s = `${s}`;
+     for (i = 0; i < s.length; i++) {
+       if (s.charCodeAt(i) > 255) {
+         return null;
+       }
+     }
+     let out = "";
+     for (i = 0; i < s.length; i += 3) {
+       const groupsOfSix = [undefined, undefined, undefined, undefined];
+       groupsOfSix[0] = s.charCodeAt(i) >> 2;
+       groupsOfSix[1] = (s.charCodeAt(i) & 0x03) << 4;
+       if (s.length > i + 1) {
+         groupsOfSix[1] |= s.charCodeAt(i + 1) >> 4;
+         groupsOfSix[2] = (s.charCodeAt(i + 1) & 0x0f) << 2;
+       }
+       if (s.length > i + 2) {
+         groupsOfSix[2] |= s.charCodeAt(i + 2) >> 6;
+         groupsOfSix[3] = s.charCodeAt(i + 2) & 0x3f;
+       }
+       for (let j = 0; j < groupsOfSix.length; j++) {
+         if (typeof groupsOfSix[j] === "undefined") {
+           out += "=";
+         } else {
+           out += btoaLookup(groupsOfSix[j]);
+         }
+       }
+     }
+     return out;
+ }
+
+// provided by: @Trung0246 on Github
+
+/**
+ * Readable refactor of crc_vrf() with identical output to the original.
+ * - Uses byte arrays throughout.
+ * - Consolidates repeated transform logic.
+ * - Adds clear naming and comments.
+ *
+ * Example check (must be true):
+ * console.log(
+ *   crc_vrf("67890@ The quick brown fox jumps over the lazy dog @12345")
+ *   === "ZBYeRCjYBk0tkZnKW4kTuWBYw-81e-csvu6v17UY4zchviixt67VJ_tjpFEsOXB-a8X4ZFpDoDbPq8ms-7IyN95vmLVdP5vWSoTAl4ZbIBE8xijci8emrkdEYmArOPMUq5KAc3KEabUzHkNwjBtwvs0fQR7nDpI"
+ * );
+ */
+
+// Node/browser-friendly base64 helpers
+const atob_ = typeof atob === "function"
+	? atob
+	: (b64) => Buffer.from(b64, "base64").toString("binary");
+
+const btoa_ = typeof btoa === "function"
+	? btoa
+	: (bin) => Buffer.from(bin, "binary").toString("base64");
+
+// Byte helpers
+const toBytes = (str) => Array.from(str, (c) => c.charCodeAt(0) & 0xff);
+const fromBytes = (bytes) => bytes.map((b) => String.fromCharCode(b & 0xff)).join("");
+
+// RC4 over byte arrays (key is a binary string)
+function rc4Bytes(key, input) {
+	const s = Array.from({ length: 256 }, (_, i) => i);
+	let j = 0;
+
+	// KSA
+	for (let i = 0; i < 256; i++) {
+		j = (j + s[i] + key.charCodeAt(i % key.length)) & 0xff;
+		[s[i], s[j]] = [s[j], s[i]];
+	}
+
+	// PRGA
+	const out = new Array(input.length);
+	let i = 0;
+	j = 0;
+	for (let y = 0; y < input.length; y++) {
+		i = (i + 1) & 0xff;
+		j = (j + s[i]) & 0xff;
+		[s[i], s[j]] = [s[j], s[i]];
+		const k = s[(s[i] + s[j]) & 0xff];
+		out[y] = (input[y] ^ k) & 0xff;
+	}
+	return out;
+}
+
+// One generic “step” to remove repeated boilerplate.
+function transform(input, initSeedBytes, prefixKeyString, prefixLen, schedule) {
+	const out = [];
+	for (let i = 0; i < input.length; i++) {
+		if (i < prefixLen) out.push(prefixKeyString.charCodeAt(i) & 0xff);
+
+		out.push(schedule[i % 10]((input[i] ^ initSeedBytes[i % 32]) & 0xff) & 0xff);
+	}
+	return out;
+}
+
+// 8-bit ops
+const add8 = (n) => (c) => (c + n) & 0xff;
+const sub8 = (n) => (c) => (c - n + 256) & 0xff;
+const xor8 = (n) => (c) => (c ^ n) & 0xff;
+const rotl8 = (n) => (c) => ((c << n) | (c >>> (8 - n))) & 0xff;
+
+// Schedules for each step (10 ops each, indexed by i % 10)
+const scheduleC = [
+	sub8(48), sub8(19), xor8(241), sub8(19), add8(223),
+	sub8(19), sub8(170), sub8(19), sub8(48), xor8(8),
+];
+
+const scheduleY = [
+	rotl8(4), add8(223), rotl8(4), xor8(163), sub8(48),
+	add8(82), add8(223), sub8(48), xor8(83), rotl8(4),
+];
+
+const scheduleB = [
+	sub8(19), add8(82), sub8(48), sub8(170), rotl8(4),
+	sub8(48), sub8(170), xor8(8), add8(82), xor8(163),
+];
+
+const scheduleJ = [
+	add8(223), rotl8(4), add8(223), xor8(83), sub8(19),
+	add8(223), sub8(170), add8(223), sub8(170), xor8(83),
+];
+
+const scheduleE = [
+	add8(82), xor8(83), xor8(163), add8(82), sub8(170),
+	xor8(8), xor8(241), add8(82), add8(176), rotl8(4),
+];
+
+function base64UrlEncodeBytes(bytes) {
+	const std = btoa_(fromBytes(bytes));
+	return std.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
+}
+
+function bytesFromBase64(b64) {
+	return toBytes(atob_(b64));
+}
+
+// Constants — grouped logically and left as-is (base64) for clarity.
+const CONST = {
+	rc4Keys: {
+		l: "u8cBwTi1CM4XE3BkwG5Ble3AxWgnhKiXD9Cr279yNW0=",
+		g: "t00NOJ/Fl3wZtez1xU6/YvcWDoXzjrDHJLL2r/IWgcY=",
+		B: "S7I+968ZY4Fo3sLVNH/ExCNq7gjuOHjSRgSqh6SsPJc=",
+		m: "7D4Q8i8dApRj6UWxXbIBEa1UqvjI+8W0UvPH9talJK8=",
+		F: "0JsmfWZA1kwZeWLk5gfV5g41lwLL72wHbam5ZPfnOVE=",
+	},
+	seeds32: {
+		A: "pGjzSCtS4izckNAOhrY5unJnO2E1VbrU+tXRYG24vTo=",
+		V: "dFcKX9Qpu7mt/AD6mb1QF4w+KqHTKmdiqp7penubAKI=",
+		N: "owp1QIY/kBiRWrRn9TLN2CdZsLeejzHhfJwdiQMjg3w=",
+		P: "H1XbRvXOvZAhyyPaO68vgIUgdAHn68Y6mrwkpIpEue8=",
+		k: "2Nmobf/mpQ7+Dxq1/olPSDj3xV8PZkPbKaucJvVckL0=",
+	},
+	prefixKeys: {
+		O: "Rowe+rg/0g==",
+		v: "8cULcnOMJVY8AA==",
+		L: "n2+Og2Gth8Hh",
+		p: "aRpvzH+yoA==",
+		W: "ZB4oBi0=",
+	},
+};
+
+function crc_vrf(input) {
+	// Stage 0: normalize to URI-encoded bytes
+	let bytes = toBytes(encodeURIComponent(input));
+
+	// RC4 1
+	bytes = rc4Bytes(atob_(CONST.rc4Keys.l), bytes);
+
+	// Step C1
+	bytes = transform(
+		bytes,
+		bytesFromBase64(CONST.seeds32.A),
+		atob_(CONST.prefixKeys.O),
+		7,
+		scheduleC
+	);
+
+	// RC4 2
+	bytes = rc4Bytes(atob_(CONST.rc4Keys.g), bytes);
+
+	// Step Y
+	bytes = transform(
+		bytes,
+		bytesFromBase64(CONST.seeds32.V),
+		atob_(CONST.prefixKeys.v),
+		10,
+		scheduleY
+	);
+
+	// RC4 3
+	bytes = rc4Bytes(atob_(CONST.rc4Keys.B), bytes);
+
+	// Step B
+	bytes = transform(
+		bytes,
+		bytesFromBase64(CONST.seeds32.N),
+		atob_(CONST.prefixKeys.L),
+		9,
+		scheduleB
+	);
+
+	// RC4 4
+	bytes = rc4Bytes(atob_(CONST.rc4Keys.m), bytes);
+
+	// Step J
+	bytes = transform(
+		bytes,
+		bytesFromBase64(CONST.seeds32.P),
+		atob_(CONST.prefixKeys.p),
+		7,
+		scheduleJ
+	);
+
+	// RC4 5
+	bytes = rc4Bytes(atob_(CONST.rc4Keys.F), bytes);
+
+	// Step E
+	bytes = transform(
+		bytes,
+		bytesFromBase64(CONST.seeds32.k),
+		atob_(CONST.prefixKeys.W),
+		5,
+		scheduleE
+	);
+
+	// Base64URL
+	return base64UrlEncodeBytes(bytes);
+}

src/all/mangafire/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'MangaFire'
     extClass = '.MangaFireFactory'
-    extVersionCode = 12
+    extVersionCode = 13
     isNsfw = true
 }
 

src/all/mangafire/src/eu/kanade/tachiyomi/extension/all/mangafire/MangaFire.kt

@@ -1,7 +1,17 @@
 package eu.kanade.tachiyomi.extension.all.mangafire
 
+import android.annotation.SuppressLint
+import android.app.Application
+import android.os.Handler
+import android.os.Looper
+import android.util.Log
+import android.webkit.WebResourceRequest
+import android.webkit.WebResourceResponse
+import android.webkit.WebView
+import android.webkit.WebViewClient
 import androidx.preference.PreferenceScreen
 import androidx.preference.SwitchPreferenceCompat
+import app.cash.quickjs.QuickJs
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -12,9 +22,9 @@ import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
 import keiyoushi.utils.getPreferencesLazy
+import keiyoushi.utils.parseAs
+import keiyoushi.utils.tryParse
 import kotlinx.serialization.Serializable
-import kotlinx.serialization.decodeFromString
-import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonPrimitive
 import kotlinx.serialization.json.int
 import okhttp3.HttpUrl.Companion.toHttpUrl
@@ -23,11 +33,13 @@ import okhttp3.Response
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
-import rx.Observable
+import uy.kohesive.injekt.Injekt
-import uy.kohesive.injekt.injectLazy
+import uy.kohesive.injekt.api.get
-import java.text.ParseException
+import java.io.ByteArrayInputStream
 import java.text.SimpleDateFormat
 import java.util.Locale
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
 
 class MangaFire(
     override val lang: String,
@@ -38,9 +50,6 @@ class MangaFire(
     override val baseUrl = "https://mangafire.to"
 
     override val supportsLatest = true
-
-    private val json: Json by injectLazy()
-
     private val preferences by getPreferencesLazy()
 
     override val client = network.cloudflareClient.newBuilder().addInterceptor(ImageInterceptor).build()
@@ -48,6 +57,10 @@ class MangaFire(
     override fun headersBuilder() = super.headersBuilder()
         .add("Referer", "$baseUrl/")
 
+    private val context = Injekt.get<Application>()
+    private val handler = Handler(Looper.getMainLooper())
+    private val emptyWebViewResponse = WebResourceResponse("text/html", "utf-8", ByteArrayInputStream(" ".toByteArray()))
+
     // ============================== Popular ===============================
 
     override fun popularMangaRequest(page: Int): Request {
@@ -73,13 +86,22 @@ class MangaFire(
     override fun latestUpdatesParse(response: Response) = searchMangaParse(response)
 
     // =============================== Search ===============================
+    private val vrfScript by lazy {
+        val vrf = this::class.java.getResourceAsStream("/assets/vrf.js")!!
+            .bufferedReader()
+            .readText()
+
+        QuickJs.create().use {
+            it.compile(vrf, "vrf")
+        }
+    }
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
         val url = baseUrl.toHttpUrl().newBuilder().apply {
             addPathSegment("filter")
 
             if (query.isNotBlank()) {
-                addQueryParameter("keyword", query)
+                addQueryParameter("keyword", query.trim())
             }
 
             val filterList = filters.ifEmpty { getFilterList() }
@@ -89,6 +111,14 @@ class MangaFire(
 
             addQueryParameter("language[]", langCode)
             addQueryParameter("page", page.toString())
+
+            if (query.isNotBlank()) {
+                val vrf = QuickJs.create().use {
+                    it.execute(vrfScript)
+                    it.evaluate("crc_vrf(\"${query.trim()}\")") as String
+                }
+                addQueryParameter("vrf", vrf)
+            }
         }.build()
 
         return GET(url, headers)
@@ -185,90 +215,137 @@ class MangaFire(
         return baseUrl + chapter.url.substringBeforeLast("#")
     }
 
-    private fun getAjaxRequest(ajaxType: String, mangaId: String, chapterType: String): Request {
+    override fun chapterListRequest(manga: SManga): Request {
-        return GET("$baseUrl/ajax/$ajaxType/$mangaId/$chapterType/$langCode", headers)
+        val mangaId = manga.url.removeSuffix(VOLUME_URL_SUFFIX).substringAfterLast(".")
-    }
+        val type = if (manga.url.endsWith(VOLUME_URL_SUFFIX)) "volume" else "chapter"
 
-    @Serializable
+        return GET("$baseUrl/ajax/manga/$mangaId/$type/$langCode", headers)
-    class AjaxReadDto(
+    }
-        val html: String,
-    )
 
     override fun chapterListParse(response: Response): List<SChapter> {
-        throw UnsupportedOperationException()
+        val isVolume = response.request.url.pathSegments.contains("volume")
-    }
 
-    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        val mangaList = response.parseAs<ResponseDto<String>>().result
-        val path = manga.url
-        val mangaId = path.removeSuffix(VOLUME_URL_SUFFIX).substringAfterLast(".")
-        val isVolume = path.endsWith(VOLUME_URL_SUFFIX)
-
-        val type = if (isVolume) "volume" else "chapter"
-        val abbrPrefix = if (isVolume) "Vol" else "Chap"
-        val fullPrefix = if (isVolume) "Volume" else "Chapter"
-
-        val ajaxMangaList = client.newCall(getAjaxRequest("manga", mangaId, type))
-            .execute().parseAs<ResponseDto<String>>().result
             .toBodyFragment()
             .select(if (isVolume) ".vol-list > .item" else "li")
 
-        val ajaxReadList = client.newCall(getAjaxRequest("read", mangaId, type))
+        val abbrPrefix = if (isVolume) "Vol" else "Chap"
-            .execute().parseAs<ResponseDto<AjaxReadDto>>().result.html
+        val fullPrefix = if (isVolume) "Volume" else "Chapter"
-            .toBodyFragment()
-            .select("ul a")
 
-        val chapterList = ajaxMangaList.zip(ajaxReadList) { m, r ->
+        return mangaList.map { m ->
-            val link = r.selectFirst("a")!!
+            val link = m.selectFirst("a")!!
-            if (!r.attr("abs:href").toHttpUrl().pathSegments.last().contains(type)) {
-                return Observable.just(emptyList())
-            }
-
-            assert(m.attr("data-number") == r.attr("data-number")) {
-                "Chapter count doesn't match. Try updating again."
-            }
 
             val number = m.attr("data-number")
             val dateStr = m.select("span").getOrNull(1)?.text() ?: ""
 
             SChapter.create().apply {
-                setUrlWithoutDomain("${link.attr("href")}#$type/${r.attr("data-id")}")
+                setUrlWithoutDomain(link.attr("href"))
                 chapter_number = number.toFloatOrNull() ?: -1f
                 name = run {
-                    val name = link.text()
+                    val name = m.selectFirst("span")!!.text()
                     val prefix = "$abbrPrefix $number: "
                     if (!name.startsWith(prefix)) return@run name
                     val realName = name.removePrefix(prefix)
                     if (realName.contains(number)) realName else "$fullPrefix $number: $realName"
                 }
-
+                date_upload = dateFormat.tryParse(dateStr)
-                date_upload = try {
-                    dateFormat.parse(dateStr)!!.time
-                } catch (_: ParseException) {
-                    0L
-                }
             }
         }
-
-        return Observable.just(chapterList)
     }
 
     // =============================== Pages ================================
 
-    override fun pageListRequest(chapter: SChapter): Request {
+    @SuppressLint("SetJavaScriptEnabled")
-        val typeAndId = chapter.url.substringAfterLast('#')
-        return GET("$baseUrl/ajax/read/$typeAndId", headers)
-    }
-
     override fun pageListParse(response: Response): List<Page> {
-        val result = response.parseAs<ResponseDto<PageListDto>>().result
+        val document = response.asJsoup()
+        var ajaxUrl: String? = null
+        var errorMessage: String? = null
 
-        return result.pages.mapIndexed { index, image ->
+        val latch = CountDownLatch(1)
-            val url = image.url
+        var webView: WebView? = null
-            val offset = image.offset
-            val imageUrl = if (offset > 0) "$url#${ImageInterceptor.SCRAMBLED}_$offset" else url
 
-            Page(index, imageUrl = imageUrl)
+        handler.post {
+            val webview = WebView(context)
+                .also { webView = it }
+            with(webview.settings) {
+                javaScriptEnabled = true
+                domStorageEnabled = true
+                databaseEnabled = true
+                blockNetworkImage = true
+                userAgentString = headers["User-Agent"]
+            }
+
+            webview.webViewClient = object : WebViewClient() {
+                private val ajaxCalls = setOf("ajax/read/chapter", "ajax/read/volume")
+
+                override fun shouldInterceptRequest(
+                    view: WebView,
+                    request: WebResourceRequest,
+                ): WebResourceResponse? {
+                    val url = request.url
+
+                    // allow script from their cdn
+                    if (url.host.orEmpty().contains("mfcdn.cc") && url.pathSegments.lastOrNull().orEmpty().contains("js")) {
+                        Log.d(name, "allowed: $url")
+
+                        return super.shouldInterceptRequest(view, request)
+                    }
+
+                    // allow jquery script
+                    if (url.host.orEmpty().contains("cloudflare.com") && url.encodedPath.orEmpty().contains("jquery")) {
+                        Log.d(name, "allowed: $url")
+
+                        return super.shouldInterceptRequest(view, request)
+                    }
+
+                    // allow ajax/read calls and intercept ajax/read/chapter or ajax/read/volume
+                    if (url.host == "mangafire.to" && url.encodedPath.orEmpty().contains("ajax/read")) {
+                        if (ajaxCalls.any { url.encodedPath!!.contains(it) }) {
+                            Log.d(name, "found: $url")
+
+                            if (url.getQueryParameter("vrf") != null) {
+                                ajaxUrl = url.toString()
+                            } else {
+                                errorMessage = "vrf not found"
+                            }
+
+                            latch.countDown()
+                        } else {
+                            // need to allow other call to ajax/read
+                            Log.d(name, "allowed: $url")
+                            return super.shouldInterceptRequest(view, request)
+                        }
+                    }
+
+                    Log.d(name, "denied: $url")
+                    return emptyWebViewResponse
+                }
+            }
+
+            webview.loadDataWithBaseURL(document.location(), document.outerHtml(), "text/html", "utf-8", "")
         }
+
+        latch.await(20, TimeUnit.SECONDS)
+        handler.post {
+            webView?.stopLoading()
+            webView?.destroy()
+        }
+
+        if (latch.count == 1L) {
+            throw Exception("Timeout getting vrf token")
+        } else if (ajaxUrl == null) {
+            throw Exception(errorMessage ?: "Unknown Error")
+        }
+
+        return client.newCall(GET(ajaxUrl!!, headers)).execute()
+            .parseAs<ResponseDto<PageListDto>>().result
+            .pages.mapIndexed { index, image ->
+                val url = image.url
+                val offset = image.offset
+                val imageUrl = if (offset > 0) "$url#${ImageInterceptor.SCRAMBLED}_$offset" else url
+
+                Page(index, imageUrl = imageUrl)
+            }
     }
 
     @Serializable
@@ -302,10 +379,6 @@ class MangaFire(
         val result: T,
     )
 
-    private inline fun <reified T> Response.parseAs(): T {
-        return json.decodeFromString(body.string())
-    }
-
     private fun String.toBodyFragment(): Document {
         return Jsoup.parseBodyFragment(this, baseUrl)
     }