FMReader Fixes: ManhuaScan, HeroScan, etc... (#7047)

* FMReader: Added to chapterDateSelector

* FMReader: Add fetchPageListEncrypted + Overrides for heroscan + manhuascan

* change manhwasmut to manhwahot + override

Co-authored-by: OncePunchedMan <64155117+OncePunchedMan@users.noreply.github.com>

multisrc/overrides/fmreader/heroscan/src/HeroScan.kt

@@ -1,6 +1,7 @@
 package eu.kanade.tachiyomi.extension.en.heroscan
 
 import eu.kanade.tachiyomi.multisrc.fmreader.FMReader
+import eu.kanade.tachiyomi.source.model.SChapter
 import okhttp3.OkHttpClient
 
 class HeroScan : FMReader("HeroScan", "https://heroscan.com", "en") {
@@ -17,4 +18,5 @@ class HeroScan : FMReader("HeroScan", "https://heroscan.com", "en") {
                 }
             }
             .build()
+    override fun fetchPageList(chapter: SChapter) = fetchPageListEncrypted(chapter)
 }

multisrc/overrides/fmreader/manhuascan/src/ManhuaScan.kt

@@ -0,0 +1,12 @@
+package eu.kanade.tachiyomi.extension.en.manhuascan
+
+import eu.kanade.tachiyomi.multisrc.fmreader.FMReader
+import eu.kanade.tachiyomi.source.model.SChapter
+
+import eu.kanade.tachiyomi.annotations.Nsfw
+
+
+@Nsfw
+class ManhuaScan : FMReader("ManhuaScan", "https://manhuascan.com", "en") {
+    override fun fetchPageList(chapter: SChapter) = fetchPageListEncrypted(chapter)
+}

multisrc/overrides/fmreader/manhwahot/src/ManhwaHot.kt

@@ -0,0 +1,13 @@
+package eu.kanade.tachiyomi.extension.en.manhwahot
+
+import eu.kanade.tachiyomi.annotations.Nsfw
+import eu.kanade.tachiyomi.multisrc.fmreader.FMReader
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.model.Page
+import okhttp3.Request
+import eu.kanade.tachiyomi.source.model.SChapter
+
+@Nsfw
+class ManhwaHot : FMReader("ManhwaHot", "https://manhwahot.com", "en") {
+    override fun fetchPageList(chapter: SChapter) = fetchPageListEncrypted(chapter)
+}

multisrc/overrides/fmreader/manhwasmut/src/ManhwaSmut.kt

@@ -1,13 +0,0 @@
-package eu.kanade.tachiyomi.extension.en.manhwasmut
-
-import eu.kanade.tachiyomi.annotations.Nsfw
-import eu.kanade.tachiyomi.multisrc.fmreader.FMReader
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.Page
-import okhttp3.Request
-
-@Nsfw
-class ManhwaSmut : FMReader("ManhwaSmut", "https://manhwasmut.com", "en") {
-    private val noReferer = headersBuilder().removeAll("Referer").build()
-    override fun imageRequest(page: Page): Request = GET(page.imageUrl!!, if (page.imageUrl!!.contains("toonily")) noReferer else headers)
-}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/fmreader/FMReader.kt

@@ -1,7 +1,10 @@
 package eu.kanade.tachiyomi.multisrc.fmreader
 
 import android.util.Base64
+import com.google.gson.JsonParser
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
@@ -12,14 +15,24 @@ import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import eu.kanade.tachiyomi.util.asJsoup
 import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.OkHttpClient
 import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
+import okio.ByteString
+import okio.ByteString.Companion.decodeBase64
+import okio.ByteString.Companion.decodeHex
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import org.jsoup.select.Elements
+import rx.Observable
 import java.nio.charset.Charset
+import java.security.MessageDigest
 import java.util.Calendar
+import javax.crypto.Cipher
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
 
 /**
  * For sites based on the Flat-Manga CMS
@@ -209,7 +222,7 @@ abstract class FMReader(
 
     open val chapterUrlSelector = "a"
 
-    open val chapterTimeSelector = "time, .chapter-time"
+    open val chapterTimeSelector = "time, .chapter-time, .publishedDate"
 
     open val chapterNameAttrSelector = "title"
 
@@ -314,6 +327,36 @@ abstract class FMReader(
         }
     }
 
+        /**
+     * fetches the pageList by decrypting and parsing the json response of a "chapter server"
+     * Meant to be used by relevant subclasses to replace their implementation of fetchPageList as needed
+     * e.g ManhuaScan, HeroScan
+     */
+    protected fun fetchPageListEncrypted(chapter: SChapter): Observable<List<Page>> {
+        fun stringAssignment(varname: String, script: String) = Regex("""var\s+$varname\s*=\s*"([^"]*)"""").find(script)?.groups?.get(1)?.value
+        fun pageList(s: String) = Regex("https.+?(?=https|\"$)").findAll(s).map { it.groups[0]!!.value.replace("\\/", "/") }
+        fun pageListRequest(id: String, server: Int = 1) = POST("$baseUrl/app/manga/controllers/cont.chapterServer$server.php", headers, "id=$id".toRequestBody("application/x-www-form-urlencoded; charset=UTF-8".toMediaTypeOrNull()))
+        return client.newCall(GET("$baseUrl${chapter.url}", headers)).asObservableSuccess().concatMap { htmlResponse ->
+            val soup = htmlResponse.asJsoup()
+            soup.selectFirst("head > script[type='text/javascript']")?.data()?.let { params ->
+                val chapterId = stringAssignment("chapter_id", params)
+                val csrfToken = stringAssignment("csrf_token", params)
+                if (chapterId == null || csrfToken == null)
+                    null
+                else
+                    client.newCall(pageListRequest(chapterId)).asObservableSuccess()
+                        .map { jsonResponse ->
+                            pageList(
+                                crypto.aes_decrypt(
+                                    jsonResponse.body!!.string(),
+                                    crypto.md5("$csrfToken$csrfToken").toByteArray()
+                                )
+                            ).mapIndexed { i, imgUrl -> Page(i, "", imgUrl) }.toList()
+                        }
+            } ?: Observable.just(emptyList())
+        }
+    }
+
     override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not used")
 
     private class TextField(name: String, val key: String) : Filter.Text(name)
@@ -471,4 +514,75 @@ abstract class FMReader(
         Genre("Western"),
         Genre("Zombies")
     )
+
+    companion object {
+        interface Crypto {
+            fun md5(s: String): String;
+            fun aes_decrypt(context: String, key: ByteArray): String;
+        }
+
+        /* cryptography utilities leveraged by fetchPageListEncrypted */
+        val crypto = object : Crypto {
+            private fun parseCryptoCT(s: String): Map<String, ByteString> {
+                // https://cryptojs.gitbook.io/docs/
+                val jsonObj = JsonParser.parseString(s).asJsonObject
+                val cipherParams = mutableMapOf("ciphertext" to jsonObj["ct"].asString.decodeBase64()!!)
+                jsonObj["iv"]?.let { cipherParams.put("iv", it.asString.decodeHex()) }
+                jsonObj["s"]?.let { cipherParams.put("salt", it.asString.decodeHex()) }
+                return cipherParams
+            }
+
+            private fun decryptAES(encrypted: ByteString, password: ByteArray, iv: ByteString, salt: ByteString): String {
+                // https://stackoverflow.com/questions/29151211/29152379#29152379
+                val (key, _iv) = EvpKDF(password, 256, 128, salt.toByteArray(), 1, "MD5")
+                val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
+                cipher.init(Cipher.DECRYPT_MODE, SecretKeySpec(key, "AES"), IvParameterSpec(_iv))
+                return String(cipher.doFinal(encrypted.toByteArray()), Charset.forName("UTF-8"))
+            }
+
+            private fun EvpKDF(password: ByteArray, _keySize: Int, _ivSize: Int, salt: ByteArray, iterations: Int, hashAlgorithm: String): Pair<ByteArray, ByteArray> {
+                // Key Derivation Function
+                val keySize = _keySize / 32
+                val ivSize = _ivSize / 32
+                val targetKeySize = keySize + ivSize
+                val derivedBytes = ByteArray(targetKeySize * 4)
+                var numberOfDerivedWords = 0
+                var block = ByteArray(0)
+                val hasher = MessageDigest.getInstance(hashAlgorithm)
+                val key = ByteArray(keySize * 4)
+                val iv = ByteArray(ivSize * 4)
+                while (numberOfDerivedWords < targetKeySize) {
+                    if (block != null) {
+                        hasher.update(block)
+                    }
+                    hasher.update(password)
+                    block = hasher.digest(salt)
+                    hasher.reset()
+
+                    // Iterations
+                    for (i in 1 until iterations) {
+                        block = hasher.digest(block)
+                        hasher.reset()
+                    }
+
+                    System.arraycopy(block, 0, derivedBytes, numberOfDerivedWords * 4,
+                        block.size.coerceAtMost((targetKeySize - numberOfDerivedWords) * 4))
+
+                    numberOfDerivedWords += block.size / 4
+                }
+
+                System.arraycopy(derivedBytes, 0, key, 0, key.size)
+                System.arraycopy(derivedBytes, key.size, iv, 0, iv.size)
+
+                return key to iv // key + iv
+            }
+
+            override fun md5(s: String): String = MessageDigest.getInstance("MD5").digest(s.toByteArray()).joinToString("") { String.format("%02x", it) }
+
+            override fun aes_decrypt(context: String, key: ByteArray): String {
+                val cipherParams = parseCryptoCT(JsonParser.parseString(context).asJsonObject["content"].asString)
+                return decryptAES(cipherParams["ciphertext"]!!, key, cipherParams["iv"]!!, cipherParams["salt"]!!)
+            }
+        }
+    }
 }

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/fmreader/FMReaderGenerator.kt

@@ -10,24 +10,24 @@ class FMReaderGenerator : ThemeSourceGenerator {
 
     override val themeClass = "FMReader"
 
-    override val baseVersionCode: Int = 3
+    override val baseVersionCode: Int = 4
 
     /** For future sources: when testing and popularMangaRequest() returns a Jsoup error instead of results
      *  most likely the fix is to override popularMangaNextPageSelector()   */
 
     override val sources = listOf(
         SingleLang("Epik Manga", "https://www.epikmanga.com", "tr"),
-        SingleLang("HeroScan", "https://heroscan.com", "en"),
+        SingleLang("HeroScan", "https://heroscan.com", "en", overrideVersionCode = 1),
         SingleLang("KissLove", "https://kissaway.net", "ja"),
         SingleLang("LHTranslation", "https://lhtranslation.net", "en", overrideVersionCode = 1),
         SingleLang("Manga-TR", "https://manga-tr.com", "tr", className = "MangaTR"),
-        SingleLang("ManhuaScan", "https://manhuascan.com", "en", isNsfw = true, overrideVersionCode = 1),
+        SingleLang("ManhuaScan", "https://manhuascan.com", "en", isNsfw = true, overrideVersionCode = 2),
         SingleLang("Manhwa18", "https://manhwa18.com", "en", isNsfw = true),
         MultiLang("Manhwa18.net", "https://manhwa18.net", listOf("en", "ko"), className = "Manhwa18NetFactory", isNsfw = true),
-        SingleLang("ManhwaSmut", "https://manhwasmut.com", "en", isNsfw = true, overrideVersionCode = 2),
         SingleLang("RawLH", "https://lovehug.net", "ja"),
         SingleLang("Say Truyen", "https://saytruyen.com", "vi"),
         SingleLang("KSGroupScans", "https://ksgroupscans.com", "en"),
+        SingleLang("ManhwaHot", "https://manhwahot.com", "en", isNsfw = true),
         // Sites that went down
         //SingleLang("18LHPlus", "https://18lhplus.com", "en", className = "EighteenLHPlus"),
         //SingleLang("HanaScan (RawQQ)", "https://hanascan.com", "ja", className = "HanaScanRawQQ"),