MCCMS: add, remove and fix sources (#14535)

* MCCMS: add, remove and fix sources

* add decrypt interceptor

* mark captcha entry as initialized

* Add more NSFW sources and some tweaks

multisrc/overrides/mccms/haoman6/src/Haoman6.kt

@@ -1,12 +1,13 @@
 package eu.kanade.tachiyomi.extension.zh.haoman6
 
-import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
+import eu.kanade.tachiyomi.multisrc.mccms.MCCMSWeb
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 
-class Haoman6 : MCCMS("好漫6", "https://www.haoman6.com") {
+class Haoman6 : MCCMSWeb("好漫6", "https://www.haoman6.com") {
     override fun SManga.cleanup() = apply {
+        description = description?.substringBefore(title)
         title = title.removeSuffix("(最新在线)").removeSuffix("-")
     }
 

multisrc/overrides/mccms/haoman6_glens/src/Haoman6_glens.kt

@@ -1,11 +1,11 @@
 package eu.kanade.tachiyomi.extension.zh.haoman6_glens
 
-import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
+import eu.kanade.tachiyomi.multisrc.mccms.MCCMSWeb
 import eu.kanade.tachiyomi.source.model.SManga
 
-class Haoman6_glens : MCCMS("好漫6 (g-lens)", "https://www.g-lens.com") {
+class Haoman6_glens : MCCMSWeb("好漫6 (g-lens)", "https://www.g-lens.com") {
     override fun SManga.cleanup() = apply {
-        title = title.removeSuffix("_").removeSuffix("-")
+        title = title.removeSuffix("_").removeSuffix("-").removeSuffix("漫画")
     }
 
     override val lazyLoadImageAttr = "pc-ec"

multisrc/overrides/mccms/haoman8/src/Haoman8.kt

@@ -2,6 +2,6 @@ package eu.kanade.tachiyomi.extension.zh.haoman8
 
 import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
 
-class Haoman8 : MCCMS("好漫8", "https://www.haoman8.com") {
-    override val lazyLoadImageAttr = "data-echo"
+class Haoman8 : MCCMS("好漫8", "https://caiji.haoman8.com", hasCategoryPage = true) {
+    override val lazyLoadImageAttr = "data-original"
 }

multisrc/overrides/mccms/manhuaorg/src/Manhuaorg.kt

@@ -1,20 +0,0 @@
-package eu.kanade.tachiyomi.extension.zh.manhuaorg
-
-import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.Page
-import eu.kanade.tachiyomi.source.model.SManga
-import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.Request
-
-class Manhuaorg : MCCMS("朴朴漫画", "https://app.manhuaorg.com", hasCategoryPage = false) {
-
-    override fun mangaDetailsRequest(manga: SManga) = throw UnsupportedOperationException()
-
-    override fun imageRequest(page: Page): Request {
-        val url = page.imageUrl!!
-        val host = url.toHttpUrl().host
-        val headers = headers.newBuilder().set("Referer", "https://$host/").build()
-        return GET(url, headers)
-    }
-}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/DecryptInterceptor.kt

@@ -0,0 +1,37 @@
+package eu.kanade.tachiyomi.multisrc.mccms
+
+import okhttp3.Interceptor
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.Response
+import okhttp3.ResponseBody.Companion.toResponseBody
+import javax.crypto.Cipher
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+
+object DecryptInterceptor : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val request = chain.request()
+        val host = request.url.host
+        val response = chain.proceed(request)
+        val key = when {
+            "bcebos.com" in host -> key1
+            "103.107.190.121" in host -> key2
+            else -> return response
+        }
+        val data = decrypt(response.body!!.bytes(), key)
+        val body = data.toResponseBody("image/jpeg".toMediaType())
+        return response.newBuilder().body(body).build()
+    }
+
+    @Synchronized
+    private fun decrypt(input: ByteArray, key: SecretKeySpec): ByteArray {
+        val cipher = cipher
+        cipher.init(Cipher.DECRYPT_MODE, key, iv)
+        return cipher.doFinal(input)
+    }
+
+    private val cipher by lazy(LazyThreadSafetyMode.NONE) { Cipher.getInstance("DESede/CBC/PKCS5Padding") }
+    private val key1 by lazy(LazyThreadSafetyMode.NONE) { SecretKeySpec("OW84U8Eerdb99rtsTXWSILDO".toByteArray(), "DESede") }
+    private val key2 by lazy(LazyThreadSafetyMode.NONE) { SecretKeySpec("OW84U8Eerdb99rtsTXWSILEC".toByteArray(), "DESede") }
+    private val iv by lazy(LazyThreadSafetyMode.NONE) { IvParameterSpec("SK8bncVu".toByteArray()) }
+}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMS.kt

@@ -29,7 +29,7 @@ open class MCCMS(
     override val name: String,
     override val baseUrl: String,
     override val lang: String = "zh",
-    hasCategoryPage: Boolean = true
+    hasCategoryPage: Boolean = false
 ) : HttpSource() {
     override val supportsLatest = true
 
@@ -38,10 +38,11 @@ open class MCCMS(
     override val client by lazy {
         network.client.newBuilder()
             .rateLimitHost(baseUrl.toHttpUrl(), 2)
+            .addInterceptor(DecryptInterceptor)
             .build()
     }
 
-    private val pcHeaders by lazy { super.headersBuilder().build() }
+    val pcHeaders by lazy { super.headersBuilder().build() }
 
     override fun headersBuilder() = Headers.Builder()
         .add("User-Agent", System.getProperty("http.agent")!!)
@@ -84,12 +85,16 @@ open class MCCMS(
     override fun searchMangaParse(response: Response) = popularMangaParse(response)
 
     // preserve mangaDetailsRequest for WebView
-    override fun fetchMangaDetails(manga: SManga): Observable<SManga> =
-        client.newCall(GET("$baseUrl/api/data/comic?key=${manga.title}", headers))
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        val url = "$baseUrl/api/data/comic".toHttpUrl().newBuilder()
+            .addQueryParameter("key", manga.title)
+            .toString()
+        return client.newCall(GET(url, headers))
             .asObservableSuccess().map { response ->
                 val list: List<MangaDto> = response.parseAs()
                 list.find { it.url == manga.url }!!.toSManga().cleanup()
             }
+    }
 
     override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException("Not used.")
 
@@ -123,13 +128,15 @@ open class MCCMS(
 
     override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Not used.")
 
+    override fun imageRequest(page: Page) = GET(page.imageUrl!!, pcHeaders)
+
     private inline fun <reified T> Response.parseAs(): T = use {
         json.decodeFromStream<ResultDto<T>>(it.body!!.byteStream()).data
     }
 
-    private val genreData = GenreData(hasCategoryPage)
+    val genreData = GenreData(hasCategoryPage)
 
-    private fun fetchGenres() {
+    fun fetchGenres() {
         if (genreData.status != GenreData.NOT_FETCHED) return
         genreData.status = GenreData.FETCHING
         thread {

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMSFilters.kt

@@ -14,17 +14,31 @@ open class MCCMSFilter(
 }
 
 class SortFilter : MCCMSFilter("排序", SORT_NAMES, SORT_QUERIES)
+class WebSortFilter : MCCMSFilter("排序", SORT_NAMES, SORT_QUERIES_WEB)
 
 private val SORT_NAMES = arrayOf("热门人气", "更新时间", "评分")
 private val SORT_QUERIES = arrayOf("order=hits", "order=addtime", "order=score")
+private val SORT_QUERIES_WEB = arrayOf("order/hits", "order/addtime", "order/score")
 
 class StatusFilter : MCCMSFilter("进度", STATUS_NAMES, STATUS_QUERIES)
+class WebStatusFilter : MCCMSFilter("进度", STATUS_NAMES, STATUS_QUERIES_WEB)
 
-private val STATUS_NAMES = arrayOf("全部", "连载(有缺漏)", "完结(有缺漏)")
+private val STATUS_NAMES = arrayOf("全部", "连载", "完结")
 private val STATUS_QUERIES = arrayOf("", "serialize=连载", "serialize=完结")
+private val STATUS_QUERIES_WEB = arrayOf("", "finish/1", "finish/2")
 
 class GenreFilter(private val values: Array<String>, private val queries: Array<String>) {
-    val filter get() = MCCMSFilter("标签(搜索文本时无效)", values, queries, isTypeQuery = true)
+
+    private val apiQueries get() = queries.run {
+        Array(size) { i -> "type[tags]=" + this[i] }
+    }
+
+    private val webQueries get() = queries.run {
+        Array(size) { i -> "tags/" + this[i] }
+    }
+
+    val filter get() = MCCMSFilter("标签(搜索文本时无效)", values, apiQueries, isTypeQuery = true)
+    val webFilter get() = MCCMSFilter("标签", values, webQueries, isTypeQuery = true)
 }
 
 class GenreData(hasCategoryPage: Boolean) {
@@ -49,7 +63,7 @@ internal fun parseGenres(document: Document, genreData: GenreData) {
         add(Pair("全部", ""))
         genres.mapTo(this) {
             val tagId = it.attr("href").substringAfterLast('/')
-            Pair(it.text(), "type[tags]=$tagId")
+            Pair(it.text(), tagId)
         }
     }
     genreData.genreFilter = GenreFilter(
@@ -73,3 +87,17 @@ internal fun getFilters(genreData: GenreData): FilterList {
     }
     return FilterList(list)
 }
+
+internal fun getWebFilters(genreData: GenreData): FilterList {
+    val list = buildList(4) {
+        add(Filter.Header("分类筛选（搜索时无效）"))
+        add(WebStatusFilter())
+        add(WebSortFilter())
+        when (genreData.status) {
+            GenreData.NO_DATA -> return@buildList
+            GenreData.FETCHED -> add(genreData.genreFilter.webFilter)
+            else -> add(Filter.Header("点击“重置”尝试刷新标签分类"))
+        }
+    }
+    return FilterList(list)
+}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMSGenerator.kt

@@ -6,26 +6,63 @@ import generator.ThemeSourceGenerator
 class MCCMSGenerator : ThemeSourceGenerator {
     override val themeClass = "MCCMS"
     override val themePkg = "mccms"
-    override val baseVersionCode = 3
+    override val baseVersionCode = 4
     override val sources = listOf(
         SingleLang(
             name = "Haoman6", baseUrl = "https://www.haoman6.com", lang = "zh",
             className = "Haoman6", sourceName = "好漫6", overrideVersionCode = 3
         ),
-        SingleLang(
+        SingleLang( // previously: app2.haoman6.com, app2.haomanwu.com
+            name = "Haomanwu", baseUrl = "https://move.bookcomic.org", lang = "zh",
+            className = "Haomanwu", sourceName = "好漫屋", overrideVersionCode = 3
+        ),
+        SingleLang( // same as: www.haoman6.cc
             name = "Haoman6 (g-lens)", baseUrl = "https://www.g-lens.com", lang = "zh",
             className = "Haoman6_glens", sourceName = "好漫6 (g-lens)", overrideVersionCode = 0
         ),
-        SingleLang( // 与 caiji.haoman8.com 相同
-            name = "Haoman8", baseUrl = "https://www.haoman8.com", lang = "zh",
+        SingleLang( // same as: www.haoman8.com
+            name = "Haoman8", baseUrl = "https://caiji.haoman8.com", lang = "zh",
             className = "Haoman8", sourceName = "好漫8", overrideVersionCode = 0
         ),
         SingleLang(
-            name = "Pupu Manhua", baseUrl = "https://app.manhuaorg.com", lang = "zh",
-            className = "Manhuaorg", sourceName = "朴朴漫画", overrideVersionCode = 2
+            name = "Kuaikuai Manhua", baseUrl = "https://mobile.manhuaorg.com", lang = "zh",
+            className = "Kuaikuai", sourceName = "快快漫画", overrideVersionCode = 0
+        ),
+        SingleLang(
+            name = "bz Manhua", baseUrl = "https://www2.pupumanhua.com", lang = "zh",
+            className = "bzManhua", sourceName = "包子漫画搬运", overrideVersionCode = 0
+        ),
+        // The following sources are from https://www.yy123.cyou/ and are configured to use MCCMSNsfw
+        SingleLang( // 103=寄宿日记, same as: www.hanman.top (different URL format)
+            name = "Damao Manhua", baseUrl = "https://www.hanman.cyou", lang = "zh", isNsfw = true,
+            className = "DamaoManhua", sourceName = "大猫漫画", overrideVersionCode = 0
+        ),
+        SingleLang( // 103=诡秘的姐妹
+            name = "Heihei Manhua", baseUrl = "https://www.hhmh.cyou", lang = "zh", isNsfw = true,
+            className = "HHMH", sourceName = "嘿嘿漫画", overrideVersionCode = 0
+        ),
+        SingleLang( // 103=望月仙女傳說, same as: www.hanman.men
+            name = "Tudou Manhua", baseUrl = "https://www.ptcomic.com", lang = "zh", isNsfw = true,
+            className = "PtComic", sourceName = "土豆漫画", overrideVersionCode = 0
+        ),
+        SingleLang( // 103=校园梦精记, same as: www.hmanwang.com, www.quanman8.com, www.lmmh.cc, www.xinmanba.com
+            name = "Dida Manhua", baseUrl = "https://www.didamanhua.com", lang = "zh", isNsfw = true,
+            className = "DidaManhua", sourceName = "嘀嗒漫画", overrideVersionCode = 0
+        ),
+        SingleLang( // 103=脱身之法, same as: www.quanmanba.com, www.999mh.net
+            name = "Dimanba", baseUrl = "https://www.dimanba.com", lang = "zh", isNsfw = true,
+            className = "Dimanba", sourceName = "滴漫吧", overrideVersionCode = 0
         ),
     )
 
+    override fun createAll() {
+        val userDir = System.getProperty("user.dir")!!
+        sources.forEach {
+            val themeClass = if (it.isNsfw) "MCCMSNsfw" else themeClass
+            ThemeSourceGenerator.createGradleProject(it, themePkg, themeClass, baseVersionCode, userDir)
+        }
+    }
+
     companion object {
         @JvmStatic
         fun main(args: Array<String>) {

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMSNsfw.kt

@@ -0,0 +1,36 @@
+package eu.kanade.tachiyomi.multisrc.mccms
+
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.select.Evaluator
+
+open class MCCMSNsfw(
+    name: String,
+    baseUrl: String,
+    lang: String = "zh",
+) : MCCMSWeb(name, baseUrl, lang, hasCategoryPage = false) {
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
+        if (query.isNotBlank()) {
+            GET("$baseUrl/search/$query/$page", pcHeaders)
+        } else {
+            super.searchMangaRequest(page, query, filters)
+        }
+
+    override fun searchMangaParse(response: Response) = parseListing(response.asJsoup())
+
+    override fun pageListRequest(chapter: SChapter): Request =
+        GET(baseUrl + chapter.url, headers)
+
+    override fun pageListParse(response: Response): List<Page> {
+        val container = response.asJsoup().selectFirst(Evaluator.Class("comic-list"))
+        return container.select(Evaluator.Tag("img")).mapIndexed { index, img ->
+            Page(index, imageUrl = img.attr("src"))
+        }
+    }
+}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMSWeb.kt

@@ -0,0 +1,115 @@
+package eu.kanade.tachiyomi.multisrc.mccms
+
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import org.jsoup.select.Evaluator
+import rx.Observable
+
+// https://github.com/tachiyomiorg/tachiyomi-extensions/blob/e0b4fcbce8aa87742da22e7fa60b834313f53533/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMS.kt
+open class MCCMSWeb(
+    name: String,
+    baseUrl: String,
+    lang: String = "zh",
+    hasCategoryPage: Boolean = true,
+) : MCCMS(name, baseUrl, lang, hasCategoryPage) {
+
+    fun parseListing(document: Document): MangasPage {
+        val mangas = document.select(Evaluator.Class("common-comic-item")).map {
+            SManga.create().apply {
+                val titleElement = it.selectFirst(Evaluator.Class("comic__title")).child(0)
+                url = titleElement.attr("href")
+                title = titleElement.ownText()
+                thumbnail_url = it.selectFirst(Evaluator.Tag("img")).attr("data-original")
+            }.cleanup()
+        }
+        val hasNextPage = run { // default pagination
+            val buttons = document.selectFirst(Evaluator.Id("Pagination")).select(Evaluator.Tag("a"))
+            val count = buttons.size
+            // Next page != Last page
+            buttons[count - 1].attr("href") != buttons[count - 2].attr("href")
+        }
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    override fun popularMangaRequest(page: Int) = GET("$baseUrl/category/order/hits/page/$page", pcHeaders)
+
+    override fun popularMangaParse(response: Response) = parseListing(response.asJsoup())
+
+    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/category/order/addtime/page/$page", pcHeaders)
+
+    override fun latestUpdatesParse(response: Response) = parseListing(response.asJsoup())
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
+        if (query.isNotBlank()) {
+            val url = "$baseUrl/index.php/search".toHttpUrl().newBuilder()
+                .addQueryParameter("key", query)
+                .toString()
+            GET(url, pcHeaders)
+        } else {
+            val url = buildString {
+                append(baseUrl).append("/category/")
+                filters.filterIsInstance<MCCMSFilter>().map { it.query }.filter { it.isNotEmpty() }
+                    .joinTo(this, "/")
+                append("/page/").append(page)
+            }
+            GET(url, pcHeaders)
+        }
+
+    override fun searchMangaParse(response: Response): MangasPage {
+        val document = response.asJsoup()
+        if (document.selectFirst(Evaluator.Id("code-div")) != null) {
+            val manga = SManga.create().apply {
+                url = "/index.php/search"
+                title = "验证码"
+                description = "请点击 WebView 按钮输入验证码，完成后返回重新搜索"
+                initialized = true
+            }
+            return MangasPage(listOf(manga), false)
+        }
+        val result = parseListing(document)
+        if (document.location().contains("search")) {
+            return MangasPage(result.mangas, false)
+        }
+        return result
+    }
+
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        if (manga.url == "/index.php/search") return Observable.just(manga)
+        return client.newCall(GET(baseUrl + manga.url, pcHeaders)).asObservableSuccess().map { response ->
+            SManga.create().apply {
+                val document = response.asJsoup().selectFirst(Evaluator.Class("de-info__box"))
+                title = document.selectFirst(Evaluator.Class("comic-title")).ownText()
+                thumbnail_url = document.selectFirst(Evaluator.Tag("img")).attr("src")
+                author = document.selectFirst(Evaluator.Class("name")).text()
+                genre = document.selectFirst(Evaluator.Class("comic-status")).select(Evaluator.Tag("a")).joinToString { it.ownText() }
+                description = document.selectFirst(Evaluator.Class("intro-total")).text()
+            }.cleanup()
+        }
+    }
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        if (manga.url == "/index.php/search") return Observable.just(emptyList())
+        return client.newCall(GET(baseUrl + manga.url, pcHeaders)).asObservableSuccess().map { response ->
+            response.asJsoup().selectFirst(Evaluator.Class("chapter__list-box")).children().map {
+                val link = it.child(0)
+                SChapter.create().apply {
+                    url = link.attr("href")
+                    name = link.ownText()
+                }
+            }.asReversed()
+        }
+    }
+
+    override fun getFilterList(): FilterList {
+        fetchGenres()
+        return getWebFilters(genreData)
+    }
+}

multisrc/src/main/java/generator/ThemeSourceGenerator.kt

@@ -114,7 +114,7 @@ interface ThemeSourceGenerator {
             }
         }
 
-        private fun createGradleProject(source: ThemeSourceData, themePkg: String, themeClass: String, baseVersionCode: Int, userDir: String) {
+        fun createGradleProject(source: ThemeSourceData, themePkg: String, themeClass: String, baseVersionCode: Int, userDir: String) {
             // userDir = tachiyomi-extensions project root path
             val projectRootPath = "$userDir/generated-src/${pkgNameSuffix(source, "/")}"
             val projectSrcPath = "$projectRootPath/src/eu/kanade/tachiyomi/extension/${pkgNameSuffix(source, "/")}"