MCCMS: rewrite and add new sources (#12531)

multisrc/overrides/mccms/haoman6/src/Haoman6.kt

@@ -1,7 +1,10 @@
 package eu.kanade.tachiyomi.extension.zh.haoman6
 
 import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
+import eu.kanade.tachiyomi.source.model.SManga
 
 class Haoman6 : MCCMS("好漫6", "https://www.haoman6.com") {
-    override fun transformTitle(title: String) = title.removeSuffix("(最新在线)").removeSuffix("-")
+    override fun SManga.cleanup() = apply {
+        title = title.removeSuffix("(最新在线)").removeSuffix("-")
+    }
 }

multisrc/overrides/mccms/haoman6_glens/src/Haoman6_glens.kt

@@ -1,13 +1,12 @@
 package eu.kanade.tachiyomi.extension.zh.haoman6_glens
 
 import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
-import eu.kanade.tachiyomi.network.GET
-import okhttp3.Response
+import eu.kanade.tachiyomi.source.model.SManga
 
 class Haoman6_glens : MCCMS("好漫6 (g-lens)", "https://www.g-lens.com") {
-    override fun transformTitle(title: String) = title.removeSuffix("_").removeSuffix("-")
-    override val lazyLoadImageAttr = "pc-ec"
+    override fun SManga.cleanup() = apply {
+        title = title.removeSuffix("_").removeSuffix("-")
+    }
 
-    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/category/order/addtime", headers)
-    override fun latestUpdatesParse(response: Response) = searchMangaParse(response)
+    override val lazyLoadImageAttr = "pc-ec"
 }

multisrc/overrides/mccms/haoman8/src/Haoman8.kt

@@ -1,14 +1,7 @@
 package eu.kanade.tachiyomi.extension.zh.haoman8
 
 import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
-import eu.kanade.tachiyomi.network.GET
 
-class Haoman8 : MCCMS("好漫8", "https://caiji.haoman8.com") {
-
-    // Search: 此站点nginx配置有问题，只能用以下格式搜索第一页
-
-    override fun textSearchRequest(page: Int, query: String) =
-        GET("$baseUrl/index.php/search?key=$query", headers)
-
-    override fun searchMangaNextPageSelector(): String? = null
+class Haoman8 : MCCMS("好漫8", "https://www.haoman8.com") {
+    override val lazyLoadImageAttr = "data-echo"
 }

multisrc/overrides/mccms/haomanwu/src/Haomanwu.kt

@@ -1,32 +1,15 @@
 package eu.kanade.tachiyomi.extension.zh.haomanwu
 
 import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
-import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.Page
-import eu.kanade.tachiyomi.source.model.SManga
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
+import okhttp3.Response
 
-class Haomanwu : MCCMS("好漫屋", "https://app2.haoman6.com") {
-
-    // Search
-
-    override fun searchMangaNextPageSelector() = "li:nth-child(30) > a" // 有30项则可能有下一页
-    override fun searchMangaSelector() = "li > a"
-    override fun searchMangaFromElement(element: Element) = SManga.create().apply {
-        title = element.text()
-        setUrlWithoutDomain(element.attr("abs:href"))
-    }
-
-    override fun pageListParse(document: Document): List<Page> {
-        val pages = super.pageListParse(document)
+class Haomanwu : MCCMS("好漫屋", "https://app2.haoman6.com", hasCategoryPage = false) {
+    override fun pageListParse(response: Response): List<Page> {
+        val pages = super.pageListParse(response)
         if (pages.any { it.imageUrl!!.endsWith("tianjia.jpg") }) {
             throw Exception("该章节有图片尚未添加")
         }
         return pages
     }
-
-    // 分类页面缺失
-    override fun fetchCategories() = Unit
-    override fun getFilterList() = FilterList(emptyList())
 }

multisrc/overrides/mccms/haomanwu_www/src/Haomanwu_www.kt

@@ -0,0 +1,7 @@
+package eu.kanade.tachiyomi.extension.zh.haomanwu_www
+
+import eu.kanade.tachiyomi.multisrc.mccms.MCCMS
+
+class Haomanwu_www : MCCMS("好漫屋 (网页)", "https://www.haomanwu.com") {
+    override val lazyLoadImageAttr = "data-echo"
+}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMS.kt

@@ -2,166 +2,148 @@ package eu.kanade.tachiyomi.multisrc.mccms
 
 import android.util.Log
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.interceptor.rateLimitHost
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
-import eu.kanade.tachiyomi.source.online.ParsedHttpSource
+import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.decodeFromStream
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Request
 import okhttp3.Response
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
+import rx.Observable
+import rx.Single
+import uy.kohesive.injekt.injectLazy
 import kotlin.concurrent.thread
 
 /**
  * 漫城CMS http://mccms.cn/
  */
-abstract class MCCMS(
+open class MCCMS(
     override val name: String,
     override val baseUrl: String,
     override val lang: String = "zh",
-) : ParsedHttpSource() {
-    override val supportsLatest: Boolean = true
+    hasCategoryPage: Boolean = true
+) : HttpSource() {
+    override val supportsLatest = true
 
-    protected open fun transformTitle(title: String) = title
+    private val json: Json by injectLazy()
 
-    // Popular
-
-    override fun popularMangaRequest(page: Int) = GET("$baseUrl/custom/hot", headers)
-    override fun popularMangaNextPageSelector(): String? = null
-    override fun popularMangaSelector() = ".top-list__box-item"
-    override fun popularMangaFromElement(element: Element) = SManga.create().apply {
-        val titleElement = element.select("p.comic__title > a")
-        title = transformTitle(titleElement.text().trim())
-        setUrlWithoutDomain(titleElement.attr("abs:href"))
-        thumbnail_url = element.select("img").attr("abs:data-original")
+    override val client by lazy {
+        network.client.newBuilder()
+            .rateLimitHost(baseUrl.toHttpUrl(), 2)
+            .build()
     }
 
-    // Latest
+    private val pcHeaders by lazy { super.headersBuilder().build() }
 
-    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/custom/update", headers)
-    override fun latestUpdatesNextPageSelector(): String? = null
-    override fun latestUpdatesSelector() = "div.common-comic-item"
-    override fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element)
+    override fun headersBuilder() = Headers.Builder()
+        .add("User-Agent", System.getProperty("http.agent")!!)
+        .add("Referer", baseUrl)
 
-    // Search
+    protected open fun SManga.cleanup(): SManga = this
 
-    protected open fun textSearchRequest(page: Int, query: String) =
-        GET("$baseUrl/search/$query/$page", headers)
+    override fun popularMangaRequest(page: Int): Request =
+        GET("$baseUrl/api/data/comic?page=$page&size=$PAGE_SIZE&order=hits", headers)
 
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
-        if (query.isNotBlank()) {
-            textSearchRequest(page, query)
-        } else {
-            val categories = filters.filterIsInstance<UriPartFilter>()
-                .map { it.toUriPart() }.filter { it.isNotEmpty() }.joinToString("/")
-            GET("$baseUrl/category/$categories/page/$page", headers)
+    override fun popularMangaParse(response: Response): MangasPage {
+        val list: List<MangaDto> = response.parseAs()
+        return MangasPage(list.map { it.toSManga().cleanup() }, list.size >= PAGE_SIZE)
+    }
+
+    override fun latestUpdatesRequest(page: Int): Request =
+        GET("$baseUrl/api/data/comic?page=$page&size=$PAGE_SIZE&order=addtime", headers)
+
+    override fun latestUpdatesParse(response: Response) = popularMangaParse(response)
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        val queries = buildList {
+            add("page=$page")
+            add("size=$PAGE_SIZE")
+            val isTextSearch = query.isNotBlank()
+            if (isTextSearch) add("key=$query")
+            for (filter in filters) if (filter is MCCMSFilter) {
+                if (isTextSearch && filter.isTypeQuery) continue
+                val part = filter.query
+                if (part.isNotEmpty()) add(part)
+            }
         }
-
-    override fun searchMangaNextPageSelector(): String? = "" // empty string means default pagination
-    override fun searchMangaSelector() = latestUpdatesSelector()
-    override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
-
-    override fun searchMangaParse(response: Response): MangasPage {
-        val document = response.asJsoup()
-        val isTextSearch = document.location().contains("search")
-        val mangas = if (isTextSearch) {
-            document.select(searchMangaSelector()).map { searchMangaFromElement(it) }
-        } else {
-            document.select(latestUpdatesSelector()).map { popularMangaFromElement(it) }
+        val url = buildString {
+            append(baseUrl).append("/api/data/comic?")
+            queries.joinTo(this, separator = "&")
         }
-        val hasNextPage = if (isTextSearch && searchMangaNextPageSelector() != "") {
-            searchMangaNextPageSelector()?.let { document.selectFirst(it) } != null
-        } else { // default pagination
-            val buttons = document.select("#Pagination a")
-            val count = buttons.size
-            // Next page != Last page
-            buttons[count - 1].attr("href") != buttons[count - 2].attr("href")
-        }
-        return MangasPage(mangas, hasNextPage)
+        return GET(url, headers)
     }
 
-    // Details
+    override fun searchMangaParse(response: Response) = popularMangaParse(response)
 
-    override fun mangaDetailsParse(document: Document) = SManga.create().apply {
-        title = transformTitle(document.select("div.de-info__box > p.comic-title").text().trim())
-        thumbnail_url = document.select("div.de-info__cover > img").attr("abs:src")
-        author = document.select("div.comic-author > span.name > a").text()
-        artist = author
-        genre = document.select("div.comic-status > span.text:nth-child(1) a").eachText().joinToString(", ")
-        description = document.select("div.comic-intro > p.intro-total").text()
-    }
+    // preserve mangaDetailsRequest for WebView
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> =
+        client.newCall(GET("$baseUrl/api/data/comic?key=${manga.title}", headers))
+            .asObservableSuccess().map { response ->
+                val list: List<MangaDto> = response.parseAs()
+                list.find { it.url == manga.url }!!.toSManga().cleanup()
+            }
 
-    // Chapters
+    override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException("Not used.")
 
-    override fun chapterListSelector() = "ul.chapter__list-box > li"
-    override fun chapterFromElement(element: Element) = SChapter.create().apply {
-        setUrlWithoutDomain(element.select("a").attr("abs:href"))
-        name = element.select("a").text()
-    }
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> = Single.create<List<SChapter>> { subscriber ->
+        val id = manga.url.substringAfterLast('/')
+        val dataResponse = client.newCall(GET("$baseUrl/api/data/chapter?mid=$id", headers)).execute()
+        val dataList: List<ChapterDataDto> = dataResponse.parseAs() // unordered
+        val dateMap = HashMap<Int, Long>(dataList.size * 2)
+        dataList.forEach { dateMap[it.id.toInt()] = it.date }
+        val response = client.newCall(GET("$baseUrl/api/comic/chapter?mid=$id", headers)).execute()
+        val list: List<ChapterDto> = response.parseAs()
+        val result = list.map { it.toSChapter(date = dateMap[it.id.toInt()] ?: 0) }.asReversed()
+        subscriber.onSuccess(result)
+    }.toObservable()
 
-    override fun chapterListParse(response: Response) = super.chapterListParse(response).reversed()
+    override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Not used.")
 
-    // Pages
+    override fun pageListRequest(chapter: SChapter): Request =
+        GET(baseUrl + chapter.url, pcHeaders)
 
     protected open val lazyLoadImageAttr = "data-original"
 
-    override fun pageListParse(document: Document) = document.select("div.rd-article__pic > img")
-        .mapIndexed { i, el -> Page(i, "", el.attr("abs:$lazyLoadImageAttr")) }
-
-    override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("Not used.")
-
-    protected class UriPartFilter(name: String, values: Array<String>, private val uriParts: Array<String>) :
-        Filter.Select<String>(name, values) {
-        fun toUriPart(): String = uriParts[state]
+    override fun pageListParse(response: Response): List<Page> {
+        val document = response.asJsoup()
+        return document.select("img[$lazyLoadImageAttr]").mapIndexed { i, element ->
+            Page(i, imageUrl = element.attr(lazyLoadImageAttr))
+        }
     }
 
-    protected data class Category(val name: String, val values: Array<String>, val uriParts: Array<String>) {
-        fun toUriPartFilter() = UriPartFilter(name, values, uriParts)
+    override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Not used.")
+
+    private inline fun <reified T> Response.parseAs(): T = use {
+        @Suppress("OPT_IN_USAGE")
+        json.decodeFromStream<ResultDto<T>>(it.body!!.byteStream()).data
     }
 
-    private val sortCategory = Category("排序", arrayOf("热门人气", "更新时间"), arrayOf("order/hits", "order/addtime"))
-    private lateinit var categories: List<Category>
-    private var isFetchingCategories = false
+    private val genreData = GenreData(hasCategoryPage)
 
-    private fun tryFetchCategories() {
-        if (isFetchingCategories) return
-        isFetchingCategories = true
+    private fun fetchGenres() {
+        if (genreData.status != GenreData.NOT_FETCHED) return
+        genreData.status = GenreData.FETCHING
         thread {
             try {
-                fetchCategories()
+                val response = client.newCall(GET("$baseUrl/category/", pcHeaders)).execute()
+                parseGenres(response.asJsoup(), genreData)
             } catch (e: Exception) {
-                Log.e("MCCMS/$name", "Failed to fetch categories ($e)")
-            } finally {
-                isFetchingCategories = false
+                genreData.status = GenreData.NOT_FETCHED
+                Log.e("MCCMS/$name", "failed to fetch genres", e)
             }
         }
     }
 
-    protected open fun fetchCategories() {
-        val document = client.newCall(GET("$baseUrl/category/", headers)).execute().asJsoup()
-        categories = document.select("div.cate-col").map { element ->
-            val name = element.select("p.cate-title").text().removeSuffix("：")
-            val tags = element.select("li.cate-item > a")
-            val values = tags.map { it.text() }.toTypedArray()
-            val uriParts = tags.map { it.attr("href").removePrefix("/category/") }.toTypedArray()
-            Category(name, values, uriParts)
-        }
-    }
-
     override fun getFilterList(): FilterList {
-        val result = mutableListOf(
-            Filter.Header("如果使用文本搜索，将会忽略分类筛选"),
-            sortCategory.toUriPartFilter(),
-        )
-        if (::categories.isInitialized) {
-            categories.forEach { result.add(it.toUriPartFilter()) }
-        } else {
-            tryFetchCategories()
-            result.add(Filter.Header("其他分类正在获取，请返回上一页后重试"))
-        }
-        return FilterList(result)
+        fetchGenres()
+        return getFilters(genreData)
     }
 }

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMSDto.kt

@@ -0,0 +1,65 @@
+package eu.kanade.tachiyomi.multisrc.mccms
+
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import kotlinx.serialization.Serializable
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+internal const val PAGE_SIZE = 30
+
+@Serializable
+class MangaDto(
+    private val name: String,
+    private val pic: String,
+    private val serialize: String,
+    private val author: String,
+    private val content: String,
+    private val addtime: String,
+    val url: String,
+    private val tags: List<String>,
+) {
+    fun toSManga() = SManga.create().apply {
+        url = this@MangaDto.url
+        title = name
+        author = this@MangaDto.author
+        description = content
+        genre = tags.joinToString()
+        val date = dateFormat.parse(addtime)?.time ?: 0
+        val isUpdating = System.currentTimeMillis() - date <= 30L * 24 * 3600 * 1000 // a month
+        status = when {
+            serialize.startsWith('连') || isUpdating -> SManga.ONGOING
+            serialize.startsWith('完') -> SManga.COMPLETED
+            else -> SManga.UNKNOWN
+        }
+        thumbnail_url = pic
+        initialized = true
+    }
+
+    companion object {
+        private val dateFormat by lazy { getDateFormat() }
+    }
+}
+
+@Serializable
+class ChapterDto(val id: String, private val name: String, private val link: String) {
+    fun toSChapter(date: Long) = SChapter.create().apply {
+        url = link
+        name = this@ChapterDto.name
+        date_upload = date
+    }
+}
+
+@Serializable
+class ChapterDataDto(val id: String, private val addtime: String) {
+    val date get() = dateFormat.parse(addtime)?.time ?: 0
+
+    companion object {
+        private val dateFormat by lazy { getDateFormat() }
+    }
+}
+
+@Serializable
+class ResultDto<T>(val data: T)
+
+fun getDateFormat() = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH)

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMSFilters.kt

@@ -0,0 +1,75 @@
+package eu.kanade.tachiyomi.multisrc.mccms
+
+import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.source.model.FilterList
+import org.jsoup.nodes.Document
+
+open class MCCMSFilter(
+    name: String,
+    values: Array<String>,
+    private val queries: Array<String>,
+    val isTypeQuery: Boolean = false,
+) : Filter.Select<String>(name, values) {
+    val query get() = queries[state]
+}
+
+class SortFilter : MCCMSFilter("排序", SORT_NAMES, SORT_QUERIES)
+
+private val SORT_NAMES = arrayOf("热门人气", "更新时间", "评分")
+private val SORT_QUERIES = arrayOf("order=hits", "order=addtime", "order=score")
+
+class StatusFilter : MCCMSFilter("进度", STATUS_NAMES, STATUS_QUERIES)
+
+private val STATUS_NAMES = arrayOf("全部", "连载(有缺漏)", "完结(有缺漏)")
+private val STATUS_QUERIES = arrayOf("", "serialize=连载", "serialize=完结")
+
+class GenreFilter(private val values: Array<String>, private val queries: Array<String>) {
+    val filter get() = MCCMSFilter("标签(搜索文本时无效)", values, queries, isTypeQuery = true)
+}
+
+class GenreData(hasCategoryPage: Boolean) {
+    var status = if (hasCategoryPage) NOT_FETCHED else NO_DATA
+    lateinit var genreFilter: GenreFilter
+
+    companion object {
+        const val NOT_FETCHED = 0
+        const val FETCHING = 1
+        const val FETCHED = 2
+        const val NO_DATA = 3
+    }
+}
+
+internal fun parseGenres(document: Document, genreData: GenreData) {
+    val genres = document.select("a[href^=/category/tags/]")
+    if (genres.isEmpty()) {
+        genreData.status = GenreData.NO_DATA
+        return
+    }
+    val result = buildList(genres.size + 1) {
+        add(Pair("全部", ""))
+        genres.mapTo(this) {
+            val tagId = it.attr("href").substringAfterLast('/')
+            Pair(it.text(), "type[tags]=$tagId")
+        }
+    }
+    genreData.genreFilter = GenreFilter(
+        values = result.map { it.first }.toTypedArray(),
+        queries = result.map { it.second }.toTypedArray(),
+    )
+    genreData.status = GenreData.FETCHED
+}
+
+internal fun getFilters(genreData: GenreData): FilterList {
+    val list = buildList(4) {
+        add(StatusFilter())
+        add(SortFilter())
+        if (genreData.status == GenreData.NO_DATA) return@buildList
+        add(Filter.Separator())
+        if (genreData.status == GenreData.FETCHED) {
+            add(genreData.genreFilter.filter)
+        } else {
+            add(Filter.Header("点击“重置”尝试刷新标签分类"))
+        }
+    }
+    return FilterList(list)
+}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mccms/MCCMSGenerator.kt

@@ -6,13 +6,13 @@ import generator.ThemeSourceGenerator
 class MCCMSGenerator : ThemeSourceGenerator {
     override val themeClass = "MCCMS"
     override val themePkg = "mccms"
-    override val baseVersionCode = 2
+    override val baseVersionCode = 3
     override val sources = listOf(
         SingleLang(
             name = "Haoman6", baseUrl = "https://www.haoman6.com", lang = "zh",
             className = "Haoman6", sourceName = "好漫6", overrideVersionCode = 2
         ),
-        SingleLang(
+        SingleLang( // 与 app2.haomanwu.com 相同
             name = "Haomanwu", baseUrl = "https://app2.haoman6.com", lang = "zh",
             className = "Haomanwu", sourceName = "好漫屋", overrideVersionCode = 3
         ),
@@ -20,10 +20,18 @@ class MCCMSGenerator : ThemeSourceGenerator {
             name = "Haoman6 (g-lens)", baseUrl = "https://www.g-lens.com", lang = "zh",
             className = "Haoman6_glens", sourceName = "好漫6 (g-lens)", overrideVersionCode = 0
         ),
-        SingleLang(
-            name = "Haoman8", baseUrl = "https://caiji.haoman8.com", lang = "zh",
+        SingleLang( // 与 caiji.haoman8.com 相同
+            name = "Haoman8", baseUrl = "https://www.haoman8.com", lang = "zh",
             className = "Haoman8", sourceName = "好漫8", overrideVersionCode = 0
         ),
+        SingleLang(
+            name = "Haomanwu (www)", baseUrl = "https://www.haomanwu.com", lang = "zh",
+            className = "Haomanwu_www", sourceName = "好漫屋 (网页)", overrideVersionCode = 0
+        ),
+        SingleLang( // 与 app.manhuaorg.com 相同（部分渠道记为“好漫2”）
+            name = "Pupu Manhua", baseUrl = "https://www.manhuaorg.com", lang = "zh",
+            className = "Manhuaorg", sourceName = "朴朴漫画", overrideVersionCode = 0
+        ),
     )
 
     companion object {