Yidan: Rewrite and rework the extension. (#8091)

* Yidan: Rewrite and rework the extension.

* Yidan: Remove logs.

* Yidan: Lint?

* Yidan: Apply review suggestions

Co-Authored-By: Vetle Ledaal <13540478+vetleledaal@users.noreply.github.com>

* Apply review suggestions

Co-Authored-By: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>

---------

Co-authored-by: Vetle Ledaal <13540478+vetleledaal@users.noreply.github.com>
Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>

src/zh/yidan/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'Yidan Girl'
     extClass = '.Yidan'
-    extVersionCode = 4
+    extVersionCode = 5
     isNsfw = true
 }
 

src/zh/yidan/src/eu/kanade/tachiyomi/extension/zh/yidan/Dto.kt

@@ -1,62 +1,90 @@
 package eu.kanade.tachiyomi.extension.zh.yidan
 
-import eu.kanade.tachiyomi.source.model.SChapter
+import kotlinx.serialization.SerialName
-import eu.kanade.tachiyomi.source.model.SManga
 import kotlinx.serialization.Serializable
-import org.jsoup.nodes.Entities
 
 @Serializable
-class MangaDto(
+class ComicFetchRequest(
-    private val title: String,
+    val column: String,
-    private val mhcate: String?,
+    val page: Int,
-    private val cateids: String?,
+    val limit: Int,
-    private val author: String?,
+)
-    private val summary: String?,
+
-    private val coverPic: String?,
+@Serializable
-    private val id: Int,
+class ComicDetailRequest(
+    val comicId: String,
+    val userId: String,
+    val limit: Int = 5,
+)
+
+@Serializable
+class ChapterContentRequest(
+    val chapterId: String,
+    val userId: String,
+    val type: Int = 1,
+)
+
+@Serializable
+class KeywordSearchRequest(
+    val key: String,
+    val type: Int = 1,
+)
+
+@Serializable
+class FilterRequest(
+    val page: Int,
+    val limit: Int,
+    val categoryId: Int,
+    val orderType: Int,
+    val overType: Int,
 ) {
-    fun toSManga(baseUrl: String) = SManga.create().apply {
+    @SerialName("updated_recent")
-        url = id.toString()
+    val updatedRecent: Int? = if (orderType == 3) {
-        title = this@MangaDto.title
+        1
-        author = this@MangaDto.author
+    } else {
-        description = summary?.trim()
+        null
-        genre = when {
-            cateids.isNullOrEmpty() -> null
-            else -> cateids.split(",").joinToString { GENRES[it.toInt()] }
-        }
-        status = when {
-            mhcate.isNullOrEmpty() -> SManga.ONGOING
-            "5" in mhcate.split(",") -> SManga.COMPLETED
-            else -> SManga.ONGOING
-        }
-        thumbnail_url = if (coverPic?.startsWith("http") == true) coverPic else baseUrl + coverPic
-        initialized = true
     }
 }
 
 @Serializable
-class ChapterDto(
+class CommonResponse<T>(val result: T)
-    private val createTime: Long,
-    private val mhid: String,
-    private val title: String,
-    private val jiNo: Int,
-) {
-    fun toSChapter() = SChapter.create().apply {
-        url = "$mhid/$jiNo"
-        name = Entities.unescape(title)
-        date_upload = createTime * 1000L
-    }
-}
 
 @Serializable
-class PageListDto(private val pics: String) {
+class RecordResult(val records: List<Record>, val total: Int)
-    val images get() = pics.split(",")
-}
 
 @Serializable
-class ListingDto(val list: List<MangaDto>, private val total: String) {
+class FilterResult(val list: List<Record>, val total: Int)
-    val totalCount get() = total.toInt()
-}
 
 @Serializable
-class ResponseDto<T>(val data: T)
+class Record(
+    val id: Long,
+    val novelTitle: String,
+    val imgUrl: String,
+)
+
+@Serializable
+class ComicInfoResult(val comic: Comic, val chapterList: List<Chapter>)
+
+@Serializable
+class Comic(
+    val id: Long,
+    val novelTitle: String,
+    val author: String,
+    val tags: String,
+    val bigImgUrl: String,
+    val introduction: String,
+    val overType: Int,
+)
+
+@Serializable
+class Chapter(
+    val id: Long,
+    val chapterName: String,
+    val createTime: String,
+)
+
+@Serializable
+class ChapterContentResult(val content: List<Content>)
+
+@Serializable
+class Content(val url: String)

src/zh/yidan/src/eu/kanade/tachiyomi/extension/zh/yidan/Filters.kt

@@ -1,54 +1,54 @@
 package eu.kanade.tachiyomi.extension.zh.yidan
 
 import eu.kanade.tachiyomi.source.model.Filter
-import eu.kanade.tachiyomi.source.model.FilterList
-import okhttp3.HttpUrl
 
-fun getFilterListInternal() = FilterList(ListingFilter(), GenreFilter())
+open class PairFilter(name: String, private val pairs: List<Pair<String, Int>>) :
-
+    Filter.Select<String>(name, pairs.map { it.first }.toTypedArray()) {
-fun parseFilters(filters: FilterList, builder: HttpUrl.Builder) {
+    val selected: Int
-    for (filter in filters) when (filter) {
+        get() = pairs[state].second
-        is ListingFilter -> {
-            if (filter.state > 0) {
-                builder.addEncodedQueryParameter("mhcate", LISTING_VALUES[filter.state].toString())
-            }
-        }
-        is GenreFilter -> {
-            if (filter.state > 0) {
-                builder.addEncodedQueryParameter("cateid", String.format("%02d", filter.state))
-            }
-        }
-        else -> {}
-    }
 }
 
-class ListingFilter : Filter.Select<String>("分类", LISTINGS)
+class CategoryFilter : PairFilter(
-
+    "分类",
-val LISTINGS = arrayOf("全部", "排行榜", "新作", "完结漫", "分类0", "分类1", "分类3", "分类7")
+    listOf(
-val LISTING_VALUES = arrayOf(0, 2, 4, 5, 0, 1, 3, 7)
+        "青梅竹马" to 18,
-
+        "办公室" to 19,
-class GenreFilter : Filter.Select<String>("标签", GENRES)
+        "娱乐圈" to 20,
-
+        "高H" to 21,
-val GENRES = arrayOf(
+        "韩国版单" to 22,
-    "全部",
+        "NP/SM" to 23,
-    "短漫", // 01
+        "校园" to 24,
-    "甜漫", // 02
+        "财阀" to 25,
-    "强强", // 03
+        "重生/重逢" to 26,
-    "年下攻", // 04
+        "ABO" to 27,
-    "诱受", // 05
+        "调教" to 28,
-    "骨科", // 06
+        "骨科" to 29,
-    "调教", // 07
+        "诱受" to 30,
-    "健气受", // 08
+        "年下攻" to 31,
-    "ABO", // 09
+        "强强" to 32,
-    "重生/重逢", // 10
+        "甜漫" to 33,
-    "财阀", // 11
+        "短漫" to 34,
-    "校园", // 12
+        "女王受" to 35,
-    "女王受", // 13
+        "健气受" to 36,
-    "NP/SM", // 14
+        "架空" to 37,
-    "韩国榜单", // 15
+    ),
-    "高H", // 16
+)
-    "架空", // 17
+
-    "娱乐圈", // 18
+class StatusFilter : PairFilter(
-    "办公室", // 19
+    "状态",
-    "青梅竹马", // 20
+    listOf(
+        "全部" to 0,
+        "连载" to 1,
+        "完结" to 2,
+    ),
+)
+
+class SortFilter : PairFilter(
+    "排序",
+    listOf(
+        "最新上架" to 0,
+        "推荐" to 1,
+        "一周人气" to 2,
+        "最近更新" to 3,
+    ),
 )

src/zh/yidan/src/eu/kanade/tachiyomi/extension/zh/yidan/Yidan.kt

@@ -1,9 +1,19 @@
 package eu.kanade.tachiyomi.extension.zh.yidan
 
+import android.annotation.SuppressLint
+import android.app.Application
+import android.os.Handler
+import android.os.Looper
+import android.webkit.WebResourceRequest
+import android.webkit.WebResourceResponse
+import android.webkit.WebView
+import android.webkit.WebViewClient
+import android.widget.Toast
+import androidx.annotation.MainThread
+import androidx.preference.EditTextPreference
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
-import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
@@ -11,125 +21,326 @@ import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
+import keiyoushi.utils.firstInstance
 import keiyoushi.utils.getPreferences
+import keiyoushi.utils.parseAs
+import keiyoushi.utils.tryParse
+import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.decodeFromStream
-import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
 import okhttp3.Request
+import okhttp3.RequestBody
+import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
-import rx.Observable
+import okhttp3.ResponseBody.Companion.asResponseBody
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
 import uy.kohesive.injekt.injectLazy
+import java.text.SimpleDateFormat
+import java.util.Locale
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
 
 class Yidan : HttpSource(), ConfigurableSource {
     override val name get() = "一耽女孩"
     override val lang get() = "zh"
     override val supportsLatest get() = true
-
+    private val apiUrl = "https://yd-api.hangtech.cn"
-    override val baseUrl: String
+    override val baseUrl: String = getPreferences().run {
-
+        val customBaseUrl = getString(PREF_KEY_CUSTOM_HOST, "")
-    init {
+        if (customBaseUrl.isNullOrEmpty()) {
-        val mirrors = MIRRORS
+            val mirrors = MIRRORS
-        val index = getPreferences()
+            val index = getPreferences()
-            .getString(MIRROR_PREF, "0")!!.toInt().coerceAtMost(mirrors.size - 1)
+                .getString(PREF_KEY_MIRROR, "0")!!.toInt().coerceAtMost(mirrors.size - 1)
-        baseUrl = "https://" + mirrors[index]
+            "https://${mirrors[index]}"
+        } else {
+            customBaseUrl.removeSuffix("/")
+        }
     }
 
-    override fun headersBuilder() = Headers.Builder()
+    override val client: OkHttpClient = network.cloudflareClient.newBuilder().addInterceptor { chain ->
-        .add("User-Agent", System.getProperty("http.agent")!!)
+        val request = chain.request()
+        val response = chain.proceed(request)
+        val requestUrl = request.url.toString()
+        if (requestUrl.contains("images/mhtp/yidan")) {
+            // remove first two bytes for image response
+            val ext = requestUrl.substringAfterLast(".", "png")
+            response.newBuilder().body(
+                response.body.source().apply { skip(2) }
+                    .asResponseBody("image/$ext".toMediaType()),
+            ).build()
+        } else {
+            response
+        }
+    }.build()
 
     private val json: Json by injectLazy()
+    private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ROOT)
 
-    override fun popularMangaRequest(page: Int) =
+    override fun popularMangaRequest(page: Int) = POST(
-        GET("$baseUrl/prod-api/app-api/vv/mh-list/page?mhcate=2&pageSize=50&pageNo=$page", headers)
+        "$baseUrl/api/getByComicByRow",
+        headers,
+        ComicFetchRequest("29", page, PAGE_SIZE).toJsonRequestBody(),
+    )
 
     override fun popularMangaParse(response: Response): MangasPage {
-        val listing: ListingDto = response.parseAs()
+        val records = response.parseAs<CommonResponse<RecordResult>>().result.records
-        val mangas = listing.list.map { it.toSManga(baseUrl) }
+        return createMangasPage(records)
-        val hasNextPage = run {
-            val url = response.request.url
-            val pageSize = url.queryParameter("pageSize")!!.toInt()
-            val pageNumber = url.queryParameter("pageNo")!!.toInt()
-            pageSize * pageNumber < listing.totalCount
-        }
-        return MangasPage(mangas, hasNextPage)
     }
 
-    override fun latestUpdatesRequest(page: Int) =
+    override fun latestUpdatesRequest(page: Int) = POST(
-        GET("$baseUrl/prod-api/app-api/vv/mh-list/page?mhcate=4&pageSize=50&pageNo=$page", headers)
+        "$baseUrl/api/getByComicByRow",
+        headers,
+        ComicFetchRequest("34", page, PAGE_SIZE).toJsonRequestBody(),
+    )
 
     override fun latestUpdatesParse(response: Response) = popularMangaParse(response)
 
+    private fun searchByKeyword(page: Int, query: String): Request {
+        return POST(
+            "$apiUrl/api/searchNovel",
+            headers,
+            KeywordSearchRequest(query).toJsonRequestBody(),
+        )
+    }
+
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val url = "$baseUrl/prod-api/app-api/vv/mh-list/page".toHttpUrl().newBuilder()
+        if (query.isNotEmpty()) {
-            .apply { if (query.isNotBlank()) addQueryParameter("word", query) }
+            return searchByKeyword(page, query)
-            .apply { parseFilters(filters, this) }
-            .addEncodedQueryParameter("pageSize", "50")
-            .addEncodedQueryParameter("pageNo", page.toString())
-            .build()
-        return Request.Builder().url(url).headers(headers).build()
-    }
-
-    override fun searchMangaParse(response: Response) = popularMangaParse(response)
-
-    // for WebView
-    override fun mangaDetailsRequest(manga: SManga) =
-        GET("$baseUrl/#/pages/detail/detail?id=${manga.url}")
-
-    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
-        val request = GET("$baseUrl/prod-api/app-api/vv/mh-list/get?id=${manga.url}", headers)
-        return client.newCall(request).asObservableSuccess().map { mangaDetailsParse(it) }
-    }
-
-    override fun mangaDetailsParse(response: Response) =
-        response.parseAs<MangaDto>().toSManga(baseUrl)
-
-    override fun chapterListRequest(manga: SManga) =
-        GET("$baseUrl/prod-api/app-api/vv/mh-episodes/list?mhid=${manga.url}", headers)
-
-    override fun chapterListParse(response: Response) =
-        response.parseAs<List<ChapterDto>>().map { it.toSChapter() }
-
-    // for WebView
-    override fun pageListRequest(chapter: SChapter): Request {
-        val (mangaId, chapterIndex) = chapter.url.split("/")
-        return GET("$baseUrl/#/pages/read/read?no=$chapterIndex&id=$mangaId")
-    }
-
-    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
-        val (mangaId, chapterIndex) = chapter.url.split("/")
-        val url = "$baseUrl/prod-api/app-api/vv/mh-episodes/get?jiNo=$chapterIndex&mhid=$mangaId"
-        return client.newCall(GET(url, headers)).asObservableSuccess().map { pageListParse(it) }
-    }
-
-    override fun pageListParse(response: Response) =
-        response.parseAs<PageListDto>().images.mapIndexed { index, url ->
-            val imageUrl = if (url.startsWith("http")) url else baseUrl + url
-            Page(index, imageUrl = imageUrl)
         }
+        return POST(
+            "$apiUrl/api/getByComicCategoryId",
+            headers,
+            FilterRequest(
+                page = page,
+                limit = PAGE_SIZE,
+                categoryId = filters.firstInstance<CategoryFilter>().selected,
+                orderType = filters.firstInstance<SortFilter>().selected,
+                overType = filters.firstInstance<StatusFilter>().selected,
+            ).toJsonRequestBody(),
+        )
+    }
+
+    override fun searchMangaParse(response: Response): MangasPage {
+        val searchByKeyword = response.request.url.toString().contains("searchNovel")
+        val records = when {
+            searchByKeyword -> response.parseAs<CommonResponse<List<Record>>>().result
+            else -> response.parseAs<CommonResponse<FilterResult>>().result.list
+        }
+        return createMangasPage(records, paginated = !searchByKeyword)
+    }
+
+    private fun createMangasPage(records: List<Record>, paginated: Boolean = true): MangasPage {
+        return MangasPage(
+            records.map {
+                SManga.create().apply {
+                    url = "${it.id}"
+                    title = it.novelTitle
+                    thumbnail_url = it.imgUrl
+                }
+            },
+            paginated && records.size >= PAGE_SIZE,
+        )
+    }
+
+    override fun getMangaUrl(manga: SManga): String {
+        return "$baseUrl/pages/comic/info".toHttpUrl().newBuilder()
+            .addQueryParameter("id", manga.url)
+            .toString()
+    }
+
+    override fun mangaDetailsRequest(manga: SManga) = chapterListRequest(manga)
+
+    override fun mangaDetailsParse(response: Response): SManga {
+        val comic = response.parseAs<CommonResponse<ComicInfoResult>>().result.comic
+        return SManga.create().apply {
+            url = "${comic.id}"
+            title = comic.novelTitle
+            thumbnail_url = comic.bigImgUrl
+            genre = comic.tags
+            author = comic.author
+            description = comic.introduction
+            status = when (comic.overType) {
+                1 -> SManga.ONGOING
+                2 -> SManga.COMPLETED
+                else -> SManga.UNKNOWN
+            }
+        }
+    }
+
+    override fun getChapterUrl(chapter: SChapter): String {
+        return "$baseUrl/pages/comic/content".toHttpUrl().newBuilder()
+            .addQueryParameter("f", "1")
+            .addQueryParameter("s", chapter.chapter_number.toInt().toString())
+            .toString()
+    }
+
+    override fun chapterListRequest(manga: SManga) = withUserId { userId ->
+        POST(
+            "$apiUrl/api/getComicInfo",
+            headers,
+            ComicDetailRequest(manga.url, userId).toJsonRequestBody(),
+        )
+    }
+
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val chapterList = response.parseAs<CommonResponse<ComicInfoResult>>().result.chapterList
+        return chapterList.mapIndexed { index, chapter ->
+            SChapter.create().apply {
+                url = "${chapter.id}"
+                name = chapter.chapterName
+                date_upload = dateFormat.tryParse(chapter.createTime)
+                // used to get the real chapter url
+                chapter_number = index.toFloat()
+            }
+        }.reversed()
+    }
+
+    override fun pageListRequest(chapter: SChapter): Request = withUserId { userId ->
+        POST(
+            "$apiUrl/api/getComicChapter",
+            headers,
+            ChapterContentRequest(chapter.url, userId).toJsonRequestBody(),
+        )
+    }
+
+    override fun pageListParse(response: Response): List<Page> {
+        val contentList = response.parseAs<CommonResponse<ChapterContentResult>>().result.content
+        return contentList.mapIndexed { index, content ->
+            Page(index, imageUrl = content.url)
+        }
+    }
 
     override fun imageUrlParse(response: Response) = throw UnsupportedOperationException()
 
-    private inline fun <reified T> Response.parseAs(): T = use {
+    override fun getFilterList() = FilterList(
-        json.decodeFromStream<ResponseDto<T>>(body.byteStream()).data
+        SortFilter(),
-    }
+        StatusFilter(),
-
+        CategoryFilter(),
-    override fun getFilterList() = getFilterListInternal()
+    )
 
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
-        ListPreference(screen.context).apply {
+        screen.apply {
-            val mirrors = MIRRORS
+            addPreference(
-            key = MIRROR_PREF
+                ListPreference(context).apply {
-            title = "镜像网址（重启生效）"
+                    val mirrors = MIRRORS
-            summary = "%s"
+                    key = PREF_KEY_MIRROR
-            entries = mirrors
+                    title = "镜像网址（重启生效）"
-            entryValues = Array(mirrors.size, Int::toString)
+                    summary = "%s"
-            setDefaultValue("0")
+                    entries = mirrors
-        }.let(screen::addPreference)
+                    entryValues = Array(mirrors.size, Int::toString)
+                    setDefaultValue("0")
+                },
+            )
+            addPreference(
+                EditTextPreference(context).apply {
+                    key = PREF_KEY_CUSTOM_HOST
+                    val customUrl = this@Yidan.getPreferences().getString(PREF_KEY_CUSTOM_HOST, "")
+                    title = "自定义网址：$customUrl"
+                    summary =
+                        "请点击后输入自定义网址（例如：https://yidan1.club），如果不需要自定义时请设置为空"
+                    setOnPreferenceChangeListener { _, _ ->
+                        Toast.makeText(context, "重启应用后生效", Toast.LENGTH_LONG).show()
+                        true
+                    }
+                },
+            )
+        }
     }
 
+    //region utils functions
+
+    private lateinit var _userId: String
+
+    @MainThread
+    private fun WebView.readUserId(block: (userId: String) -> Unit) {
+        val script = "javascript:localStorage['uc']"
+        evaluateJavascript(script) { uc ->
+            if (uc.isNotEmpty() && uc != "null" && uc != "undefined") {
+                block(uc.removeSurrounding("'").removeSurrounding("\""))
+            }
+        }
+    }
+
+    @SuppressLint("SetJavaScriptEnabled")
+    private fun <T> withUserId(block: (userId: String) -> T): T {
+        return if (this::_userId.isInitialized) {
+            block(_userId)
+        } else {
+            val mainHandler = Handler(Looper.getMainLooper())
+            var latch = CountDownLatch(1)
+            var webView: WebView? = null
+            mainHandler.post {
+                webView = WebView(Injekt.get<Application>()).apply {
+                    with(settings) {
+                        javaScriptEnabled = true
+                        domStorageEnabled = true
+                        databaseEnabled = true
+                        blockNetworkImage = true
+                    }
+                }
+                webView?.webViewClient = object : WebViewClient() {
+                    override fun onPageFinished(view: WebView?, url: String?) {
+                        view?.readUserId {
+                            _userId = it
+                            latch.countDown()
+                        }
+                    }
+
+                    override fun shouldInterceptRequest(
+                        view: WebView?,
+                        request: WebResourceRequest?,
+                    ): WebResourceResponse? {
+                        // wait the auto register request
+                        if (request?.url?.encodedPath?.contains("api/regUser") == true) {
+                            latch.countDown()
+                        }
+                        return super.shouldInterceptRequest(view, request)
+                    }
+                }
+                webView?.loadUrl(baseUrl)
+            }
+            latch.await(15, TimeUnit.SECONDS)
+            if (!this::_userId.isInitialized) {
+                latch = CountDownLatch(1)
+                mainHandler.postDelayed(
+                    {
+                        webView?.readUserId {
+                            _userId = it
+                            latch.countDown()
+                        }
+                    },
+                    500L,
+                )
+                latch.await(5, TimeUnit.SECONDS)
+            }
+            mainHandler.post {
+                webView?.apply {
+                    stopLoading()
+                    destroy()
+                }
+                webView = null
+            }
+            if (!this::_userId.isInitialized) {
+                throw Exception("无法自动获取UserId，请先尝试通过内置WebView进入网站")
+            }
+            block(_userId)
+        }
+    }
+
+    private inline fun <reified T : Any> T.toJsonRequestBody(): RequestBody =
+        json.encodeToString(this)
+            .toRequestBody("application/json".toMediaType())
+    //endregion
+
     companion object {
-        private const val MIRROR_PREF = "MIRROR"
+        private const val PREF_KEY_MIRROR = "MIRROR"
-        private val MIRRORS get() = arrayOf("yidan1.club", "yidan22.club", "yidan10.club", "yidan9.club")
+        private const val PREF_KEY_CUSTOM_HOST = "CUSTOM_HOST"
+
+        private val MIRRORS = arrayOf("yidan1.club", "yidan22.club", "yidan10.club", "yidan9.club")
+
+        private const val PAGE_SIZE = 16
     }
 }