Pufei Manhua: rewrite, fix bugs, add mirrors and filters (#12360)

* Pufei: rewrite, fix bugs, add mirrors and filters

* intercept some weird responses

* move rate limiter forward

src/zh/pufei/build.gradle

@@ -2,10 +2,10 @@ apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 
 ext {
-    extName = 'Pufei'
+    extName = 'Pufei Manhua'
     pkgNameSuffix = 'zh.pufei'
     extClass = '.Pufei'
-    extVersionCode = 8
+    extVersionCode = 9
 }
 
 apply from: "$rootDir/common.gradle"

src/zh/pufei/src/eu/kanade/tachiyomi/extension/zh/pufei/NonblockingRateLimiter.kt

@@ -0,0 +1,58 @@
+package eu.kanade.tachiyomi.extension.zh.pufei
+
+import android.os.SystemClock
+import okhttp3.Interceptor
+import okhttp3.Response
+import java.io.IOException
+import java.util.concurrent.TimeUnit
+
+// See https://github.com/tachiyomiorg/tachiyomi/pull/7389
+internal class NonblockingRateLimiter(
+    private val permits: Int,
+    period: Long = 1,
+    unit: TimeUnit = TimeUnit.SECONDS,
+) : Interceptor {
+
+    private val requestQueue = ArrayList<Long>(permits)
+    private val rateLimitMillis = unit.toMillis(period)
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        // Ignore canceled calls, otherwise they would jam the queue
+        if (chain.call().isCanceled()) {
+            throw IOException()
+        }
+
+        synchronized(requestQueue) {
+            val now = SystemClock.elapsedRealtime()
+            val waitTime = if (requestQueue.size < permits) {
+                0
+            } else {
+                val oldestReq = requestQueue[0]
+                val newestReq = requestQueue[permits - 1]
+
+                if (newestReq - oldestReq > rateLimitMillis) {
+                    0
+                } else {
+                    oldestReq + rateLimitMillis - now // Remaining time
+                }
+            }
+
+            // Final check
+            if (chain.call().isCanceled()) {
+                throw IOException()
+            }
+
+            if (requestQueue.size == permits) {
+                requestQueue.removeAt(0)
+            }
+            if (waitTime > 0) {
+                requestQueue.add(now + waitTime)
+                Thread.sleep(waitTime) // Sleep inside synchronized to pause queued requests
+            } else {
+                requestQueue.add(now)
+            }
+        }
+
+        return chain.proceed(chain.request())
+    }
+}

src/zh/pufei/src/eu/kanade/tachiyomi/extension/zh/pufei/OctetStreamInterceptor.kt

@@ -0,0 +1,41 @@
+package eu.kanade.tachiyomi.extension.zh.pufei
+
+import eu.kanade.tachiyomi.network.GET
+import okhttp3.Interceptor
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.Response
+import okhttp3.ResponseBody.Companion.asResponseBody
+
+object OctetStreamInterceptor : Interceptor {
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val request = chain.request()
+        val response = chain.proceed(request)
+
+        if (response.header("Content-Type") != "application/octet-stream") {
+            return response
+        }
+
+        if (response.header("Content-Length")!!.toInt() < 100) { // usually 96
+            // The actual URL is '/.../xxx.jpg/0'.
+            val peek = response.peekBody(100).string()
+            if (peek.startsWith("The actual URL")) {
+                response.body!!.close()
+                val actualPath = peek.substringAfter('\'').substringBeforeLast('\'')
+                return chain.proceed(GET("https://manhua.acimg.cn$actualPath"))
+            }
+        }
+
+        val url = request.url.encodedPath
+        val mediaType = when {
+            url.endsWith(".h") -> webpMediaType
+            url.contains(".jpg") -> jpegMediaType
+            else -> return response
+        }
+        val body = response.body!!.source().asResponseBody(mediaType)
+        return response.newBuilder().body(body).build()
+    }
+
+    private val jpegMediaType = "image/jpeg".toMediaType()
+    private val webpMediaType = "image/webp".toMediaType()
+}

src/zh/pufei/src/eu/kanade/tachiyomi/extension/zh/pufei/Pufei.kt

@@ -1,214 +1,228 @@
 package eu.kanade.tachiyomi.extension.zh.pufei
 
-// temp patch:
+import android.app.Application
-// https://github.com/tachiyomiorg/tachiyomi/pull/2031
+import android.content.SharedPreferences
-
 import android.util.Base64
-import com.squareup.duktape.Duktape
+import androidx.preference.ListPreference
+import androidx.preference.PreferenceScreen
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
-import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import okhttp3.FormBody
-import okhttp3.Interceptor
-import okhttp3.MediaType.Companion.toMediaTypeOrNull
-import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
-import okhttp3.ResponseBody.Companion.toResponseBody
-import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
+import org.jsoup.select.Evaluator
+import rx.Observable
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
 
-fun asJsoup(response: Response, html: String? = null): Document {
+// Uses www733dm/IMH/dm456 theme
-    return Jsoup.parse(html ?: bodyWithAutoCharset(response), response.request.url.toString())
+class Pufei : ParsedHttpSource(), ConfigurableSource {
-}
-
-fun bodyWithAutoCharset(response: Response, _charset: String? = null): String {
-    val htmlBytes: ByteArray = response.body!!.bytes()
-    var c = _charset
-
-    if (c == null) {
-        val regexPat = Regex("""charset=(\w+)""")
-        val match = regexPat.find(String(htmlBytes))
-        c = match?.groups?.get(1)?.value
-    }
-
-    return String(htmlBytes, charset(c ?: "utf8"))
-}
-
-// patch finish
-
-fun ByteArray.toHexString() = joinToString("%") { "%02x".format(it) }
-
-class Pufei : ParsedHttpSource() {
 
     override val name = "扑飞漫画"
-    override val baseUrl = "http://m.pufei8.com"
     override val lang = "zh"
     override val supportsLatest = true
-    val imageServer = "http://res.img.shengda0769.com/"
 
-    override val client: OkHttpClient
+    private val preferences: SharedPreferences =
-        get() = network.client.newBuilder()
+        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
-            .addNetworkInterceptor(rewriteOctetStream)
-            .build()
 
-    private val rewriteOctetStream: Interceptor = Interceptor { chain ->
+    private val domain = preferences.getString(MIRROR_PREF, "0")!!.toInt()
-        val originalResponse: Response = chain.proceed(chain.request())
+        .coerceIn(0, MIRRORS.size - 1).let { MIRRORS[it] }
-        if (originalResponse.headers("Content-Type").contains("application/octet-stream") && originalResponse.request.url.toString().contains(".jpg")) {
-            val orgBody = originalResponse.body!!.bytes()
-            val newBody = orgBody.toResponseBody("image/jpeg".toMediaTypeOrNull())
-            originalResponse.newBuilder()
-                .body(newBody)
-                .build()
-        } else originalResponse
-    }
 
-    override fun popularMangaSelector() = "ul#detail li"
+    override val baseUrl = "http://m.$domain"
+    private val pcUrl = "http://www.$domain"
 
-    override fun latestUpdatesSelector() = popularMangaSelector()
+    override val client = network.client.newBuilder()
+        .addInterceptor(NonblockingRateLimiter(2))
+        .addInterceptor(OctetStreamInterceptor)
+        .build()
 
-    override fun headersBuilder() = super.headersBuilder()
+    private val searchClient = network.client.newBuilder()
-        .add("Referer", baseUrl)
+        .followRedirects(false)
+        .build()
 
     override fun popularMangaRequest(page: Int) = GET("$baseUrl/manhua/paihang.html", headers)
-
+    override fun popularMangaNextPageSelector() = throw UnsupportedOperationException("Not used.")
-    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/manhua/update.html", headers)
+    override fun popularMangaSelector() = "ul#detail > li > a"
-
+    override fun popularMangaFromElement(element: Element) = SManga.create().apply {
-    private fun mangaFromElement(element: Element): SManga {
+        url = element.attr("href").removeSuffix("/index.html")
-        val manga = SManga.create()
+        title = element.selectFirst(Evaluator.Tag("h3")).text()
-        element.select("a").first().let {
+        thumbnail_url = element.selectFirst(Evaluator.Tag("img")).attr("data-src")
-            manga.setUrlWithoutDomain(it.attr("href"))
-            manga.title = it.select("h3").text().trim()
-            manga.thumbnail_url = it.select("div.thumb img").attr("data-src")
-        }
-        return manga
-    }
-
-    override fun popularMangaFromElement(element: Element) = mangaFromElement(element)
-    override fun latestUpdatesFromElement(element: Element) = mangaFromElement(element)
-    override fun searchMangaFromElement(element: Element) = mangaFromElement(element)
-
-    override fun popularMangaNextPageSelector() = null
-
-    override fun latestUpdatesNextPageSelector() = null
-
-    override fun mangaDetailsParse(document: Document): SManga {
-        val infoElement = document.select("div.book-detail")
-
-        val manga = SManga.create()
-        manga.description = infoElement.select("div#bookIntro > p").text().trim()
-        manga.thumbnail_url = infoElement.select("div.thumb > img").first()?.attr("src")
-        manga.author = infoElement.select(":nth-child(4) dd").first()?.text()
-        return manga
-    }
-
-    override fun searchMangaNextPageSelector() = throw UnsupportedOperationException("Not used")
-
-    override fun searchMangaSelector() = "ul#detail > li"
-
-    private fun encodeGBK(str: String) = "%" + str.toByteArray(charset("gb2312")).toHexString()
-
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val url = ("$baseUrl/e/search/?searchget=1&tbname=mh&show=title,player,playadmin,bieming,pinyin,playadmin&tempid=4&keyboard=" + encodeGBK(query)).toHttpUrlOrNull()
-            ?.newBuilder()
-        return GET(url.toString(), headers)
-    }
-
-    override fun searchMangaParse(response: Response): MangasPage {
-//        val document = response.asJsoup()
-        val document = asJsoup(response)
-        val mangas = document.select(searchMangaSelector()).map { element ->
-            searchMangaFromElement(element)
-        }
-
-        return MangasPage(mangas, false)
-    }
-
-    override fun chapterListSelector() = "div.chapter-list > ul > li"
-
-    override fun chapterListRequest(manga: SManga) = mangaDetailsRequest(manga)
-
-    override fun chapterFromElement(element: Element): SChapter {
-        val urlElement = element.select("a")
-
-        val chapter = SChapter.create()
-        chapter.setUrlWithoutDomain(urlElement.attr("href"))
-        chapter.name = urlElement.text().trim()
-        return chapter
-    }
-
-    override fun mangaDetailsRequest(manga: SManga) = GET(baseUrl + manga.url, headers)
-
-    override fun pageListRequest(chapter: SChapter) = GET(baseUrl + chapter.url, headers)
-
-    override fun pageListParse(document: Document): List<Page> {
-        val html = document.html()
-        val re = Regex("cp=\"(.*?)\"")
-        val imgbase64 = re.find(html)?.groups?.get(1)?.value
-        val imgCode = String(Base64.decode(imgbase64, Base64.DEFAULT))
-        val imgArrStr = Duktape.create().use {
-            it.evaluate("$imgCode.join('|')") as String
-        }
-        val hasHost = imgArrStr.startsWith("http")
-        return imgArrStr.split('|').mapIndexed { i, imgStr ->
-            Page(i, "", if (hasHost) imgStr else imageServer + imgStr)
-        }
-    }
-
-    override fun imageUrlParse(document: Document) = ""
-
-    private class GenreFilter(genres: Array<String>) : Filter.Select<String>("Genre", genres)
-
-    override fun getFilterList() = FilterList(
-        GenreFilter(getGenreList())
-    )
-
-    private fun getGenreList() = arrayOf(
-        "All"
-    )
-
-    // temp patch
-    override fun latestUpdatesParse(response: Response): MangasPage {
-        val document = asJsoup(response)
-
-        val mangas = document.select(latestUpdatesSelector()).map { element ->
-            latestUpdatesFromElement(element)
-        }
-
-        return MangasPage(mangas, false)
     }
 
     override fun popularMangaParse(response: Response): MangasPage {
-        val document = asJsoup(response)
+        val document = response.asPufeiJsoup()
-
+        val mangas = document.select(popularMangaSelector()).map { popularMangaFromElement(it) }
-        val mangas = document.select(popularMangaSelector()).map { element ->
-            popularMangaFromElement(element)
-        }
-
         return MangasPage(mangas, false)
     }
 
-    override fun mangaDetailsParse(response: Response): SManga {
+    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/manhua/update.html", headers)
-        return mangaDetailsParse(asJsoup(response))
+    override fun latestUpdatesNextPageSelector() = throw UnsupportedOperationException("Not used.")
+    override fun latestUpdatesSelector() = popularMangaSelector()
+    override fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element)
+
+    override fun latestUpdatesParse(response: Response): MangasPage {
+        val document = response.asPufeiJsoup()
+        val mangas = document.select(latestUpdatesSelector()).map { latestUpdatesFromElement(it) }
+        return MangasPage(mangas, false)
+    }
+
+    private val searchCache = HashMap<String, String>(0)
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        return if (query.isNotBlank()) {
+            val path = searchCache.getOrPut(query) {
+                val formBody = FormBody.Builder(GB2312)
+                    .addEncoded("tempid", "4")
+                    .addEncoded("show", "title,player,playadmin,bieming,pinyin")
+                    .add("keyboard", query)
+                    .build()
+                val request = POST("$baseUrl/e/search/index.php", headers, formBody)
+                searchClient.newCall(request).execute().header("location")!!
+            }
+            val sortQuery = parseSearchSort(filters)
+            GET("$baseUrl/e/search/$path$sortQuery&page=${page - 1}")
+        } else {
+            val path = parseFilters(page, filters)
+            if (path.isEmpty())
+                popularMangaRequest(page)
+            else
+                GET("$baseUrl$path", headers)
+        }
+    }
+
+    override fun searchMangaNextPageSelector() = throw UnsupportedOperationException("Not used.")
+    override fun searchMangaSelector() = popularMangaSelector()
+    override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
+
+    override fun searchMangaParse(response: Response): MangasPage {
+        val document = response.asPufeiJsoup()
+        val mangas = document.select(searchMangaSelector()).map { searchMangaFromElement(it) }
+        val hasNextPage = run {
+            for (element in document.body().children().asReversed()) {
+                if (element.tagName() == "a") return@run true
+                else if (element.tagName() == "b") return@run false
+            }
+            false
+        }
+        return MangasPage(mangas, hasNextPage)
+    }
+
+    override fun getFilterList() = getFilters()
+
+    // 让 WebView 显示移动端页面
+    override fun mangaDetailsRequest(manga: SManga) = GET(baseUrl + manga.url, headers)
+
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> =
+        client.newCall(GET(pcUrl + manga.urlWithCheck(), headers)).asObservableSuccess()
+            .map { mangaDetailsParse(it.asPufeiJsoup()) }
+
+    override fun mangaDetailsParse(document: Document) = SManga.create().apply {
+        val details = document.selectFirst(Evaluator.Class("detailInfo")).children()
+        title = details[0].child(0).text() // div.titleInfo > h1
+        val genreList = mutableListOf<String>()
+        for (item in details[1].children()) { // ul > li
+            when (item.child(0).text()) { // span
+                "作者：" -> author = item.ownText()
+                "类别：" -> item.ownText().let { if (it.isNotEmpty()) genreList.add(it) }
+                "关键词：" -> item.ownText().let { if (it.isNotEmpty()) genreList.addAll(it.split(',')) }
+            }
+        }
+        author = author ?: details[0].ownText().removePrefix("作者：")
+        if (genreList.isEmpty()) {
+            genreList.add(document.selectFirst(Evaluator.Class("position")).child(1).text())
+        }
+        genre = genreList.joinToString()
+        description = document.selectFirst("div.introduction")?.text() ?: details[2].ownText()
+        status = SManga.UNKNOWN // 所有漫画的标记都是连载，所以没有意义，参见 baseUrl/manhua/wanjie.html
+        thumbnail_url = document.selectFirst("img.pic").attr("src")
+    }
+
+    override fun chapterListRequest(manga: SManga) = GET(pcUrl + manga.urlWithCheck(), headers)
+
+    override fun chapterListSelector() = "div.plistBox ul > li > a"
+
+    override fun chapterFromElement(element: Element) = SChapter.create().apply {
+        url = element.attr("href")
+        name = element.attr("title")
     }
 
     override fun chapterListParse(response: Response): List<SChapter> {
-        val document = asJsoup(response)
+        val document = response.asPufeiJsoup()
-        return document.select(chapterListSelector()).map { chapterFromElement(it) }
+        val list = document.select(chapterListSelector()).map { chapterFromElement(it) }
+        if (isNewDateLogic && list.isNotEmpty()) {
+            val date = document.selectFirst("li.twoCol:contains(更新时间)").text().removePrefix("更新时间：").trim()
+            list[0].date_upload = dateFormat.parse(date)?.time ?: 0
+        }
+        return list
     }
 
+    override fun pageListRequest(chapter: SChapter) = GET(baseUrl + chapter.url, headers)
+
+    // Reference: https://github.com/evanw/packer/blob/master/packer.js
     override fun pageListParse(response: Response): List<Page> {
-        return pageListParse(asJsoup(response))
+        val html = String(response.body!!.bytes(), GB2312).let(::ProgressiveParser)
+        val base64 = html.substringBetween("cp=\"", "\"")
+        val packed = String(Base64.decode(base64, Base64.DEFAULT)).let(::ProgressiveParser)
+        packed.consumeUntil("p}('")
+        val imageList = packed.substringBetween("[", "]").replace("\\", "")
+        if (imageList.isEmpty()) return emptyList()
+        packed.consumeUntil("',")
+        val dictionary = packed.substringBetween("'", "'").split('|')
+        val result = unpack(imageList, dictionary).removeSurrounding("'").split("','")
+        // baseUrl/skin/2014mh/view.js (imgserver), mobileUrl/skin/main.js (IMH.reader)
+        return result.mapIndexed { i, image ->
+            val imageUrl = if (image.startsWith("http")) image else IMAGE_SERVER + image
+            Page(i, imageUrl = imageUrl)
+        }
     }
 
-    override fun imageUrlParse(response: Response): String {
+    override fun pageListParse(document: Document) = throw UnsupportedOperationException("Not used.")
-        return imageUrlParse(asJsoup(response))
+
+    override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("Not used.")
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        ListPreference(screen.context).apply {
+            key = MIRROR_PREF
+            title = "使用镜像网站"
+            summary = "选择要使用的镜像网站，重启生效"
+            entries = MIRRORS_DESCRIPTION
+            entryValues = MIRROR_VALUES
+            setDefaultValue("0")
+            setOnPreferenceChangeListener { _, newValue ->
+                preferences.edit().putString(MIRROR_PREF, newValue as String).apply()
+                true
+            }
+        }.let { screen.addPreference(it) }
+    }
+
+    companion object {
+        private const val MIRROR_PREF = "MIRROR"
+        private val MIRROR_VALUES = arrayOf("0", "1", "2", "3", "4")
+        private val MIRRORS = arrayOf(
+            "pufei.cc",
+            "pfmh.net",
+            "alimanhua.com",
+            "8nfw.com",
+            "pufei5.com",
+        )
+        private val MIRRORS_DESCRIPTION = arrayOf(
+            "pufei.cc",
+            "pfmh.net",
+            "alimanhua.com (阿狸漫画)",
+            "8nfw.com (风之动漫)",
+            "pufei5.com (不推荐)",
+        )
+
+        private const val IMAGE_SERVER = "http://res.img.tueqi.com/"
     }
-    // patch finish
 }

src/zh/pufei/src/eu/kanade/tachiyomi/extension/zh/pufei/PufeiFilters.kt

@@ -0,0 +1,51 @@
+package eu.kanade.tachiyomi.extension.zh.pufei
+
+import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.source.model.FilterList
+
+internal fun getFilters() = FilterList(
+    Filter.Header("排序只对文本搜索和分类筛选有效"),
+    SortFilter(),
+    Filter.Separator(),
+    Filter.Header("以下筛选最多使用一个，使用文本搜索时将会忽略"),
+    CategoryFilter(),
+    AlphabetFilter(),
+)
+
+internal fun parseSearchSort(filters: FilterList): String =
+    filters.filterIsInstance<SortFilter>().firstOrNull()?.let { SORT_QUERIES[it.state] } ?: ""
+
+internal fun parseFilters(page: Int, filters: FilterList): String {
+    val pageStr = if (page == 1) "" else "_$page"
+    var category = 0
+    var categorySort = 0
+    var alphabet = 0
+    for (filter in filters) when (filter) {
+        is SortFilter -> categorySort = filter.state
+        is CategoryFilter -> category = filter.state
+        is AlphabetFilter -> alphabet = filter.state
+        else -> {}
+    }
+    return if (category > 0) {
+        "/${CATEGORY_KEYS[category]}/${SORT_KEYS[categorySort]}$pageStr.html"
+    } else if (alphabet > 0) {
+        "/mh/${ALPHABET[alphabet].lowercase()}/index$pageStr.html"
+    } else {
+        ""
+    }
+}
+
+internal class SortFilter : Filter.Select<String>("排序", SORT_NAMES)
+
+private val SORT_NAMES = arrayOf("添加时间", "更新时间", "点击次数")
+private val SORT_KEYS = arrayOf("index", "update", "view")
+private val SORT_QUERIES = arrayOf("&orderby=newstime", "&orderby=lastdotime", "&orderby=onclick")
+
+internal class CategoryFilter : Filter.Select<String>("分类", CATEGORY_NAMES)
+
+private val CATEGORY_NAMES = arrayOf("全部", "少年热血", "少女爱情", "武侠格斗", "科幻魔幻", "竞技体育", "搞笑喜剧", "耽美人生", "侦探推理", "恐怖灵异")
+private val CATEGORY_KEYS = arrayOf("", "shaonianrexue", "shaonvaiqing", "wuxiagedou", "kehuan", "jingjitiyu", "gaoxiaoxiju", "danmeirensheng", "zhentantuili", "kongbulingyi")
+
+internal class AlphabetFilter : Filter.Select<String>("字母", ALPHABET)
+
+private val ALPHABET = arrayOf("全部", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z")

src/zh/pufei/src/eu/kanade/tachiyomi/extension/zh/pufei/PufeiUtils.kt

@@ -0,0 +1,64 @@
+package eu.kanade.tachiyomi.extension.zh.pufei
+
+import eu.kanade.tachiyomi.AppInfo
+import eu.kanade.tachiyomi.source.model.SManga
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+internal val GB2312 = charset("GB2312")
+
+internal fun Response.asPufeiJsoup(): Document =
+    Jsoup.parse(String(body!!.bytes(), GB2312), request.url.toString())
+
+internal fun SManga.urlWithCheck(): String {
+    val result = url
+    if (result.endsWith("/index.html")) {
+        throw Exception("作品地址格式过期，请迁移更新")
+    }
+    return result
+}
+
+internal val isNewDateLogic = AppInfo.getVersionCode() >= 81
+
+internal val dateFormat by lazy {
+    SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ENGLISH)
+}
+
+internal class ProgressiveParser(private val text: String) {
+    private var startIndex = 0
+    fun consumeUntil(string: String) = with(text) { startIndex = indexOf(string, startIndex) + string.length }
+    fun substringBetween(left: String, right: String): String = with(text) {
+        val leftIndex = indexOf(left, startIndex) + left.length
+        val rightIndex = indexOf(right, leftIndex)
+        startIndex = rightIndex + right.length
+        return substring(leftIndex, rightIndex)
+    }
+}
+
+internal fun unpack(data: String, dictionary: List<String>): String {
+    val size = dictionary.size
+    return Regex("""\b\w+\b""").replace(data) {
+        with(it.value) {
+            val key = parseRadix62()
+            if (key >= size) return@replace this
+            val value = dictionary[key]
+            if (value.isEmpty()) return@replace this
+            return@replace value
+        }
+    }
+}
+
+private fun String.parseRadix62(): Int {
+    var result = 0
+    for (char in this) {
+        result = result * 62 + when {
+            char <= '9' -> char.code - '0'.code
+            char >= 'a' -> char.code - 'a'.code + 10
+            else -> char.code - 'A'.code + 36
+        }
+    }
+    return result
+}