Pufei Manhua: rewrite, fix bugs, add mirrors and filters (#12360)

* Pufei: rewrite, fix bugs, add mirrors and filters

* intercept some weird responses

* move rate limiter forward
This commit is contained in:
stevenyomi 2022-06-30 03:31:34 +08:00 committed by GitHub
parent e7f6fd0330
commit 7bc4e3f647
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 402 additions and 174 deletions

View File

@ -2,10 +2,10 @@ apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
ext {
extName = 'Pufei'
extName = 'Pufei Manhua'
pkgNameSuffix = 'zh.pufei'
extClass = '.Pufei'
extVersionCode = 8
extVersionCode = 9
}
apply from: "$rootDir/common.gradle"

View File

@ -0,0 +1,58 @@
package eu.kanade.tachiyomi.extension.zh.pufei
import android.os.SystemClock
import okhttp3.Interceptor
import okhttp3.Response
import java.io.IOException
import java.util.concurrent.TimeUnit
// See https://github.com/tachiyomiorg/tachiyomi/pull/7389
internal class NonblockingRateLimiter(
private val permits: Int,
period: Long = 1,
unit: TimeUnit = TimeUnit.SECONDS,
) : Interceptor {
private val requestQueue = ArrayList<Long>(permits)
private val rateLimitMillis = unit.toMillis(period)
override fun intercept(chain: Interceptor.Chain): Response {
// Ignore canceled calls, otherwise they would jam the queue
if (chain.call().isCanceled()) {
throw IOException()
}
synchronized(requestQueue) {
val now = SystemClock.elapsedRealtime()
val waitTime = if (requestQueue.size < permits) {
0
} else {
val oldestReq = requestQueue[0]
val newestReq = requestQueue[permits - 1]
if (newestReq - oldestReq > rateLimitMillis) {
0
} else {
oldestReq + rateLimitMillis - now // Remaining time
}
}
// Final check
if (chain.call().isCanceled()) {
throw IOException()
}
if (requestQueue.size == permits) {
requestQueue.removeAt(0)
}
if (waitTime > 0) {
requestQueue.add(now + waitTime)
Thread.sleep(waitTime) // Sleep inside synchronized to pause queued requests
} else {
requestQueue.add(now)
}
}
return chain.proceed(chain.request())
}
}

View File

@ -0,0 +1,41 @@
package eu.kanade.tachiyomi.extension.zh.pufei
import eu.kanade.tachiyomi.network.GET
import okhttp3.Interceptor
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.Response
import okhttp3.ResponseBody.Companion.asResponseBody
object OctetStreamInterceptor : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (response.header("Content-Type") != "application/octet-stream") {
return response
}
if (response.header("Content-Length")!!.toInt() < 100) { // usually 96
// The actual URL is '/.../xxx.jpg/0'.
val peek = response.peekBody(100).string()
if (peek.startsWith("The actual URL")) {
response.body!!.close()
val actualPath = peek.substringAfter('\'').substringBeforeLast('\'')
return chain.proceed(GET("https://manhua.acimg.cn$actualPath"))
}
}
val url = request.url.encodedPath
val mediaType = when {
url.endsWith(".h") -> webpMediaType
url.contains(".jpg") -> jpegMediaType
else -> return response
}
val body = response.body!!.source().asResponseBody(mediaType)
return response.newBuilder().body(body).build()
}
private val jpegMediaType = "image/jpeg".toMediaType()
private val webpMediaType = "image/webp".toMediaType()
}

View File

@ -1,214 +1,228 @@
package eu.kanade.tachiyomi.extension.zh.pufei
// temp patch:
// https://github.com/tachiyomiorg/tachiyomi/pull/2031
import android.app.Application
import android.content.SharedPreferences
import android.util.Base64
import com.squareup.duktape.Duktape
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.ConfigurableSource
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.Interceptor
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.OkHttpClient
import okhttp3.FormBody
import okhttp3.Request
import okhttp3.Response
import okhttp3.ResponseBody.Companion.toResponseBody
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Evaluator
import rx.Observable
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
fun asJsoup(response: Response, html: String? = null): Document {
return Jsoup.parse(html ?: bodyWithAutoCharset(response), response.request.url.toString())
}
fun bodyWithAutoCharset(response: Response, _charset: String? = null): String {
val htmlBytes: ByteArray = response.body!!.bytes()
var c = _charset
if (c == null) {
val regexPat = Regex("""charset=(\w+)""")
val match = regexPat.find(String(htmlBytes))
c = match?.groups?.get(1)?.value
}
return String(htmlBytes, charset(c ?: "utf8"))
}
// patch finish
fun ByteArray.toHexString() = joinToString("%") { "%02x".format(it) }
class Pufei : ParsedHttpSource() {
// Uses www733dm/IMH/dm456 theme
class Pufei : ParsedHttpSource(), ConfigurableSource {
override val name = "扑飞漫画"
override val baseUrl = "http://m.pufei8.com"
override val lang = "zh"
override val supportsLatest = true
val imageServer = "http://res.img.shengda0769.com/"
override val client: OkHttpClient
get() = network.client.newBuilder()
.addNetworkInterceptor(rewriteOctetStream)
.build()
private val preferences: SharedPreferences =
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
private val rewriteOctetStream: Interceptor = Interceptor { chain ->
val originalResponse: Response = chain.proceed(chain.request())
if (originalResponse.headers("Content-Type").contains("application/octet-stream") && originalResponse.request.url.toString().contains(".jpg")) {
val orgBody = originalResponse.body!!.bytes()
val newBody = orgBody.toResponseBody("image/jpeg".toMediaTypeOrNull())
originalResponse.newBuilder()
.body(newBody)
.build()
} else originalResponse
}
private val domain = preferences.getString(MIRROR_PREF, "0")!!.toInt()
.coerceIn(0, MIRRORS.size - 1).let { MIRRORS[it] }
override fun popularMangaSelector() = "ul#detail li"
override val baseUrl = "http://m.$domain"
private val pcUrl = "http://www.$domain"
override fun latestUpdatesSelector() = popularMangaSelector()
override val client = network.client.newBuilder()
.addInterceptor(NonblockingRateLimiter(2))
.addInterceptor(OctetStreamInterceptor)
.build()
override fun headersBuilder() = super.headersBuilder()
.add("Referer", baseUrl)
private val searchClient = network.client.newBuilder()
.followRedirects(false)
.build()
override fun popularMangaRequest(page: Int) = GET("$baseUrl/manhua/paihang.html", headers)
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/manhua/update.html", headers)
private fun mangaFromElement(element: Element): SManga {
val manga = SManga.create()
element.select("a").first().let {
manga.setUrlWithoutDomain(it.attr("href"))
manga.title = it.select("h3").text().trim()
manga.thumbnail_url = it.select("div.thumb img").attr("data-src")
}
return manga
}
override fun popularMangaFromElement(element: Element) = mangaFromElement(element)
override fun latestUpdatesFromElement(element: Element) = mangaFromElement(element)
override fun searchMangaFromElement(element: Element) = mangaFromElement(element)
override fun popularMangaNextPageSelector() = null
override fun latestUpdatesNextPageSelector() = null
override fun mangaDetailsParse(document: Document): SManga {
val infoElement = document.select("div.book-detail")
val manga = SManga.create()
manga.description = infoElement.select("div#bookIntro > p").text().trim()
manga.thumbnail_url = infoElement.select("div.thumb > img").first()?.attr("src")
manga.author = infoElement.select(":nth-child(4) dd").first()?.text()
return manga
}
override fun searchMangaNextPageSelector() = throw UnsupportedOperationException("Not used")
override fun searchMangaSelector() = "ul#detail > li"
private fun encodeGBK(str: String) = "%" + str.toByteArray(charset("gb2312")).toHexString()
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = ("$baseUrl/e/search/?searchget=1&tbname=mh&show=title,player,playadmin,bieming,pinyin,playadmin&tempid=4&keyboard=" + encodeGBK(query)).toHttpUrlOrNull()
?.newBuilder()
return GET(url.toString(), headers)
}
override fun searchMangaParse(response: Response): MangasPage {
// val document = response.asJsoup()
val document = asJsoup(response)
val mangas = document.select(searchMangaSelector()).map { element ->
searchMangaFromElement(element)
}
return MangasPage(mangas, false)
}
override fun chapterListSelector() = "div.chapter-list > ul > li"
override fun chapterListRequest(manga: SManga) = mangaDetailsRequest(manga)
override fun chapterFromElement(element: Element): SChapter {
val urlElement = element.select("a")
val chapter = SChapter.create()
chapter.setUrlWithoutDomain(urlElement.attr("href"))
chapter.name = urlElement.text().trim()
return chapter
}
override fun mangaDetailsRequest(manga: SManga) = GET(baseUrl + manga.url, headers)
override fun pageListRequest(chapter: SChapter) = GET(baseUrl + chapter.url, headers)
override fun pageListParse(document: Document): List<Page> {
val html = document.html()
val re = Regex("cp=\"(.*?)\"")
val imgbase64 = re.find(html)?.groups?.get(1)?.value
val imgCode = String(Base64.decode(imgbase64, Base64.DEFAULT))
val imgArrStr = Duktape.create().use {
it.evaluate("$imgCode.join('|')") as String
}
val hasHost = imgArrStr.startsWith("http")
return imgArrStr.split('|').mapIndexed { i, imgStr ->
Page(i, "", if (hasHost) imgStr else imageServer + imgStr)
}
}
override fun imageUrlParse(document: Document) = ""
private class GenreFilter(genres: Array<String>) : Filter.Select<String>("Genre", genres)
override fun getFilterList() = FilterList(
GenreFilter(getGenreList())
)
private fun getGenreList() = arrayOf(
"All"
)
// temp patch
override fun latestUpdatesParse(response: Response): MangasPage {
val document = asJsoup(response)
val mangas = document.select(latestUpdatesSelector()).map { element ->
latestUpdatesFromElement(element)
}
return MangasPage(mangas, false)
override fun popularMangaNextPageSelector() = throw UnsupportedOperationException("Not used.")
override fun popularMangaSelector() = "ul#detail > li > a"
override fun popularMangaFromElement(element: Element) = SManga.create().apply {
url = element.attr("href").removeSuffix("/index.html")
title = element.selectFirst(Evaluator.Tag("h3")).text()
thumbnail_url = element.selectFirst(Evaluator.Tag("img")).attr("data-src")
}
override fun popularMangaParse(response: Response): MangasPage {
val document = asJsoup(response)
val mangas = document.select(popularMangaSelector()).map { element ->
popularMangaFromElement(element)
}
val document = response.asPufeiJsoup()
val mangas = document.select(popularMangaSelector()).map { popularMangaFromElement(it) }
return MangasPage(mangas, false)
}
override fun mangaDetailsParse(response: Response): SManga {
return mangaDetailsParse(asJsoup(response))
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/manhua/update.html", headers)
override fun latestUpdatesNextPageSelector() = throw UnsupportedOperationException("Not used.")
override fun latestUpdatesSelector() = popularMangaSelector()
override fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element)
override fun latestUpdatesParse(response: Response): MangasPage {
val document = response.asPufeiJsoup()
val mangas = document.select(latestUpdatesSelector()).map { latestUpdatesFromElement(it) }
return MangasPage(mangas, false)
}
private val searchCache = HashMap<String, String>(0)
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
return if (query.isNotBlank()) {
val path = searchCache.getOrPut(query) {
val formBody = FormBody.Builder(GB2312)
.addEncoded("tempid", "4")
.addEncoded("show", "title,player,playadmin,bieming,pinyin")
.add("keyboard", query)
.build()
val request = POST("$baseUrl/e/search/index.php", headers, formBody)
searchClient.newCall(request).execute().header("location")!!
}
val sortQuery = parseSearchSort(filters)
GET("$baseUrl/e/search/$path$sortQuery&page=${page - 1}")
} else {
val path = parseFilters(page, filters)
if (path.isEmpty())
popularMangaRequest(page)
else
GET("$baseUrl$path", headers)
}
}
override fun searchMangaNextPageSelector() = throw UnsupportedOperationException("Not used.")
override fun searchMangaSelector() = popularMangaSelector()
override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
override fun searchMangaParse(response: Response): MangasPage {
val document = response.asPufeiJsoup()
val mangas = document.select(searchMangaSelector()).map { searchMangaFromElement(it) }
val hasNextPage = run {
for (element in document.body().children().asReversed()) {
if (element.tagName() == "a") return@run true
else if (element.tagName() == "b") return@run false
}
false
}
return MangasPage(mangas, hasNextPage)
}
override fun getFilterList() = getFilters()
// 让 WebView 显示移动端页面
override fun mangaDetailsRequest(manga: SManga) = GET(baseUrl + manga.url, headers)
override fun fetchMangaDetails(manga: SManga): Observable<SManga> =
client.newCall(GET(pcUrl + manga.urlWithCheck(), headers)).asObservableSuccess()
.map { mangaDetailsParse(it.asPufeiJsoup()) }
override fun mangaDetailsParse(document: Document) = SManga.create().apply {
val details = document.selectFirst(Evaluator.Class("detailInfo")).children()
title = details[0].child(0).text() // div.titleInfo > h1
val genreList = mutableListOf<String>()
for (item in details[1].children()) { // ul > li
when (item.child(0).text()) { // span
"作者:" -> author = item.ownText()
"类别:" -> item.ownText().let { if (it.isNotEmpty()) genreList.add(it) }
"关键词:" -> item.ownText().let { if (it.isNotEmpty()) genreList.addAll(it.split(',')) }
}
}
author = author ?: details[0].ownText().removePrefix("作者:")
if (genreList.isEmpty()) {
genreList.add(document.selectFirst(Evaluator.Class("position")).child(1).text())
}
genre = genreList.joinToString()
description = document.selectFirst("div.introduction")?.text() ?: details[2].ownText()
status = SManga.UNKNOWN // 所有漫画的标记都是连载,所以没有意义,参见 baseUrl/manhua/wanjie.html
thumbnail_url = document.selectFirst("img.pic").attr("src")
}
override fun chapterListRequest(manga: SManga) = GET(pcUrl + manga.urlWithCheck(), headers)
override fun chapterListSelector() = "div.plistBox ul > li > a"
override fun chapterFromElement(element: Element) = SChapter.create().apply {
url = element.attr("href")
name = element.attr("title")
}
override fun chapterListParse(response: Response): List<SChapter> {
val document = asJsoup(response)
return document.select(chapterListSelector()).map { chapterFromElement(it) }
val document = response.asPufeiJsoup()
val list = document.select(chapterListSelector()).map { chapterFromElement(it) }
if (isNewDateLogic && list.isNotEmpty()) {
val date = document.selectFirst("li.twoCol:contains(更新时间)").text().removePrefix("更新时间:").trim()
list[0].date_upload = dateFormat.parse(date)?.time ?: 0
}
return list
}
override fun pageListRequest(chapter: SChapter) = GET(baseUrl + chapter.url, headers)
// Reference: https://github.com/evanw/packer/blob/master/packer.js
override fun pageListParse(response: Response): List<Page> {
return pageListParse(asJsoup(response))
val html = String(response.body!!.bytes(), GB2312).let(::ProgressiveParser)
val base64 = html.substringBetween("cp=\"", "\"")
val packed = String(Base64.decode(base64, Base64.DEFAULT)).let(::ProgressiveParser)
packed.consumeUntil("p}('")
val imageList = packed.substringBetween("[", "]").replace("\\", "")
if (imageList.isEmpty()) return emptyList()
packed.consumeUntil("',")
val dictionary = packed.substringBetween("'", "'").split('|')
val result = unpack(imageList, dictionary).removeSurrounding("'").split("','")
// baseUrl/skin/2014mh/view.js (imgserver), mobileUrl/skin/main.js (IMH.reader)
return result.mapIndexed { i, image ->
val imageUrl = if (image.startsWith("http")) image else IMAGE_SERVER + image
Page(i, imageUrl = imageUrl)
}
}
override fun imageUrlParse(response: Response): String {
return imageUrlParse(asJsoup(response))
override fun pageListParse(document: Document) = throw UnsupportedOperationException("Not used.")
override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("Not used.")
override fun setupPreferenceScreen(screen: PreferenceScreen) {
ListPreference(screen.context).apply {
key = MIRROR_PREF
title = "使用镜像网站"
summary = "选择要使用的镜像网站,重启生效"
entries = MIRRORS_DESCRIPTION
entryValues = MIRROR_VALUES
setDefaultValue("0")
setOnPreferenceChangeListener { _, newValue ->
preferences.edit().putString(MIRROR_PREF, newValue as String).apply()
true
}
}.let { screen.addPreference(it) }
}
companion object {
private const val MIRROR_PREF = "MIRROR"
private val MIRROR_VALUES = arrayOf("0", "1", "2", "3", "4")
private val MIRRORS = arrayOf(
"pufei.cc",
"pfmh.net",
"alimanhua.com",
"8nfw.com",
"pufei5.com",
)
private val MIRRORS_DESCRIPTION = arrayOf(
"pufei.cc",
"pfmh.net",
"alimanhua.com (阿狸漫画)",
"8nfw.com (风之动漫)",
"pufei5.com (不推荐)",
)
private const val IMAGE_SERVER = "http://res.img.tueqi.com/"
}
// patch finish
}

View File

@ -0,0 +1,51 @@
package eu.kanade.tachiyomi.extension.zh.pufei
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList
internal fun getFilters() = FilterList(
Filter.Header("排序只对文本搜索和分类筛选有效"),
SortFilter(),
Filter.Separator(),
Filter.Header("以下筛选最多使用一个,使用文本搜索时将会忽略"),
CategoryFilter(),
AlphabetFilter(),
)
internal fun parseSearchSort(filters: FilterList): String =
filters.filterIsInstance<SortFilter>().firstOrNull()?.let { SORT_QUERIES[it.state] } ?: ""
internal fun parseFilters(page: Int, filters: FilterList): String {
val pageStr = if (page == 1) "" else "_$page"
var category = 0
var categorySort = 0
var alphabet = 0
for (filter in filters) when (filter) {
is SortFilter -> categorySort = filter.state
is CategoryFilter -> category = filter.state
is AlphabetFilter -> alphabet = filter.state
else -> {}
}
return if (category > 0) {
"/${CATEGORY_KEYS[category]}/${SORT_KEYS[categorySort]}$pageStr.html"
} else if (alphabet > 0) {
"/mh/${ALPHABET[alphabet].lowercase()}/index$pageStr.html"
} else {
""
}
}
internal class SortFilter : Filter.Select<String>("排序", SORT_NAMES)
private val SORT_NAMES = arrayOf("添加时间", "更新时间", "点击次数")
private val SORT_KEYS = arrayOf("index", "update", "view")
private val SORT_QUERIES = arrayOf("&orderby=newstime", "&orderby=lastdotime", "&orderby=onclick")
internal class CategoryFilter : Filter.Select<String>("分类", CATEGORY_NAMES)
private val CATEGORY_NAMES = arrayOf("全部", "少年热血", "少女爱情", "武侠格斗", "科幻魔幻", "竞技体育", "搞笑喜剧", "耽美人生", "侦探推理", "恐怖灵异")
private val CATEGORY_KEYS = arrayOf("", "shaonianrexue", "shaonvaiqing", "wuxiagedou", "kehuan", "jingjitiyu", "gaoxiaoxiju", "danmeirensheng", "zhentantuili", "kongbulingyi")
internal class AlphabetFilter : Filter.Select<String>("字母", ALPHABET)
private val ALPHABET = arrayOf("全部", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z")

View File

@ -0,0 +1,64 @@
package eu.kanade.tachiyomi.extension.zh.pufei
import eu.kanade.tachiyomi.AppInfo
import eu.kanade.tachiyomi.source.model.SManga
import okhttp3.Response
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import java.text.SimpleDateFormat
import java.util.Locale
internal val GB2312 = charset("GB2312")
internal fun Response.asPufeiJsoup(): Document =
Jsoup.parse(String(body!!.bytes(), GB2312), request.url.toString())
internal fun SManga.urlWithCheck(): String {
val result = url
if (result.endsWith("/index.html")) {
throw Exception("作品地址格式过期,请迁移更新")
}
return result
}
internal val isNewDateLogic = AppInfo.getVersionCode() >= 81
internal val dateFormat by lazy {
SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.ENGLISH)
}
internal class ProgressiveParser(private val text: String) {
private var startIndex = 0
fun consumeUntil(string: String) = with(text) { startIndex = indexOf(string, startIndex) + string.length }
fun substringBetween(left: String, right: String): String = with(text) {
val leftIndex = indexOf(left, startIndex) + left.length
val rightIndex = indexOf(right, leftIndex)
startIndex = rightIndex + right.length
return substring(leftIndex, rightIndex)
}
}
internal fun unpack(data: String, dictionary: List<String>): String {
val size = dictionary.size
return Regex("""\b\w+\b""").replace(data) {
with(it.value) {
val key = parseRadix62()
if (key >= size) return@replace this
val value = dictionary[key]
if (value.isEmpty()) return@replace this
return@replace value
}
}
}
private fun String.parseRadix62(): Int {
var result = 0
for (char in this) {
result = result * 62 + when {
char <= '9' -> char.code - '0'.code
char >= 'a' -> char.code - 'a'.code + 10
else -> char.code - 'A'.code + 36
}
}
return result
}