[ZH-dmzj] Move some logic to use dmzj's new v4api, fix #6858 (#7369)

* [ZH-dmzj] Move some logic to use dmzj's new v4api, fix #6858

* Remove usage of desugar libs
This commit is contained in:
Oldwangtouchtouchdoge 2021-06-02 01:54:48 +08:00 committed by GitHub
parent 0452d87b2f
commit fb892a4158
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 244 additions and 82 deletions

View File

@ -1,16 +1,18 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply plugin: 'kotlinx-serialization'
ext {
extName = 'Dmzj'
pkgNameSuffix = 'zh.dmzj'
extClass = '.Dmzj'
extVersionCode = 16
extVersionCode = 17
libVersion = '1.2'
}
dependencies {
implementation project(':lib-ratelimit')
implementation 'org.jetbrains.kotlinx:kotlinx-serialization-protobuf:1.2.0'
}
apply from: "$rootDir/common.gradle"

View File

@ -3,8 +3,11 @@ package eu.kanade.tachiyomi.extension.zh.dmzj
import android.app.Application
import android.content.SharedPreferences
import android.net.Uri
import android.util.Base64
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.extension.zh.dmzj.protobuf.ComicDetailResponse
import eu.kanade.tachiyomi.extension.zh.dmzj.utils.RSA
import eu.kanade.tachiyomi.lib.ratelimit.SpecificHostRateLimitInterceptor
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
@ -17,7 +20,10 @@ import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.protobuf.ProtoBuf
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
import okhttp3.Request
@ -27,8 +33,8 @@ import org.json.JSONObject
import rx.Observable
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.net.URLDecoder
import java.net.URLEncoder
import java.util.ArrayList
/**
* Dmzj source
@ -40,8 +46,12 @@ class Dmzj : ConfigurableSource, HttpSource() {
override val name = "动漫之家"
override val baseUrl = "https://m.dmzj1.com"
private val v3apiUrl = "https://v3api.dmzj1.com"
private val v3ChapterApiUrl = "https://nnv3api.dmzj1.com"
// v3api now shutdown the functionality to fetch manga detail and chapter list, so move these logic to v4api
private val v4apiUrl = "https://nnv4api.dmzj1.com" // https://v4api.dmzj1.com
private val apiUrl = "https://api.dmzj.com"
private val oldPageListApiUrl = "https://m.dmzj.com/chapinfo"
private val oldPageListApiUrl = "https://api.m.dmzj1.com"
private val webviewPageListApiUrl = "https://m.dmzj1.com/chapinfo"
private val imageCDNUrl = "https://images.dmzj1.com"
private fun cleanUrl(url: String) = if (url.startsWith("//"))
@ -56,6 +66,10 @@ class Dmzj : ConfigurableSource, HttpSource() {
v3apiUrl.toHttpUrlOrNull()!!,
preferences.getString(API_RATELIMIT_PREF, "5")!!.toInt()
)
private val v4apiRateLimitInterceptor = SpecificHostRateLimitInterceptor(
v4apiUrl.toHttpUrlOrNull()!!,
preferences.getString(API_RATELIMIT_PREF, "5")!!.toInt()
)
private val apiRateLimitInterceptor = SpecificHostRateLimitInterceptor(
apiUrl.toHttpUrlOrNull()!!,
preferences.getString(API_RATELIMIT_PREF, "5")!!.toInt()
@ -68,6 +82,7 @@ class Dmzj : ConfigurableSource, HttpSource() {
override val client: OkHttpClient = network.client.newBuilder()
.addNetworkInterceptor(apiRateLimitInterceptor)
.addNetworkInterceptor(v3apiRateLimitInterceptor)
.addNetworkInterceptor(v4apiRateLimitInterceptor)
.addNetworkInterceptor(imageCDNRateLimitInterceptor)
.build()
@ -126,6 +141,18 @@ class Dmzj : ConfigurableSource, HttpSource() {
return MangasPage(ret, arr.length() != 0)
}
private fun customUrlBuilder(baseUrl: String): HttpUrl.Builder {
val rightNow = System.currentTimeMillis() / 1000
return baseUrl.toHttpUrlOrNull()!!.newBuilder()
.addQueryParameter("channel", "android")
.addQueryParameter("version", "3.0.0")
.addQueryParameter("timestamp", rightNow.toInt().toString())
}
private fun decryptProtobufData(rawData: String): ByteArray {
return RSA.decrypt(Base64.decode(rawData, Base64.DEFAULT), privateKey)
}
override fun popularMangaRequest(page: Int) = GET("$v3apiUrl/classify/0/0/${page - 1}.json")
override fun popularMangaParse(response: Response) = searchMangaParse(response)
@ -138,18 +165,24 @@ class Dmzj : ConfigurableSource, HttpSource() {
val comicNumberID = if (checkComicIdIsNumericalRegex.matches(id)) {
id
} else {
// Chinese Pinyin ID
val document = client.newCall(GET("$baseUrl/info/$id.html", headers)).execute().asJsoup()
extractComicIdFromWebpageRegex.find(document.select("#Subscribe").attr("onclick"))!!.groups[1]!!.value // onclick="addSubscribe('{comicNumberID}')"
extractComicIdFromWebpageRegex.find(
document.select("#Subscribe").attr("onclick")
)!!.groups[1]!!.value // onclick="addSubscribe('{comicNumberID}')"
}
val sManga = try {
val r = client.newCall(GET("$v3apiUrl/comic/comic_$comicNumberID.json", headers)).execute()
val r = client.newCall(GET("$v4apiUrl/comic/detail/$comicNumberID.json", headers)).execute()
mangaDetailsParse(r)
} catch (_: Exception) {
val r = client.newCall(GET("$apiUrl/dynamic/comicinfo/$comicNumberID.json", headers)).execute()
mangaDetailsParse(r)
}
sManga.url = "$baseUrl/info/$comicNumberID.html"
// Change url format to as same as mangaFromJSON, which used by popularMangaParse and latestUpdatesParse.
// manga.url being used as key to identity a manga in tachiyomi, so if url format don't match popularMangaParse and latestUpdatesParse,
// tachiyomi will mark them as unsubscribe in popularManga and latestUpdates page.
sManga.url = "/comic/comic_$comicNumberID.json?version=2.7.019"
return MangasPage(listOf(sManga), false)
}
@ -204,7 +237,11 @@ class Dmzj : ConfigurableSource, HttpSource() {
val cid = extractComicIdFromMangaUrlRegex.find(manga.url)!!.groups[1]!!.value
return try {
// Not using client.newCall().asObservableSuccess() to ensure we can catch exception here.
val response = client.newCall(GET("$v3apiUrl/comic/comic_$cid.json", headers)).execute()
val response = client.newCall(
GET(
customUrlBuilder("$v4apiUrl/comic/detail/$cid").build().toString(), headers
)
).execute()
val sManga = mangaDetailsParse(response).apply { initialized = true }
Observable.just(sManga)
} catch (e: Exception) {
@ -223,32 +260,23 @@ class Dmzj : ConfigurableSource, HttpSource() {
}
override fun mangaDetailsParse(response: Response) = SManga.create().apply {
val obj = JSONObject(response.body!!.string())
val responseBody = response.body!!.string()
if (response.request.url.toString().startsWith(v4apiUrl)) {
val pb = ProtoBuf.decodeFromByteArray<ComicDetailResponse>(decryptProtobufData(responseBody))
val pbData = pb.Data
title = pbData.Title
thumbnail_url = pbData.Cover
author = pbData.Authors.joinToString(separator = ", ") { it.TagName }
genre = pbData.TypesTypes.joinToString(separator = ", ") { it.TagName }
if (response.request.url.toString().startsWith(v3apiUrl)) {
title = obj.getString("title")
thumbnail_url = obj.getString("cover")
var arr = obj.getJSONArray("authors")
val tmparr = ArrayList<String>(arr.length())
for (i in 0 until arr.length()) {
tmparr.add(arr.getJSONObject(i).getString("tag_name"))
}
author = tmparr.joinToString(", ")
arr = obj.getJSONArray("types")
tmparr.clear()
for (i in 0 until arr.length()) {
tmparr.add(arr.getJSONObject(i).getString("tag_name"))
}
genre = tmparr.joinToString(", ")
status = when (obj.getJSONArray("status").getJSONObject(0).getInt("tag_id")) {
2310 -> SManga.COMPLETED
2309 -> SManga.ONGOING
status = when (pbData.Status[0].TagName) {
"已完结" -> SManga.COMPLETED
"连载中" -> SManga.ONGOING
else -> SManga.UNKNOWN
}
description = obj.getString("description")
description = pbData.Description
} else {
val obj = JSONObject(responseBody)
val data = obj.getJSONObject("data").getJSONObject("info")
title = data.getString("title")
thumbnail_url = data.getString("cover")
@ -269,13 +297,17 @@ class Dmzj : ConfigurableSource, HttpSource() {
val cid = extractComicIdFromMangaUrlRegex.find(manga.url)!!.groups[1]!!.value
return if (manga.status != SManga.LICENSED) {
try {
val response = client.newCall(GET("$v3apiUrl/comic/comic_$cid.json", headers)).execute()
val sChapter = chapterListParse(response)
Observable.just(sChapter)
val response =
client.newCall(
GET(
customUrlBuilder("$v4apiUrl/comic/detail/$cid").build().toString(),
headers
)
).execute()
Observable.just(chapterListParse(response))
} catch (e: Exception) {
val response = client.newCall(GET("$apiUrl/dynamic/comicinfo/$cid.json", headers)).execute()
val sChapter = chapterListParse(response)
Observable.just(sChapter)
Observable.just(chapterListParse(response))
} catch (e: Exception) {
Observable.error(e)
}
@ -285,29 +317,25 @@ class Dmzj : ConfigurableSource, HttpSource() {
}
override fun chapterListParse(response: Response): List<SChapter> {
val obj = JSONObject(response.body!!.string())
val ret = ArrayList<SChapter>()
if (response.request.url.toString().startsWith(v3apiUrl)) {
val cid = obj.getString("id")
val chaptersList = obj.getJSONArray("chapters")
for (i in 0 until chaptersList.length()) {
val chapterObj = chaptersList.getJSONObject(i)
val chapterData = chapterObj.getJSONArray("data")
val prefix = chapterObj.getString("title")
for (j in 0 until chapterData.length()) {
val chapter = chapterData.getJSONObject(j)
ret.add(
SChapter.create().apply {
name = "$prefix: ${chapter.getString("chapter_title")}"
date_upload = chapter.getString("updatetime").toLong() * 1000 // milliseconds
url = "https://api.m.dmzj1.com/comic/chapter/$cid/${chapter.getString("chapter_id")}.html"
}
)
}
val responseBody = response.body!!.string()
if (response.request.url.toString().startsWith(v4apiUrl)) {
val pb = ProtoBuf.decodeFromByteArray<ComicDetailResponse>(decryptProtobufData(responseBody))
val mangaPBData = pb.Data
val chapterPBData = mangaPBData.Chapters[0]
for (i in chapterPBData.Data.indices) {
val chapter = chapterPBData.Data[i]
ret.add(
SChapter.create().apply {
name = chapter.ChapterTitle
date_upload = chapter.Updatetime * 1000
url = "${mangaPBData.Id}/${chapter.ChapterId}"
}
)
}
} else {
// Fallback to old api
// get chapter info from old api
val obj = JSONObject(responseBody)
val chaptersList = obj.getJSONObject("data").getJSONArray("list")
for (i in 0 until chaptersList.length()) {
val chapter = chaptersList.getJSONObject(i)
@ -315,7 +343,7 @@ class Dmzj : ConfigurableSource, HttpSource() {
SChapter.create().apply {
name = chapter.getString("chapter_name")
date_upload = chapter.getString("updatetime").toLong() * 1000
url = "$oldPageListApiUrl/${chapter.getString("comic_id")}/${chapter.getString("id")}.html"
url = "${chapter.getString("comic_id")}/${chapter.getString("id")}"
}
)
}
@ -323,40 +351,59 @@ class Dmzj : ConfigurableSource, HttpSource() {
return ret
}
override fun pageListRequest(chapter: SChapter) = GET(chapter.url, headers) // Bypass base url
override fun pageListRequest(chapter: SChapter) = throw UnsupportedOperationException("Not used.")
override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
return try {
// webpage api
val response = client.newCall(GET("$webviewPageListApiUrl/${chapter.url}.html", headers)).execute()
Observable.just(pageListParse(response))
} catch (e: Exception) {
// api.m.dmzj1.com
val response = client.newCall(GET("$oldPageListApiUrl/comic/chapter/${chapter.url}.html", headers)).execute()
Observable.just(pageListParse(response))
} catch (e: Exception) {
// v3api
val response = client.newCall(
GET(
customUrlBuilder("$v3ChapterApiUrl/chapter/${chapter.url}.json").build().toString(),
headers
)
).execute()
Observable.just(pageListParse(response))
} catch (e: Exception) {
Observable.error(e)
}
}
override fun pageListParse(response: Response): List<Page> {
val arr = if (response.request.url.toString().startsWith(oldPageListApiUrl)) {
JSONObject(response.body!!.string()).getJSONArray("page_url")
} else {
// some chapters are hidden and won't return a JSONObject from api.m.dmzj, have to get them through v3api (but images won't be as HQ)
val requestUrl = response.request.url.toString()
val responseBody = response.body!!.string()
val arr = if (
requestUrl.startsWith(webviewPageListApiUrl) ||
requestUrl.startsWith(v3ChapterApiUrl)
) {
// webpage api or v3api
JSONObject(responseBody).getJSONArray("page_url")
} else if (requestUrl.startsWith(oldPageListApiUrl)) {
try {
val obj = JSONObject(response.body!!.string())
obj.getJSONObject("chapter").getJSONArray("page_url") // api.m.dmzj1.com already return HD image url
} catch (_: Exception) {
// example url: http://v3api.dmzj.com/chapter/44253/101852.json
val url = response.request.url.toString()
.replace("api.m", "v3api")
.replace("comic/", "")
.replace(".html", ".json")
val obj = client.newCall(GET(url, headers)).execute().let { JSONObject(it.body!!.string()) }
obj.getJSONArray("page_url_hd") // page_url in v3api.dmzj1.com will return compressed image, page_url_hd will return HD image url as api.m.dmzj1.com does.
} catch (_: Exception) {
// Fallback to old api
// example url: https://m.dmzj.com/chapinfo/44253/101852.html
val url = response.request.url.toString()
.replaceFirst("api.", "")
.replaceFirst(".dmzj1.", ".dmzj.")
.replaceFirst("comic/chapter", "chapinfo")
val obj = client.newCall(GET(url, headers)).execute().let { JSONObject(it.body!!.string()) }
obj.getJSONArray("page_url")
val obj = JSONObject(responseBody)
obj.getJSONObject("chapter").getJSONArray("page_url")
} catch (e: org.json.JSONException) {
// JSON data from api.m.dmzj1.com may be incomplete, extract page_url list using regex
val extractPageList = extractPageListRegex.find(responseBody)!!.value
JSONObject("{$extractPageList}").getJSONArray("page_url")
}
} else {
throw Exception("can't parse response")
}
val ret = ArrayList<Page>(arr.length())
for (i in 0 until arr.length()) {
ret.add(
Page(i, "", arr.getString(i).replace("http:", "https:").replace("dmzj.com", "dmzj1.com"))
)
// Seems image urls from webpage api and api.m.dmzj1.com may be URL encoded multiple times
val url = URLDecoder.decode(URLDecoder.decode(arr.getString(i), "UTF-8"), "UTF-8")
.replace("http:", "https:")
.replace("dmzj.com", "dmzj1.com")
ret.add(Page(i, "", url))
}
return ret
}
@ -535,8 +582,12 @@ class Dmzj : ConfigurableSource, HttpSource() {
private val extractComicIdFromWebpageRegex = Regex("""addSubscribe\((\d+)\)""")
private val checkComicIdIsNumericalRegex = Regex("""^\d+$""")
private val extractComicIdFromMangaUrlRegex = Regex("""(\d+)\.(json|html)""") // Get comic ID from manga.url
private val extractPageListRegex = Regex("""\"page_url\".+?\]""")
private val ENTRIES_ARRAY = (1..10).map { i -> i.toString() }.toTypedArray()
const val PREFIX_ID_SEARCH = "id:"
private const val privateKey =
"MIICeAIBADANBgkqhkiG9w0BAQEFAASCAmIwggJeAgEAAoGBAK8nNR1lTnIfIes6oRWJNj3mB6OssDGx0uGMpgpbVCpf6+VwnuI2stmhZNoQcM417Iz7WqlPzbUmu9R4dEKmLGEEqOhOdVaeh9Xk2IPPjqIu5TbkLZRxkY3dJM1htbz57d/roesJLkZXqssfG5EJauNc+RcABTfLb4IiFjSMlTsnAgMBAAECgYEAiz/pi2hKOJKlvcTL4jpHJGjn8+lL3wZX+LeAHkXDoTjHa47g0knYYQteCbv+YwMeAGupBWiLy5RyyhXFoGNKbbnvftMYK56hH+iqxjtDLnjSDKWnhcB7089sNKaEM9Ilil6uxWMrMMBH9v2PLdYsqMBHqPutKu/SigeGPeiB7VECQQDizVlNv67go99QAIv2n/ga4e0wLizVuaNBXE88AdOnaZ0LOTeniVEqvPtgUk63zbjl0P/pzQzyjitwe6HoCAIpAkEAxbOtnCm1uKEp5HsNaXEJTwE7WQf7PrLD4+BpGtNKkgja6f6F4ld4QZ2TQ6qvsCizSGJrjOpNdjVGJ7bgYMcczwJBALvJWPLmDi7ToFfGTB0EsNHZVKE66kZ/8Stx+ezueke4S556XplqOflQBjbnj2PigwBN/0afT+QZUOBOjWzoDJkCQClzo+oDQMvGVs9GEajS/32mJ3hiWQZrWvEzgzYRqSf3XVcEe7PaXSd8z3y3lACeeACsShqQoc8wGlaHXIJOHTcCQQCZw5127ZGs8ZDTSrogrH73Kw/HvX55wGAeirKYcv28eauveCG7iyFR0PFB/P/EDZnyb+ifvyEFlucPUI0+Y87F"
}
}

View File

@ -0,0 +1,66 @@
package eu.kanade.tachiyomi.extension.zh.dmzj.protobuf
/*
* Created by reference to https://github.com/xiaoyaocz/dmzj_flutter/blob/23b04c2af930cb7c18a74665e8ec0bf1ccc6f09b/lib/protobuf/comic/detail_response.proto
* All credit goes to their outstanding work.
*/
import kotlinx.serialization.Serializable
import kotlinx.serialization.protobuf.ProtoNumber
@Serializable
data class ComicDetailResponse(
@ProtoNumber(1) val Errno: Int = 0,
@ProtoNumber(2) val Errmsg: String = "",
@ProtoNumber(3) val Data: ComicDetailInfoResponse,
)
@Serializable
data class ComicDetailInfoResponse(
@ProtoNumber(1) val Id: Int,
@ProtoNumber(2) val Title: String,
@ProtoNumber(3) val Direction: Int? = null,
@ProtoNumber(4) val Islong: Int? = null,
@ProtoNumber(5) val IsDmzj: Int? = null,
@ProtoNumber(6) val Cover: String,
@ProtoNumber(7) val Description: String,
@ProtoNumber(8) val LastUpdatetime: Long,
@ProtoNumber(9) val LastUpdateChapterName: String,
@ProtoNumber(10) val Copyright: Int? = null,
@ProtoNumber(11) val FirstLetter: String? = null,
@ProtoNumber(12) val ComicPy: String? = null,
@ProtoNumber(13) val Hidden: Int? = null,
@ProtoNumber(14) val HotNum: Int? = null,
@ProtoNumber(15) val HitNum: Int? = null,
@ProtoNumber(16) val Uid: Int? = null,
@ProtoNumber(17) val IsLock: Int? = null,
@ProtoNumber(18) val LastUpdateChapterId: Int,
@ProtoNumber(19) val TypesTypes: List<ComicDetailTypeItemResponse> = emptyList(),
@ProtoNumber(20) val Status: List<ComicDetailTypeItemResponse> = emptyList(),
@ProtoNumber(21) val Authors: List<ComicDetailTypeItemResponse> = emptyList(),
@ProtoNumber(22) val SubscribeNum: Int? = null,
@ProtoNumber(23) val Chapters: List<ComicDetailChapterResponse> = emptyList(),
@ProtoNumber(24) val IsNeedLogin: Int? = null,
@ProtoNumber(26) val IsHideChapter: Int? = null,
)
@Serializable
data class ComicDetailTypeItemResponse(
@ProtoNumber(1) val TagId: Int,
@ProtoNumber(2) val TagName: String,
)
@Serializable
data class ComicDetailChapterResponse(
@ProtoNumber(1) val Title: String,
@ProtoNumber(2) val Data: List<ComicDetailChapterInfoResponse> = emptyList(),
)
@Serializable
data class ComicDetailChapterInfoResponse(
@ProtoNumber(1) val ChapterId: Int,
@ProtoNumber(2) val ChapterTitle: String,
@ProtoNumber(3) val Updatetime: Long,
@ProtoNumber(4) val Filesize: Int,
@ProtoNumber(5) val ChapterOrder: Int,
)

View File

@ -0,0 +1,43 @@
package eu.kanade.tachiyomi.extension.zh.dmzj.utils
import android.util.Base64
import java.io.ByteArrayOutputStream
import java.security.KeyFactory
import java.security.spec.PKCS8EncodedKeySpec
import javax.crypto.Cipher
object RSA {
private const val MAX_DECRYPT_BLOCK = 128
fun decrypt(encryptedData: ByteArray, privateKey: String): ByteArray {
val keyBytes = Base64.decode(privateKey, Base64.DEFAULT)
val pkcs8KeySpec = PKCS8EncodedKeySpec(keyBytes)
val keyFactory = KeyFactory.getInstance("RSA")
val privateK = keyFactory.generatePrivate(pkcs8KeySpec)
val cipher = Cipher.getInstance("RSA/ECB/PKCS1Padding")
cipher.init(Cipher.DECRYPT_MODE, privateK)
return doFinal(encryptedData, cipher)
}
private fun doFinal(encryptedData: ByteArray, cipher: Cipher): ByteArray {
val inputLen = encryptedData.size
ByteArrayOutputStream().use { out ->
var offSet = 0
var cache: ByteArray
var i = 0
val block = MAX_DECRYPT_BLOCK
while (inputLen - offSet > 0) {
cache = if (inputLen - offSet > block) {
cipher.doFinal(encryptedData, offSet, block)
} else {
cipher.doFinal(encryptedData, offSet, inputLen - offSet)
}
out.write(cache, 0, cache.size)
i++
offSet = i * block
}
return out.toByteArray()
}
}
}