From 1a6774af591ec271bae2ea968d64012adabe5b3a Mon Sep 17 00:00:00 2001 From: Jake Date: Sun, 27 Apr 2025 10:07:35 +0800 Subject: [PATCH] MangaHub (multisrc) - Fixes and Improvements (#8586) * remove rate limit * Fixes and improvements * Version bump * Review changes, more improvements --- lib-multisrc/mangahub/build.gradle.kts | 5 +- .../tachiyomi/multisrc/mangahub/MangaHub.kt | 135 ++++++++++++------ .../multisrc/mangahub/MangaHubDto.kt | 3 + .../multisrc/mangahub/MangaHubQueries.kt | 7 +- 4 files changed, 102 insertions(+), 48 deletions(-) diff --git a/lib-multisrc/mangahub/build.gradle.kts b/lib-multisrc/mangahub/build.gradle.kts index 722908d9f..7168a68d8 100644 --- a/lib-multisrc/mangahub/build.gradle.kts +++ b/lib-multisrc/mangahub/build.gradle.kts @@ -2,8 +2,9 @@ plugins { id("lib-multisrc") } -baseVersionCode = 32 +baseVersionCode = 33 dependencies { - api(project(":lib:randomua")) + //noinspection UseTomlInstead + implementation("org.brotli:dec:0.1.2") } diff --git a/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHub.kt b/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHub.kt index efa8bc9ef..e2b17a6d7 100644 --- a/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHub.kt +++ b/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHub.kt @@ -3,11 +3,8 @@ package eu.kanade.tachiyomi.multisrc.mangahub import android.content.SharedPreferences import androidx.preference.PreferenceScreen import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.lib.randomua.UserAgentType -import eu.kanade.tachiyomi.lib.randomua.setRandomUserAgent import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.POST -import eu.kanade.tachiyomi.network.interceptor.rateLimit import eu.kanade.tachiyomi.source.ConfigurableSource import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList @@ -30,12 +27,16 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.brotli.dec.BrotliInputStream import rx.Observable +import java.io.ByteArrayOutputStream import java.io.IOException import java.net.URLEncoder import java.text.SimpleDateFormat import java.util.Calendar import java.util.Locale +import java.util.zip.GZIPInputStream abstract class MangaHub( override val name: String, @@ -50,7 +51,8 @@ abstract class MangaHub( private val baseApiUrl = "https://api.mghcdn.com" private val baseCdnUrl = "https://imgx.mghcdn.com" private val baseThumbCdnUrl = "https://thumb.mghcdn.com" - private val regex = Regex("mhub_access=([^;]+)") + private val apiRegex = Regex("mhub_access=([^;]+)") + private val spaceRegex = Regex("\\s+") private val preferences: SharedPreferences by getPreferencesLazy() @@ -60,13 +62,9 @@ abstract class MangaHub( ) override val client: OkHttpClient = network.cloudflareClient.newBuilder() - .setRandomUserAgent( - userAgentType = UserAgentType.DESKTOP, - filterInclude = listOf("chrome"), - ) .addInterceptor(::apiAuthInterceptor) .addInterceptor(::graphQLApiInterceptor) - .rateLimit(1) + .addNetworkInterceptor(::compatEncodingInterceptor) .build() override fun headersBuilder(): Headers.Builder = super.headersBuilder() @@ -119,6 +117,44 @@ abstract class MangaHub( return chain.proceed(request) } + // Normally this gets handled properly but in older forks such as TachiyomiJ2K, we have to manually intercept it + // as they have an outdated implementation of NetworkHelper. + private fun compatEncodingInterceptor(chain: Interceptor.Chain): Response { + var response = chain.proceed(chain.request()) + val contentEncoding = response.header("Content-Encoding") + + if (contentEncoding == "gzip") { + val parsedBody = response.body.byteStream().let { gzipInputStream -> + GZIPInputStream(gzipInputStream).use { inputStream -> + val outputStream = ByteArrayOutputStream() + inputStream.copyTo(outputStream) + outputStream.toByteArray() + } + } + + response = response.createNewWithCompatBody(parsedBody) + } else if (contentEncoding == "br") { + val parsedBody = response.body.byteStream().let { brotliInputStream -> + BrotliInputStream(brotliInputStream).use { inputStream -> + val outputStream = ByteArrayOutputStream() + inputStream.copyTo(outputStream) + outputStream.toByteArray() + } + } + + response = response.createNewWithCompatBody(parsedBody) + } + + return response + } + + private fun Response.createNewWithCompatBody(outputStream: ByteArray): Response { + return this.newBuilder() + .body(outputStream.toResponseBody(this.body.contentType())) + .removeHeader("Content-Encoding") + .build() + } + private fun graphQLApiInterceptor(chain: Interceptor.Chain): Response { val request = chain.request() @@ -147,33 +183,15 @@ abstract class MangaHub( } private fun refreshApiKey(chapter: SChapter) { - val now = Calendar.getInstance().time.time - val url = "$baseUrl/chapter${chapter.url}".toHttpUrl() val oldKey = client.cookieJar .loadForRequest(baseUrl.toHttpUrl()) .firstOrNull { it.name == "mhub_access" && it.value.isNotEmpty() }?.value - // With the recent changes on how refresh API token works, we are now apparently required to have - // a cookie for recently when requesting for a new one. Not having this will result in a hit or miss. - val recently = buildJsonObject { - putJsonObject((now - (0..3600).random()).toString()) { - put("mangaID", (1..42_000).random()) - put("number", (1..20).random()) - } - }.toString() - - val recentlyCookie = Cookie.Builder() - .domain(url.host) - .name("recently") - .value(URLEncoder.encode(recently, "utf-8")) - .expiresAt(now + 2 * 60 * 60 * 24 * 31) // +2 months - .build() - for (i in 1..2) { // Clear key cookie val cookie = Cookie.parse(url, "mhub_access=; Max-Age=0; Path=/")!! - client.cookieJar.saveFromResponse(url, listOf(cookie, recentlyCookie)) + client.cookieJar.saveFromResponse(url, listOf(cookie)) // We try requesting again with param if the first one fails val query = if (i == 2) "?reloadKey=1" else "" @@ -187,7 +205,7 @@ abstract class MangaHub( .build(), ), ).execute() - val returnedKey = response.headers["set-cookie"]?.let { regex.find(it)?.groupValues?.get(1) } + val returnedKey = response.headers["set-cookie"]?.let { apiRegex.find(it)?.groupValues?.get(1) } response.close() // Avoid potential resource leaks if (returnedKey != oldKey) break // Break out of loop since we got an allegedly valid API key @@ -195,16 +213,6 @@ abstract class MangaHub( throw IOException("An error occurred while obtaining a new API key") // Show error } } - - // Sometimes, the new API key is still invalid. To ensure that the token will be fresh and available to use, - // we have to mimic how the browser site works. To put it simply, we will send a GET request that indicates what - // manga and chapter were browsing. If this succeeded, the API key that we use will be revalidated (assuming that we got an expired one.) - // We first need to obtain our public IP first since it is required as a query. - val ipRequest = client.newCall(GET("https://api.ipify.org?format=json")).execute() - val ip = ipRequest.parseAs().ip - - // We'll log our action to the site to revalidate the API key in case we got an expired one - client.newCall(GET("$baseUrl/action/logHistory2/${url.pathSegments[1]}/${chapter.chapter_number}?browserID=$ip")).execute() } data class SMangaDTO( @@ -339,7 +347,7 @@ abstract class MangaHub( val numberString = "${if (it.number % 1 == 0f) it.number.toInt() else it.number}" name = if (!useGenericTitle) { - generateChapterName(it.title.trim().replace("\n", " "), numberString) + generateChapterName(it.title.trim().replace(spaceRegex, " "), numberString) } else { generateGenericChapterName(numberString) } @@ -374,15 +382,52 @@ abstract class MangaHub( return postRequestGraphQL(pagesQuery(mangaSource, chapterUrl[1], chapterUrl[2].substringAfter("-").toFloat())) } - override fun fetchPageList(chapter: SChapter): Observable> = - super.fetchPageList(chapter) - .doOnError { refreshApiKey(chapter) } + override fun fetchPageList(chapter: SChapter): Observable> { + var doApiRefresh = true + + return super.fetchPageList(chapter) + .doOnError { + // Ensure that the api refresh call will happen only once + if (doApiRefresh) { + refreshApiKey(chapter) + doApiRefresh = false + } + } .retry(1) + } override fun pageListParse(response: Response): List { val chapterObject = response.parseAs() val pages = chapterObject.data.chapter.pages.parseAs() + // We'll update the cookie here to match the browser's "recently" opened chapter. + // This mimics how the browser works and gives us more chance to receive a valid API key upon refresh + val now = Calendar.getInstance().time.time + val baseHttpUrl = baseUrl.toHttpUrl() + val recently = buildJsonObject { + putJsonObject((now).toString()) { + put("mangaID", chapterObject.data.chapter.mangaID) + put("number", chapterObject.data.chapter.chapterNumber) + } + }.toString() + + val recentlyCookie = Cookie.Builder() + .domain(baseHttpUrl.host) + .name("recently") + .value(URLEncoder.encode(recently, "utf-8")) + .expiresAt(now + 2 * 60 * 60 * 24 * 31) // +2 months + .build() + + // Add/update the cookie + client.cookieJar.saveFromResponse(baseHttpUrl, listOf(recentlyCookie)) + + // We'll log our action to the site to further increase the chance of valid API key + val ipRequest = client.newCall(GET("https://api.ipify.org?format=json")).execute() + val ip = ipRequest.parseAs().ip + + client.newCall(GET("$baseUrl/action/logHistory2/${chapterObject.data.chapter.manga.slug}/${chapterObject.data.chapter.chapterNumber}?browserID=$ip")).execute().close() + ipRequest.close() + return pages.images.mapIndexed { i, page -> Page(i, "", "$baseCdnUrl/${pages.page}$page") } @@ -517,7 +562,7 @@ abstract class MangaHub( Genre("Video Games", "video-games"), Genre("Monsters", "monsters"), Genre("Office Workers", "office-workers"), - Genre("system", "system"), + Genre("System", "system"), Genre("Villainess", "villainess"), Genre("Zombies", "zombies"), Genre("Vampires", "vampires"), @@ -540,7 +585,7 @@ abstract class MangaHub( Genre("Cheat Systems", "cheat-systems"), Genre("Dungeons", "dungeons"), Genre("Overpowered", "overpowered"), - ) + ).sortedBy { it.toString() } override fun setupPreferenceScreen(screen: PreferenceScreen) { SwitchPreferenceCompat(screen.context).apply { diff --git a/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubDto.kt b/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubDto.kt index f94ab4eec..205bc11e2 100644 --- a/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubDto.kt +++ b/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubDto.kt @@ -37,6 +37,9 @@ class ApiChapterData( @Serializable class ApiChapter( val pages: String, + val mangaID: Int, + @SerialName("number") val chapterNumber: Float, + val manga: ApiMangaData, ) @Serializable diff --git a/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubQueries.kt b/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubQueries.kt index 68139cac9..9aab384e6 100644 --- a/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubQueries.kt +++ b/lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubQueries.kt @@ -56,7 +56,12 @@ val pagesQuery = { mangaSource: String, slug: String, number: Float -> """ { chapter(x: $mangaSource, slug: "$slug", number: $number) { - pages + pages, + mangaID, + number, + manga { + slug + } } } """.trimIndent()