MangaHub (multisrc) - Fixes and Improvements (#8586)

* remove rate limit

* Fixes and improvements

* Version bump

* Review changes, more improvements

lib-multisrc/mangahub/build.gradle.kts

@@ -2,8 +2,9 @@ plugins {
     id("lib-multisrc")
 }
 
-baseVersionCode = 32
+baseVersionCode = 33
 
 dependencies {
-    api(project(":lib:randomua"))
+    //noinspection UseTomlInstead
+    implementation("org.brotli:dec:0.1.2")
 }

lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHub.kt

@@ -3,11 +3,8 @@ package eu.kanade.tachiyomi.multisrc.mangahub
 import android.content.SharedPreferences
 import androidx.preference.PreferenceScreen
 import androidx.preference.SwitchPreferenceCompat
-import eu.kanade.tachiyomi.lib.randomua.UserAgentType
-import eu.kanade.tachiyomi.lib.randomua.setRandomUserAgent
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.network.interceptor.rateLimit
 import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -30,12 +27,16 @@ import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
+import okhttp3.ResponseBody.Companion.toResponseBody
+import org.brotli.dec.BrotliInputStream
 import rx.Observable
+import java.io.ByteArrayOutputStream
 import java.io.IOException
 import java.net.URLEncoder
 import java.text.SimpleDateFormat
 import java.util.Calendar
 import java.util.Locale
+import java.util.zip.GZIPInputStream
 
 abstract class MangaHub(
     override val name: String,
@@ -50,7 +51,8 @@ abstract class MangaHub(
     private val baseApiUrl = "https://api.mghcdn.com"
     private val baseCdnUrl = "https://imgx.mghcdn.com"
     private val baseThumbCdnUrl = "https://thumb.mghcdn.com"
-    private val regex = Regex("mhub_access=([^;]+)")
+    private val apiRegex = Regex("mhub_access=([^;]+)")
+    private val spaceRegex = Regex("\\s+")
 
     private val preferences: SharedPreferences by getPreferencesLazy()
 
@@ -60,13 +62,9 @@ abstract class MangaHub(
     )
 
     override val client: OkHttpClient = network.cloudflareClient.newBuilder()
-        .setRandomUserAgent(
-            userAgentType = UserAgentType.DESKTOP,
-            filterInclude = listOf("chrome"),
-        )
         .addInterceptor(::apiAuthInterceptor)
         .addInterceptor(::graphQLApiInterceptor)
-        .rateLimit(1)
+        .addNetworkInterceptor(::compatEncodingInterceptor)
         .build()
 
     override fun headersBuilder(): Headers.Builder = super.headersBuilder()
@@ -119,6 +117,44 @@ abstract class MangaHub(
         return chain.proceed(request)
     }
 
+    // Normally this gets handled properly but in older forks such as TachiyomiJ2K, we have to manually intercept it
+    // as they have an outdated implementation of NetworkHelper.
+    private fun compatEncodingInterceptor(chain: Interceptor.Chain): Response {
+        var response = chain.proceed(chain.request())
+        val contentEncoding = response.header("Content-Encoding")
+
+        if (contentEncoding == "gzip") {
+            val parsedBody = response.body.byteStream().let { gzipInputStream ->
+                GZIPInputStream(gzipInputStream).use { inputStream ->
+                    val outputStream = ByteArrayOutputStream()
+                    inputStream.copyTo(outputStream)
+                    outputStream.toByteArray()
+                }
+            }
+
+            response = response.createNewWithCompatBody(parsedBody)
+        } else if (contentEncoding == "br") {
+            val parsedBody = response.body.byteStream().let { brotliInputStream ->
+                BrotliInputStream(brotliInputStream).use { inputStream ->
+                    val outputStream = ByteArrayOutputStream()
+                    inputStream.copyTo(outputStream)
+                    outputStream.toByteArray()
+                }
+            }
+
+            response = response.createNewWithCompatBody(parsedBody)
+        }
+
+        return response
+    }
+
+    private fun Response.createNewWithCompatBody(outputStream: ByteArray): Response {
+        return this.newBuilder()
+            .body(outputStream.toResponseBody(this.body.contentType()))
+            .removeHeader("Content-Encoding")
+            .build()
+    }
+
     private fun graphQLApiInterceptor(chain: Interceptor.Chain): Response {
         val request = chain.request()
 
@@ -147,33 +183,15 @@ abstract class MangaHub(
     }
 
     private fun refreshApiKey(chapter: SChapter) {
-        val now = Calendar.getInstance().time.time
-
         val url = "$baseUrl/chapter${chapter.url}".toHttpUrl()
         val oldKey = client.cookieJar
             .loadForRequest(baseUrl.toHttpUrl())
             .firstOrNull { it.name == "mhub_access" && it.value.isNotEmpty() }?.value
 
-        // With the recent changes on how refresh API token works, we are now apparently required to have
-        // a cookie for recently when requesting for a new one. Not having this will result in a hit or miss.
-        val recently = buildJsonObject {
-            putJsonObject((now - (0..3600).random()).toString()) {
-                put("mangaID", (1..42_000).random())
-                put("number", (1..20).random())
-            }
-        }.toString()
-
-        val recentlyCookie = Cookie.Builder()
-            .domain(url.host)
-            .name("recently")
-            .value(URLEncoder.encode(recently, "utf-8"))
-            .expiresAt(now + 2 * 60 * 60 * 24 * 31) // +2 months
-            .build()
-
         for (i in 1..2) {
             // Clear key cookie
             val cookie = Cookie.parse(url, "mhub_access=; Max-Age=0; Path=/")!!
-            client.cookieJar.saveFromResponse(url, listOf(cookie, recentlyCookie))
+            client.cookieJar.saveFromResponse(url, listOf(cookie))
 
             // We try requesting again with param if the first one fails
             val query = if (i == 2) "?reloadKey=1" else ""
@@ -187,7 +205,7 @@ abstract class MangaHub(
                             .build(),
                     ),
                 ).execute()
-                val returnedKey = response.headers["set-cookie"]?.let { regex.find(it)?.groupValues?.get(1) }
+                val returnedKey = response.headers["set-cookie"]?.let { apiRegex.find(it)?.groupValues?.get(1) }
                 response.close() // Avoid potential resource leaks
 
                 if (returnedKey != oldKey) break // Break out of loop since we got an allegedly valid API key
@@ -195,16 +213,6 @@ abstract class MangaHub(
                 throw IOException("An error occurred while obtaining a new API key") // Show error
             }
         }
-
-        // Sometimes, the new API key is still invalid. To ensure that the token will be fresh and available to use,
-        // we have to mimic how the browser site works. To put it simply, we will send a GET request that indicates what
-        // manga and chapter were browsing. If this succeeded, the API key that we use will be revalidated (assuming that we got an expired one.)
-        // We first need to obtain our public IP first since it is required as a query.
-        val ipRequest = client.newCall(GET("https://api.ipify.org?format=json")).execute()
-        val ip = ipRequest.parseAs<PublicIPResponse>().ip
-
-        // We'll log our action to the site to revalidate the API key in case we got an expired one
-        client.newCall(GET("$baseUrl/action/logHistory2/${url.pathSegments[1]}/${chapter.chapter_number}?browserID=$ip")).execute()
     }
 
     data class SMangaDTO(
@@ -339,7 +347,7 @@ abstract class MangaHub(
                 val numberString = "${if (it.number % 1 == 0f) it.number.toInt() else it.number}"
 
                 name = if (!useGenericTitle) {
-                    generateChapterName(it.title.trim().replace("\n", " "), numberString)
+                    generateChapterName(it.title.trim().replace(spaceRegex, " "), numberString)
                 } else {
                     generateGenericChapterName(numberString)
                 }
@@ -374,15 +382,52 @@ abstract class MangaHub(
         return postRequestGraphQL(pagesQuery(mangaSource, chapterUrl[1], chapterUrl[2].substringAfter("-").toFloat()))
     }
 
-    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> =
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
-        super.fetchPageList(chapter)
+        var doApiRefresh = true
-            .doOnError { refreshApiKey(chapter) }
+
+        return super.fetchPageList(chapter)
+            .doOnError {
+                // Ensure that the api refresh call will happen only once
+                if (doApiRefresh) {
+                    refreshApiKey(chapter)
+                    doApiRefresh = false
+                }
+            }
             .retry(1)
+    }
 
     override fun pageListParse(response: Response): List<Page> {
         val chapterObject = response.parseAs<ApiChapterPagesResponse>()
         val pages = chapterObject.data.chapter.pages.parseAs<ApiChapterPages>()
 
+        // We'll update the cookie here to match the browser's "recently" opened chapter.
+        // This mimics how the browser works and gives us more chance to receive a valid API key upon refresh
+        val now = Calendar.getInstance().time.time
+        val baseHttpUrl = baseUrl.toHttpUrl()
+        val recently = buildJsonObject {
+            putJsonObject((now).toString()) {
+                put("mangaID", chapterObject.data.chapter.mangaID)
+                put("number", chapterObject.data.chapter.chapterNumber)
+            }
+        }.toString()
+
+        val recentlyCookie = Cookie.Builder()
+            .domain(baseHttpUrl.host)
+            .name("recently")
+            .value(URLEncoder.encode(recently, "utf-8"))
+            .expiresAt(now + 2 * 60 * 60 * 24 * 31) // +2 months
+            .build()
+
+        // Add/update the cookie
+        client.cookieJar.saveFromResponse(baseHttpUrl, listOf(recentlyCookie))
+
+        // We'll log our action to the site to further increase the chance of valid API key
+        val ipRequest = client.newCall(GET("https://api.ipify.org?format=json")).execute()
+        val ip = ipRequest.parseAs<PublicIPResponse>().ip
+
+        client.newCall(GET("$baseUrl/action/logHistory2/${chapterObject.data.chapter.manga.slug}/${chapterObject.data.chapter.chapterNumber}?browserID=$ip")).execute().close()
+        ipRequest.close()
+
         return pages.images.mapIndexed { i, page ->
             Page(i, "", "$baseCdnUrl/${pages.page}$page")
         }
@@ -517,7 +562,7 @@ abstract class MangaHub(
         Genre("Video Games", "video-games"),
         Genre("Monsters", "monsters"),
         Genre("Office Workers", "office-workers"),
-        Genre("system", "system"),
+        Genre("System", "system"),
         Genre("Villainess", "villainess"),
         Genre("Zombies", "zombies"),
         Genre("Vampires", "vampires"),
@@ -540,7 +585,7 @@ abstract class MangaHub(
         Genre("Cheat Systems", "cheat-systems"),
         Genre("Dungeons", "dungeons"),
         Genre("Overpowered", "overpowered"),
-    )
+    ).sortedBy { it.toString() }
 
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
         SwitchPreferenceCompat(screen.context).apply {

lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubDto.kt

@@ -37,6 +37,9 @@ class ApiChapterData(
 @Serializable
 class ApiChapter(
     val pages: String,
+    val mangaID: Int,
+    @SerialName("number") val chapterNumber: Float,
+    val manga: ApiMangaData,
 )
 
 @Serializable

lib-multisrc/mangahub/src/eu/kanade/tachiyomi/multisrc/mangahub/MangaHubQueries.kt

@@ -56,7 +56,12 @@ val pagesQuery = { mangaSource: String, slug: String, number: Float ->
     """
         {
             chapter(x: $mangaSource, slug: "$slug", number: $number) {
-                    pages
+                    pages,
+                    mangaID,
+                    number,
+                    manga {
+                        slug
+                    }
                 }
         }
     """.trimIndent()