From 8ae6bdca689c68cf2086b04983b735783e0c0797 Mon Sep 17 00:00:00 2001 From: beerpsi <92439990+beerpiss@users.noreply.github.com> Date: Sun, 22 Jan 2023 19:22:16 +0700 Subject: [PATCH] make requests as browser-like as possible (#15067) * make requests as browser-like as possible * bump version * upgrade-insecure-requests * useragent fuckeries * don't use tachi's user-agents.json * trim dead code * fix funky AS formatting * filter out unwated UAs * randomizer randomizer * mom look at this funny file * filter user agents * Change revision of UA_DB_URL --- .../constellarscans/src/ConstellarScans.kt | 50 +++++++++++++++---- .../mangathemesia/MangaThemesiaGenerator.kt | 2 +- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/multisrc/overrides/mangathemesia/constellarscans/src/ConstellarScans.kt b/multisrc/overrides/mangathemesia/constellarscans/src/ConstellarScans.kt index 68735995b..ac1f187c4 100644 --- a/multisrc/overrides/mangathemesia/constellarscans/src/ConstellarScans.kt +++ b/multisrc/overrides/mangathemesia/constellarscans/src/ConstellarScans.kt @@ -15,6 +15,7 @@ import eu.kanade.tachiyomi.source.model.SChapter import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive +import okhttp3.CacheControl import okhttp3.Headers import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.Request @@ -49,30 +50,44 @@ class ConstellarScans : MangaThemesia("Constellar Scans", "https://constellarsca } .build() - override fun headersBuilder(): Headers.Builder = super.headersBuilder() + override fun headersBuilder(): Headers.Builder = Headers.Builder() .add("Referer", "$baseUrl/") + .add("Accept-Language", "en-US,en;q=0.9") + .add("DNT", "1") + .add("User-Agent", mobileUserAgent) + .add("Upgrade-Insecure-Requests", "1") override val seriesStatusSelector = ".status" private val mobileUserAgent by lazy { val req = GET(UA_DB_URL) - val resp = client.newCall(req).execute() - val mobileUaList = resp.body!!.use { - json.parseToJsonElement(it.string()).jsonObject["mobile"]!!.jsonArray.map { - it.jsonPrimitive.content + val data = client.newCall(req).execute().body!!.use { + json.parseToJsonElement(it.string()).jsonArray + }.mapNotNull { + it.jsonObject["user-agent"]?.jsonPrimitive?.content?.takeIf { + it.startsWith("Mozilla/5.0") && + ( + it.contains("iPhone") && + (it.contains("FxiOS") || it.contains("CriOS")) || + it.contains("Android") && + (it.contains("EdgA") || it.contains("Chrome") || it.contains("Firefox")) + ) } } - - mobileUaList.random().trim() + data.random() } override fun pageListRequest(chapter: SChapter): Request = super.pageListRequest(chapter).newBuilder() - .header("User-Agent", mobileUserAgent) + .header( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ) .header("Sec-Fetch-Site", "same-origin") .header("Sec-Fetch-Mode", "navigate") .header("Sec-Fetch-Dest", "document") .header("Sec-Fetch-User", "?1") + .cacheControl(CacheControl.FORCE_NETWORK) .build() override fun pageListParse(document: Document): List { @@ -88,6 +103,13 @@ class ConstellarScans : MangaThemesia("Constellar Scans", "https://constellarsca } } + override fun imageRequest(page: Page): Request = super.imageRequest(page).newBuilder() + .header("Accept", "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8") + .header("Sec-Fetch-Dest", "image") + .header("Sec-Fetch-Mode", "no-cors") + .header("Sec-Fetch-Site", "same-origin") + .build() + private fun descramblePageUrls(pages: List): List { return pages.map { val lastSegment = it.imageUrl!!.split('/').last() @@ -119,10 +141,11 @@ class ConstellarScans : MangaThemesia("Constellar Scans", "https://constellarsca val pages = mutableListOf() for (i in 1..imageCount) { + val filename = i.toString().padStart(5, '0') pages.add( Page( i, - imageUrl = "$encodedUploadsPath/$fragment/${i.toString().padStart(5, '0')}.webp#$DESCRAMBLE" + imageUrl = "$encodedUploadsPath/$fragment/$filename.webp#$DESCRAMBLE" ) ) } @@ -181,14 +204,19 @@ class ConstellarScans : MangaThemesia("Constellar Scans", "https://constellarsca companion object { const val DESCRAMBLE = "descramble" - const val UA_DB_URL = "https://tachiyomiorg.github.io/user-agents/user-agents.json" + + const val UA_DB_URL = + "https://cdn.jsdelivr.net/gh/mimmi20/browscap-helper@30a83c095688f40b9eaca0165a479c661e5a7fbe/tests/0002999.json" + const val LOOKUP_STRING = " !\"#${'$'}%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}" const val LOOKUP_STRING_ALNUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + val NOT_DIGIT_RE = Regex("""\D""") // The decoding algorithm looks for a hex number in 32..33, so we write our regex accordingly - val DESCRAMBLING_KEY_RE = Regex("""'([\da-z]{32}[\da-f]{2}[\da-z]+)'""", RegexOption.IGNORE_CASE) + val DESCRAMBLING_KEY_RE = + Regex("""'([\da-z]{32}[\da-f]{2}[\da-z]+)'""", RegexOption.IGNORE_CASE) } } diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangathemesia/MangaThemesiaGenerator.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangathemesia/MangaThemesiaGenerator.kt index 92bb0b7f2..eab61d21b 100644 --- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangathemesia/MangaThemesiaGenerator.kt +++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangathemesia/MangaThemesiaGenerator.kt @@ -25,7 +25,7 @@ class MangaThemesiaGenerator : ThemeSourceGenerator { SingleLang("Azure Scans", "https://azuremanga.com", "en", overrideVersionCode = 1), SingleLang("Boosei", "https://boosei.net", "id", overrideVersionCode = 2), SingleLang("Clayrer", "https://clayrer.net", "es"), - SingleLang("Constellar Scans", "https://constellarscans.com", "en", isNsfw = true, overrideVersionCode = 4), + SingleLang("Constellar Scans", "https://constellarscans.com", "en", isNsfw = true, overrideVersionCode = 5), SingleLang("Cosmic Scans", "https://cosmicscans.com", "en", overrideVersionCode = 1), SingleLang("Diskus Scan", "https://diskusscan.com", "pt-BR", overrideVersionCode = 7), SingleLang("Dojing.net", "https://dojing.net", "id", isNsfw = true, className = "DojingNet"),