From 4f0481f71d5135b1c9584c0576e17005650ac258 Mon Sep 17 00:00:00 2001 From: Chopper <156493704+choppeh@users.noreply.github.com> Date: Fri, 10 Jan 2025 20:48:26 -0300 Subject: [PATCH] SussyToons: Fix loading pages (#7093) * Fix loading pages * Remove 'named capture groups' and add configuration * Fix * Use dynamic api url --- src/pt/sussyscan/build.gradle | 2 +- .../extension/pt/sussyscan/SussyToons.kt | 332 ++++++++++++++++-- 2 files changed, 295 insertions(+), 39 deletions(-) diff --git a/src/pt/sussyscan/build.gradle b/src/pt/sussyscan/build.gradle index 0773631a4..6c7440046 100644 --- a/src/pt/sussyscan/build.gradle +++ b/src/pt/sussyscan/build.gradle @@ -1,7 +1,7 @@ ext { extName = 'Sussy Toons' extClass = '.SussyToons' - extVersionCode = 45 + extVersionCode = 46 isNsfw = true } diff --git a/src/pt/sussyscan/src/eu/kanade/tachiyomi/extension/pt/sussyscan/SussyToons.kt b/src/pt/sussyscan/src/eu/kanade/tachiyomi/extension/pt/sussyscan/SussyToons.kt index 021b63891..66c6b354b 100644 --- a/src/pt/sussyscan/src/eu/kanade/tachiyomi/extension/pt/sussyscan/SussyToons.kt +++ b/src/pt/sussyscan/src/eu/kanade/tachiyomi/extension/pt/sussyscan/SussyToons.kt @@ -2,45 +2,51 @@ package eu.kanade.tachiyomi.extension.pt.sussyscan import android.annotation.SuppressLint import android.app.Application +import android.content.SharedPreferences import android.os.Handler import android.os.Looper import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView import android.webkit.WebViewClient +import android.widget.Toast +import androidx.preference.EditTextPreference +import androidx.preference.PreferenceScreen import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.interceptor.rateLimit +import eu.kanade.tachiyomi.source.ConfigurableSource import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.HttpSource +import eu.kanade.tachiyomi.util.asJsoup import kotlinx.serialization.json.Json import kotlinx.serialization.json.decodeFromStream import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Interceptor +import okhttp3.Protocol import okhttp3.Request import okhttp3.Response +import okhttp3.internal.http.HTTP_BAD_GATEWAY import org.jsoup.Jsoup -import rx.Observable +import org.jsoup.select.Elements import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import uy.kohesive.injekt.injectLazy +import java.io.IOException +import java.net.SocketTimeoutException import java.text.SimpleDateFormat import java.util.Locale import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit -class SussyToons : HttpSource() { +class SussyToons : HttpSource(), ConfigurableSource { override val name = "Sussy Toons" - override val baseUrl = "https://new.sussytoons.site" - - private val apiUrl = "https://api-dev.sussytoons.site" - override val lang = "pt-BR" override val supportsLatest = true @@ -52,12 +58,60 @@ class SussyToons : HttpSource() { private val json: Json by injectLazy() + private val isCi = System.getenv("CI") == "true" + + private val preferences: SharedPreferences = + Injekt.get().getSharedPreferences("source_$id", 0x0000) + + private var _apiUrlCache: String? = null + + private var apiUrl: String + get() = _apiUrlCache ?: preferences.prefApiUrl.also { _apiUrlCache = it } + set(value) { _apiUrlCache = value } + + override val baseUrl: String get() = when { + isCi -> defaultBaseUrl + else -> preferences.prefBaseUrl + } + + private val SharedPreferences.prefBaseUrl: String get() = getString(BASE_URL_PREF, defaultBaseUrl)!! + private val SharedPreferences.prefApiUrl: String get() = getString(API_BASE_URL_PREF, defaultApiUrl)!! + private fun SharedPreferences.prefApiUrlUpSet(url: String): String { + edit().putString(API_BASE_URL_PREF, url) + .apply() + return url + } + + private val defaultBaseUrl: String = "https://www.sussytoons.site" + private val defaultApiUrl: String = "https://api-dev.sussytoons.site" + override val client = network.cloudflareClient.newBuilder() - .rateLimit(1, 2, TimeUnit.SECONDS) + .rateLimit(2) + .addInterceptor(::findApiUrl) + .addInterceptor(::findChapterUrl) .addInterceptor(::chapterPages) .addInterceptor(::imageLocation) .build() + init { + preferences.getString(DEFAULT_BASE_URL_PREF, null).let { domain -> + if (domain != defaultBaseUrl) { + preferences.edit() + .putString(BASE_URL_PREF, defaultBaseUrl) + .putString(DEFAULT_BASE_URL_PREF, defaultBaseUrl) + .apply() + } + } + preferences.getString(API_DEFAULT_BASE_URL_PREF, null).let { domain -> + if (domain != defaultApiUrl) { + preferences.edit() + .putString(API_BASE_URL_PREF, defaultApiUrl) + .putString(API_DEFAULT_BASE_URL_PREF, defaultApiUrl) + .apply() + } + } + } + override fun headersBuilder() = super.headersBuilder() .set("scan-id", "1") // Required header for requests @@ -109,6 +163,7 @@ class SussyToons : HttpSource() { override fun mangaDetailsRequest(manga: SManga): Request { val url = "$apiUrl/obras".toHttpUrl().newBuilder() .addPathSegment(manga.id) + .fragment("$mangaPagePrefix${getMangaUrl(manga)}") .build() return GET(url, headers) } @@ -125,13 +180,6 @@ class SussyToons : HttpSource() { // ============================= Chapters ================================= - override fun getChapterUrl(chapter: SChapter): String { - return "$baseUrl/capitulo".toHttpUrl().newBuilder() - .addPathSegment(chapter.id) - .build() - .toString() - } - override fun chapterListRequest(manga: SManga) = mangaDetailsRequest(manga) override fun chapterListParse(response: Response): List { @@ -141,47 +189,75 @@ class SussyToons : HttpSource() { it.chapterNumber?.let { chapter_number = it } - val chapterApiUrl = "$apiUrl/capitulos".toHttpUrl().newBuilder() + val chapterApiUrl = apiUrl.toHttpUrl().newBuilder() + .addEncodedPathSegments(chapterUrl!!) .addPathSegment(it.id.toString()) .build() setUrlWithoutDomain(chapterApiUrl.toString()) date_upload = it.updateAt.toDate() } - } - } - - override fun fetchChapterList(manga: SManga): Observable> { - return super.fetchChapterList(manga) - .map { it.sortedBy(SChapter::chapter_number).reversed() } - } - - private val SChapter.id: String get() { - val chapterApiUrl = apiUrl.toHttpUrl().newBuilder() - .addPathSegments(url) - .build() - return chapterApiUrl.pathSegments.last() + }.sortedBy(SChapter::chapter_number).reversed() } // ============================= Pages ==================================== override fun pageListRequest(chapter: SChapter): Request { - val url = "$apiUrl${chapter.url}".toHttpUrl().newBuilder() - .fragment(getChapterUrl(chapter)) - .build() - return GET(url, headers) + return super.pageListRequest(chapter).let { request -> + val url = request.url.newBuilder() + .fragment("$pageImagePrefix${chapter.url}") + .build() + + request.newBuilder() + .url(url) + .build() + } } + private var pageUrl: String? = null + override fun pageListParse(response: Response): List { - val dto = response.parseAs>().results + pageUrl = pageUrl ?: findPageUrl(response) + val chapterPageId = response.request.url.pathSegments.last() + + val chapterUrl = response.request.url.fragment + ?.substringAfter(pageImagePrefix) + ?: throw Exception("Não foi possivel carregar as páginas") + + val url = apiUrl.toHttpUrl().newBuilder() + .addEncodedPathSegments(pageUrl!!) + .addPathSegment(chapterPageId) + .fragment( + "$chapterPagePrefix${"$baseUrl$chapterUrl"}", + ) + .build() + + val res = client.newCall(GET(url, headers)).execute() + + val dto = res.parseAs>().results return dto.pages.mapIndexed { index, image -> val imageUrl = CDN_URL.toHttpUrl().newBuilder() .addPathSegments("wp-content/uploads/WP-manga/data") - .addPathSegments(image.src) + .addPathSegments(image.src.toPathSegment()) .build().toString() Page(index, imageUrl = imageUrl) } } + /** + * Get the “dynamic” path segment of the chapter page + */ + private fun findPageUrl(response: Response): String { + val document = response.asJsoup() + val scriptUrl = document.select("script[src]") + .map { it.absUrl("src") } + .firstOrNull { it.contains("app/capitulo", ignoreCase = true) } + ?: throw IOException("Não foi possivel encontrar a URL da página") + + return client.newCall(GET(scriptUrl, headers)).execute().use { + pageUrlRegex.find(it.body.string())?.groups?.get(1)?.value?.toPathSegment() + } ?: throw IOException("Não foi possivel extrair a URL da página") + } + override fun imageUrlParse(response: Response): String = "" override fun imageUrlRequest(page: Page): Request { @@ -195,6 +271,109 @@ class SussyToons : HttpSource() { private var chapterPageHeaders: Headers? = null + private var chapterUrl: String? = null + + private fun findApiUrl(chain: Interceptor.Chain): Response { + val request = chain.request() + val response: Response = try { + chain.proceed(request) + } catch (ex: SocketTimeoutException) { + chain.createTimeoutResponse(request) + } + + if (request.url.toString().contains(apiUrl).not()) { + return response + } + + if (response.isSuccessful) { + return response + } + + response.close() + + fetchApiUrl(chain).forEach { urlCandidate -> + val url = request.url.toString() + .replace(apiUrl, urlCandidate) + .toHttpUrl() + + val newRequest = request.newBuilder() + .url(url) + .build() + + return chain.proceed(newRequest).takeIf(Response::isSuccessful).also { + apiUrl = preferences.prefApiUrlUpSet(urlCandidate) + } ?: return@forEach + } + + throw IOException( + buildString { + append("Não foi possível encontrar a URL da API.") + append("Troque manualmente nas configurações da extensão") + }, + ) + } + private fun Interceptor.Chain.createTimeoutResponse(request: Request): Response { + return Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .message("") + .code(HTTP_BAD_GATEWAY) + .build() + } + + private fun fetchApiUrl(chain: Interceptor.Chain): List { + val scripts = chain.proceed(GET(baseUrl, headers)).asJsoup() + .select("script[src*=next]:not([nomodule]):not([src*=app])") + + val script = getScriptBodyWithUrls(scripts, chain) + ?: throw Exception("Não foi possivel localizar a URL da API") + + return apiUrlRegex.findAll(script) + .flatMap { stringsRegex.findAll(it.value).map { match -> match.groupValues[1] } } + .filter(urlRegex::containsMatchIn) + .toList() + } + + private fun getScriptBodyWithUrls(scripts: Elements, chain: Interceptor.Chain): String? { + val elements = scripts.toList().reversed() + for (element in elements) { + val scriptUrl = element.absUrl("src") + val script = chain.proceed(GET(scriptUrl, headers)).body.string() + if (apiUrlRegex.containsMatchIn(script)) { + return script + } + } + return null + } + + /** + * Get the “dynamic” path segment of the chapter list + */ + private fun findChapterUrl(chain: Interceptor.Chain): Response { + val request = chain.request() + + val mangaUrl = request.url.fragment + ?.takeIf { + it.contains(mangaPagePrefix, ignoreCase = true) && chapterUrl.isNullOrBlank() + }?.substringAfter(mangaPagePrefix) + ?: return chain.proceed(request) + + val document = chain.proceed(GET(mangaUrl, headers)).asJsoup() + + val scriptUrl = document.select("script[src]") + .map { it.absUrl("src") } + .firstOrNull { it.contains("app/obra", ignoreCase = true) } + ?: throw IOException("Não foi possivel encontrar a URL do capitulo") + + chapterUrl = chain.proceed(GET(scriptUrl, headers)).use { response -> + response.body.string().let { + chapterUrlRegex.find(it)?.groups?.get(1)?.value?.toPathSegment() + } ?: throw IOException("Não foi possivel extrair a URL do capitulo") + } + + return chain.proceed(request) + } + private fun imageLocation(chain: Interceptor.Chain): Response { val request = chain.request() val response = chain.proceed(request) @@ -215,9 +394,15 @@ class SussyToons : HttpSource() { return response } + /** + * Resolve the “dynamic” headers of the chapter page + */ private fun chapterPages(chain: Interceptor.Chain): Response { val request = chain.request() - val chapterUrl = request.url.fragment?.takeIf { it.contains("capitulo") } + val chapterUrl = request.url.fragment + ?.takeIf { it.contains(chapterPagePrefix) } + ?.substringAfter(chapterPagePrefix)?.toHttpUrl()?.newBuilder()?.fragment(null) + ?.build() ?: return chain.proceed(request) val originUrl = request.url.newBuilder() @@ -249,7 +434,6 @@ class SussyToons : HttpSource() { val headers = originRequest.headers.newBuilder() var webView: WebView? = null val looper = Handler(Looper.getMainLooper()) - looper.post { webView = WebView(Injekt.get()) webView?.let { @@ -279,7 +463,7 @@ class SussyToons : HttpSource() { private fun emptyResource() = WebResourceResponse(null, null, null) } - webView?.loadUrl(baseRequest.url.toString()) + webView?.loadUrl(baseRequest.url.toString(), headers.build().toMap()) } latch.await(client.readTimeoutMillis.toLong(), TimeUnit.MILLISECONDS) @@ -304,6 +488,48 @@ class SussyToons : HttpSource() { } } + // ============================= Settings ==================================== + + override fun setupPreferenceScreen(screen: PreferenceScreen) { + val fields = listOf( + EditTextPreference(screen.context).apply { + key = BASE_URL_PREF + title = BASE_URL_PREF_TITLE + summary = URL_PREF_SUMMARY + + dialogTitle = BASE_URL_PREF_TITLE + dialogMessage = "URL padrão:\n$defaultBaseUrl" + + setDefaultValue(defaultBaseUrl) + setOnPreferenceChangeListener { _, _ -> + Toast.makeText(screen.context, RESTART_APP_MESSAGE, Toast.LENGTH_LONG).show() + true + } + }, + EditTextPreference(screen.context).apply { + key = API_BASE_URL_PREF + title = API_BASE_URL_PREF_TITLE + summary = buildString { + append("Se não souber como verificar a URL da API, ") + append("busque suporte no Discord do repositório de extensões.") + appendLine(URL_PREF_SUMMARY) + append("\n⚠ A fonte não oferece suporte para essa extensão.") + } + + dialogTitle = BASE_URL_PREF_TITLE + dialogMessage = "URL da API padrão:\n$defaultApiUrl" + + setDefaultValue(defaultApiUrl) + setOnPreferenceChangeListener { _, _ -> + Toast.makeText(screen.context, RESTART_APP_MESSAGE, Toast.LENGTH_LONG).show() + true + } + }, + ) + + fields.forEach(screen::addPreference) + } + // ============================= Utilities ==================================== private fun MangaDto.toSManga(): SManga { @@ -331,9 +557,39 @@ class SussyToons : HttpSource() { private fun String.toDate() = try { dateFormat.parse(this)!!.time } catch (_: Exception) { 0L } + /** + * Normalizes path segments: + * Ex: [ "/a/b/", "/a/b", "a/b/", "a/b" ] + * Result: "a/b" + */ + private fun String.toPathSegment() = this.trim().split("/") + .filter(String::isNotEmpty) + .joinToString("/") + companion object { - const val CDN_URL = "https://usc1.contabostorage.com/23b45111d96c42c18a678c1d8cba7123:cdn" + const val CDN_URL = "https://cdn.sussytoons.site" const val OLDI_URL = "https://oldi.sussytoons.site" + const val mangaPagePrefix = "mangaPage:" + const val chapterPagePrefix = "chapterPage:" + const val pageImagePrefix = "pageImage:" + + private const val URL_PREF_SUMMARY = "Para uso temporário, se a extensão for atualizada, a alteração será perdida." + + private const val BASE_URL_PREF = "overrideBaseUrl" + private const val BASE_URL_PREF_TITLE = "Editar URL da fonte" + private const val DEFAULT_BASE_URL_PREF = "defaultBaseUrl" + private const val RESTART_APP_MESSAGE = "Reinicie o aplicativo para aplicar as alterações" + + private const val API_BASE_URL_PREF = "overrideApiUrl" + private const val API_BASE_URL_PREF_TITLE = "Editar URL da API da fonte" + private const val API_DEFAULT_BASE_URL_PREF = "defaultApiUrl" + + val chapterUrlRegex = """push\("([^"]*capitulo[^"]*)/?"\.concat""".toRegex() + val pageUrlRegex = """\.get\("([^"]*capitulo[^(/?")]*)/?"\.concat""".toRegex() + + val apiUrlRegex = """(?<=production",)(.*?)(?=;function)""".toRegex() + val urlRegex = """https?://[\w\-]+(\.[\w\-]+)+[/#?]?.*$""".toRegex() + val stringsRegex = """"(.*?)"""".toRegex() @SuppressLint("SimpleDateFormat") val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ROOT)