From 1e6fb1ea3f1881d9cec99f5725d5a147b010fdfe Mon Sep 17 00:00:00 2001 From: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com> Date: Tue, 19 Sep 2023 19:41:10 -0300 Subject: [PATCH] Some fixes for two sources (#18053) * Add support to URL guess and add harsh rate limit on GM. * Fix the checks and add more paths. * Remove the log statement. * Use their current URL as of this moment. --------- Co-authored-by: Alessandro Jean --- src/pt/goldenmangas/build.gradle | 3 +- .../extension/pt/goldenmangas/GoldenMangas.kt | 82 +++++++++++++---- .../ObsoleteExtensionInterceptor.kt | 43 +++++++++ .../SpecificPathRateLimitInterceptor.kt | 90 +++++++++++++++++++ src/pt/mundowebtoon/build.gradle | 3 +- .../extension/pt/mundowebtoon/MundoWebtoon.kt | 3 +- .../ObsoleteExtensionInterceptor.kt | 43 +++++++++ 7 files changed, 249 insertions(+), 18 deletions(-) create mode 100644 src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/ObsoleteExtensionInterceptor.kt create mode 100644 src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/SpecificPathRateLimitInterceptor.kt create mode 100644 src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/ObsoleteExtensionInterceptor.kt diff --git a/src/pt/goldenmangas/build.gradle b/src/pt/goldenmangas/build.gradle index 948ec22a7..6e70477db 100644 --- a/src/pt/goldenmangas/build.gradle +++ b/src/pt/goldenmangas/build.gradle @@ -1,11 +1,12 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' +apply plugin: 'kotlinx-serialization' ext { extName = 'Golden Mangás' pkgNameSuffix = 'pt.goldenmangas' extClass = '.GoldenMangas' - extVersionCode = 21 + extVersionCode = 22 isNsfw = true } diff --git a/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/GoldenMangas.kt b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/GoldenMangas.kt index b08c99e1c..1c1f69a2d 100644 --- a/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/GoldenMangas.kt +++ b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/GoldenMangas.kt @@ -13,7 +13,6 @@ import eu.kanade.tachiyomi.lib.randomua.getPrefCustomUA import eu.kanade.tachiyomi.lib.randomua.getPrefUAType import eu.kanade.tachiyomi.lib.randomua.setRandomUserAgent import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.interceptor.rateLimit import eu.kanade.tachiyomi.source.ConfigurableSource import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.Page @@ -21,7 +20,9 @@ import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.ParsedHttpSource import okhttp3.Headers +import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response @@ -32,6 +33,7 @@ import uy.kohesive.injekt.api.get import java.text.SimpleDateFormat import java.util.Locale import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.seconds class GoldenMangas : ParsedHttpSource(), ConfigurableSource { @@ -40,7 +42,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { override val name = "Golden Mangás" - override val baseUrl = "https://www.goldenmangas.top" + override val baseUrl = "https://www.goldenmanga.top" override val lang = "pt-BR" @@ -50,15 +52,23 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { Injekt.get().getSharedPreferences("source_$id", 0x0000) } + private val baseHttpUrl: HttpUrl + get() = preferences.baseUrl.toHttpUrl() + override val client: OkHttpClient = network.cloudflareClient.newBuilder() .connectTimeout(1, TimeUnit.MINUTES) .readTimeout(1, TimeUnit.MINUTES) .writeTimeout(1, TimeUnit.MINUTES) + .addInterceptor(ObsoleteExtensionInterceptor()) .setRandomUserAgent( userAgentType = preferences.getPrefUAType(), customUA = preferences.getPrefCustomUA(), ) - .rateLimit(1, 3, TimeUnit.SECONDS) + .rateLimitPath("/mangas", 1, 8.seconds) + .rateLimitPath("/mm-admin/uploads", 1, 8.seconds) + .rateLimitPath("/timthumb.php", 1, 3.seconds) + .rateLimitPath("/index.php", 1, 3.seconds) + .addInterceptor(::guessNewUrlIntercept) .build() override fun headersBuilder(): Headers.Builder = Headers.Builder() @@ -68,7 +78,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { override fun popularMangaRequest(page: Int): Request = GET(baseUrl, headers) - override fun popularMangaSelector(): String = "div#maisLidos div.itemmanga" + override fun popularMangaSelector(): String = "div#maisLidos div.itemmanga:not(:contains(Avisos e Recados))" override fun popularMangaFromElement(element: Element): SManga = SManga.create().apply { title = element.selectFirst("h3")!!.text().withoutLanguage() @@ -83,7 +93,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { return GET("$baseUrl$path", headers) } - override fun latestUpdatesSelector() = "div.col-sm-12.atualizacao > div.row" + override fun latestUpdatesSelector() = "div.col-sm-12.atualizacao > div.row:not(:contains(Avisos e Recados))" override fun latestUpdatesFromElement(element: Element): SManga = SManga.create().apply { val infoElement = element.selectFirst("div.col-sm-10.col-xs-8 h3")!! @@ -123,7 +133,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { val mangaResult = runCatching { super.mangaDetailsParse(response) } val manga = mangaResult.getOrNull() - if (manga?.title.isNullOrEmpty()) { + if (manga?.title.isNullOrEmpty() && !response.hasChangedDomain) { throw Exception(MIGRATE_WARNING) } @@ -150,11 +160,11 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { val chaptersResult = runCatching { super.chapterListParse(response) } val chapterList = chaptersResult.getOrNull() - if (chapterList.isNullOrEmpty()) { + if (chapterList.isNullOrEmpty() && !response.hasChangedDomain) { throw Exception(MIGRATE_WARNING) } - return chapterList + return chapterList.orEmpty() } override fun chapterListSelector() = "ul#capitulos li.row" @@ -177,6 +187,17 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { return GET(baseUrl + chapter.url, newHeaders) } + override fun pageListParse(response: Response): List { + val pagesResult = runCatching { super.pageListParse(response) } + val pageList = pagesResult.getOrNull() + + if (pageList.isNullOrEmpty() && !response.hasChangedDomain) { + throw Exception(MIGRATE_WARNING) + } + + return pageList.orEmpty() + } + override fun pageListParse(document: Document): List { val chapterImages = document.selectFirst("div.col-sm-12[id^='capitulos_images']:has(img[pag])") @@ -186,11 +207,8 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { throw Exception(CHAPTER_IS_NOVEL_ERROR) } - if (chapterImages == null) { - throw Exception(MIGRATE_WARNING) - } - - return chapterImages.select("img[pag]") + return chapterImages?.select("img[pag]") + .orEmpty() .mapIndexed { i, element -> Page(i, document.location(), element.attr("abs:src")) } @@ -208,7 +226,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { } override fun setupPreferenceScreen(screen: PreferenceScreen) { - val uaPreferece = ListPreference(screen.context).apply { + val uaPreference = ListPreference(screen.context).apply { key = PREF_KEY_RANDOM_UA title = "User Agent aleatório" summary = "%s" @@ -239,15 +257,48 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { } } - screen.addPreference(uaPreferece) + screen.addPreference(uaPreference) screen.addPreference(customUaPreference) } + private fun guessNewUrlIntercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + + if (chain.request().url.host == "raw.githubusercontent.com") { + return response + } + + if (response.hasChangedDomain && preferences.baseUrl == baseUrl) { + return response + } + + preferences.baseUrl = "https://${response.request.url.host}" + + val newUrl = chain.request().url.toString() + .replaceFirst(baseUrl, preferences.baseUrl) + .toHttpUrl() + val newRequest = chain.request().newBuilder() + .url(newUrl) + .build() + + response.close() + + return chain.proceed(newRequest) + } + + private var SharedPreferences.baseUrl: String + get() = getString(BASE_URL_PREF, this@GoldenMangas.baseUrl)!! + set(newValue) = edit().putString(BASE_URL_PREF, newValue).apply() + private fun String.toDate(): Long { return runCatching { DATE_FORMATTER.parse(trim())?.time } .getOrNull() ?: 0L } + private val Response.hasChangedDomain: Boolean + get() = request.url.host != baseHttpUrl.host && + request.url.host.contains("goldenmang") + private fun String.toStatus() = when (this) { "Ativo" -> SManga.ONGOING "Completo" -> SManga.COMPLETED @@ -277,5 +328,6 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource { } private const val RESTART_APP_MESSAGE = "Reinicie o aplicativo para aplicar as alterações." + private const val BASE_URL_PREF = "base_url" } } diff --git a/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/ObsoleteExtensionInterceptor.kt b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/ObsoleteExtensionInterceptor.kt new file mode 100644 index 000000000..ba99930cf --- /dev/null +++ b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/ObsoleteExtensionInterceptor.kt @@ -0,0 +1,43 @@ +package eu.kanade.tachiyomi.extension.pt.goldenmangas + +import eu.kanade.tachiyomi.network.GET +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json +import okhttp3.Interceptor +import okhttp3.Response +import uy.kohesive.injekt.injectLazy +import java.io.IOException + +class ObsoleteExtensionInterceptor : Interceptor { + + private val json: Json by injectLazy() + private var isObsolete: Boolean? = null + + override fun intercept(chain: Interceptor.Chain): Response { + if (isObsolete == null) { + val extRepoResponse = chain.proceed(GET(REPO_URL)) + val extRepo = json.decodeFromString>(extRepoResponse.body.string()) + + isObsolete = !extRepo.any { ext -> + ext.pkg == this.javaClass.`package`?.name && ext.lang == "pt-BR" + } + } + + if (isObsolete == true) { + throw IOException("Extensão obsoleta. Desinstale e migre para outras fontes.") + } + + return chain.proceed(chain.request()) + } + + @Serializable + private data class ExtensionJsonObject( + val pkg: String, + val lang: String, + ) + + companion object { + private const val REPO_URL = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/index.min.json" + } +} diff --git a/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/SpecificPathRateLimitInterceptor.kt b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/SpecificPathRateLimitInterceptor.kt new file mode 100644 index 000000000..ea48849f6 --- /dev/null +++ b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/SpecificPathRateLimitInterceptor.kt @@ -0,0 +1,90 @@ +package eu.kanade.tachiyomi.extension.pt.goldenmangas + +import android.os.SystemClock +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Response +import java.io.IOException +import java.util.ArrayDeque +import java.util.concurrent.Semaphore +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") +class SpecificPathRateLimitInterceptor( + private val path: String, + private val permits: Int, + period: Duration, +) : Interceptor { + + private val requestQueue = ArrayDeque(permits) + private val rateLimitMillis = period.inWholeMilliseconds + private val fairLock = Semaphore(1, true) + + override fun intercept(chain: Interceptor.Chain): Response { + val call = chain.call() + if (call.isCanceled()) throw IOException("Canceled") + + val request = chain.request() + + if (!request.url.encodedPath.startsWith(path)) { + return chain.proceed(request) + } + + try { + fairLock.acquire() + } catch (e: InterruptedException) { + throw IOException(e) + } + + val requestQueue = this.requestQueue + val timestamp: Long + + try { + synchronized(requestQueue) { + while (requestQueue.size >= permits) { // queue is full, remove expired entries + val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis + var hasRemovedExpired = false + while (requestQueue.isEmpty().not() && requestQueue.first <= periodStart) { + requestQueue.removeFirst() + hasRemovedExpired = true + } + if (call.isCanceled()) { + throw IOException("Canceled") + } else if (hasRemovedExpired) { + break + } else { + try { // wait for the first entry to expire, or notified by cached response + (requestQueue as Object).wait(requestQueue.first - periodStart) + } catch (_: InterruptedException) { + continue + } + } + } + + // add request to queue + timestamp = SystemClock.elapsedRealtime() + requestQueue.addLast(timestamp) + } + } finally { + fairLock.release() + } + + val response = chain.proceed(request) + if (response.networkResponse == null) { // response is cached, remove it from queue + synchronized(requestQueue) { + if (requestQueue.isEmpty() || timestamp < requestQueue.first) return@synchronized + requestQueue.removeFirstOccurrence(timestamp) + (requestQueue as Object).notifyAll() + } + } + + return response + } +} + +fun OkHttpClient.Builder.rateLimitPath( + path: String, + permits: Int, + period: Duration = 1.seconds, +) = addInterceptor(SpecificPathRateLimitInterceptor(path, permits, period)) diff --git a/src/pt/mundowebtoon/build.gradle b/src/pt/mundowebtoon/build.gradle index 11a4d79ff..10963ad32 100644 --- a/src/pt/mundowebtoon/build.gradle +++ b/src/pt/mundowebtoon/build.gradle @@ -1,11 +1,12 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' +apply plugin: 'kotlinx-serialization' ext { extName = 'Mundo Webtoon' pkgNameSuffix = 'pt.mundowebtoon' extClass = '.MundoWebtoon' - extVersionCode = 7 + extVersionCode = 8 isNsfw = true } diff --git a/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/MundoWebtoon.kt b/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/MundoWebtoon.kt index 7cb35ef45..09cce66e6 100644 --- a/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/MundoWebtoon.kt +++ b/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/MundoWebtoon.kt @@ -35,8 +35,9 @@ class MundoWebtoon : ParsedHttpSource() { override val supportsLatest = true override val client: OkHttpClient = network.cloudflareClient.newBuilder() + .addInterceptor(ObsoleteExtensionInterceptor()) .addInterceptor(::sanitizeHtmlIntercept) - .rateLimit(1, 2, TimeUnit.SECONDS) + .rateLimit(1, 3, TimeUnit.SECONDS) .build() override fun headersBuilder(): Headers.Builder = Headers.Builder() diff --git a/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/ObsoleteExtensionInterceptor.kt b/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/ObsoleteExtensionInterceptor.kt new file mode 100644 index 000000000..a16f98ffe --- /dev/null +++ b/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/ObsoleteExtensionInterceptor.kt @@ -0,0 +1,43 @@ +package eu.kanade.tachiyomi.extension.pt.mundowebtoon + +import eu.kanade.tachiyomi.network.GET +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json +import okhttp3.Interceptor +import okhttp3.Response +import uy.kohesive.injekt.injectLazy +import java.io.IOException + +class ObsoleteExtensionInterceptor : Interceptor { + + private val json: Json by injectLazy() + private var isObsolete: Boolean? = null + + override fun intercept(chain: Interceptor.Chain): Response { + if (isObsolete == null) { + val extRepoResponse = chain.proceed(GET(REPO_URL)) + val extRepo = json.decodeFromString>(extRepoResponse.body.string()) + + isObsolete = !extRepo.any { ext -> + ext.pkg == this.javaClass.`package`?.name && ext.lang == "pt-BR" + } + } + + if (isObsolete == true) { + throw IOException("Extensão obsoleta. Desinstale e migre para outras fontes.") + } + + return chain.proceed(chain.request()) + } + + @Serializable + private data class ExtensionJsonObject( + val pkg: String, + val lang: String, + ) + + companion object { + private const val REPO_URL = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/index.min.json" + } +}