Some fixes for two sources (#18053)

* Add support to URL guess and add harsh rate limit on GM. * Fix the checks and add more paths. * Remove the log statement. * Use their current URL as of this moment. --------- Co-authored-by: Alessandro Jean <alessandrojean@users.noreply.github.com>
2023-09-19 19:41:10 -03:00 · 2023-09-19 19:41:10 -03:00 · 1e6fb1ea3f
parent 82ca70bfd1
commit 1e6fb1ea3f
7 changed files with 249 additions and 18 deletions
--- a/src/pt/goldenmangas/build.gradle
+++ b/src/pt/goldenmangas/build.gradle
@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'

 ext {
    extName = 'Golden Mangás'
    pkgNameSuffix = 'pt.goldenmangas'
    extClass = '.GoldenMangas'
-    extVersionCode = 21
+    extVersionCode = 22
    isNsfw = true
 }

--- a/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/GoldenMangas.kt
+++ b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/GoldenMangas.kt
@ -13,7 +13,6 @@ import eu.kanade.tachiyomi.lib.randomua.getPrefCustomUA
 import eu.kanade.tachiyomi.lib.randomua.getPrefUAType
 import eu.kanade.tachiyomi.lib.randomua.setRandomUserAgent
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.interceptor.rateLimit
 import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.Page
@ -21,7 +20,9 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import okhttp3.Headers
+import okhttp3.HttpUrl
 import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Interceptor
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
@ -32,6 +33,7 @@ import uy.kohesive.injekt.api.get
 import java.text.SimpleDateFormat
 import java.util.Locale
 import java.util.concurrent.TimeUnit
+import kotlin.time.Duration.Companion.seconds

 class GoldenMangas : ParsedHttpSource(), ConfigurableSource {

@ -40,7 +42,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {

    override val name = "Golden Mangás"

-    override val baseUrl = "https://www.goldenmangas.top"
+    override val baseUrl = "https://www.goldenmanga.top"

    override val lang = "pt-BR"

@ -50,15 +52,23 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
    }

+    private val baseHttpUrl: HttpUrl
+        get() = preferences.baseUrl.toHttpUrl()
+
    override val client: OkHttpClient = network.cloudflareClient.newBuilder()
        .connectTimeout(1, TimeUnit.MINUTES)
        .readTimeout(1, TimeUnit.MINUTES)
        .writeTimeout(1, TimeUnit.MINUTES)
+        .addInterceptor(ObsoleteExtensionInterceptor())
        .setRandomUserAgent(
            userAgentType = preferences.getPrefUAType(),
            customUA = preferences.getPrefCustomUA(),
        )
-        .rateLimit(1, 3, TimeUnit.SECONDS)
+        .rateLimitPath("/mangas", 1, 8.seconds)
+        .rateLimitPath("/mm-admin/uploads", 1, 8.seconds)
+        .rateLimitPath("/timthumb.php", 1, 3.seconds)
+        .rateLimitPath("/index.php", 1, 3.seconds)
+        .addInterceptor(::guessNewUrlIntercept)
        .build()

    override fun headersBuilder(): Headers.Builder = Headers.Builder()
@ -68,7 +78,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {

    override fun popularMangaRequest(page: Int): Request = GET(baseUrl, headers)

-    override fun popularMangaSelector(): String = "div#maisLidos div.itemmanga"
+    override fun popularMangaSelector(): String = "div#maisLidos div.itemmanga:not(:contains(Avisos e Recados))"

    override fun popularMangaFromElement(element: Element): SManga = SManga.create().apply {
        title = element.selectFirst("h3")!!.text().withoutLanguage()
@ -83,7 +93,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
        return GET("$baseUrl$path", headers)
    }

-    override fun latestUpdatesSelector() = "div.col-sm-12.atualizacao > div.row"
+    override fun latestUpdatesSelector() = "div.col-sm-12.atualizacao > div.row:not(:contains(Avisos e Recados))"

    override fun latestUpdatesFromElement(element: Element): SManga = SManga.create().apply {
        val infoElement = element.selectFirst("div.col-sm-10.col-xs-8 h3")!!
@ -123,7 +133,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
        val mangaResult = runCatching { super.mangaDetailsParse(response) }
        val manga = mangaResult.getOrNull()

-        if (manga?.title.isNullOrEmpty()) {
+        if (manga?.title.isNullOrEmpty() && !response.hasChangedDomain) {
            throw Exception(MIGRATE_WARNING)
        }

@ -150,11 +160,11 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
        val chaptersResult = runCatching { super.chapterListParse(response) }
        val chapterList = chaptersResult.getOrNull()

-        if (chapterList.isNullOrEmpty()) {
+        if (chapterList.isNullOrEmpty() && !response.hasChangedDomain) {
            throw Exception(MIGRATE_WARNING)
        }

-        return chapterList
+        return chapterList.orEmpty()
    }

    override fun chapterListSelector() = "ul#capitulos li.row"
@ -177,6 +187,17 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
        return GET(baseUrl + chapter.url, newHeaders)
    }

+    override fun pageListParse(response: Response): List<Page> {
+        val pagesResult = runCatching { super.pageListParse(response) }
+        val pageList = pagesResult.getOrNull()
+
+        if (pageList.isNullOrEmpty() && !response.hasChangedDomain) {
+            throw Exception(MIGRATE_WARNING)
+        }
+
+        return pageList.orEmpty()
+    }
+
    override fun pageListParse(document: Document): List<Page> {
        val chapterImages = document.selectFirst("div.col-sm-12[id^='capitulos_images']:has(img[pag])")

@ -186,11 +207,8 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
            throw Exception(CHAPTER_IS_NOVEL_ERROR)
        }

-        if (chapterImages == null) {
-            throw Exception(MIGRATE_WARNING)
-        }
-
-        return chapterImages.select("img[pag]")
+        return chapterImages?.select("img[pag]")
+            .orEmpty()
            .mapIndexed { i, element ->
                Page(i, document.location(), element.attr("abs:src"))
            }
@ -208,7 +226,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
    }

    override fun setupPreferenceScreen(screen: PreferenceScreen) {
-        val uaPreferece = ListPreference(screen.context).apply {
+        val uaPreference = ListPreference(screen.context).apply {
            key = PREF_KEY_RANDOM_UA
            title = "User Agent aleatório"
            summary = "%s"
@ -239,15 +257,48 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
            }
        }

-        screen.addPreference(uaPreferece)
+        screen.addPreference(uaPreference)
        screen.addPreference(customUaPreference)
    }

+    private fun guessNewUrlIntercept(chain: Interceptor.Chain): Response {
+        val response = chain.proceed(chain.request())
+
+        if (chain.request().url.host == "raw.githubusercontent.com") {
+            return response
+        }
+
+        if (response.hasChangedDomain && preferences.baseUrl == baseUrl) {
+            return response
+        }
+
+        preferences.baseUrl = "https://${response.request.url.host}"
+
+        val newUrl = chain.request().url.toString()
+            .replaceFirst(baseUrl, preferences.baseUrl)
+            .toHttpUrl()
+        val newRequest = chain.request().newBuilder()
+            .url(newUrl)
+            .build()
+
+        response.close()
+
+        return chain.proceed(newRequest)
+    }
+
+    private var SharedPreferences.baseUrl: String
+        get() = getString(BASE_URL_PREF, this@GoldenMangas.baseUrl)!!
+        set(newValue) = edit().putString(BASE_URL_PREF, newValue).apply()
+
    private fun String.toDate(): Long {
        return runCatching { DATE_FORMATTER.parse(trim())?.time }
            .getOrNull() ?: 0L
    }

+    private val Response.hasChangedDomain: Boolean
+        get() = request.url.host != baseHttpUrl.host &&
+            request.url.host.contains("goldenmang")
+
    private fun String.toStatus() = when (this) {
        "Ativo" -> SManga.ONGOING
        "Completo" -> SManga.COMPLETED
@ -277,5 +328,6 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
        }

        private const val RESTART_APP_MESSAGE = "Reinicie o aplicativo para aplicar as alterações."
+        private const val BASE_URL_PREF = "base_url"
    }
 }
--- a/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/ObsoleteExtensionInterceptor.kt
+++ b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/ObsoleteExtensionInterceptor.kt
@ -0,0 +1,43 @@
+package eu.kanade.tachiyomi.extension.pt.goldenmangas
+
+import eu.kanade.tachiyomi.network.GET
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import okhttp3.Interceptor
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+import java.io.IOException
+
+class ObsoleteExtensionInterceptor : Interceptor {
+
+    private val json: Json by injectLazy()
+    private var isObsolete: Boolean? = null
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        if (isObsolete == null) {
+            val extRepoResponse = chain.proceed(GET(REPO_URL))
+            val extRepo = json.decodeFromString<List<ExtensionJsonObject>>(extRepoResponse.body.string())
+
+            isObsolete = !extRepo.any { ext ->
+                ext.pkg == this.javaClass.`package`?.name && ext.lang == "pt-BR"
+            }
+        }
+
+        if (isObsolete == true) {
+            throw IOException("Extensão obsoleta. Desinstale e migre para outras fontes.")
+        }
+
+        return chain.proceed(chain.request())
+    }
+
+    @Serializable
+    private data class ExtensionJsonObject(
+        val pkg: String,
+        val lang: String,
+    )
+
+    companion object {
+        private const val REPO_URL = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/index.min.json"
+    }
+}
--- a/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/SpecificPathRateLimitInterceptor.kt
+++ b/src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/SpecificPathRateLimitInterceptor.kt
@ -0,0 +1,90 @@
+package eu.kanade.tachiyomi.extension.pt.goldenmangas
+
+import android.os.SystemClock
+import okhttp3.Interceptor
+import okhttp3.OkHttpClient
+import okhttp3.Response
+import java.io.IOException
+import java.util.ArrayDeque
+import java.util.concurrent.Semaphore
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds
+
+@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
+class SpecificPathRateLimitInterceptor(
+    private val path: String,
+    private val permits: Int,
+    period: Duration,
+) : Interceptor {
+
+    private val requestQueue = ArrayDeque<Long>(permits)
+    private val rateLimitMillis = period.inWholeMilliseconds
+    private val fairLock = Semaphore(1, true)
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val call = chain.call()
+        if (call.isCanceled()) throw IOException("Canceled")
+
+        val request = chain.request()
+
+        if (!request.url.encodedPath.startsWith(path)) {
+            return chain.proceed(request)
+        }
+
+        try {
+            fairLock.acquire()
+        } catch (e: InterruptedException) {
+            throw IOException(e)
+        }
+
+        val requestQueue = this.requestQueue
+        val timestamp: Long
+
+        try {
+            synchronized(requestQueue) {
+                while (requestQueue.size >= permits) { // queue is full, remove expired entries
+                    val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis
+                    var hasRemovedExpired = false
+                    while (requestQueue.isEmpty().not() && requestQueue.first <= periodStart) {
+                        requestQueue.removeFirst()
+                        hasRemovedExpired = true
+                    }
+                    if (call.isCanceled()) {
+                        throw IOException("Canceled")
+                    } else if (hasRemovedExpired) {
+                        break
+                    } else {
+                        try { // wait for the first entry to expire, or notified by cached response
+                            (requestQueue as Object).wait(requestQueue.first - periodStart)
+                        } catch (_: InterruptedException) {
+                            continue
+                        }
+                    }
+                }
+
+                // add request to queue
+                timestamp = SystemClock.elapsedRealtime()
+                requestQueue.addLast(timestamp)
+            }
+        } finally {
+            fairLock.release()
+        }
+
+        val response = chain.proceed(request)
+        if (response.networkResponse == null) { // response is cached, remove it from queue
+            synchronized(requestQueue) {
+                if (requestQueue.isEmpty() || timestamp < requestQueue.first) return@synchronized
+                requestQueue.removeFirstOccurrence(timestamp)
+                (requestQueue as Object).notifyAll()
+            }
+        }
+
+        return response
+    }
+}
+
+fun OkHttpClient.Builder.rateLimitPath(
+    path: String,
+    permits: Int,
+    period: Duration = 1.seconds,
+) = addInterceptor(SpecificPathRateLimitInterceptor(path, permits, period))
--- a/src/pt/mundowebtoon/build.gradle
+++ b/src/pt/mundowebtoon/build.gradle
@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'

 ext {
    extName = 'Mundo Webtoon'
    pkgNameSuffix = 'pt.mundowebtoon'
    extClass = '.MundoWebtoon'
-    extVersionCode = 7
+    extVersionCode = 8
    isNsfw = true
 }

--- a/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/MundoWebtoon.kt
+++ b/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/MundoWebtoon.kt
@ -35,8 +35,9 @@ class MundoWebtoon : ParsedHttpSource() {
    override val supportsLatest = true

    override val client: OkHttpClient = network.cloudflareClient.newBuilder()
+        .addInterceptor(ObsoleteExtensionInterceptor())
        .addInterceptor(::sanitizeHtmlIntercept)
-        .rateLimit(1, 2, TimeUnit.SECONDS)
+        .rateLimit(1, 3, TimeUnit.SECONDS)
        .build()

    override fun headersBuilder(): Headers.Builder = Headers.Builder()
--- a/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/ObsoleteExtensionInterceptor.kt
+++ b/src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/ObsoleteExtensionInterceptor.kt
@ -0,0 +1,43 @@
+package eu.kanade.tachiyomi.extension.pt.mundowebtoon
+
+import eu.kanade.tachiyomi.network.GET
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import okhttp3.Interceptor
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+import java.io.IOException
+
+class ObsoleteExtensionInterceptor : Interceptor {
+
+    private val json: Json by injectLazy()
+    private var isObsolete: Boolean? = null
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        if (isObsolete == null) {
+            val extRepoResponse = chain.proceed(GET(REPO_URL))
+            val extRepo = json.decodeFromString<List<ExtensionJsonObject>>(extRepoResponse.body.string())
+
+            isObsolete = !extRepo.any { ext ->
+                ext.pkg == this.javaClass.`package`?.name && ext.lang == "pt-BR"
+            }
+        }
+
+        if (isObsolete == true) {
+            throw IOException("Extensão obsoleta. Desinstale e migre para outras fontes.")
+        }
+
+        return chain.proceed(chain.request())
+    }
+
+    @Serializable
+    private data class ExtensionJsonObject(
+        val pkg: String,
+        val lang: String,
+    )
+
+    companion object {
+        private const val REPO_URL = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/index.min.json"
+    }
+}