Some fixes for two sources (#18053)

* Add support to URL guess and add harsh rate limit on GM.

* Fix the checks and add more paths.

* Remove the log statement.

* Use their current URL as of this moment.

---------

Co-authored-by: Alessandro Jean <alessandrojean@users.noreply.github.com>

src/pt/goldenmangas/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'Golden Mangás'
     pkgNameSuffix = 'pt.goldenmangas'
     extClass = '.GoldenMangas'
-    extVersionCode = 21
+    extVersionCode = 22
     isNsfw = true
 }
 

src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/GoldenMangas.kt

@@ -13,7 +13,6 @@ import eu.kanade.tachiyomi.lib.randomua.getPrefCustomUA
 import eu.kanade.tachiyomi.lib.randomua.getPrefUAType
 import eu.kanade.tachiyomi.lib.randomua.setRandomUserAgent
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.interceptor.rateLimit
 import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.Page
@@ -21,7 +20,9 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import okhttp3.Headers
+import okhttp3.HttpUrl
 import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Interceptor
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
@@ -32,6 +33,7 @@ import uy.kohesive.injekt.api.get
 import java.text.SimpleDateFormat
 import java.util.Locale
 import java.util.concurrent.TimeUnit
+import kotlin.time.Duration.Companion.seconds
 
 class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
 
@@ -40,7 +42,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
 
     override val name = "Golden Mangás"
 
-    override val baseUrl = "https://www.goldenmangas.top"
+    override val baseUrl = "https://www.goldenmanga.top"
 
     override val lang = "pt-BR"
 
@@ -50,15 +52,23 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
         Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
     }
 
+    private val baseHttpUrl: HttpUrl
+        get() = preferences.baseUrl.toHttpUrl()
+
     override val client: OkHttpClient = network.cloudflareClient.newBuilder()
         .connectTimeout(1, TimeUnit.MINUTES)
         .readTimeout(1, TimeUnit.MINUTES)
         .writeTimeout(1, TimeUnit.MINUTES)
+        .addInterceptor(ObsoleteExtensionInterceptor())
         .setRandomUserAgent(
             userAgentType = preferences.getPrefUAType(),
             customUA = preferences.getPrefCustomUA(),
         )
-        .rateLimit(1, 3, TimeUnit.SECONDS)
+        .rateLimitPath("/mangas", 1, 8.seconds)
+        .rateLimitPath("/mm-admin/uploads", 1, 8.seconds)
+        .rateLimitPath("/timthumb.php", 1, 3.seconds)
+        .rateLimitPath("/index.php", 1, 3.seconds)
+        .addInterceptor(::guessNewUrlIntercept)
         .build()
 
     override fun headersBuilder(): Headers.Builder = Headers.Builder()
@@ -68,7 +78,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
 
     override fun popularMangaRequest(page: Int): Request = GET(baseUrl, headers)
 
-    override fun popularMangaSelector(): String = "div#maisLidos div.itemmanga"
+    override fun popularMangaSelector(): String = "div#maisLidos div.itemmanga:not(:contains(Avisos e Recados))"
 
     override fun popularMangaFromElement(element: Element): SManga = SManga.create().apply {
         title = element.selectFirst("h3")!!.text().withoutLanguage()
@@ -83,7 +93,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
         return GET("$baseUrl$path", headers)
     }
 
-    override fun latestUpdatesSelector() = "div.col-sm-12.atualizacao > div.row"
+    override fun latestUpdatesSelector() = "div.col-sm-12.atualizacao > div.row:not(:contains(Avisos e Recados))"
 
     override fun latestUpdatesFromElement(element: Element): SManga = SManga.create().apply {
         val infoElement = element.selectFirst("div.col-sm-10.col-xs-8 h3")!!
@@ -123,7 +133,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
         val mangaResult = runCatching { super.mangaDetailsParse(response) }
         val manga = mangaResult.getOrNull()
 
-        if (manga?.title.isNullOrEmpty()) {
+        if (manga?.title.isNullOrEmpty() && !response.hasChangedDomain) {
             throw Exception(MIGRATE_WARNING)
         }
 
@@ -150,11 +160,11 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
         val chaptersResult = runCatching { super.chapterListParse(response) }
         val chapterList = chaptersResult.getOrNull()
 
-        if (chapterList.isNullOrEmpty()) {
+        if (chapterList.isNullOrEmpty() && !response.hasChangedDomain) {
             throw Exception(MIGRATE_WARNING)
         }
 
-        return chapterList
+        return chapterList.orEmpty()
     }
 
     override fun chapterListSelector() = "ul#capitulos li.row"
@@ -177,6 +187,17 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
         return GET(baseUrl + chapter.url, newHeaders)
     }
 
+    override fun pageListParse(response: Response): List<Page> {
+        val pagesResult = runCatching { super.pageListParse(response) }
+        val pageList = pagesResult.getOrNull()
+
+        if (pageList.isNullOrEmpty() && !response.hasChangedDomain) {
+            throw Exception(MIGRATE_WARNING)
+        }
+
+        return pageList.orEmpty()
+    }
+
     override fun pageListParse(document: Document): List<Page> {
         val chapterImages = document.selectFirst("div.col-sm-12[id^='capitulos_images']:has(img[pag])")
 
@@ -186,11 +207,8 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
             throw Exception(CHAPTER_IS_NOVEL_ERROR)
         }
 
-        if (chapterImages == null) {
+        return chapterImages?.select("img[pag]")
-            throw Exception(MIGRATE_WARNING)
+            .orEmpty()
-        }
-
-        return chapterImages.select("img[pag]")
             .mapIndexed { i, element ->
                 Page(i, document.location(), element.attr("abs:src"))
             }
@@ -208,7 +226,7 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
     }
 
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
-        val uaPreferece = ListPreference(screen.context).apply {
+        val uaPreference = ListPreference(screen.context).apply {
             key = PREF_KEY_RANDOM_UA
             title = "User Agent aleatório"
             summary = "%s"
@@ -239,15 +257,48 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
             }
         }
 
-        screen.addPreference(uaPreferece)
+        screen.addPreference(uaPreference)
         screen.addPreference(customUaPreference)
     }
 
+    private fun guessNewUrlIntercept(chain: Interceptor.Chain): Response {
+        val response = chain.proceed(chain.request())
+
+        if (chain.request().url.host == "raw.githubusercontent.com") {
+            return response
+        }
+
+        if (response.hasChangedDomain && preferences.baseUrl == baseUrl) {
+            return response
+        }
+
+        preferences.baseUrl = "https://${response.request.url.host}"
+
+        val newUrl = chain.request().url.toString()
+            .replaceFirst(baseUrl, preferences.baseUrl)
+            .toHttpUrl()
+        val newRequest = chain.request().newBuilder()
+            .url(newUrl)
+            .build()
+
+        response.close()
+
+        return chain.proceed(newRequest)
+    }
+
+    private var SharedPreferences.baseUrl: String
+        get() = getString(BASE_URL_PREF, this@GoldenMangas.baseUrl)!!
+        set(newValue) = edit().putString(BASE_URL_PREF, newValue).apply()
+
     private fun String.toDate(): Long {
         return runCatching { DATE_FORMATTER.parse(trim())?.time }
             .getOrNull() ?: 0L
     }
 
+    private val Response.hasChangedDomain: Boolean
+        get() = request.url.host != baseHttpUrl.host &&
+            request.url.host.contains("goldenmang")
+
     private fun String.toStatus() = when (this) {
         "Ativo" -> SManga.ONGOING
         "Completo" -> SManga.COMPLETED
@@ -277,5 +328,6 @@ class GoldenMangas : ParsedHttpSource(), ConfigurableSource {
         }
 
         private const val RESTART_APP_MESSAGE = "Reinicie o aplicativo para aplicar as alterações."
+        private const val BASE_URL_PREF = "base_url"
     }
 }

src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/ObsoleteExtensionInterceptor.kt

@@ -0,0 +1,43 @@
+package eu.kanade.tachiyomi.extension.pt.goldenmangas
+
+import eu.kanade.tachiyomi.network.GET
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import okhttp3.Interceptor
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+import java.io.IOException
+
+class ObsoleteExtensionInterceptor : Interceptor {
+
+    private val json: Json by injectLazy()
+    private var isObsolete: Boolean? = null
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        if (isObsolete == null) {
+            val extRepoResponse = chain.proceed(GET(REPO_URL))
+            val extRepo = json.decodeFromString<List<ExtensionJsonObject>>(extRepoResponse.body.string())
+
+            isObsolete = !extRepo.any { ext ->
+                ext.pkg == this.javaClass.`package`?.name && ext.lang == "pt-BR"
+            }
+        }
+
+        if (isObsolete == true) {
+            throw IOException("Extensão obsoleta. Desinstale e migre para outras fontes.")
+        }
+
+        return chain.proceed(chain.request())
+    }
+
+    @Serializable
+    private data class ExtensionJsonObject(
+        val pkg: String,
+        val lang: String,
+    )
+
+    companion object {
+        private const val REPO_URL = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/index.min.json"
+    }
+}

src/pt/goldenmangas/src/eu/kanade/tachiyomi/extension/pt/goldenmangas/SpecificPathRateLimitInterceptor.kt

@@ -0,0 +1,90 @@
+package eu.kanade.tachiyomi.extension.pt.goldenmangas
+
+import android.os.SystemClock
+import okhttp3.Interceptor
+import okhttp3.OkHttpClient
+import okhttp3.Response
+import java.io.IOException
+import java.util.ArrayDeque
+import java.util.concurrent.Semaphore
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds
+
+@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
+class SpecificPathRateLimitInterceptor(
+    private val path: String,
+    private val permits: Int,
+    period: Duration,
+) : Interceptor {
+
+    private val requestQueue = ArrayDeque<Long>(permits)
+    private val rateLimitMillis = period.inWholeMilliseconds
+    private val fairLock = Semaphore(1, true)
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        val call = chain.call()
+        if (call.isCanceled()) throw IOException("Canceled")
+
+        val request = chain.request()
+
+        if (!request.url.encodedPath.startsWith(path)) {
+            return chain.proceed(request)
+        }
+
+        try {
+            fairLock.acquire()
+        } catch (e: InterruptedException) {
+            throw IOException(e)
+        }
+
+        val requestQueue = this.requestQueue
+        val timestamp: Long
+
+        try {
+            synchronized(requestQueue) {
+                while (requestQueue.size >= permits) { // queue is full, remove expired entries
+                    val periodStart = SystemClock.elapsedRealtime() - rateLimitMillis
+                    var hasRemovedExpired = false
+                    while (requestQueue.isEmpty().not() && requestQueue.first <= periodStart) {
+                        requestQueue.removeFirst()
+                        hasRemovedExpired = true
+                    }
+                    if (call.isCanceled()) {
+                        throw IOException("Canceled")
+                    } else if (hasRemovedExpired) {
+                        break
+                    } else {
+                        try { // wait for the first entry to expire, or notified by cached response
+                            (requestQueue as Object).wait(requestQueue.first - periodStart)
+                        } catch (_: InterruptedException) {
+                            continue
+                        }
+                    }
+                }
+
+                // add request to queue
+                timestamp = SystemClock.elapsedRealtime()
+                requestQueue.addLast(timestamp)
+            }
+        } finally {
+            fairLock.release()
+        }
+
+        val response = chain.proceed(request)
+        if (response.networkResponse == null) { // response is cached, remove it from queue
+            synchronized(requestQueue) {
+                if (requestQueue.isEmpty() || timestamp < requestQueue.first) return@synchronized
+                requestQueue.removeFirstOccurrence(timestamp)
+                (requestQueue as Object).notifyAll()
+            }
+        }
+
+        return response
+    }
+}
+
+fun OkHttpClient.Builder.rateLimitPath(
+    path: String,
+    permits: Int,
+    period: Duration = 1.seconds,
+) = addInterceptor(SpecificPathRateLimitInterceptor(path, permits, period))

src/pt/mundowebtoon/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'Mundo Webtoon'
     pkgNameSuffix = 'pt.mundowebtoon'
     extClass = '.MundoWebtoon'
-    extVersionCode = 7
+    extVersionCode = 8
     isNsfw = true
 }
 

src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/MundoWebtoon.kt

@@ -35,8 +35,9 @@ class MundoWebtoon : ParsedHttpSource() {
     override val supportsLatest = true
 
     override val client: OkHttpClient = network.cloudflareClient.newBuilder()
+        .addInterceptor(ObsoleteExtensionInterceptor())
         .addInterceptor(::sanitizeHtmlIntercept)
-        .rateLimit(1, 2, TimeUnit.SECONDS)
+        .rateLimit(1, 3, TimeUnit.SECONDS)
         .build()
 
     override fun headersBuilder(): Headers.Builder = Headers.Builder()

src/pt/mundowebtoon/src/eu/kanade/tachiyomi/extension/pt/mundowebtoon/ObsoleteExtensionInterceptor.kt

@@ -0,0 +1,43 @@
+package eu.kanade.tachiyomi.extension.pt.mundowebtoon
+
+import eu.kanade.tachiyomi.network.GET
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import okhttp3.Interceptor
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+import java.io.IOException
+
+class ObsoleteExtensionInterceptor : Interceptor {
+
+    private val json: Json by injectLazy()
+    private var isObsolete: Boolean? = null
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        if (isObsolete == null) {
+            val extRepoResponse = chain.proceed(GET(REPO_URL))
+            val extRepo = json.decodeFromString<List<ExtensionJsonObject>>(extRepoResponse.body.string())
+
+            isObsolete = !extRepo.any { ext ->
+                ext.pkg == this.javaClass.`package`?.name && ext.lang == "pt-BR"
+            }
+        }
+
+        if (isObsolete == true) {
+            throw IOException("Extensão obsoleta. Desinstale e migre para outras fontes.")
+        }
+
+        return chain.proceed(chain.request())
+    }
+
+    @Serializable
+    private data class ExtensionJsonObject(
+        val pkg: String,
+        val lang: String,
+    )
+
+    companion object {
+        private const val REPO_URL = "https://raw.githubusercontent.com/tachiyomiorg/tachiyomi-extensions/repo/index.min.json"
+    }
+}