From 6a12eab8b0614556b31fbad777e43f028bd9d5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Mej=C3=ADa?= Date: Tue, 31 Aug 2021 04:51:12 -0600 Subject: [PATCH] Fix LectorManga loading pages, add rate limit for new image CDN hosts (#8901) --- src/es/lectormanga/build.gradle | 2 +- .../extension/es/lectormanga/LectorManga.kt | 57 ++++++++++++------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/es/lectormanga/build.gradle b/src/es/lectormanga/build.gradle index 390f8a918..a67b79b8d 100755 --- a/src/es/lectormanga/build.gradle +++ b/src/es/lectormanga/build.gradle @@ -5,7 +5,7 @@ ext { extName = 'LectorManga' pkgNameSuffix = 'es.lectormanga' extClass = '.LectorManga' - extVersionCode = 19 + extVersionCode = 20 libVersion = '1.2' } diff --git a/src/es/lectormanga/src/eu/kanade/tachiyomi/extension/es/lectormanga/LectorManga.kt b/src/es/lectormanga/src/eu/kanade/tachiyomi/extension/es/lectormanga/LectorManga.kt index f2eb87bd3..347d8b933 100755 --- a/src/es/lectormanga/src/eu/kanade/tachiyomi/extension/es/lectormanga/LectorManga.kt +++ b/src/es/lectormanga/src/eu/kanade/tachiyomi/extension/es/lectormanga/LectorManga.kt @@ -46,7 +46,7 @@ class LectorManga : ConfigurableSource, ParsedHttpSource() { .add("Referer", "$baseUrl/") } - private val imageCDNUrl = "https://img1.followmanga.com" + private val imageCDNUrls = arrayOf("https://img1.followmanga.com", "https://img1.biggestchef.com", "https://img1.indalchef.com") private val preferences: SharedPreferences by lazy { Injekt.get().getSharedPreferences("source_$id", 0x0000) @@ -54,19 +54,29 @@ class LectorManga : ConfigurableSource, ParsedHttpSource() { private val webRateLimitInterceptor = SpecificHostRateLimitInterceptor( baseUrl.toHttpUrlOrNull()!!, - preferences.getString(WEB_RATELIMIT_PREF, WEB_RATELIMIT_PREF_DEFAULT_VALUE)!!.toInt(), - 60 + preferences.getString(WEB_RATELIMIT_PREF, WEB_RATELIMIT_PREF_DEFAULT_VALUE)!!.toInt(), 60 ) private val imageCDNRateLimitInterceptor = SpecificHostRateLimitInterceptor( - imageCDNUrl.toHttpUrlOrNull()!!, - preferences.getString(IMAGE_CDN_RATELIMIT_PREF, IMAGE_CDN_RATELIMIT_PREF_DEFAULT_VALUE)!!.toInt(), - 60 + imageCDNUrls[0].toHttpUrlOrNull()!!, + preferences.getString(IMAGE_CDN_RATELIMIT_PREF, IMAGE_CDN_RATELIMIT_PREF_DEFAULT_VALUE)!!.toInt(), 60 + ) + + private val imageCDNRateLimitInterceptor1 = SpecificHostRateLimitInterceptor( + imageCDNUrls[1].toHttpUrlOrNull()!!, + preferences.getString(IMAGE_CDN_RATELIMIT_PREF, IMAGE_CDN_RATELIMIT_PREF_DEFAULT_VALUE)!!.toInt(), 60 + ) + + private val imageCDNRateLimitInterceptor2 = SpecificHostRateLimitInterceptor( + imageCDNUrls[2].toHttpUrlOrNull()!!, + preferences.getString(IMAGE_CDN_RATELIMIT_PREF, IMAGE_CDN_RATELIMIT_PREF_DEFAULT_VALUE)!!.toInt(), 60 ) override val client: OkHttpClient = network.client.newBuilder() .addNetworkInterceptor(webRateLimitInterceptor) .addNetworkInterceptor(imageCDNRateLimitInterceptor) + .addNetworkInterceptor(imageCDNRateLimitInterceptor1) + .addNetworkInterceptor(imageCDNRateLimitInterceptor2) .build() override fun popularMangaRequest(page: Int) = GET("$baseUrl/library?order_item=likes_count&order_dir=desc&type=&filter_by=title&page=$page", headers) @@ -244,25 +254,25 @@ class LectorManga : ConfigurableSource, ParsedHttpSource() { val currentUrl = client.newCall(GET(chapter.url, headers)).execute().asJsoup().body().baseUri() // Get /cascade instead of /paginate to get all pages at once - val newUrl = if (getPageMethodPref() == "cascade" && currentUrl.contains("paginated")) { - currentUrl.substringBefore("paginated") + "cascade" - } else if (getPageMethodPref() == "paginated" && currentUrl.contains("cascade")) { - currentUrl.substringBefore("cascade") + "paginated" + val newUrl = if (getPageMethodPref() == PAGE_METHOD_PREF_CASCADE && currentUrl.contains(PAGE_METHOD_PREF_PAGINATED)) { + currentUrl.substringBefore(PAGE_METHOD_PREF_PAGINATED) + PAGE_METHOD_PREF_CASCADE + } else if (getPageMethodPref() == PAGE_METHOD_PREF_PAGINATED && currentUrl.contains(PAGE_METHOD_PREF_CASCADE)) { + currentUrl.substringBefore(PAGE_METHOD_PREF_CASCADE) + PAGE_METHOD_PREF_PAGINATED } else currentUrl return GET(newUrl, headers) } override fun pageListParse(document: Document): List = mutableListOf().apply { - if (getPageMethodPref() == "cascade") { + if (getPageMethodPref() == PAGE_METHOD_PREF_CASCADE) { document.select("div.viewer-image-container img").forEach { add( Page( size, - "", + document.baseUri(), it.let { - if (it.hasAttr("data-src")) - it.attr("abs:data-src") else it.attr("abs:src") + if (it.hasAttr("data-src")) it.attr("abs:data-src") + else it.attr("abs:src") } ) ) @@ -276,10 +286,15 @@ class LectorManga : ConfigurableSource, ParsedHttpSource() { } } - // Note: At this moment (26/08/2021) it's necessary to make the image request with headers to prevent 403. - override fun imageRequest(page: Page) = GET(page.imageUrl!!, headers) + override fun imageRequest(page: Page) = GET( + url = page.imageUrl!!, + headers = headers.newBuilder() + .removeAll("Referer") + .add("Referer", page.url.substringBefore("news/")) + .build() + ) - override fun imageUrlParse(document: Document): String = document.select("img.viewer-image").attr("src") + override fun imageUrlParse(document: Document): String = document.select("img.viewer-image").attr("data-src") private fun searchMangaByIdRequest(id: String) = GET("$baseUrl/$MANGA_URL_CHUNK/$id", headers) @@ -373,7 +388,7 @@ class LectorManga : ConfigurableSource, ParsedHttpSource() { // Array.from(document.querySelectorAll('#advancedSearch .custom-checkbox')) // .map(a => `Genre("${a.querySelector('label').innerText}", "${a.querySelector('input').value}")`).join(',\n') // on https://lectormanga.com/library - // Last revision 13/07/2020 + // Last revision 30/08/2021 private fun getGenreList() = listOf( Genre("Acción", "1"), Genre("Aventura", "2"), @@ -448,7 +463,7 @@ class LectorManga : ConfigurableSource, ParsedHttpSource() { key = PAGE_METHOD_PREF title = PAGE_METHOD_PREF_TITLE entries = arrayOf("Cascada", "Páginado") - entryValues = arrayOf("cascade", "paginated") + entryValues = arrayOf(PAGE_METHOD_PREF_CASCADE, PAGE_METHOD_PREF_PAGINATED) summary = PAGE_METHOD_PREF_SUMMARY setDefaultValue(PAGE_METHOD_PREF_DEFAULT_VALUE) @@ -521,7 +536,9 @@ class LectorManga : ConfigurableSource, ParsedHttpSource() { private const val PAGE_METHOD_PREF = "pageMethodPref" private const val PAGE_METHOD_PREF_TITLE = "Método para descargar imágenes" private const val PAGE_METHOD_PREF_SUMMARY = "Previene ser banneado por el servidor cuando se usa la configuración \"Cascada\" ya que esta reduce la cantidad de solicitudes.\nPuedes usar \"Páginado\" cuando las imágenes no carguen usando \"Cascada\".\nConfiguración actual: %s" - private const val PAGE_METHOD_PREF_DEFAULT_VALUE = "cascade" + private const val PAGE_METHOD_PREF_CASCADE = "cascade" + private const val PAGE_METHOD_PREF_PAGINATED = "paginated" + private const val PAGE_METHOD_PREF_DEFAULT_VALUE = PAGE_METHOD_PREF_CASCADE private const val WEB_RATELIMIT_PREF = "webRatelimitPreference" // Ratelimit permits per second for main website