From 93e6774da021afc10049bda8915ce09fb96add58 Mon Sep 17 00:00:00 2001 From: arkon Date: Sun, 5 Jun 2022 17:24:05 -0400 Subject: [PATCH] Refactor to use app-provided rate limiting interceptors (#12088) --- CONTRIBUTING.md | 10 ---- src/all/mangadex/build.gradle | 4 +- .../all/mangadex/MangaDexInterceptors.kt | 1 - .../all/mangadex/RateLimitInterceptor.kt | 58 +++++++++++++++++++ src/en/digitalcomicmuseum/build.gradle | 6 +- src/zh/baimangu/build.gradle | 6 +- .../extension/zh/baimangu/Baimangu.kt | 17 +++--- src/zh/jinmantiantang/build.gradle | 6 +- .../zh/jinmantiantang/Jinmantiantang.kt | 12 ++-- 9 files changed, 77 insertions(+), 43 deletions(-) create mode 100644 src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/RateLimitInterceptor.kt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8ec3383b5..51f4a76cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -135,16 +135,6 @@ The extension's version name is generated automatically by concatenating `libVer Extensions rely on [extensions-lib](https://github.com/tachiyomiorg/extensions-lib), which provides some interfaces and stubs from the [app](https://github.com/tachiyomiorg/tachiyomi) for compilation purposes. The actual implementations can be found [here](https://github.com/tachiyomiorg/tachiyomi/tree/master/app/src/main/java/eu/kanade/tachiyomi/source). Referencing the actual implementation will help with understanding extensions' call flow. -#### Rate limiting library - -[`lib-ratelimit`](https://github.com/tachiyomiorg/tachiyomi-extensions/tree/master/lib/ratelimit) is a library for adding rate limiting functionality as an [OkHttp interceptor](https://square.github.io/okhttp/interceptors/). - -```gradle -dependencies { - implementation(project(':lib-ratelimit')) -} -``` - #### DataImage library [`lib-dataimage`](https://github.com/tachiyomiorg/tachiyomi-extensions/tree/master/lib/dataimage) is a library for handling [base 64 encoded image data](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs) using an [OkHttp interceptor](https://square.github.io/okhttp/interceptors/). diff --git a/src/all/mangadex/build.gradle b/src/all/mangadex/build.gradle index cb3044b17..48dabf0e6 100644 --- a/src/all/mangadex/build.gradle +++ b/src/all/mangadex/build.gradle @@ -6,12 +6,12 @@ ext { extName = 'MangaDex' pkgNameSuffix = 'all.mangadex' extClass = '.MangaDexFactory' - extVersionCode = 159 + extVersionCode = 160 isNsfw = true } dependencies { - implementation project(':lib-ratelimit') + compileOnly(libs.okhttp) } apply from: "$rootDir/common.gradle" diff --git a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexInterceptors.kt b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexInterceptors.kt index 3dd0900fc..d32d90cfc 100644 --- a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexInterceptors.kt +++ b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexInterceptors.kt @@ -2,7 +2,6 @@ package eu.kanade.tachiyomi.extension.all.mangadex import android.util.Log import eu.kanade.tachiyomi.extension.all.mangadex.dto.ImageReportDto -import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor import eu.kanade.tachiyomi.network.POST import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json diff --git a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/RateLimitInterceptor.kt b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/RateLimitInterceptor.kt new file mode 100644 index 000000000..b6152f62a --- /dev/null +++ b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/RateLimitInterceptor.kt @@ -0,0 +1,58 @@ +package eu.kanade.tachiyomi.extension.all.mangadex + +import android.os.SystemClock +import okhttp3.Interceptor +import okhttp3.Response +import java.util.concurrent.TimeUnit + +/** + * An OkHttp interceptor that handles rate limiting. + * + * Examples: + * + * permits = 5, period = 1, unit = seconds => 5 requests per second + * permits = 10, period = 2, unit = minutes => 10 requests per 2 minutes + * + * @param permits {Int} Number of requests allowed within a period of units. + * @param period {Long} The limiting duration. Defaults to 1. + * @param unit {TimeUnit} The unit of time for the period. Defaults to seconds. + */ +class RateLimitInterceptor( + private val permits: Int, + private val period: Long = 1, + private val unit: TimeUnit = TimeUnit.SECONDS) : Interceptor { + + private val requestQueue = ArrayList(permits) + private val rateLimitMillis = unit.toMillis(period) + + override fun intercept(chain: Interceptor.Chain): Response { + synchronized(requestQueue) { + val now = SystemClock.elapsedRealtime() + val waitTime = if (requestQueue.size < permits) { + 0 + } else { + val oldestReq = requestQueue[0] + val newestReq = requestQueue[permits - 1] + + if (newestReq - oldestReq > rateLimitMillis) { + 0 + } else { + oldestReq + rateLimitMillis - now // Remaining time + } + } + + if (requestQueue.size == permits) { + requestQueue.removeAt(0) + } + if (waitTime > 0) { + requestQueue.add(now + waitTime) + Thread.sleep(waitTime) // Sleep inside synchronized to pause queued requests + } else { + requestQueue.add(now) + } + } + + return chain.proceed(chain.request()) + } + +} diff --git a/src/en/digitalcomicmuseum/build.gradle b/src/en/digitalcomicmuseum/build.gradle index 610b78c69..34ae77ef4 100644 --- a/src/en/digitalcomicmuseum/build.gradle +++ b/src/en/digitalcomicmuseum/build.gradle @@ -6,11 +6,7 @@ ext { extName = 'Digital Comic Museum' pkgNameSuffix = 'en.digitalcomicmuseum' extClass = '.DigitalComicMuseum' - extVersionCode = 1 -} - -dependencies { - implementation project(':lib-ratelimit') + extVersionCode = 2 } apply from: "$rootDir/common.gradle" diff --git a/src/zh/baimangu/build.gradle b/src/zh/baimangu/build.gradle index 43076ba37..6a261db9f 100644 --- a/src/zh/baimangu/build.gradle +++ b/src/zh/baimangu/build.gradle @@ -5,11 +5,7 @@ ext { extName = 'Baimangu (Darpou)' pkgNameSuffix = 'zh.baimangu' extClass = '.Baimangu' - extVersionCode = 2 -} - -dependencies { - implementation project(path: ':lib-ratelimit') + extVersionCode = 3 } apply from: "$rootDir/common.gradle" diff --git a/src/zh/baimangu/src/eu/kanade/tachiyomi/extension/zh/baimangu/Baimangu.kt b/src/zh/baimangu/src/eu/kanade/tachiyomi/extension/zh/baimangu/Baimangu.kt index 5b17e76f5..f6b373c8e 100644 --- a/src/zh/baimangu/src/eu/kanade/tachiyomi/extension/zh/baimangu/Baimangu.kt +++ b/src/zh/baimangu/src/eu/kanade/tachiyomi/extension/zh/baimangu/Baimangu.kt @@ -3,8 +3,8 @@ package eu.kanade.tachiyomi.extension.zh.baimangu import android.app.Application import android.content.SharedPreferences import android.widget.Toast -import eu.kanade.tachiyomi.lib.ratelimit.SpecificHostRateLimitInterceptor import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.interceptor.rateLimitHost import eu.kanade.tachiyomi.source.ConfigurableSource import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList @@ -36,16 +36,13 @@ class Baimangu : ConfigurableSource, ParsedHttpSource() { override val baseUrl = preferences.getString(MAINSITE_URL_PREF, MAINSITE_URL_PREF_DEFAULT)!! - // Client configs - private val mainSiteRateLimitInterceptor = SpecificHostRateLimitInterceptor( - baseUrl.toHttpUrlOrNull()!!, - preferences.getString(MAINSITE_RATEPERMITS_PREF, MAINSITE_RATEPERMITS_PREF_DEFAULT)!!.toInt(), - preferences.getString(MAINSITE_RATEPERIOD_PREF, MAINSITE_RATEPERIOD_PREF_DEFAULT)!!.toLong(), - TimeUnit.MILLISECONDS, - ) - override val client: OkHttpClient = network.cloudflareClient.newBuilder() - .addNetworkInterceptor(mainSiteRateLimitInterceptor) + .rateLimitHost( + baseUrl.toHttpUrlOrNull()!!, + preferences.getString(MAINSITE_RATEPERMITS_PREF, MAINSITE_RATEPERMITS_PREF_DEFAULT)!!.toInt(), + preferences.getString(MAINSITE_RATEPERIOD_PREF, MAINSITE_RATEPERIOD_PREF_DEFAULT)!!.toLong(), + TimeUnit.MILLISECONDS, + ) .build() override fun headersBuilder(): Headers.Builder = Headers.Builder() diff --git a/src/zh/jinmantiantang/build.gradle b/src/zh/jinmantiantang/build.gradle index e28e4f4c2..7e5f4d4e1 100644 --- a/src/zh/jinmantiantang/build.gradle +++ b/src/zh/jinmantiantang/build.gradle @@ -6,12 +6,8 @@ ext { extName = 'Jinmantiantang' pkgNameSuffix = 'zh.jinmantiantang' extClass = '.Jinmantiantang' - extVersionCode = 25 + extVersionCode = 26 isNsfw = true } -dependencies { - implementation project(':lib-ratelimit') -} - apply from: "$rootDir/common.gradle" diff --git a/src/zh/jinmantiantang/src/eu/kanade/tachiyomi/extension/zh/jinmantiantang/Jinmantiantang.kt b/src/zh/jinmantiantang/src/eu/kanade/tachiyomi/extension/zh/jinmantiantang/Jinmantiantang.kt index 4dfa921bb..0ffbc1f18 100644 --- a/src/zh/jinmantiantang/src/eu/kanade/tachiyomi/extension/zh/jinmantiantang/Jinmantiantang.kt +++ b/src/zh/jinmantiantang/src/eu/kanade/tachiyomi/extension/zh/jinmantiantang/Jinmantiantang.kt @@ -4,9 +4,9 @@ import android.app.Application import android.content.SharedPreferences import androidx.preference.EditTextPreference import androidx.preference.PreferenceScreen -import eu.kanade.tachiyomi.lib.ratelimit.SpecificHostRateLimitInterceptor import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess +import eu.kanade.tachiyomi.network.interceptor.rateLimitHost import eu.kanade.tachiyomi.source.ConfigurableSource import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList @@ -44,13 +44,15 @@ class Jinmantiantang : ConfigurableSource, ParsedHttpSource() { SITE_ENTRIES_ARRAY[preferences.getString(USE_MIRROR_URL_PREF, "0")!!.toInt()] private val baseHttpUrl: HttpUrl = baseUrl.toHttpUrlOrNull()!! - // Add rate limit to fix manga thumbnail load failure - private val mainSiteRateLimitInterceptor = SpecificHostRateLimitInterceptor(baseHttpUrl, preferences.getString(MAINSITE_RATELIMIT_PREF, "1")!!.toInt(), preferences.getString(MAINSITE_RATELIMIT_PERIOD, "3")!!.toLong()) - // 处理URL请求 override val client: OkHttpClient = network.cloudflareClient .newBuilder() - .addNetworkInterceptor(mainSiteRateLimitInterceptor) + // Add rate limit to fix manga thumbnail load failure + .rateLimitHost( + baseHttpUrl, + preferences.getString(MAINSITE_RATELIMIT_PREF, "1")!!.toInt(), + preferences.getString(MAINSITE_RATELIMIT_PERIOD, "3")!!.toLong(), + ) .addInterceptor(ScrambledImageInterceptor).build() // 点击量排序(人气)