<zh-dmzj>Add ratelimit to fix HTTP 429 and return webpage url to "Open in browser" and "Share manga". (#5537)

* <zh-dmzj>Add ratelimit to fix HTTP 429 and return human readable webpage url to "Open in browser" and "Share manga".

* Add ratelimit interceptor that only handle specific url host.

lib/ratelimit/src/main/java/eu/kanade/tachiyomi/lib/ratelimit/SpecificHostRateLimitInterceptor.kt

@@ -0,0 +1,65 @@
+package eu.kanade.tachiyomi.lib.ratelimit
+
+import android.os.SystemClock
+import okhttp3.HttpUrl
+import okhttp3.Interceptor
+import okhttp3.Response
+import java.util.concurrent.TimeUnit
+
+/**
+ * An OkHttp interceptor that handles given url host's rate limiting.
+ *
+ * Examples:
+ *
+ * httpUrl = Httpurl.parse("api.manga.com"), permits = 5, period = 1, unit = seconds  =>  5 requests per second to api.manga.com
+ * httpUrl = Httpurl.parse("imagecdn.manga.com"), permits = 10, period = 2, unit = minutes  =>  10 requests per 2 minutes to imagecdn.manga.com
+ *
+ * @param httpUrl {HttpUrl} The url host that this interceptor should handle. Will get url's host by using HttpUrl.host()
+ * @param permits {Int}   Number of requests allowed within a period of units.
+ * @param period {Long}   The limiting duration. Defaults to 1.
+ * @param unit {TimeUnit} The unit of time for the period. Defaults to seconds.
+ */
+class SpecificHostRateLimitInterceptor(
+        private val httpUrl: HttpUrl,
+        private val permits: Int,
+        private val period: Long = 1,
+        private val unit: TimeUnit = TimeUnit.SECONDS
+) : Interceptor {
+
+    private val requestQueue = ArrayList<Long>(permits)
+    private val rateLimitMillis = unit.toMillis(period)
+    private val host = httpUrl.host()
+
+    override fun intercept(chain: Interceptor.Chain): Response {
+        if (chain.request().url().host() != host) {
+            return chain.proceed(chain.request())
+        }
+        synchronized(requestQueue) {
+            val now = SystemClock.elapsedRealtime()
+            val waitTime = if (requestQueue.size < permits) {
+                0
+            } else {
+                val oldestReq = requestQueue[0]
+                val newestReq = requestQueue[permits - 1]
+
+                if (newestReq - oldestReq > rateLimitMillis) {
+                    0
+                } else {
+                    oldestReq + rateLimitMillis - now // Remaining time
+                }
+            }
+
+            if (requestQueue.size == permits) {
+                requestQueue.removeAt(0)
+            }
+            if (waitTime > 0) {
+                requestQueue.add(now + waitTime)
+                Thread.sleep(waitTime) // Sleep inside synchronized to pause queued requests
+            } else {
+                requestQueue.add(now)
+            }
+        }
+
+        return chain.proceed(chain.request())
+    }
+}

src/zh/dmzj/build.gradle

@@ -5,9 +5,12 @@ ext {
     extName = 'Dmzj'
     pkgNameSuffix = 'zh.dmzj'
     extClass = '.Dmzj'
-    extVersionCode = 14
+    extVersionCode = 15
     libVersion = '1.2'
-    containsNsfw = true
+}
+
+dependencies {
+    implementation project(':lib-ratelimit')
 }
 
 apply from: "$rootDir/common.gradle"

src/zh/dmzj/src/eu/kanade/tachiyomi/extension/zh/dmzj/Dmzj.kt

@@ -1,7 +1,14 @@
 package eu.kanade.tachiyomi.extension.zh.dmzj
 
+import android.app.Application
+import android.content.SharedPreferences
 import android.net.Uri
+import android.support.v7.preference.ListPreference
+import android.support.v7.preference.PreferenceScreen
+import eu.kanade.tachiyomi.lib.ratelimit.SpecificHostRateLimitInterceptor
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
@@ -9,10 +16,15 @@ import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
+import okhttp3.HttpUrl
+import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
 import org.json.JSONArray
 import org.json.JSONObject
+import rx.Observable
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
 import java.net.URLEncoder
 import java.util.ArrayList
 
@@ -20,24 +32,44 @@ import java.util.ArrayList
  * Dmzj source
  */
 
-class Dmzj : HttpSource() {
+class Dmzj : ConfigurableSource, HttpSource() {
     override val lang = "zh"
     override val supportsLatest = true
     override val name = "动漫之家"
-    override val baseUrl = "https://v3api.dmzj1.com"
+    override val baseUrl = "https://m.dmzj1.com"
+    private val apiUrl = "https://v3api.dmzj1.com"
+    private val imageCDNUrl = "https://images.dmzj1.com"
 
     private fun cleanUrl(url: String) = if (url.startsWith("//"))
         "https:$url"
     else url
 
+    private val preferences: SharedPreferences by lazy {
+        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
+    }
+
+    private val apiRateLimitInterceptor = SpecificHostRateLimitInterceptor(
+        HttpUrl.parse(apiUrl)!!,
+        preferences.getString(API_RATELIMIT_PREF, "5")!!.toInt()
+    )
+    private val imageCDNRateLimitInterceptor = SpecificHostRateLimitInterceptor(
+        HttpUrl.parse(imageCDNUrl)!!,
+        preferences.getString(IMAGE_CDN_RATELIMIT_PREF, "5")!!.toInt()
+    )
+
+    override val client: OkHttpClient = network.client.newBuilder()
+        .addNetworkInterceptor(apiRateLimitInterceptor)
+        .addNetworkInterceptor(imageCDNRateLimitInterceptor)
+        .build()
+
     private fun myGet(url: String) = GET(url)
         .newBuilder()
         .header(
             "User-Agent",
-            "Mozilla/5.0 (X11; Linux x86_64) " +
+            "Mozilla/5.0 (Linux; Android 10) " +
                 "AppleWebKit/537.36 (KHTML, like Gecko) " +
-                "Chrome/56.0.2924.87 " +
+                "Chrome/88.0.4324.93 " +
-                "Safari/537.36 " +
+                "Mobile Safari/537.36 " +
                 "Tachiyomi/1.0"
         )
         .build()!!
@@ -85,11 +117,11 @@ class Dmzj : HttpSource() {
         return MangasPage(ret, arr.length() != 0)
     }
 
-    override fun popularMangaRequest(page: Int) = myGet("$baseUrl/classify/0/0/${page - 1}.json")
+    override fun popularMangaRequest(page: Int) = myGet("$apiUrl/classify/0/0/${page - 1}.json")
 
     override fun popularMangaParse(response: Response) = searchMangaParse(response)
 
-    override fun latestUpdatesRequest(page: Int) = myGet("$baseUrl/classify/0/1/${page - 1}.json")
+    override fun latestUpdatesRequest(page: Int) = myGet("$apiUrl/classify/0/1/${page - 1}.json")
 
     override fun latestUpdatesParse(response: Response): MangasPage = searchMangaParse(response)
 
@@ -110,7 +142,7 @@ class Dmzj : HttpSource() {
 
             val order = filters.filterIsInstance<SortFilter>().joinToString("") { (it as UriPartFilter).toUriPart() }
 
-            return myGet("$baseUrl/classify/$params/$order/${page - 1}.json")
+            return myGet("$apiUrl/classify/$params/$order/${page - 1}.json")
         }
     }
 
@@ -124,6 +156,25 @@ class Dmzj : HttpSource() {
         }
     }
 
+    // Bypass mangaDetailsRequest, fetch v3api url directly
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        return client.newCall(GET(apiUrl + manga.url, headers))
+            .asObservableSuccess()
+            .map { response ->
+                mangaDetailsParse(response).apply { initialized = true }
+            }
+    }
+
+    private val re1 = Regex("""\d+""") // Get comic ID from manga.url
+    // Workaround to allow "Open in browser" use human readable webpage url.
+    override fun mangaDetailsRequest(manga: SManga): Request {
+        return GET("$baseUrl/info/${re1.find(manga.url)!!.value}.html")
+    }
+
+    override fun chapterListRequest(manga: SManga): Request {
+        return GET(apiUrl + manga.url, headers)
+    }
+
     override fun mangaDetailsParse(response: Response) = SManga.create().apply {
         val obj = JSONObject(response.body()!!.string())
 
@@ -320,4 +371,101 @@ class Dmzj : HttpSource() {
         Filter.Select<String>(displayName, vals.map { it.first }.toTypedArray(), defaultValue) {
         open fun toUriPart() = vals[state].second
     }
+
+    override fun setupPreferenceScreen(screen: androidx.preference.PreferenceScreen) {
+        val apiRateLimitPreference = androidx.preference.ListPreference(screen.context).apply {
+            key = API_RATELIMIT_PREF
+            title = API_RATELIMIT_PREF_TITLE
+            summary = API_RATELIMIT_PREF_SUMMARY
+            entries = ENTRIES_ARRAY
+            entryValues = ENTRIES_ARRAY
+
+            setDefaultValue("5")
+            setOnPreferenceChangeListener { _, newValue ->
+                try {
+                    val setting = preferences.edit().putString(API_RATELIMIT_PREF, newValue as String).commit()
+                    setting
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                    false
+                }
+            }
+        }
+
+        val imgCDNRateLimitPreference = androidx.preference.ListPreference(screen.context).apply {
+            key = IMAGE_CDN_RATELIMIT_PREF
+            title = IMAGE_CDN_RATELIMIT_PREF_TITLE
+            summary = IMAGE_CDN_RATELIMIT_PREF_SUMMARY
+            entries = ENTRIES_ARRAY
+            entryValues = ENTRIES_ARRAY
+
+            setDefaultValue("5")
+            setOnPreferenceChangeListener { _, newValue ->
+                try {
+                    val setting = preferences.edit().putString(IMAGE_CDN_RATELIMIT_PREF, newValue as String).commit()
+                    setting
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                    false
+                }
+            }
+        }
+
+        screen.addPreference(apiRateLimitPreference)
+        screen.addPreference(imgCDNRateLimitPreference)
+    }
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        val apiRateLimitPreference = ListPreference(screen.context).apply {
+            key = API_RATELIMIT_PREF
+            title = API_RATELIMIT_PREF_TITLE
+            summary = API_RATELIMIT_PREF_SUMMARY
+            entries = ENTRIES_ARRAY
+            entryValues = ENTRIES_ARRAY
+
+            setDefaultValue("5")
+            setOnPreferenceChangeListener { _, newValue ->
+                try {
+                    val setting = preferences.edit().putString(API_RATELIMIT_PREF, newValue as String).commit()
+                    setting
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                    false
+                }
+            }
+        }
+
+        val imgCDNRateLimitPreference = ListPreference(screen.context).apply {
+            key = IMAGE_CDN_RATELIMIT_PREF
+            title = IMAGE_CDN_RATELIMIT_PREF_TITLE
+            summary = IMAGE_CDN_RATELIMIT_PREF_SUMMARY
+            entries = ENTRIES_ARRAY
+            entryValues = ENTRIES_ARRAY
+
+            setDefaultValue("5")
+            setOnPreferenceChangeListener { _, newValue ->
+                try {
+                    val setting = preferences.edit().putString(IMAGE_CDN_RATELIMIT_PREF, newValue as String).commit()
+                    setting
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                    false
+                }
+            }
+        }
+        screen.addPreference(apiRateLimitPreference)
+        screen.addPreference(imgCDNRateLimitPreference)
+    }
+
+    companion object {
+        private const val API_RATELIMIT_PREF = "apiRatelimitPreference"
+        private const val API_RATELIMIT_PREF_TITLE = "主站每秒连接数限制" // "Ratelimit permits per second for main website"
+        private const val API_RATELIMIT_PREF_SUMMARY = "此值影响向动漫之家网站发起连接请求的数量。调低此值可能减少发生HTTP 429（连接请求过多）错误的几率，但加载速度也会变慢。需要重启软件以生效。\n当前值：%s" // "This value affects network request amount to dmzj's url. Lower this value may reduce the chance to get HTTP 429 error, but loading speed will be slower too. Tachiyomi restart required. Current value: %s"
+
+        private const val IMAGE_CDN_RATELIMIT_PREF = "imgCDNRatelimitPreference"
+        private const val IMAGE_CDN_RATELIMIT_PREF_TITLE = "图片CDN每秒连接数限制" // "Ratelimit permits per second for image CDN"
+        private const val IMAGE_CDN_RATELIMIT_PREF_SUMMARY = "此值影响加载图片时发起连接请求的数量。调低此值可能减小图片加载错误的几率，但加载速度也会变慢。需要重启软件以生效。\n当前值：%s" // "This value affects network request amount for loading image. Lower this value may reduce the chance to get error when loading image, but loading speed will be slower too. Tachiyomi restart required. Current value: %s"
+
+        private val ENTRIES_ARRAY = (1..10).map { i -> i.toString() }.toTypedArray()
+    }
 }