SchaleNetwork: simplify clearance token logic (#9630)

src/all/koharu/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'SchaleNetwork'
     extClass = '.KoharuFactory'
-    extVersionCode = 14
+    extVersionCode = 15
     isNsfw = true
 }
 

src/all/koharu/src/eu/kanade/tachiyomi/extension/all/koharu/Koharu.kt

@@ -1,6 +1,12 @@
 package eu.kanade.tachiyomi.extension.all.koharu
 
+import android.annotation.SuppressLint
+import android.app.Application
 import android.content.SharedPreferences
+import android.os.Handler
+import android.os.Looper
+import android.webkit.WebView
+import android.webkit.WebViewClient
 import androidx.preference.EditTextPreference
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
@@ -40,8 +46,11 @@ import okhttp3.Request
 import okhttp3.Response
 import rx.Observable
 import uy.kohesive.injekt.injectLazy
+import java.io.IOException
 import java.text.SimpleDateFormat
 import java.util.Locale
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
 
 class Koharu(
     override val lang: String = "all",
@@ -58,8 +67,6 @@ class Koharu(
 
     private val apiBooksUrl = "$apiUrl/books"
 
-    private val authUrl = "${baseUrl.replace("://", "://auth.")}/clearance"
-
     override val supportsLatest = true
 
     private val json: Json by injectLazy()
@@ -76,7 +83,7 @@ class Koharu(
     private fun alwaysExcludeTags() = preferences.getString(PREF_EXCLUDE_TAGS, "")
 
     private var _domainUrl: String? = null
-    internal val domainUrl: String
+    private val domainUrl: String
         get() {
             return _domainUrl ?: run {
                 val domain = getDomain()
@@ -108,11 +115,63 @@ class Koharu(
         .rateLimit(3)
         .build()
 
-    private val interceptedClient: OkHttpClient
+    private val clearanceClient = network.cloudflareClient.newBuilder()
-        get() = network.cloudflareClient.newBuilder()
+        .addInterceptor { chain ->
-            .addInterceptor(TurnstileInterceptor(client, domainUrl, authUrl, lazyHeaders["User-Agent"]))
+            val request = chain.request()
-            .rateLimit(3)
+            val url = request.url
-            .build()
+            val clearance = getClearance()
+                ?: throw IOException("Open webview to refresh token")
+
+            val newUrl = url.newBuilder()
+                .setQueryParameter("crt", clearance)
+                .build()
+            val newRequest = request.newBuilder()
+                .url(newUrl)
+                .build()
+
+            val response = chain.proceed(newRequest)
+
+            if (response.code !in listOf(400, 403)) {
+                return@addInterceptor response
+            }
+            response.close()
+            _clearance = null
+            throw IOException("Open webview to refresh token")
+        }
+        .rateLimit(3)
+        .build()
+
+    private val context: Application by injectLazy()
+    private val handler by lazy { Handler(Looper.getMainLooper()) }
+    private var _clearance: String? = null
+
+    @SuppressLint("SetJavaScriptEnabled")
+    fun getClearance(): String? {
+        _clearance?.also { return it }
+        val latch = CountDownLatch(1)
+        handler.post {
+            val webview = WebView(context)
+            with(webview.settings) {
+                javaScriptEnabled = true
+                domStorageEnabled = true
+                databaseEnabled = true
+                blockNetworkImage = true
+            }
+            webview.webViewClient = object : WebViewClient() {
+                override fun onPageFinished(view: WebView?, url: String?) {
+                    view!!.evaluateJavascript("window.localStorage.getItem('clearance')") { clearance ->
+                        webview.stopLoading()
+                        webview.destroy()
+                        _clearance = clearance.takeUnless { it == "null" }?.removeSurrounding("\"")
+                        latch.countDown()
+                    }
+                }
+            }
+            webview.loadUrl("$domainUrl/")
+        }
+        latch.await(10, TimeUnit.SECONDS)
+        return _clearance
+    }
 
     private fun getManga(book: Entry) = SManga.create().apply {
         setUrlWithoutDomain("${book.id}/${book.key}")
@@ -154,7 +213,7 @@ class Koharu(
             else -> "0"
         }
 
-        val imagesResponse = interceptedClient.newCall(GET("$apiBooksUrl/data/$entryId/$entryKey/$id/$public_key/$realQuality?crt=$token", lazyHeaders)).execute()
+        val imagesResponse = clearanceClient.newCall(GET("$apiBooksUrl/data/$entryId/$entryKey/$id/$public_key/$realQuality", lazyHeaders)).execute()
         val images = imagesResponse.parseAs<ImagesInfo>() to realQuality
         return images
     }
@@ -366,7 +425,7 @@ class Koharu(
     // Page List
 
     override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
-        return interceptedClient.newCall(pageListRequest(chapter))
+        return clearanceClient.newCall(pageListRequest(chapter))
             .asObservableSuccess()
             .map { response ->
                 pageListParse(response)
@@ -374,7 +433,7 @@ class Koharu(
     }
 
     override fun pageListRequest(chapter: SChapter): Request {
-        return POST("$apiBooksUrl/detail/${chapter.url}?crt=$token", lazyHeaders)
+        return POST("$apiBooksUrl/detail/${chapter.url}", lazyHeaders)
     }
 
     override fun pageListParse(response: Response): List<Page> {
@@ -433,9 +492,6 @@ class Koharu(
         private const val PREF_REM_ADD = "pref_remove_additional"
         private const val PREF_EXCLUDE_TAGS = "pref_exclude_tags"
 
-        internal var token: String? = null
-        internal var authorization: String? = null
-
         internal val dateReformat = SimpleDateFormat("EEEE, d MMM yyyy HH:mm (z)", Locale.ENGLISH)
     }
 }

src/all/koharu/src/eu/kanade/tachiyomi/extension/all/koharu/TurnstileInterceptor.kt

@@ -1,248 +0,0 @@
-package eu.kanade.tachiyomi.extension.all.koharu
-
-import android.annotation.SuppressLint
-import android.app.Application
-import android.os.Handler
-import android.os.Looper
-import android.webkit.WebResourceRequest
-import android.webkit.WebResourceResponse
-import android.webkit.WebView
-import android.webkit.WebViewClient
-import eu.kanade.tachiyomi.extension.all.koharu.Koharu.Companion.authorization
-import eu.kanade.tachiyomi.extension.all.koharu.Koharu.Companion.token
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.POST
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.runBlocking
-import okhttp3.Headers
-import okhttp3.Interceptor
-import okhttp3.OkHttpClient
-import okhttp3.Response
-import uy.kohesive.injekt.injectLazy
-import java.io.IOException
-import java.util.concurrent.CountDownLatch
-import java.util.concurrent.TimeUnit
-
-/** Cloudflare Turnstile interceptor */
-class TurnstileInterceptor(
-    private val client: OkHttpClient,
-    private val domainUrl: String,
-    private val authUrl: String,
-    private val userAgent: String?,
-) : Interceptor {
-
-    private val context: Application by injectLazy()
-    private val handler by lazy { Handler(Looper.getMainLooper()) }
-
-    private val lazyHeaders by lazy {
-        Headers.Builder().apply {
-            set("User-Agent", userAgent ?: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
-            set("Referer", "$domainUrl/")
-            set("Origin", domainUrl)
-        }.build()
-    }
-
-    private fun authHeaders(authorization: String) =
-        Headers.Builder().apply {
-            set("User-Agent", userAgent ?: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36")
-            set("Referer", "$domainUrl/")
-            set("Origin", domainUrl)
-            set("Authorization", authorization)
-        }.build()
-
-    private val authorizedRequestRegex by lazy { Regex("""(.+\?crt=)(.*)""") }
-
-    override fun intercept(chain: Interceptor.Chain): Response {
-        val request = chain.request()
-
-        val url = request.url.toString()
-        val matchResult = authorizedRequestRegex.find(url) ?: return chain.proceed(request)
-        if (matchResult.groupValues.size == 3) {
-            val requestingUrl = matchResult.groupValues[1]
-            val crt = matchResult.groupValues[2]
-            var newResponse: Response
-
-            if (crt.isNotBlank() && crt != "null") {
-                // Token already set in URL, just make the request
-                newResponse = chain.proceed(request)
-                if (newResponse.code !in listOf(400, 403)) return newResponse
-            } else {
-                // Token doesn't include, add token then make request
-                if (token.isNullOrBlank()) resolveInWebview()
-                val newRequest = if (request.method == "POST") {
-                    POST("${requestingUrl}$token", lazyHeaders)
-                } else {
-                    GET("${requestingUrl}$token", lazyHeaders)
-                }
-                newResponse = chain.proceed(newRequest)
-                if (newResponse.code !in listOf(400, 403)) return newResponse
-            }
-            newResponse.close()
-
-            // Request failed, refresh token then try again
-            clearToken()
-            token = null
-            resolveInWebview()
-            val newRequest = if (request.method == "POST") {
-                POST("${requestingUrl}$token", lazyHeaders)
-            } else {
-                GET("${requestingUrl}$token", lazyHeaders)
-            }
-            newResponse = chain.proceed(newRequest)
-            if (newResponse.code !in listOf(400, 403)) return newResponse
-            throw IOException("Open webview once to refresh token (${newResponse.code})")
-        }
-        return chain.proceed(request)
-    }
-
-    @SuppressLint("SetJavaScriptEnabled")
-    fun resolveInWebview(): Pair<String?, String?> {
-        val latch = CountDownLatch(1)
-        var webView: WebView? = null
-
-        handler.post {
-            val webview = WebView(context)
-            webView = webview
-            with(webview.settings) {
-                javaScriptEnabled = true
-                domStorageEnabled = true
-                databaseEnabled = true
-                useWideViewPort = false
-                loadWithOverviewMode = false
-            }
-
-            webview.webViewClient = object : WebViewClient() {
-                override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? {
-                    val authHeader = request?.requestHeaders?.get("Authorization")
-                    if (request?.url.toString().contains(authUrl) && authHeader != null) {
-                        authorization = authHeader
-                        if (request.method == "POST") {
-                            // Authorize & requesting a new token.
-                            // `authorization` here should be in format: Bearer <authorization>
-                            try {
-                                val noRedirectClient = client.newBuilder().followRedirects(false).build()
-                                val authHeaders = authHeaders(authHeader)
-                                val response = runBlocking(Dispatchers.IO) {
-                                    noRedirectClient.newCall(POST(authUrl, authHeaders)).execute()
-                                }
-                                response.use {
-                                    if (response.isSuccessful) {
-                                        with(response) {
-                                            token = body.string()
-                                                .removeSurrounding("\"")
-                                        }
-                                    }
-                                }
-                            } catch (_: IOException) {
-                            } finally {
-                                latch.countDown()
-                            }
-                        }
-                        if (request.method == "GET") {
-                            // Site is trying to recheck old token validation here.
-                            // If it fails then site will request a new one using POST method.
-                            // But we will check it ourselves.
-                            // Normally this might not occur because old token should already be acquired & rechecked via onPageFinished.
-                            // `authorization` here should be in format: Bearer <token>
-                            val oldToken = authorization
-                                ?.substringAfterLast(" ")
-                            if (oldToken != null && recheckTokenValid(oldToken)) {
-                                token = oldToken
-                                latch.countDown()
-                            }
-                        }
-                    }
-                    return super.shouldInterceptRequest(view, request)
-                }
-
-                /**
-                 * Read the saved token in localStorage and use it.
-                 * This token might already expired. Normally site will check token for expiration with a GET request.
-                 * Here will will recheck it ourselves.
-                 */
-                override fun onPageFinished(view: WebView?, url: String?) {
-                    if (view == null) return
-                    val script = "javascript:localStorage['clearance']"
-                    view.evaluateJavascript(script) {
-                        // Avoid overwrite newly requested token
-                        if (!it.isNullOrBlank() && it != "null" && token.isNullOrBlank()) {
-                            val oldToken = it
-                                .removeSurrounding("\"")
-                            if (recheckTokenValid(oldToken)) {
-                                token = oldToken
-                                latch.countDown()
-                            }
-                        }
-                    }
-                }
-
-                private fun recheckTokenValid(token: String): Boolean {
-                    try {
-                        val noRedirectClient = client.newBuilder().followRedirects(false).build()
-                        val authHeaders = authHeaders("Bearer $token")
-                        val response = runBlocking(Dispatchers.IO) {
-                            noRedirectClient.newCall(GET(authUrl, authHeaders)).execute()
-                        }
-                        response.use {
-                            if (response.isSuccessful) {
-                                return true
-                            }
-                        }
-                    } catch (_: IOException) {
-                    }
-                    return false
-                }
-            }
-
-            webview.loadUrl("$domainUrl/")
-        }
-
-        latch.await(20, TimeUnit.SECONDS)
-
-        handler.post {
-            // One last try to read the token from localStorage, in case it got updated last minute.
-            if (token.isNullOrBlank()) {
-                val script = "javascript:localStorage['clearance']"
-                webView?.evaluateJavascript(script) {
-                    if (!it.isNullOrBlank() && it != "null") {
-                        token = it
-                            .removeSurrounding("\"")
-                    }
-                }
-            }
-
-            webView?.stopLoading()
-            webView?.destroy()
-            webView = null
-        }
-
-        return token to authorization
-    }
-
-    @SuppressLint("SetJavaScriptEnabled")
-    private fun clearToken() {
-        val latch = CountDownLatch(1)
-        handler.post {
-            val webView = WebView(context)
-            with(webView.settings) {
-                javaScriptEnabled = true
-                domStorageEnabled = true
-                databaseEnabled = true
-            }
-            webView.webViewClient = object : WebViewClient() {
-                override fun onPageFinished(view: WebView?, url: String?) {
-                    if (view == null) return
-                    val script = "javascript:localStorage.clear()"
-                    view.evaluateJavascript(script) {
-                        token = null
-                        view.stopLoading()
-                        view.destroy()
-                        latch.countDown()
-                    }
-                }
-            }
-            webView.loadUrl(domainUrl)
-        }
-        latch.await(20, TimeUnit.SECONDS)
-    }
-}