Japscan: Fix chapter list, cloudflare issues, and page list not loading (#10186)

Fix chapter list, cloudflare issues, and page list not loading

src/fr/japscan/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'Japscan'
     extClass = '.Japscan'
-    extVersionCode = 50
+    extVersionCode = 51
 }
 
 apply from: "$rootDir/common.gradle"

src/fr/japscan/src/eu/kanade/tachiyomi/extension/fr/japscan/Japscan.kt

@@ -1,8 +1,8 @@
 package eu.kanade.tachiyomi.extension.fr.japscan
 
-import android.annotation.SuppressLint
 import android.app.Application
 import android.content.SharedPreferences
+import android.graphics.Bitmap
 import android.os.Handler
 import android.os.Looper
 import android.view.View
@@ -35,6 +35,7 @@ import okhttp3.Request
 import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
+import rx.Observable
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
 import uy.kohesive.injekt.injectLazy
@@ -49,7 +50,10 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
 
     override val name = "Japscan"
 
-    override val baseUrl = "https://www.japscan.si"
+    // Sometimes an adblock blocker will pop up, preventing the user from opening
+    // a cloudflare protected page
+    private val internalBaseUrl = "https://www.japscan.si"
+    override val baseUrl = "$internalBaseUrl/mangas/?sort=popular&p=1"
 
     override val lang = "fr"
 
@@ -76,11 +80,11 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
     private fun chapterListPref() = preferences.getString(SHOW_SPOILER_CHAPTERS, "hide")
 
     override fun headersBuilder() = super.headersBuilder()
-        .add("referer", "$baseUrl/")
+        .add("referer", "$internalBaseUrl/")
 
     // Popular
     override fun popularMangaRequest(page: Int): Request {
-        return GET("$baseUrl/mangas/?sort=popular&p=$page", headers)
+        return GET("$internalBaseUrl/mangas/?sort=popular&p=$page", headers)
     }
 
     override fun popularMangaNextPageSelector() = ".pagination > li:last-child:not(.disabled)"
@@ -99,7 +103,7 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
 
     // Latest
     override fun latestUpdatesRequest(page: Int): Request {
-        return GET("$baseUrl/mangas/?sort=updated&p=$page", headers)
+        return GET("$internalBaseUrl/mangas/?sort=updated&p=$page", headers)
     }
 
     override fun latestUpdatesSelector() = popularMangaSelector()
@@ -111,7 +115,7 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
     // Search
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
         if (query.isEmpty()) {
-            val url = baseUrl.toHttpUrl().newBuilder().apply {
+            val url = internalBaseUrl.toHttpUrl().newBuilder().apply {
                 addPathSegment("mangas")
 
                 filters.forEach { filter ->
@@ -132,7 +136,7 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
                 .add("X-Requested-With", "XMLHttpRequest")
                 .build()
 
-            return POST("$baseUrl/ls/", searchHeaders, formBody)
+            return POST("$internalBaseUrl/ls/", searchHeaders, formBody)
         }
     }
 
@@ -149,7 +153,7 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
             return MangasPage(mangaList, hasNextPage = false)
         }
 
-        val baseUrlHost = baseUrl.toHttpUrl().host
+        val baseUrlHost = internalBaseUrl.toHttpUrl().host
         val document = response.asJsoup()
         val manga = document
             .select(searchMangaSelector())
@@ -174,7 +178,11 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
     private fun searchMangaFromJson(jsonObj: JsonObject): SManga = SManga.create().apply {
         url = jsonObj["url"]!!.jsonPrimitive.content
         title = jsonObj["name"]!!.jsonPrimitive.content
-        thumbnail_url = baseUrl + jsonObj["image"]!!.jsonPrimitive.content
+        thumbnail_url = internalBaseUrl + jsonObj["image"]!!.jsonPrimitive.content
+    }
+
+    override fun mangaDetailsRequest(manga: SManga): Request {
+        return GET(internalBaseUrl + manga.url, headers)
     }
 
     override fun mangaDetailsParse(document: Document): SManga {
@@ -194,7 +202,7 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
                 }
             }
         }
-        manga.description = infoElement.selectFirst("#synopsis")?.ownText().orEmpty()
+        manga.description = infoElement.selectFirst("div:contains(Synopsis) + p")?.ownText().orEmpty()
 
         return manga
     }
@@ -205,13 +213,21 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
         else -> SManga.UNKNOWN
     }
 
+    override fun getChapterUrl(chapter: SChapter): String {
+        return internalBaseUrl + chapter.url
+    }
+
+    override fun chapterListRequest(manga: SManga): Request {
+        return GET(internalBaseUrl + manga.url, headers)
+    }
+
     override fun chapterListSelector() = "#list_chapters > div.collapse > div.list_chapters" +
         if (chapterListPref() == "hide") { ":not(:has(.badge:contains(SPOILER),.badge:contains(RAW),.badge:contains(VUS)))" } else { "" }
     // JapScan sometimes uploads some "spoiler preview" chapters, containing 2 or 3 untranslated pictures taken from a raw. Sometimes they also upload full RAWs/US versions and replace them with a translation as soon as available.
     // Those have a span.badge "SPOILER" or "RAW". The additional pseudo selector makes sure to exclude these from the chapter list.
 
     override fun chapterFromElement(element: Element): SChapter {
-        val urlElement = element.selectFirst("a")!!
+        val urlElement = element.selectFirst("*[href~=manga]")!!
 
         val chapter = SChapter.create()
         chapter.setUrlWithoutDomain(urlElement.attr("href"))
@@ -225,12 +241,30 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
         dateFormat.parse(date)!!.time
     }.getOrDefault(0L)
 
-    @SuppressLint("SetJavaScriptEnabled")
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
-    override fun pageListParse(document: Document): List<Page> {
         val interfaceName = randomString()
-        document.body().prepend(
+
+        val handler = Handler(Looper.getMainLooper())
+        val latch = CountDownLatch(1)
+        val jsInterface = JsInterface(latch)
+        var webView: WebView? = null
+
+        handler.post {
+            val innerWv = WebView(Injekt.get<Application>())
+
+            webView = innerWv
+            innerWv.settings.domStorageEnabled = true
+            innerWv.settings.javaScriptEnabled = true
+            innerWv.settings.blockNetworkImage = true
+            innerWv.settings.userAgentString = headers["User-Agent"]
+            innerWv.setLayerType(View.LAYER_TYPE_SOFTWARE, null)
+            innerWv.addJavascriptInterface(jsInterface, interfaceName)
+
+            innerWv.webViewClient = object : WebViewClient() {
+                override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) {
+                    super.onPageStarted(view, url, favicon)
+                    view?.evaluateJavascript(
                         """
-            <script>
                             Object.defineProperty(Object.prototype, 'imagesLink', {
                                 set: function(value) {
                                     window.$interfaceName.passPayload(JSON.stringify(value));
@@ -247,40 +281,14 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
                                 enumerable: false,
                                 configurable: true
                             });
-            </script>
                         """.trimIndent(),
-        )
+                    ) {}
-
-        val handler = Handler(Looper.getMainLooper())
-        val latch = CountDownLatch(1)
-        val jsInterface = JsInterface(latch)
-        var webView: WebView? = null
-
-        handler.post {
-            val innerWv = WebView(Injekt.get<Application>())
-
-            webView = innerWv
-            innerWv.settings.javaScriptEnabled = true
-            innerWv.settings.blockNetworkImage = true
-            innerWv.setLayerType(View.LAYER_TYPE_SOFTWARE, null)
-            innerWv.addJavascriptInterface(jsInterface, interfaceName)
-
-            innerWv.webViewClient = object : WebViewClient() {
-                override fun shouldOverrideUrlLoading(
-                    view: WebView?,
-                    url: String?,
-                ): Boolean {
-                    url ?: return true
-                    return !url.contains("/zjs/")
                 }
             }
 
-            innerWv.loadDataWithBaseURL(
+            innerWv.loadUrl(
-                document.location(),
+                "$internalBaseUrl${chapter.url}",
-                document.outerHtml(),
+                headers.toMap(),
-                "text/html",
-                "UTF-8",
-                null,
             )
         }
 
@@ -291,16 +299,19 @@ class Japscan : ConfigurableSource, ParsedHttpSource() {
             throw Exception("Timed out decrypting image links")
         }
 
-        val baseUrlHost = baseUrl.toHttpUrl().host.substringAfter("www.")
+        val baseUrlHost = internalBaseUrl.toHttpUrl().host.substringAfter("www.")
-
+        val images = jsInterface
-        return jsInterface
             .images
             .filter { it.toHttpUrl().host.endsWith(baseUrlHost) } // Pages not served through their CDN are probably ads
             .mapIndexed { i, url ->
                 Page(i, imageUrl = url)
             }
+
+        return Observable.just(images)
     }
 
+    override fun pageListParse(document: Document) = throw Exception("Not used")
+
     override fun imageUrlParse(document: Document): String = ""
 
     // Filters