Asura: attempt to find new url (#16350)

* Asura Scans: attempt to find new url

* bump

* store slugMap in SharedPreference

* use `MutableMap.getOrPut()`

* use existing instance of json

* fix no page on Asura Tr

* Don't use rng slug to for search

* retry ci

* use dbSlug as key

permaSlug could end up causing problems when a user migrates to update the url as the extension would still end up using the storedSlug from the Map instead of the more updated one from db.

* attempt to make search better

their search is really dogshit
`player who returned 10000 years later` doesn't work while `player who returned 10 000 years later` works

* suggested changes

* remove initialized check in `fetchMangaDetails`

ddos time

* fix deep links

* use updated slug for WebView

* make sure random user-agent is applied

multisrc/overrides/mangathemesia/asurascans/src/AsuraScans.kt

@@ -1,22 +1,24 @@
 package eu.kanade.tachiyomi.extension.all.asurascans
 
 import android.app.Application
-import android.content.SharedPreferences
-import androidx.preference.PreferenceScreen
-import androidx.preference.SwitchPreferenceCompat
 import eu.kanade.tachiyomi.multisrc.mangathemesia.MangaThemesia
 import eu.kanade.tachiyomi.network.interceptor.rateLimit
-import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import okhttp3.Interceptor
 import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import rx.Observable
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
+import java.io.IOException
 import java.text.SimpleDateFormat
 import java.util.concurrent.TimeUnit
 
@@ -29,31 +31,46 @@ open class AsuraScans(
     baseUrl,
     lang,
     dateFormat = dateFormat,
-),
+) {
-    ConfigurableSource {
+    private val preferences = Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
-
-    private val preferences: SharedPreferences by lazy {
-        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
-    }
 
     override val client: OkHttpClient = network.cloudflareClient.newBuilder()
+        .addInterceptor(::urlChangeInterceptor)
         .addInterceptor(uaIntercept)
         .connectTimeout(10, TimeUnit.SECONDS)
         .readTimeout(30, TimeUnit.SECONDS)
         .rateLimit(1, 3, TimeUnit.SECONDS)
         .build()
 
-    // Permanent Url for Manga/Chapter End
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
-    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
+        val newManga = manga.apply {
-        return super.fetchPopularManga(page).tempUrlToPermIfNeeded()
+            url = "$url#chapters#${title.toSearchQuery()}"
+        }
+        return super.fetchChapterList(newManga)
     }
 
-    override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
-        return super.fetchLatestUpdates(page).tempUrlToPermIfNeeded()
+        val newManga = try {
+            manga.apply {
+                url = "$url#details#${title.toSearchQuery()}"
+            }
+        } catch (e: UninitializedPropertyAccessException) {
+            // when called from deep link, title is not present
+            manga
+        }
+
+        return super.fetchMangaDetails(newManga)
     }
 
-    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+    // use updated url for webView
-        return super.fetchSearchManga(page, query, filters).tempUrlToPermIfNeeded()
+    override fun getMangaUrl(manga: SManga): String {
+        val dbSlug = manga.url
+            .removeSuffix("/")
+            .substringAfterLast("/")
+
+        val storedSlug = getSlugMap()[dbSlug] ?: dbSlug
+
+        return "$baseUrl$mangaUrlDirectory/$storedSlug/"
     }
 
     // Skip scriptPages
@@ -70,55 +87,100 @@ open class AsuraScans(
         else -> attr("abs:src")
     }
 
-    private fun Observable<MangasPage>.tempUrlToPermIfNeeded(): Observable<MangasPage> {
+    private fun urlChangeInterceptor(chain: Interceptor.Chain): Response {
-        return this.map { mangasPage ->
+        val request = chain.request()
-            MangasPage(
+        val frag = request.url.fragment
-                mangasPage.mangas.map { it.tempUrlToPermIfNeeded() },
+
-                mangasPage.hasNextPage,
+        if (frag.isNullOrEmpty()) {
-            )
+            return chain.proceed(request)
         }
+
+        val search = frag.substringAfter("#")
+
+        val dbSlug = request.url.toString()
+            .substringBefore("#")
+            .removeSuffix("/")
+            .substringAfterLast("/")
+
+        val slugMap = getSlugMap().toMutableMap()
+
+        // make sure db slug key is present in the slugMap
+        val storedSlug = slugMap[dbSlug] ?: dbSlug
+
+        val response = chain.proceed(newRequest(frag, storedSlug))
+
+        if (!response.isSuccessful && response.code == 404) {
+            response.close()
+
+            val newSlug = getNewSlug(storedSlug, search)
+                ?: throw IOException("Migrate from Asura to Asura")
+
+            slugMap[dbSlug] = newSlug
+            putSlugMap(slugMap)
+
+            return chain.proceed(newRequest(frag, newSlug))
+        }
+
+        return response
     }
 
-    private fun SManga.tempUrlToPermIfNeeded(): SManga {
+    private fun getNewSlug(existingSlug: String, search: String): String? {
-        val turnTempUrlToPerm = preferences.getBoolean(getPermanentMangaUrlPreferenceKey(), true)
+        val permaSlug = existingSlug
-        if (!turnTempUrlToPerm) return this
+            .replaceFirst(TEMP_TO_PERM_REGEX, "")
 
-        val sMangaTitleFirstWord = this.title.split(" ")[0]
+        val mangas = client.newCall(searchMangaRequest(1, search, FilterList()))
-        if (!this.url.contains("/$sMangaTitleFirstWord", ignoreCase = true)) {
+            .execute()
-            this.url = this.url.replaceFirst(TEMP_TO_PERM_URL_REGEX, "$1")
+            .use {
-        }
+                searchMangaParse(it)
-        return this
-    }
-
-    override fun setupPreferenceScreen(screen: PreferenceScreen) {
-        val permanentMangaUrlPref = SwitchPreferenceCompat(screen.context).apply {
-            key = getPermanentMangaUrlPreferenceKey()
-            title = PREF_PERM_MANGA_URL_TITLE
-            summary = PREF_PERM_MANGA_URL_SUMMARY
-            setDefaultValue(true)
-
-            setOnPreferenceChangeListener { _, newValue ->
-                val checkValue = newValue as Boolean
-                preferences.edit()
-                    .putBoolean(getPermanentMangaUrlPreferenceKey(), checkValue)
-                    .commit()
             }
+
+        return mangas.mangas.firstOrNull { newManga ->
+            newManga.url.contains(permaSlug, true)
+        }
+            ?.url
+            ?.removeSuffix("/")
+            ?.substringAfterLast("/")
+    }
+
+    private fun newRequest(frag: String, slug: String): Request {
+        val manga = SManga.create().apply {
+            this.url = "$mangaUrlDirectory/$slug/"
         }
 
-        screen.addPreference(permanentMangaUrlPref)
+        return when (frag.substringBefore("#")) {
-        addRandomAndCustomUserAgentPreferences(screen)
+            "chapters" -> chapterListRequest(manga)
+            "details" -> mangaDetailsRequest(manga)
+            else -> throw IOException("unknown url fragment for urlChangeInterceptor")
+        }
     }
 
-    private fun getPermanentMangaUrlPreferenceKey(): String {
+    private fun putSlugMap(slugMap: MutableMap<String, String>) {
-        return PREF_PERM_MANGA_URL_KEY_PREFIX + lang
+        val serialized = json.encodeToString(slugMap)
+
+        preferences.edit().putString(PREF_URL_MAP, serialized).commit()
+    }
+
+    private fun getSlugMap(): Map<String, String> {
+        val serialized = preferences.getString(PREF_URL_MAP, null) ?: return emptyMap()
+
+        return try {
+            json.decodeFromString(serialized)
+        } catch (e: Exception) {
+            emptyMap()
+        }
+    }
+
+    private fun String.toSearchQuery(): String {
+        return this.trim()
+            .lowercase()
+            .replace(titleSpecialCharactersRegex, "+")
+            .replace(trailingPlusRegex, "")
     }
-    // Permanent Url for Manga/Chapter End
 
     companion object {
-        private const val PREF_PERM_MANGA_URL_KEY_PREFIX = "pref_permanent_manga_url_"
+        private const val PREF_URL_MAP = "pref_url_map"
-        private const val PREF_PERM_MANGA_URL_TITLE = "Permanent Manga URL"
+        private val TEMP_TO_PERM_REGEX = Regex("""^\d+-""")
-        private const val PREF_PERM_MANGA_URL_SUMMARY = "Turns all manga urls into permanent ones."
+        private val titleSpecialCharactersRegex = Regex("""[^a-z0-9]+""")
-
+        private val trailingPlusRegex = Regex("""\++$""")
-        private val TEMP_TO_PERM_URL_REGEX = Regex("""(/)\d+-""")
     }
 }

multisrc/overrides/mangathemesia/asurascans/src/AsuraScansFactory.kt

@@ -1,7 +1,11 @@
 package eu.kanade.tachiyomi.extension.all.asurascans
 
 import eu.kanade.tachiyomi.source.SourceFactory
+import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SManga
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import org.jsoup.nodes.Document
 import java.text.SimpleDateFormat
 import java.util.Locale
 
@@ -12,7 +16,7 @@ class AsuraScansFactory : SourceFactory {
     )
 }
 
-class AsuraScansEn : AsuraScans("https://asura.gg", "en", SimpleDateFormat("MMM d, yyyy", Locale.US)) {
+class AsuraScansEn : AsuraScans("https://www.asurascans.com", "en", SimpleDateFormat("MMM d, yyyy", Locale.US)) {
 
     override val seriesDescriptionSelector = "div.desc p, div.entry-content p, div[itemprop=description]:not(:has(p))"
 
@@ -35,4 +39,23 @@ class AsuraScansTr : AsuraScans("https://asurascanstr.com", "tr", SimpleDateForm
         this.contains("Tamamlandı", ignoreCase = true) -> SManga.COMPLETED
         else -> SManga.UNKNOWN
     }
+
+    override fun pageListParse(document: Document): List<Page> {
+        val scriptContent = document.selectFirst("script:containsData(ts_reader)")!!.data()
+        val jsonString = scriptContent.substringAfter("ts_reader.run(").substringBefore(");")
+        val tsReader = json.decodeFromString<TSReader>(jsonString)
+        val imageUrls = tsReader.sources.firstOrNull()?.images ?: return emptyList()
+        return imageUrls.mapIndexed { index, imageUrl -> Page(index, imageUrl = imageUrl) }
+    }
+
+    @Serializable
+    data class TSReader(
+        val sources: List<ReaderImageSource>,
+    )
+
+    @Serializable
+    data class ReaderImageSource(
+        val source: String,
+        val images: List<String>,
+    )
 }

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangathemesia/MangaThemesiaGenerator.kt

@@ -14,7 +14,7 @@ class MangaThemesiaGenerator : ThemeSourceGenerator {
     override val baseVersionCode: Int = 25
 
     override val sources = listOf(
-        MultiLang("Asura Scans", "https://www.asurascans.com", listOf("en", "tr"), className = "AsuraScansFactory", pkgName = "asurascans", overrideVersionCode = 20),
+        MultiLang("Asura Scans", "https://www.asurascans.com", listOf("en", "tr"), className = "AsuraScansFactory", pkgName = "asurascans", overrideVersionCode = 21),
         MultiLang("Flame Scans", "https://flamescans.org", listOf("en"), className = "FlameScansFactory", pkgName = "flamescans", overrideVersionCode = 4),
         MultiLang("Komik Lab", "https://komiklab.com", listOf("en", "id"), className = "KomikLabFactory", pkgName = "komiklab", overrideVersionCode = 1),
         MultiLang("Miau Scan", "https://miauscan.com", listOf("es", "pt-BR")),