MMRCMS: Dynamic filter rework (#1315)

* MMRCMS: Dynamic filter rework, remove Last updated sort in Mangas.in

* Formatting

* Show the reset message when filters are fetching

* Linting

* Dynamically fetch sort options

* Add i18n support

* Remove unused import

lib-multisrc/mmrcms/assets/i18n/messages_en.properties

@@ -0,0 +1,10 @@
+filter_warning=Ignored if using text search
+filter_missing_warning=Press 'Reset' to attempt to show filters
+category_filter_title=Category
+status_filter_title=Status
+type_filter_title=Type
+year_filter_title=Year of release
+author_filter_title=Author
+tag_filter_title=Tag
+title_begins_with_filter_title=Title begins with
+sort_by_filter_title=Sort by

lib-multisrc/mmrcms/build.gradle.kts

@@ -2,4 +2,8 @@ plugins {
     id("lib-multisrc")
 }
 
-baseVersionCode = 9
+baseVersionCode = 10
+
+dependencies {
+    api(project(":lib:i18n"))
+}

lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMS.kt

@@ -2,11 +2,11 @@ package eu.kanade.tachiyomi.multisrc.mmrcms
 
 import android.annotation.SuppressLint
 import android.util.Log
-import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMSUtils.imgAttr
+import eu.kanade.tachiyomi.lib.i18n.Intl
-import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMSUtils.textWithNewlines
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.await
 import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
@@ -15,6 +15,9 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
 import okhttp3.FormBody
@@ -23,14 +26,12 @@ import okhttp3.Request
 import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
+import org.jsoup.select.Elements
 import rx.Observable
-import rx.Single
-import rx.Subscription
-import rx.schedulers.Schedulers
 import uy.kohesive.injekt.injectLazy
+import java.text.ParseException
 import java.text.SimpleDateFormat
 import java.util.Locale
-import java.util.concurrent.locks.ReentrantLock
 
 /**
  * @param dateFormat The date format used for parsing chapter dates.
@@ -50,7 +51,7 @@ constructor(
 
     vararg useNamedArgumentsBelow: Forbidden,
 
-    private val dateFormat: SimpleDateFormat = SimpleDateFormat("d MMM. yyyy", Locale.US),
+    protected val dateFormat: SimpleDateFormat = SimpleDateFormat("d MMM. yyyy", Locale.US),
     protected val itemPath: String = "manga",
     private val fetchFilterOptions: Boolean = true,
     private val supportsAdvancedSearch: Boolean = true,
@@ -70,6 +71,13 @@ constructor(
 
     protected val json: Json by injectLazy()
 
+    protected val intl = Intl(
+        lang,
+        setOf("en"),
+        "en",
+        this::class.java.classLoader!!,
+    )
+
     override fun popularMangaRequest(page: Int) = GET("$baseUrl/filterList?page=$page&sortBy=views&asc=false")
 
     override fun popularMangaSelector() = searchMangaSelector()
@@ -117,16 +125,13 @@ constructor(
 
     protected var searchDirectory = emptyList<SuggestionDto>()
 
-    private var searchQuery = ""
-
     override fun fetchSearchManga(
         page: Int,
         query: String,
         filters: FilterList,
     ): Observable<MangasPage> {
         return if (query.isNotEmpty()) {
-            if (page == 1 && query != searchQuery) {
+            if (page == 1) {
-                searchQuery = query
                 client.newCall(searchMangaRequest(page, query, filters))
                     .asObservableSuccess()
                     .map { searchMangaParse(it) }
@@ -197,26 +202,23 @@ constructor(
 
         setUrlWithoutDomain(anchor.attr("href"))
         title = anchor.text()
-        thumbnail_url = MMRCMSUtils.guessCover(baseUrl, url, element.selectFirst("img")?.imgAttr())
+        thumbnail_url = guessCover(url, element.selectFirst("img")?.imgAttr())
     }
 
     override fun searchMangaNextPageSelector(): String? = ".pagination a[rel=next]"
 
     protected fun parseSearchDirectory(page: Int): MangasPage {
-        val manga = mutableListOf<SManga>()
+        val manga = searchDirectory.subList((page - 1) * 24, page * 24)
-        val endRange = ((page * 24) - 1).let { if (it <= searchDirectory.lastIndex) it else searchDirectory.lastIndex }
+            .map {
-
-        for (i in (((page - 1) * 24)..endRange)) {
-            manga.add(
                 SManga.create().apply {
-                    url = "/$itemPath/${searchDirectory[i].data}"
+                    url = "/$itemPath/${it.data}"
-                    title = searchDirectory[i].value
+                    title = it.value
-                    thumbnail_url = MMRCMSUtils.guessCover(baseUrl, url, null)
+                    thumbnail_url = guessCover(url, null)
-                },
-            )
                 }
+            }
+        val hasNextPage = (page + 1) * 24 <= searchDirectory.size
 
-        return MangasPage(manga, endRange < searchDirectory.lastIndex)
+        return MangasPage(manga, hasNextPage)
     }
 
     protected val detailAuthor = hashSetOf("author(s)", "autor(es)", "auteur(s)", "著作", "yazar(lar)", "mangaka(lar)", "pengarang/penulis", "pengarang", "penulis", "autor", "المؤلف", "перевод", "autor/autorzy")
@@ -230,8 +232,7 @@ constructor(
     @SuppressLint("DefaultLocale")
     override fun mangaDetailsParse(document: Document) = SManga.create().apply {
         title = document.selectFirst(detailsTitleSelector)!!.text()
-        thumbnail_url = MMRCMSUtils.guessCover(
+        thumbnail_url = guessCover(
-            baseUrl,
             document.location(),
             document.selectFirst(".row img.img-responsive")?.imgAttr(),
         )
@@ -274,16 +275,16 @@ constructor(
 
         setUrlWithoutDomain(anchor.attr("href"))
         name = cleanChapterName(mangaTitle, titleWrapper.text())
-        date_upload = runCatching {
+        date_upload = try {
             val date = element.selectFirst(".date-chapter-title-rtl")!!.text()
 
             dateFormat.parse(date)!!.time
-        }.getOrDefault(0L)
+        } catch (_: ParseException) {
+            0L
+        } catch (_: NullPointerException) {
+            0L
+        }
     }
-
-    /**
-     * The word for "Chapter" in your language.
-     */
 
     /**
      * Function to clean up chapter names. Mostly useful for sites that
@@ -309,22 +310,20 @@ constructor(
     override fun imageUrlParse(document: Document) = throw UnsupportedOperationException()
 
     override fun getFilterList(): FilterList {
-        runCatching { fetchFilterOptions() }
+        fetchFilterOptions()
 
-        val filters = buildList<Filter<*>> {
+        val filters = buildList {
-            add(Filter.Header("Note: Ignored if using text search!"))
+            add(Filter.Header(intl["filter_warning"]))
-
+            if (fetchFilterOptions && fetchFiltersStatus != FetchFilterStatus.FETCHED) {
-            if (supportsAdvancedSearch) {
+                add(Filter.Header(intl["filter_missing_warning"]))
-                if (fetchFilterOptions && (categories.isEmpty() || statuses.isEmpty())) {
-                    add(Filter.Header("Press 'Reset' to attempt to show filter options"))
             }
-
             add(Filter.Separator())
 
+            if (supportsAdvancedSearch) {
                 if (categories.isNotEmpty()) {
                     add(
                         UriMultiSelectFilter(
-                            "Categories",
+                            intl["category_filter_title"],
                             "categories[]",
                             categories.toTypedArray(),
                         ),
@@ -334,7 +333,7 @@ constructor(
                 if (statuses.isNotEmpty()) {
                     add(
                         UriMultiSelectFilter(
-                            "Statuses",
+                            intl["status_filter_title"],
                             "status[]",
                             statuses.toTypedArray(),
                         ),
@@ -344,26 +343,20 @@ constructor(
                 if (tags.isNotEmpty()) {
                     add(
                         UriMultiSelectFilter(
-                            "Types",
+                            intl["type_filter_title"],
                             "types[]",
                             tags.toTypedArray(),
                         ),
                     )
                 }
 
-                add(TextFilter("Year of release", "release"))
+                add(TextFilter(intl["year_filter_title"], "release"))
-                add(TextFilter("Author", "author"))
+                add(TextFilter(intl["author_filter_title"], "author"))
             } else {
-                if (fetchFilterOptions && categories.isEmpty()) {
-                    add(Filter.Header("Press 'Reset' to attempt to show filter options"))
-                }
-
-                add(Filter.Separator())
-
                 if (categories.isNotEmpty()) {
                     add(
                         UriPartFilter(
-                            "Category",
+                            intl["category_filter_title"],
                             "cat",
                             arrayOf(
                                 "Any" to "",
@@ -373,23 +366,12 @@ constructor(
                     )
                 }
 
-                add(
+                add(UriPartFilter(intl["title_begins_with_filter_title"], "alpha", alphaOptions))
-                    UriPartFilter(
-                        "Title begins with",
-                        "alpha",
-                        arrayOf(
-                            "Any" to "",
-                            *"#ABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray().map {
-                                Pair(it.toString(), it.toString())
-                            }.toTypedArray(),
-                        ),
-                    ),
-                )
 
                 if (tags.isNotEmpty()) {
                     add(
                         UriPartFilter(
-                            "Tag",
+                            intl["tag_filter_title"],
                             "tag",
                             arrayOf(
                                 "Any" to "",
@@ -399,40 +381,49 @@ constructor(
                     )
                 }
 
-                add(SortFilter())
+                if (sortOptions.isNotEmpty()) {
+                    add(SortFilter(intl, sortOptions))
+                }
             }
         }
 
         return FilterList(filters)
     }
 
+    private val alphaOptions by lazy {
+        arrayOf(
+            "Any" to "",
+            *"#ABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray().map {
+                Pair(it.toString(), it.toString())
+            }.toTypedArray(),
+        )
+    }
     private var categories = emptyList<Pair<String, String>>()
-
     private var statuses = emptyList<Pair<String, String>>()
-
     private var tags = emptyList<Pair<String, String>>()
+    private var sortOptions = emptyArray<Pair<String, String>>()
 
-    private var fetchFiltersFailed = false
+    private var fetchFiltersStatus = FetchFilterStatus.NOT_FETCHED
-
     private var fetchFiltersAttempts = 0
+    private val scope = CoroutineScope(Dispatchers.IO)
 
-    private val fetchFiltersLock = ReentrantLock()
+    protected open fun fetchFilterOptions() {
-
-    protected open fun fetchFilterOptions(): Subscription = Single.fromCallable {
         if (!fetchFilterOptions) {
-            return@fromCallable
+            return
         }
 
-        fetchFiltersLock.lock()
+        if (fetchFiltersStatus != FetchFilterStatus.NOT_FETCHED || fetchFiltersAttempts >= 3) {
-
+            return
-        if (fetchFiltersAttempts > 3 || (fetchFiltersAttempts > 0 && !fetchFiltersFailed)) {
-            fetchFiltersLock.unlock()
-            return@fromCallable
         }
 
-        fetchFiltersFailed = try {
+        fetchFiltersStatus = FetchFilterStatus.FETCHING
+        fetchFiltersAttempts++
+        scope.launch {
+            try {
                 if (supportsAdvancedSearch) {
-                val document = client.newCall(GET("$baseUrl/advanced-search", headers)).execute().asJsoup()
+                    val document = client.newCall(GET("$baseUrl/advanced-search", headers))
+                        .await()
+                        .asJsoup()
 
                     categories = document.select("select[name='categories[]'] option").map {
                         it.text() to it.attr("value")
@@ -444,7 +435,9 @@ constructor(
                         it.text() to it.attr("value")
                     }
                 } else {
-                val document = client.newCall(GET("$baseUrl/$itemPath-list", headers)).execute().asJsoup()
+                    val document = client.newCall(GET("$baseUrl/$itemPath-list", headers))
+                        .await()
+                        .asJsoup()
 
                     categories = document.select("a.category").map {
                         it.text() to it.attr("href").toHttpUrl().queryParameter("cat")!!
@@ -452,18 +445,43 @@ constructor(
                     tags = document.select("div.tag-links a").map {
                         it.text() to it.attr("href").toHttpUrl().pathSegments.last()
                     }
+                    sortOptions = document.select("#sort-types label:has(input)").map {
+                        it.ownText() to it.selectFirst("input")!!.id()
+                    }.toTypedArray()
                 }
 
-            false
+                fetchFiltersStatus = FetchFilterStatus.FETCHED
-        } catch (e: Throwable) {
+            } catch (e: Exception) {
-            Log.e(name, "Could not fetch filtering options", e)
+                fetchFiltersStatus = FetchFilterStatus.NOT_FETCHED
-            true
+                Log.e("MMRCMS/$name", "Could not fetch filters", e)
+            }
+        }
     }
 
-        fetchFiltersAttempts++
+    protected fun guessCover(mangaUrl: String, url: String?): String {
-        fetchFiltersLock.unlock()
+        return if (url == null || url.endsWith("no-image.png")) {
+            "$baseUrl/uploads/manga/${mangaUrl.substringAfterLast('/')}/cover/cover_250x350.jpg"
+        } else {
+            url
         }
-        .subscribeOn(Schedulers.io())
+    }
-        .observeOn(Schedulers.io())
+
-        .subscribe()
+    protected fun Element.imgAttr(): String = when {
+        hasAttr("data-background-image") -> absUrl("data-background-image")
+        hasAttr("data-cfsrc") -> absUrl("data-cfsrc")
+        hasAttr("data-lazy-src") -> absUrl("data-lazy-src")
+        hasAttr("data-src") -> absUrl("data-src")
+        else -> absUrl("src")
+    }
+
+    protected fun Elements.textWithNewlines() = run {
+        select("p, br").prepend("\\n")
+        text().replace("\\n", "\n").replace("\n ", "\n")
+    }
+}
+
+private enum class FetchFilterStatus {
+    NOT_FETCHED,
+    FETCHED,
+    FETCHING,
 }

lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSDto.kt

@@ -3,12 +3,12 @@ package eu.kanade.tachiyomi.multisrc.mmrcms
 import kotlinx.serialization.Serializable
 
 @Serializable
-data class SearchResultDto(
+class SearchResultDto(
     val suggestions: List<SuggestionDto>,
 )
 
 @Serializable
-data class SuggestionDto(
+class SuggestionDto(
     val value: String,
     val data: String,
 )

lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSFilters.kt

@@ -1,5 +1,6 @@
 package eu.kanade.tachiyomi.multisrc.mmrcms
 
+import eu.kanade.tachiyomi.lib.i18n.Intl
 import eu.kanade.tachiyomi.source.model.Filter
 import okhttp3.HttpUrl
 
@@ -47,27 +48,23 @@ class UriMultiSelectFilter(
     }
 }
 
-class SortFilter(selection: Selection = Selection(0, true)) :
+class SortFilter(
+    intl: Intl,
+    private val sortables: Array<Pair<String, String>>,
+    selection: Selection = Selection(0, true),
+) :
     Filter.Sort(
-        "Sort by",
+        intl["sort_by_filter_title"],
-        sortables.map { it.second }.toTypedArray(),
+        sortables.map { it.first }.toTypedArray(),
         selection,
     ),
     UriFilter {
     override fun addToUri(builder: HttpUrl.Builder) {
-        val state = state!!
+        val state = state ?: return
 
         builder.apply {
-            addQueryParameter("sortBy", sortables[state.index].first)
+            addQueryParameter("sortBy", sortables[state.index].second)
             addQueryParameter("asc", state.ascending.toString())
         }
     }
-
-    companion object {
-        private val sortables = arrayOf(
-            "name" to "Name",
-            "views" to "Popularity",
-            "last_release" to "Last update",
-        )
-    }
 }

lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSUtils.kt

@@ -1,27 +0,0 @@
-package eu.kanade.tachiyomi.multisrc.mmrcms
-
-import org.jsoup.nodes.Element
-import org.jsoup.select.Elements
-
-object MMRCMSUtils {
-    fun guessCover(baseUrl: String, mangaUrl: String, url: String?): String {
-        return if (url == null || url.endsWith("no-image.png")) {
-            "$baseUrl/uploads/manga/${mangaUrl.substringAfterLast('/')}/cover/cover_250x350.jpg"
-        } else {
-            url
-        }
-    }
-
-    fun Element.imgAttr(): String = when {
-        hasAttr("data-background-image") -> absUrl("data-background-image")
-        hasAttr("data-cfsrc") -> absUrl("data-cfsrc")
-        hasAttr("data-lazy-src") -> absUrl("data-lazy-src")
-        hasAttr("data-src") -> absUrl("data-src")
-        else -> absUrl("src")
-    }
-
-    fun Elements.textWithNewlines() = run {
-        select("p, br").prepend("\\n")
-        text().replace("\\n", "\n").replace("\n ", "\n")
-    }
-}

src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasIn.kt

@@ -4,7 +4,6 @@ import android.util.Base64
 import eu.kanade.tachiyomi.lib.cryptoaes.CryptoAES
 import eu.kanade.tachiyomi.lib.synchrony.Deobfuscator
 import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMS
-import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMSUtils
 import eu.kanade.tachiyomi.multisrc.mmrcms.SuggestionDto
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.interceptor.rateLimitHost
@@ -18,6 +17,7 @@ import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.Request
 import okhttp3.Response
 import org.jsoup.nodes.Document
+import java.text.ParseException
 import java.text.SimpleDateFormat
 import java.util.Locale
 
@@ -26,6 +26,7 @@ class MangasIn : MMRCMS(
     "https://mangas.in",
     "es",
     supportsAdvancedSearch = false,
+    dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US),
 ) {
     override val client = super.client.newBuilder()
         .rateLimitHost(baseUrl.toHttpUrl(), 1, 1)
@@ -44,7 +45,7 @@ class MangasIn : MMRCMS(
             SManga.create().apply {
                 url = "/$itemPath/${it.slug}"
                 title = it.name
-                thumbnail_url = MMRCMSUtils.guessCover(baseUrl, url, null)
+                thumbnail_url = guessCover(url, null)
             }
         }
         val hasNextPage = response.request.url.queryParameter("p")!!.toInt() < data.totalPages
@@ -124,7 +125,12 @@ class MangasIn : MMRCMS(
                     "Capítulo ${it.number}: ${it.name}"
                 }
 
-                date_upload = it.createdAt.parseDate()
+                date_upload = try {
+                    dateFormat.parse(it.createdAt)!!.time
+                } catch (_: ParseException) {
+                    0L
+                }
+
                 setUrlWithoutDomain("$mangaUrl/${it.slug}")
             }
         }
@@ -163,15 +169,9 @@ class MangasIn : MMRCMS(
             .map { it.toInt(16).toByte() }
             .toByteArray()
     }
+}
 
-    companion object {
+private val UNESCAPE_REGEX = """\\(.)""".toRegex()
-        val UNESCAPE_REGEX = """\\(.)""".toRegex()
+private val RECEIVED_DATA_REGEX = """receivedData\s*=\s*["'](.*)["']\s*;""".toRegex()
-        val RECEIVED_DATA_REGEX = """receivedData\s*=\s*["'](.*)["']\s*;""".toRegex()
+private val KEY_REGEX = """decrypt\(.*'(.*)'.*\)""".toRegex()
-        val KEY_REGEX = """decrypt\(.*'(.*)'.*\)""".toRegex()
+private val SALTED = "Salted__".toByteArray(Charsets.UTF_8)
-        val SALTED = "Salted__".toByteArray(Charsets.UTF_8)
-
-        val dateFormat by lazy {
-            SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-        }
-    }
-}

src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasInDto.kt

@@ -4,10 +4,10 @@ import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
 @Serializable
-data class CDT(val ct: String, val s: String)
+class CDT(val ct: String, val s: String)
 
 @Serializable
-data class Chapter(
+class Chapter(
     val slug: String,
     val name: String,
     val number: String,
@@ -15,13 +15,13 @@ data class Chapter(
 )
 
 @Serializable
-data class LatestManga(
+class LatestManga(
     @SerialName("manga_name") val name: String,
     @SerialName("manga_slug") val slug: String,
 )
 
 @Serializable
-data class LatestUpdateResponse(
+class LatestUpdateResponse(
     val data: List<LatestManga>,
     val totalPages: Int,
 )