From 93c5dbc6500ee900740cb9e349b8f8a7869947b8 Mon Sep 17 00:00:00 2001 From: beerpsi <92439990+beerpiss@users.noreply.github.com> Date: Sat, 17 Feb 2024 13:26:49 +0700 Subject: [PATCH] MMRCMS: Dynamic filter rework (#1315) * MMRCMS: Dynamic filter rework, remove Last updated sort in Mangas.in * Formatting * Show the reset message when filters are fetching * Linting * Dynamically fetch sort options * Add i18n support * Remove unused import --- .../mmrcms/assets/i18n/messages_en.properties | 10 + lib-multisrc/mmrcms/build.gradle.kts | 6 +- .../tachiyomi/multisrc/mmrcms/MMRCMS.kt | 270 ++++++++++-------- .../tachiyomi/multisrc/mmrcms/MMRCMSDto.kt | 4 +- .../multisrc/mmrcms/MMRCMSFilters.kt | 23 +- .../tachiyomi/multisrc/mmrcms/MMRCMSUtils.kt | 27 -- .../extension/es/mangasin/MangasIn.kt | 28 +- .../extension/es/mangasin/MangasInDto.kt | 8 +- 8 files changed, 189 insertions(+), 187 deletions(-) create mode 100644 lib-multisrc/mmrcms/assets/i18n/messages_en.properties delete mode 100644 lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSUtils.kt diff --git a/lib-multisrc/mmrcms/assets/i18n/messages_en.properties b/lib-multisrc/mmrcms/assets/i18n/messages_en.properties new file mode 100644 index 000000000..8a8565360 --- /dev/null +++ b/lib-multisrc/mmrcms/assets/i18n/messages_en.properties @@ -0,0 +1,10 @@ +filter_warning=Ignored if using text search +filter_missing_warning=Press 'Reset' to attempt to show filters +category_filter_title=Category +status_filter_title=Status +type_filter_title=Type +year_filter_title=Year of release +author_filter_title=Author +tag_filter_title=Tag +title_begins_with_filter_title=Title begins with +sort_by_filter_title=Sort by diff --git a/lib-multisrc/mmrcms/build.gradle.kts b/lib-multisrc/mmrcms/build.gradle.kts index d1d9afd6e..0ce8d2e28 100644 --- a/lib-multisrc/mmrcms/build.gradle.kts +++ b/lib-multisrc/mmrcms/build.gradle.kts @@ -2,4 +2,8 @@ plugins { id("lib-multisrc") } -baseVersionCode = 9 +baseVersionCode = 10 + +dependencies { + api(project(":lib:i18n")) +} diff --git a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMS.kt b/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMS.kt index 9f18871af..641df3a48 100644 --- a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMS.kt +++ b/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMS.kt @@ -2,11 +2,11 @@ package eu.kanade.tachiyomi.multisrc.mmrcms import android.annotation.SuppressLint import android.util.Log -import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMSUtils.imgAttr -import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMSUtils.textWithNewlines +import eu.kanade.tachiyomi.lib.i18n.Intl import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.POST import eu.kanade.tachiyomi.network.asObservableSuccess +import eu.kanade.tachiyomi.network.await import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage @@ -15,6 +15,9 @@ import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.ParsedHttpSource import eu.kanade.tachiyomi.util.asJsoup +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json import okhttp3.FormBody @@ -23,14 +26,12 @@ import okhttp3.Request import okhttp3.Response import org.jsoup.nodes.Document import org.jsoup.nodes.Element +import org.jsoup.select.Elements import rx.Observable -import rx.Single -import rx.Subscription -import rx.schedulers.Schedulers import uy.kohesive.injekt.injectLazy +import java.text.ParseException import java.text.SimpleDateFormat import java.util.Locale -import java.util.concurrent.locks.ReentrantLock /** * @param dateFormat The date format used for parsing chapter dates. @@ -50,7 +51,7 @@ constructor( vararg useNamedArgumentsBelow: Forbidden, - private val dateFormat: SimpleDateFormat = SimpleDateFormat("d MMM. yyyy", Locale.US), + protected val dateFormat: SimpleDateFormat = SimpleDateFormat("d MMM. yyyy", Locale.US), protected val itemPath: String = "manga", private val fetchFilterOptions: Boolean = true, private val supportsAdvancedSearch: Boolean = true, @@ -70,6 +71,13 @@ constructor( protected val json: Json by injectLazy() + protected val intl = Intl( + lang, + setOf("en"), + "en", + this::class.java.classLoader!!, + ) + override fun popularMangaRequest(page: Int) = GET("$baseUrl/filterList?page=$page&sortBy=views&asc=false") override fun popularMangaSelector() = searchMangaSelector() @@ -117,16 +125,13 @@ constructor( protected var searchDirectory = emptyList() - private var searchQuery = "" - override fun fetchSearchManga( page: Int, query: String, filters: FilterList, ): Observable { return if (query.isNotEmpty()) { - if (page == 1 && query != searchQuery) { - searchQuery = query + if (page == 1) { client.newCall(searchMangaRequest(page, query, filters)) .asObservableSuccess() .map { searchMangaParse(it) } @@ -197,26 +202,23 @@ constructor( setUrlWithoutDomain(anchor.attr("href")) title = anchor.text() - thumbnail_url = MMRCMSUtils.guessCover(baseUrl, url, element.selectFirst("img")?.imgAttr()) + thumbnail_url = guessCover(url, element.selectFirst("img")?.imgAttr()) } override fun searchMangaNextPageSelector(): String? = ".pagination a[rel=next]" protected fun parseSearchDirectory(page: Int): MangasPage { - val manga = mutableListOf() - val endRange = ((page * 24) - 1).let { if (it <= searchDirectory.lastIndex) it else searchDirectory.lastIndex } - - for (i in (((page - 1) * 24)..endRange)) { - manga.add( + val manga = searchDirectory.subList((page - 1) * 24, page * 24) + .map { SManga.create().apply { - url = "/$itemPath/${searchDirectory[i].data}" - title = searchDirectory[i].value - thumbnail_url = MMRCMSUtils.guessCover(baseUrl, url, null) - }, - ) - } + url = "/$itemPath/${it.data}" + title = it.value + thumbnail_url = guessCover(url, null) + } + } + val hasNextPage = (page + 1) * 24 <= searchDirectory.size - return MangasPage(manga, endRange < searchDirectory.lastIndex) + return MangasPage(manga, hasNextPage) } protected val detailAuthor = hashSetOf("author(s)", "autor(es)", "auteur(s)", "著作", "yazar(lar)", "mangaka(lar)", "pengarang/penulis", "pengarang", "penulis", "autor", "المؤلف", "перевод", "autor/autorzy") @@ -230,8 +232,7 @@ constructor( @SuppressLint("DefaultLocale") override fun mangaDetailsParse(document: Document) = SManga.create().apply { title = document.selectFirst(detailsTitleSelector)!!.text() - thumbnail_url = MMRCMSUtils.guessCover( - baseUrl, + thumbnail_url = guessCover( document.location(), document.selectFirst(".row img.img-responsive")?.imgAttr(), ) @@ -274,17 +275,17 @@ constructor( setUrlWithoutDomain(anchor.attr("href")) name = cleanChapterName(mangaTitle, titleWrapper.text()) - date_upload = runCatching { + date_upload = try { val date = element.selectFirst(".date-chapter-title-rtl")!!.text() dateFormat.parse(date)!!.time - }.getOrDefault(0L) + } catch (_: ParseException) { + 0L + } catch (_: NullPointerException) { + 0L + } } - /** - * The word for "Chapter" in your language. - */ - /** * Function to clean up chapter names. Mostly useful for sites that * don't know what a chapter title is and do "One Piece 1234 : Chapter 1234". @@ -309,22 +310,20 @@ constructor( override fun imageUrlParse(document: Document) = throw UnsupportedOperationException() override fun getFilterList(): FilterList { - runCatching { fetchFilterOptions() } + fetchFilterOptions() - val filters = buildList> { - add(Filter.Header("Note: Ignored if using text search!")) + val filters = buildList { + add(Filter.Header(intl["filter_warning"])) + if (fetchFilterOptions && fetchFiltersStatus != FetchFilterStatus.FETCHED) { + add(Filter.Header(intl["filter_missing_warning"])) + } + add(Filter.Separator()) if (supportsAdvancedSearch) { - if (fetchFilterOptions && (categories.isEmpty() || statuses.isEmpty())) { - add(Filter.Header("Press 'Reset' to attempt to show filter options")) - } - - add(Filter.Separator()) - if (categories.isNotEmpty()) { add( UriMultiSelectFilter( - "Categories", + intl["category_filter_title"], "categories[]", categories.toTypedArray(), ), @@ -334,7 +333,7 @@ constructor( if (statuses.isNotEmpty()) { add( UriMultiSelectFilter( - "Statuses", + intl["status_filter_title"], "status[]", statuses.toTypedArray(), ), @@ -344,26 +343,20 @@ constructor( if (tags.isNotEmpty()) { add( UriMultiSelectFilter( - "Types", + intl["type_filter_title"], "types[]", tags.toTypedArray(), ), ) } - add(TextFilter("Year of release", "release")) - add(TextFilter("Author", "author")) + add(TextFilter(intl["year_filter_title"], "release")) + add(TextFilter(intl["author_filter_title"], "author")) } else { - if (fetchFilterOptions && categories.isEmpty()) { - add(Filter.Header("Press 'Reset' to attempt to show filter options")) - } - - add(Filter.Separator()) - if (categories.isNotEmpty()) { add( UriPartFilter( - "Category", + intl["category_filter_title"], "cat", arrayOf( "Any" to "", @@ -373,23 +366,12 @@ constructor( ) } - add( - UriPartFilter( - "Title begins with", - "alpha", - arrayOf( - "Any" to "", - *"#ABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray().map { - Pair(it.toString(), it.toString()) - }.toTypedArray(), - ), - ), - ) + add(UriPartFilter(intl["title_begins_with_filter_title"], "alpha", alphaOptions)) if (tags.isNotEmpty()) { add( UriPartFilter( - "Tag", + intl["tag_filter_title"], "tag", arrayOf( "Any" to "", @@ -399,71 +381,107 @@ constructor( ) } - add(SortFilter()) + if (sortOptions.isNotEmpty()) { + add(SortFilter(intl, sortOptions)) + } } } return FilterList(filters) } - private var categories = emptyList>() - - private var statuses = emptyList>() - - private var tags = emptyList>() - - private var fetchFiltersFailed = false - - private var fetchFiltersAttempts = 0 - - private val fetchFiltersLock = ReentrantLock() - - protected open fun fetchFilterOptions(): Subscription = Single.fromCallable { - if (!fetchFilterOptions) { - return@fromCallable - } - - fetchFiltersLock.lock() - - if (fetchFiltersAttempts > 3 || (fetchFiltersAttempts > 0 && !fetchFiltersFailed)) { - fetchFiltersLock.unlock() - return@fromCallable - } - - fetchFiltersFailed = try { - if (supportsAdvancedSearch) { - val document = client.newCall(GET("$baseUrl/advanced-search", headers)).execute().asJsoup() - - categories = document.select("select[name='categories[]'] option").map { - it.text() to it.attr("value") - } - statuses = document.select("select[name='status[]'] option").map { - it.text() to it.attr("value") - } - tags = document.select("select[name='types[]'] option").map { - it.text() to it.attr("value") - } - } else { - val document = client.newCall(GET("$baseUrl/$itemPath-list", headers)).execute().asJsoup() - - categories = document.select("a.category").map { - it.text() to it.attr("href").toHttpUrl().queryParameter("cat")!! - } - tags = document.select("div.tag-links a").map { - it.text() to it.attr("href").toHttpUrl().pathSegments.last() - } - } - - false - } catch (e: Throwable) { - Log.e(name, "Could not fetch filtering options", e) - true - } - - fetchFiltersAttempts++ - fetchFiltersLock.unlock() + private val alphaOptions by lazy { + arrayOf( + "Any" to "", + *"#ABCDEFGHIJKLMNOPQRSTUVWXYZ".toCharArray().map { + Pair(it.toString(), it.toString()) + }.toTypedArray(), + ) + } + private var categories = emptyList>() + private var statuses = emptyList>() + private var tags = emptyList>() + private var sortOptions = emptyArray>() + + private var fetchFiltersStatus = FetchFilterStatus.NOT_FETCHED + private var fetchFiltersAttempts = 0 + private val scope = CoroutineScope(Dispatchers.IO) + + protected open fun fetchFilterOptions() { + if (!fetchFilterOptions) { + return + } + + if (fetchFiltersStatus != FetchFilterStatus.NOT_FETCHED || fetchFiltersAttempts >= 3) { + return + } + + fetchFiltersStatus = FetchFilterStatus.FETCHING + fetchFiltersAttempts++ + scope.launch { + try { + if (supportsAdvancedSearch) { + val document = client.newCall(GET("$baseUrl/advanced-search", headers)) + .await() + .asJsoup() + + categories = document.select("select[name='categories[]'] option").map { + it.text() to it.attr("value") + } + statuses = document.select("select[name='status[]'] option").map { + it.text() to it.attr("value") + } + tags = document.select("select[name='types[]'] option").map { + it.text() to it.attr("value") + } + } else { + val document = client.newCall(GET("$baseUrl/$itemPath-list", headers)) + .await() + .asJsoup() + + categories = document.select("a.category").map { + it.text() to it.attr("href").toHttpUrl().queryParameter("cat")!! + } + tags = document.select("div.tag-links a").map { + it.text() to it.attr("href").toHttpUrl().pathSegments.last() + } + sortOptions = document.select("#sort-types label:has(input)").map { + it.ownText() to it.selectFirst("input")!!.id() + }.toTypedArray() + } + + fetchFiltersStatus = FetchFilterStatus.FETCHED + } catch (e: Exception) { + fetchFiltersStatus = FetchFilterStatus.NOT_FETCHED + Log.e("MMRCMS/$name", "Could not fetch filters", e) + } + } + } + + protected fun guessCover(mangaUrl: String, url: String?): String { + return if (url == null || url.endsWith("no-image.png")) { + "$baseUrl/uploads/manga/${mangaUrl.substringAfterLast('/')}/cover/cover_250x350.jpg" + } else { + url + } + } + + protected fun Element.imgAttr(): String = when { + hasAttr("data-background-image") -> absUrl("data-background-image") + hasAttr("data-cfsrc") -> absUrl("data-cfsrc") + hasAttr("data-lazy-src") -> absUrl("data-lazy-src") + hasAttr("data-src") -> absUrl("data-src") + else -> absUrl("src") + } + + protected fun Elements.textWithNewlines() = run { + select("p, br").prepend("\\n") + text().replace("\\n", "\n").replace("\n ", "\n") } - .subscribeOn(Schedulers.io()) - .observeOn(Schedulers.io()) - .subscribe() +} + +private enum class FetchFilterStatus { + NOT_FETCHED, + FETCHED, + FETCHING, } diff --git a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSDto.kt b/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSDto.kt index 7791759c7..defb770fb 100644 --- a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSDto.kt +++ b/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSDto.kt @@ -3,12 +3,12 @@ package eu.kanade.tachiyomi.multisrc.mmrcms import kotlinx.serialization.Serializable @Serializable -data class SearchResultDto( +class SearchResultDto( val suggestions: List, ) @Serializable -data class SuggestionDto( +class SuggestionDto( val value: String, val data: String, ) diff --git a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSFilters.kt b/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSFilters.kt index d144cb505..e934156fd 100644 --- a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSFilters.kt +++ b/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSFilters.kt @@ -1,5 +1,6 @@ package eu.kanade.tachiyomi.multisrc.mmrcms +import eu.kanade.tachiyomi.lib.i18n.Intl import eu.kanade.tachiyomi.source.model.Filter import okhttp3.HttpUrl @@ -47,27 +48,23 @@ class UriMultiSelectFilter( } } -class SortFilter(selection: Selection = Selection(0, true)) : +class SortFilter( + intl: Intl, + private val sortables: Array>, + selection: Selection = Selection(0, true), +) : Filter.Sort( - "Sort by", - sortables.map { it.second }.toTypedArray(), + intl["sort_by_filter_title"], + sortables.map { it.first }.toTypedArray(), selection, ), UriFilter { override fun addToUri(builder: HttpUrl.Builder) { - val state = state!! + val state = state ?: return builder.apply { - addQueryParameter("sortBy", sortables[state.index].first) + addQueryParameter("sortBy", sortables[state.index].second) addQueryParameter("asc", state.ascending.toString()) } } - - companion object { - private val sortables = arrayOf( - "name" to "Name", - "views" to "Popularity", - "last_release" to "Last update", - ) - } } diff --git a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSUtils.kt b/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSUtils.kt deleted file mode 100644 index a8e460a2f..000000000 --- a/lib-multisrc/mmrcms/src/eu/kanade/tachiyomi/multisrc/mmrcms/MMRCMSUtils.kt +++ /dev/null @@ -1,27 +0,0 @@ -package eu.kanade.tachiyomi.multisrc.mmrcms - -import org.jsoup.nodes.Element -import org.jsoup.select.Elements - -object MMRCMSUtils { - fun guessCover(baseUrl: String, mangaUrl: String, url: String?): String { - return if (url == null || url.endsWith("no-image.png")) { - "$baseUrl/uploads/manga/${mangaUrl.substringAfterLast('/')}/cover/cover_250x350.jpg" - } else { - url - } - } - - fun Element.imgAttr(): String = when { - hasAttr("data-background-image") -> absUrl("data-background-image") - hasAttr("data-cfsrc") -> absUrl("data-cfsrc") - hasAttr("data-lazy-src") -> absUrl("data-lazy-src") - hasAttr("data-src") -> absUrl("data-src") - else -> absUrl("src") - } - - fun Elements.textWithNewlines() = run { - select("p, br").prepend("\\n") - text().replace("\\n", "\n").replace("\n ", "\n") - } -} diff --git a/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasIn.kt b/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasIn.kt index 219c012de..5d15c5012 100644 --- a/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasIn.kt +++ b/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasIn.kt @@ -4,7 +4,6 @@ import android.util.Base64 import eu.kanade.tachiyomi.lib.cryptoaes.CryptoAES import eu.kanade.tachiyomi.lib.synchrony.Deobfuscator import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMS -import eu.kanade.tachiyomi.multisrc.mmrcms.MMRCMSUtils import eu.kanade.tachiyomi.multisrc.mmrcms.SuggestionDto import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.interceptor.rateLimitHost @@ -18,6 +17,7 @@ import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Request import okhttp3.Response import org.jsoup.nodes.Document +import java.text.ParseException import java.text.SimpleDateFormat import java.util.Locale @@ -26,6 +26,7 @@ class MangasIn : MMRCMS( "https://mangas.in", "es", supportsAdvancedSearch = false, + dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US), ) { override val client = super.client.newBuilder() .rateLimitHost(baseUrl.toHttpUrl(), 1, 1) @@ -44,7 +45,7 @@ class MangasIn : MMRCMS( SManga.create().apply { url = "/$itemPath/${it.slug}" title = it.name - thumbnail_url = MMRCMSUtils.guessCover(baseUrl, url, null) + thumbnail_url = guessCover(url, null) } } val hasNextPage = response.request.url.queryParameter("p")!!.toInt() < data.totalPages @@ -124,7 +125,12 @@ class MangasIn : MMRCMS( "Capítulo ${it.number}: ${it.name}" } - date_upload = it.createdAt.parseDate() + date_upload = try { + dateFormat.parse(it.createdAt)!!.time + } catch (_: ParseException) { + 0L + } + setUrlWithoutDomain("$mangaUrl/${it.slug}") } } @@ -163,15 +169,9 @@ class MangasIn : MMRCMS( .map { it.toInt(16).toByte() } .toByteArray() } - - companion object { - val UNESCAPE_REGEX = """\\(.)""".toRegex() - val RECEIVED_DATA_REGEX = """receivedData\s*=\s*["'](.*)["']\s*;""".toRegex() - val KEY_REGEX = """decrypt\(.*'(.*)'.*\)""".toRegex() - val SALTED = "Salted__".toByteArray(Charsets.UTF_8) - - val dateFormat by lazy { - SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) - } - } } + +private val UNESCAPE_REGEX = """\\(.)""".toRegex() +private val RECEIVED_DATA_REGEX = """receivedData\s*=\s*["'](.*)["']\s*;""".toRegex() +private val KEY_REGEX = """decrypt\(.*'(.*)'.*\)""".toRegex() +private val SALTED = "Salted__".toByteArray(Charsets.UTF_8) diff --git a/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasInDto.kt b/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasInDto.kt index 557de9697..8adf3ccc6 100644 --- a/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasInDto.kt +++ b/src/es/mangasin/src/eu/kanade/tachiyomi/extension/es/mangasin/MangasInDto.kt @@ -4,10 +4,10 @@ import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @Serializable -data class CDT(val ct: String, val s: String) +class CDT(val ct: String, val s: String) @Serializable -data class Chapter( +class Chapter( val slug: String, val name: String, val number: String, @@ -15,13 +15,13 @@ data class Chapter( ) @Serializable -data class LatestManga( +class LatestManga( @SerialName("manga_name") val name: String, @SerialName("manga_slug") val slug: String, ) @Serializable -data class LatestUpdateResponse( +class LatestUpdateResponse( val data: List, val totalPages: Int, )