From fcba0e4efee425a0f04433482692641bf937000a Mon Sep 17 00:00:00 2001 From: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com> Date: Sat, 1 Jul 2023 19:03:51 +0500 Subject: [PATCH] comick: improve performance (#16947) * comick: improve performance for real this time * simplify logic Co-authored-by: stevenyomi <95685115+stevenyomi@users.noreply.github.com> * remove redundancy --------- Co-authored-by: stevenyomi <95685115+stevenyomi@users.noreply.github.com> --- src/all/comickfun/build.gradle | 2 +- .../extension/all/comickfun/ComickFun.kt | 279 +++++++++--------- .../extension/all/comickfun/ComickFunDto.kt | 21 +- .../all/comickfun/ComickFunFilters.kt | 2 +- .../all/comickfun/ComickFunHelper.kt | 9 +- 5 files changed, 151 insertions(+), 162 deletions(-) diff --git a/src/all/comickfun/build.gradle b/src/all/comickfun/build.gradle index 64019663e..644d588cc 100644 --- a/src/all/comickfun/build.gradle +++ b/src/all/comickfun/build.gradle @@ -6,7 +6,7 @@ ext { extName = 'Comick' pkgNameSuffix = 'all.comickfun' extClass = '.ComickFunFactory' - extVersionCode = 28 + extVersionCode = 29 isNsfw = true } diff --git a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt index c8dd7ad5f..51a7a8dd3 100644 --- a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt +++ b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFun.kt @@ -1,12 +1,8 @@ package eu.kanade.tachiyomi.extension.all.comickfun -import android.app.Application -import android.content.SharedPreferences -import androidx.preference.ListPreference -import androidx.preference.PreferenceScreen import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.network.interceptor.rateLimit -import eu.kanade.tachiyomi.source.ConfigurableSource import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page @@ -17,20 +13,18 @@ import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response import rx.Observable -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get import java.text.SimpleDateFormat import java.util.Locale import java.util.TimeZone +import kotlin.math.min abstract class ComickFun( override val lang: String, private val comickFunLang: String, -) : HttpSource(), ConfigurableSource { +) : HttpSource() { override val name = "Comick" @@ -47,157 +41,170 @@ abstract class ComickFun( explicitNulls = true } + private lateinit var searchResponse: List + override fun headersBuilder() = Headers.Builder().apply { add("Referer", "$baseUrl/") add("User-Agent", "Tachiyomi ${System.getProperty("http.agent")}") } - override val client: OkHttpClient = network.client.newBuilder() - .addNetworkInterceptor(::thumbnailIntercept) - .rateLimit(4, 1) + override val client = network.client.newBuilder() + .addInterceptor(::thumbnailIntercept) + .rateLimit(3, 1) .build() - private val preferences: SharedPreferences by lazy { - Injekt.get().getSharedPreferences("source_$id", 0x0000) - } - /** Popular Manga **/ override fun popularMangaRequest(page: Int): Request { - return searchMangaRequest( - page = page, - query = "", - filters = FilterList( - SortFilter("", getSortsList, defaultPopularSort), - ), - ) + val url = "$apiUrl/v1.0/search?sort=follow&limit=$limit&page=$page&tachiyomi=true" + return GET(url, headers) } - override fun popularMangaParse(response: Response) = searchMangaParse(response) + override fun popularMangaParse(response: Response): MangasPage { + val result = response.parseAs>() + return MangasPage( + result.map(SearchManga::toSManga), + hasNextPage = result.size >= limit, + ) + } /** Latest Manga **/ override fun latestUpdatesRequest(page: Int): Request { - return searchMangaRequest( - page = page, - query = "", - filters = FilterList( - SortFilter("", getSortsList, defaultLatestSort), - ), - ) + val url = "$apiUrl/v1.0/search?sort=uploaded&limit=$limit&page=$page&tachiyomi=true" + return GET(url, headers) } - override fun latestUpdatesParse(response: Response) = searchMangaParse(response) + override fun latestUpdatesParse(response: Response) = popularMangaParse(response) /** Manga Search **/ override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - if (!query.startsWith(SLUG_SEARCH_PREFIX)) { - return super.fetchSearchManga(page, query, filters) + return if (query.startsWith(SLUG_SEARCH_PREFIX)) { + // url deep link + val slugOrHid = query.substringAfter(SLUG_SEARCH_PREFIX) + val manga = SManga.create().apply { this.url = "/comic/$slugOrHid#" } + fetchMangaDetails(manga).map { + MangasPage(listOf(it), false) + } + } else if (query.isEmpty()) { + // regular filtering without text search + client.newCall(searchMangaRequest(page, query, filters)) + .asObservableSuccess() + .map(::searchMangaParse) + } else { + // text search, no pagination in api + if (page == 1) { + client.newCall(querySearchRequest(query)) + .asObservableSuccess() + .map(::querySearchParse) + } else { + Observable.just(paginatedSearchPage(page)) + } } + } - val slugOrHid = query.substringAfter(SLUG_SEARCH_PREFIX) - val manga = SManga.create().apply { this.url = "/comic/$slugOrHid#" } - return fetchMangaDetails(manga).map { - MangasPage(listOf(it), false) - } + private fun querySearchRequest(query: String): Request { + val url = "$apiUrl/v1.0/search?limit=300&page=1&tachiyomi=true" + .toHttpUrl().newBuilder() + .addQueryParameter("q", query.trim()) + .build() + + return GET(url, headers) + } + + private fun querySearchParse(response: Response): MangasPage { + searchResponse = response.parseAs() + + return paginatedSearchPage(1) + } + + private fun paginatedSearchPage(page: Int): MangasPage { + val end = min(page * limit, searchResponse.size) + val entries = searchResponse.subList((page - 1) * limit, end) + .map(SearchManga::toSManga) + return MangasPage(entries, end < searchResponse.size) } override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { val url = "$apiUrl/v1.0/search".toHttpUrl().newBuilder().apply { - if (query.isEmpty()) { - filters.forEach { it -> - when (it) { - is CompletedFilter -> { - if (it.state) { - addQueryParameter("completed", "true") - } + filters.forEach { it -> + when (it) { + is CompletedFilter -> { + if (it.state) { + addQueryParameter("completed", "true") } - is GenreFilter -> { - it.state.filter { (it as TriState).isIncluded() }.forEach { - addQueryParameter( - "genres", - (it as TriState).value, - ) - } - - it.state.filter { (it as TriState).isExcluded() }.forEach { - addQueryParameter( - "excludes", - (it as TriState).value, - ) - } - } - is DemographicFilter -> { - it.state.filter { (it as CheckBox).state }.forEach { - addQueryParameter( - "demographic", - (it as CheckBox).value, - ) - } - } - is TypeFilter -> { - it.state.filter { (it as CheckBox).state }.forEach { - addQueryParameter( - "country", - (it as CheckBox).value, - ) - } - } - is SortFilter -> { - addQueryParameter("sort", it.getValue()) - } - is CreatedAtFilter -> { - if (it.state > 0) { - addQueryParameter("time", it.getValue()) - } - } - is MinimumFilter -> { - if (it.state.isNotEmpty()) { - addQueryParameter("minimum", it.state) - } - } - is FromYearFilter -> { - if (it.state.isNotEmpty()) { - addQueryParameter("from", it.state) - } - } - is ToYearFilter -> { - if (it.state.isNotEmpty()) { - addQueryParameter("to", it.state) - } - } - is TagFilter -> { - if (it.state.isNotEmpty()) { - it.state.split(",").forEach { - addQueryParameter("tags", it.trim()) - } - } - } - else -> {} } + is GenreFilter -> { + it.state.filter { (it as TriState).isIncluded() }.forEach { + addQueryParameter( + "genres", + (it as TriState).value, + ) + } + + it.state.filter { (it as TriState).isExcluded() }.forEach { + addQueryParameter( + "excludes", + (it as TriState).value, + ) + } + } + is DemographicFilter -> { + it.state.filter { (it as CheckBox).state }.forEach { + addQueryParameter( + "demographic", + (it as CheckBox).value, + ) + } + } + is TypeFilter -> { + it.state.filter { (it as CheckBox).state }.forEach { + addQueryParameter( + "country", + (it as CheckBox).value, + ) + } + } + is SortFilter -> { + addQueryParameter("sort", it.getValue()) + } + is CreatedAtFilter -> { + if (it.state > 0) { + addQueryParameter("time", it.getValue()) + } + } + is MinimumFilter -> { + if (it.state.isNotEmpty()) { + addQueryParameter("minimum", it.state) + } + } + is FromYearFilter -> { + if (it.state.isNotEmpty()) { + addQueryParameter("from", it.state) + } + } + is ToYearFilter -> { + if (it.state.isNotEmpty()) { + addQueryParameter("to", it.state) + } + } + is TagFilter -> { + if (it.state.isNotEmpty()) { + it.state.split(",").forEach { + addQueryParameter("tags", it.trim()) + } + } + } + else -> {} } - } else { - addQueryParameter("q", query) } addQueryParameter("tachiyomi", "true") - addQueryParameter("limit", "50") + addQueryParameter("limit", "$limit") addQueryParameter("page", "$page") }.build() + return GET(url, headers) } - override fun searchMangaParse(response: Response): MangasPage { - val isQueryPresent = response.request.url.queryParameterNames.contains("q") - val result = response.parseAs>() - return MangasPage( - result.map { it.toSManga(useScaledCover) }, - /* - api always returns `limit` amount of results - for text search and page>=2 is always empty - so here we are checking if url has the text query parameter - to avoid false 'No result found' toasts. - */ - hasNextPage = !isQueryPresent && result.size >= 50, - ) - } + override fun searchMangaParse(response: Response) = popularMangaParse(response) /** Manga Details **/ override fun mangaDetailsRequest(manga: SManga): Request { @@ -212,7 +219,7 @@ abstract class ComickFun( override fun mangaDetailsParse(response: Response): SManga { val mangaData = response.parseAs() - return mangaData.toSManga(useScaledCover) + return mangaData.toSManga() } override fun getMangaUrl(manga: SManga): String { @@ -290,28 +297,11 @@ abstract class ComickFun( throw UnsupportedOperationException("Not used") } - protected open val defaultPopularSort: Int = 0 - protected open val defaultLatestSort: Int = 4 - override fun getFilterList() = getFilters() - override fun setupPreferenceScreen(screen: PreferenceScreen) { - ListPreference(screen.context).apply { - key = coverQualityPref - title = "Cover Quality" - entries = arrayOf("Original", "Scaled") - entryValues = arrayOf("orig", "scaled") - setDefaultValue("orig") - summary = "%s" - }.let { screen.addPreference(it) } - } - - private val useScaledCover: Boolean by lazy { - preferences.getString(coverQualityPref, "orig") != "orig" - } - companion object { const val SLUG_SEARCH_PREFIX = "id:" + private const val limit = 20 val dateFormat by lazy { SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ENGLISH).apply { timeZone = TimeZone.getTimeZone("UTC") @@ -320,6 +310,5 @@ abstract class ComickFun( val markdownLinksRegex = "\\[([^]]+)\\]\\(([^)]+)\\)".toRegex() val markdownItalicBoldRegex = "\\*+\\s*([^\\*]*)\\s*\\*+".toRegex() val markdownItalicRegex = "_+\\s*([^_]*)\\s*_+".toRegex() - private const val coverQualityPref = "pref_cover_quality" } } diff --git a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunDto.kt b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunDto.kt index aa1cc9e34..51e1742fa 100644 --- a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunDto.kt +++ b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunDto.kt @@ -13,11 +13,11 @@ data class SearchManga( val cover_url: String? = null, ) { - fun toSManga(useScaledCover: Boolean) = SManga.create().apply { + fun toSManga() = SManga.create().apply { // appennding # at end as part of migration from slug to hid url = "/comic/$hid#" title = this@SearchManga.title - thumbnail_url = parseCover(cover_url, md_covers, useScaledCover) + thumbnail_url = parseCover(cover_url, md_covers) } } @@ -28,21 +28,24 @@ data class Manga( val authors: List = emptyList(), val genres: List = emptyList(), ) { - fun toSManga(useScaledCover: Boolean) = SManga.create().apply { + fun toSManga() = SManga.create().apply { // appennding # at end as part of migration from slug to hid url = "/comic/${comic.hid}#" title = comic.title description = comic.desc.beautifyDescription() if (comic.altTitles.isNotEmpty()) { - description += comic.altTitles.joinToString( - separator = "\n", - prefix = "\n\nAlternative Titles:\n", - ) { - it.title.toString() + if (description.isNullOrEmpty()) { + description = "Alternative Titles:\n" + } else { + description += "\n\nAlternative Titles:\n" } + + description += comic.altTitles.mapNotNull { title -> + title.title?.let { "• $it" } + }.joinToString("\n") } status = comic.status.parseStatus(comic.translation_completed) - thumbnail_url = parseCover(comic.cover_url, comic.md_covers, useScaledCover) + thumbnail_url = parseCover(comic.cover_url, comic.md_covers) artist = artists.joinToString { it.name.trim() } author = authors.joinToString { it.name.trim() } genre = genres.joinToString { it.name.trim() } diff --git a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunFilters.kt b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunFilters.kt index cf70ed194..321f8173f 100644 --- a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunFilters.kt +++ b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunFilters.kt @@ -171,7 +171,7 @@ private val getCreatedAtList: Array> = arrayOf( Pair("1 year", "365"), ) -internal val getSortsList: Array> = arrayOf( +private val getSortsList: Array> = arrayOf( Pair("Most popular", "follow"), Pair("Most follows", "user_follow_count"), Pair("Most views", "view"), diff --git a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunHelper.kt b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunHelper.kt index 301a4175a..e73fc3f62 100644 --- a/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunHelper.kt +++ b/src/all/comickfun/src/eu/kanade/tachiyomi/extension/all/comickfun/ComickFunHelper.kt @@ -11,6 +11,7 @@ import org.jsoup.parser.Parser internal fun String.beautifyDescription(): String { return Parser.unescapeEntities(this, false) + .substringBefore("---") .replace(markdownLinksRegex, "") .replace(markdownItalicBoldRegex, "") .replace(markdownItalicRegex, "") @@ -33,15 +34,11 @@ internal fun Int.parseStatus(translationComplete: Boolean): Int { } } -internal fun parseCover(thumbnailUrl: String?, mdCovers: List, useScaled: Boolean): String? { +internal fun parseCover(thumbnailUrl: String?, mdCovers: List): String { val b2key = runCatching { mdCovers.first().b2key } .getOrNull() ?: "" - return if (useScaled) { - "$thumbnailUrl#$b2key" - } else { - thumbnailUrl?.replaceAfterLast("/", b2key) - } + return "$thumbnailUrl#$b2key" } internal fun thumbnailIntercept(chain: Interceptor.Chain): Response {