From d82967ea3d274f0d834da78aed3255955bfaf4a0 Mon Sep 17 00:00:00 2001 From: Carlos <2092019+CarlosEsco@users.noreply.github.com> Date: Sat, 5 Jun 2021 10:38:21 -0400 Subject: [PATCH] switch to dex covers (cherry picked from commit 28d83509dadb0ddaca2e13e98f5e5f6e3df2d2a7) # Conflicts: # app/src/main/java/eu/kanade/tachiyomi/source/online/MangaDexCache.kt # app/src/main/java/eu/kanade/tachiyomi/source/online/handlers/ApiMangaParser.kt # app/src/main/java/eu/kanade/tachiyomi/source/online/handlers/FollowsHandler.kt # app/src/main/java/eu/kanade/tachiyomi/source/online/handlers/PopularHandler.kt # app/src/main/java/eu/kanade/tachiyomi/source/online/handlers/SearchHandler.kt # app/src/main/java/eu/kanade/tachiyomi/source/online/handlers/SimilarHandler.kt # app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaDetailsPresenter.kt # app/src/main/java/exh/md/utils/MdUtil.kt --- .../java/exh/md/handlers/ApiMangaParser.kt | 14 +--- .../java/exh/md/handlers/FollowsHandler.kt | 16 +--- .../main/java/exh/md/handlers/PageHandler.kt | 16 +++- .../java/exh/md/handlers/PopularHandler.kt | 15 +--- .../java/exh/md/handlers/SearchHandler.kt | 31 +------- .../java/exh/md/handlers/SimilarHandler.kt | 33 ++++++-- .../handlers/serializers/MangaSerializer.kt | 1 + app/src/main/java/exh/md/utils/MdUtil.kt | 76 ++++++++++++++++--- 8 files changed, 118 insertions(+), 84 deletions(-) diff --git a/app/src/main/java/exh/md/handlers/ApiMangaParser.kt b/app/src/main/java/exh/md/handlers/ApiMangaParser.kt index e1befe8f9..5b7913c3f 100644 --- a/app/src/main/java/exh/md/handlers/ApiMangaParser.kt +++ b/app/src/main/java/exh/md/handlers/ApiMangaParser.kt @@ -78,18 +78,8 @@ class ApiMangaParser( title = MdUtil.cleanString(networkManga.title[lang] ?: networkManga.title["en"]!!) altTitles = networkManga.altTitles.mapNotNull { it[lang] }.nullIfEmpty() - val coverUrl = MdUtil.formThumbUrl(networkApiManga.data.id) - /*val coverUrlId = networkApiManga.relationships.firstOrNull { it.type == "cover_art" }?.id - if (coverUrlId != null) { - runCatching { - val json = client.newCall(GET(MdUtil.coverUrl(networkApiManga.data.id, coverUrlId))).await() - .parseAs(MdUtil.jsonParser) - json.results.firstOrNull()?.data?.attributes?.fileName?.let { fileName -> - coverUrl = "${MdUtil.cdnUrl}/covers/${networkApiManga.data.id}/$fileName" - } - } - }*/ - cover = coverUrl + val coverId = networkApiManga.relationships.firstOrNull { it.type.equals("cover_art", true) }?.id + cover = MdUtil.getCoverUrl(networkApiManga.data.id, coverId, client) description = MdUtil.cleanDescription(networkManga.description[lang] ?: networkManga.description["en"]!!) diff --git a/app/src/main/java/exh/md/handlers/FollowsHandler.kt b/app/src/main/java/exh/md/handlers/FollowsHandler.kt index 2413b2253..42b2dc08b 100644 --- a/app/src/main/java/exh/md/handlers/FollowsHandler.kt +++ b/app/src/main/java/exh/md/handlers/FollowsHandler.kt @@ -75,23 +75,13 @@ class FollowsHandler( val comparator = compareBy> { it.second.followStatus } .thenBy { it.first.title } - return response.map { - val coverUrl = MdUtil.formThumbUrl(it.data.id) - /*val coverUrlId = it.relationships.firstOrNull { it.type == "cover_art" }?.id - if (coverUrlId != null) { - runCatching { - val covers = client.newCall(GET(MdUtil.coverUrl(it.data.id, coverUrlId))).await() - .parseAs() - covers.results.firstOrNull()?.data?.attributes?.fileName?.let { fileName -> - coverUrl = "${MdUtil.cdnUrl}/covers/${it.data.id}/$fileName" - } - } - }*/ + val coverMap = MdUtil.getCoversFromMangaList(response, client) + return response.map { MdUtil.createMangaEntry( it, lang, - coverUrl + coverMap[it.data.id] ).toSManga() to MangaDexSearchMetadata().apply { followStatus = FollowStatus.fromDex(statuses[it.data.id]).int } diff --git a/app/src/main/java/exh/md/handlers/PageHandler.kt b/app/src/main/java/exh/md/handlers/PageHandler.kt index 0c688d74d..c07c3f2fe 100644 --- a/app/src/main/java/exh/md/handlers/PageHandler.kt +++ b/app/src/main/java/exh/md/handlers/PageHandler.kt @@ -14,9 +14,10 @@ import rx.Observable class PageHandler( private val client: OkHttpClient, private val headers: Headers, - private val dataSaver: Boolean, private val apiChapterParser: ApiChapterParser, - private val mangaPlusHandler: MangaPlusHandler + private val mangaPlusHandler: MangaPlusHandler, + private val usePort443Only: () -> Boolean, + private val dataSaver: () -> Boolean ) { fun fetchPageList(chapter: SChapter): Observable> { @@ -28,11 +29,18 @@ class PageHandler( mangaPlusHandler.fetchPageList(chapterId) } } + + val atHomeRequestUrl = if (usePort443Only()) { + "${MdUtil.atHomeUrl}/${MdUtil.getChapterId(chapter.url)}?forcePort443=true" + } else { + "${MdUtil.atHomeUrl}/${MdUtil.getChapterId(chapter.url)}" + } + return client.newCall(pageListRequest(chapter)) .asObservableSuccess() .map { response -> - val host = MdUtil.atHomeUrlHostUrl("${MdUtil.atHomeUrl}/${MdUtil.getChapterId(chapter.url)}", client) - apiChapterParser.pageListParse(response, host, dataSaver) + val host = MdUtil.atHomeUrlHostUrl(atHomeRequestUrl, client, CacheControl.FORCE_NETWORK) + apiChapterParser.pageListParse(response, host, dataSaver()) } } diff --git a/app/src/main/java/exh/md/handlers/PopularHandler.kt b/app/src/main/java/exh/md/handlers/PopularHandler.kt index b562f8031..dfdf32e53 100644 --- a/app/src/main/java/exh/md/handlers/PopularHandler.kt +++ b/app/src/main/java/exh/md/handlers/PopularHandler.kt @@ -50,19 +50,10 @@ class PopularHandler( val mlResponse = response.parseAs(MdUtil.jsonParser) val hasMoreResults = mlResponse.limit + mlResponse.offset < mlResponse.total + val coverMap = MdUtil.getCoversFromMangaList(mlResponse.results, client) + val mangaList = mlResponse.results.map { - val coverUrl = MdUtil.formThumbUrl(it.data.id) - /*val coverUrlId = it.relationships.firstOrNull { it.type == "cover_art" }?.id - if (coverUrlId != null) { - runCatching { - val covers = client.newCall(GET(MdUtil.coverUrl(it.data.id, coverUrlId))).await() - .parseAs() - covers.results.firstOrNull()?.data?.attributes?.fileName?.let { fileName -> - coverUrl = "${MdUtil.cdnUrl}/covers/${it.data.id}/$fileName" - } - } - }*/ - MdUtil.createMangaEntry(it, lang, coverUrl).toSManga() + MdUtil.createMangaEntry(it, lang, coverMap[it.data.id]).toSManga() } return MangasPage(mangaList, hasMoreResults) } diff --git a/app/src/main/java/exh/md/handlers/SearchHandler.kt b/app/src/main/java/exh/md/handlers/SearchHandler.kt index 87bc7de6b..f95f7fc57 100644 --- a/app/src/main/java/exh/md/handlers/SearchHandler.kt +++ b/app/src/main/java/exh/md/handlers/SearchHandler.kt @@ -2,13 +2,11 @@ package exh.md.handlers import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.network.await import eu.kanade.tachiyomi.network.parseAs import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.toSManga import eu.kanade.tachiyomi.util.lang.runAsObservable -import exh.md.handlers.serializers.CoverListResponse import exh.md.handlers.serializers.MangaListResponse import exh.md.handlers.serializers.MangaResponse import exh.md.utils.MdUtil @@ -36,21 +34,8 @@ class SearchHandler( .flatMap { response -> runAsObservable({ val mangaResponse = response.parseAs(MdUtil.jsonParser) - - val coverUrl = MdUtil.formThumbUrl(mangaResponse.data.id) - /*val coverUrlId = mangaResponse.relationships.firstOrNull { it.type == "cover_art" }?.id - if (coverUrlId != null) { - runCatching { - val covers = client.newCall(GET(MdUtil.coverUrl(mangaResponse.data.id, coverUrlId))).await() - .parseAs() - covers.results.firstOrNull()?.data?.attributes?.fileName?.let { fileName -> - coverUrl = "${MdUtil.cdnUrl}/covers/${mangaResponse.data.id}/$fileName" - } - } - }*/ - val details = apiMangaParser - .parseToManga(MdUtil.createMangaEntry(mangaResponse, lang, coverUrl), response, sourceId).toSManga() + .parseToManga(MdUtil.createMangaEntry(mangaResponse, lang, null), response, sourceId).toSManga() MangasPage(listOf(details), false) }) } @@ -67,20 +52,10 @@ class SearchHandler( private suspend fun searchMangaParse(response: Response): MangasPage { val mlResponse = response.parseAs(MdUtil.jsonParser) + val coverMap = MdUtil.getCoversFromMangaList(mlResponse.results, client) val hasMoreResults = mlResponse.limit + mlResponse.offset < mlResponse.total val mangaList = mlResponse.results.map { - var coverUrl = MdUtil.formThumbUrl(it.data.id) - val coverUrlId = it.relationships.firstOrNull { it.type == "cover_art" }?.id - if (coverUrlId != null) { - runCatching { - val covers = client.newCall(GET(MdUtil.coverUrl(it.data.id, coverUrlId))).await() - .parseAs() - covers.results.firstOrNull()?.data?.attributes?.fileName?.let { fileName -> - coverUrl = "${MdUtil.cdnUrl}/covers/${it.data.id}/$fileName" - } - } - } - MdUtil.createMangaEntry(it, lang, coverUrl).toSManga() + MdUtil.createMangaEntry(it, lang, coverMap[it.data.id]).toSManga() } return MangasPage(mangaList, hasMoreResults) } diff --git a/app/src/main/java/exh/md/handlers/SimilarHandler.kt b/app/src/main/java/exh/md/handlers/SimilarHandler.kt index 4c618f0e9..f8bcc04d7 100644 --- a/app/src/main/java/exh/md/handlers/SimilarHandler.kt +++ b/app/src/main/java/exh/md/handlers/SimilarHandler.kt @@ -5,13 +5,14 @@ import eu.kanade.tachiyomi.network.await import eu.kanade.tachiyomi.network.parseAs import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.SManga +import exh.md.handlers.serializers.CoverListResponse import exh.md.handlers.serializers.SimilarMangaResponse import exh.md.utils.MdUtil import okhttp3.CacheControl import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient import okhttp3.Request -import okhttp3.Response import tachiyomi.source.model.MangaInfo class SimilarHandler( @@ -21,7 +22,29 @@ class SimilarHandler( suspend fun getSimilar(manga: MangaInfo): MangasPage { val response = client.newCall(similarMangaRequest(manga)).await() - return similarMangaParse(response) + .parseAs() + + val ids = response.matches.map { it.id } + + val coverUrl = MdUtil.coverUrl.toHttpUrl().newBuilder().apply { + ids.forEach { mangaId -> + addQueryParameter("manga[]", mangaId) + } + addQueryParameter("limit", ids.size.toString()) + }.build().toString() + val coverListResponse = client.newCall(GET(coverUrl)).await() + .parseAs() + + val unique = coverListResponse.results.distinctBy { it.relationships[0].id } + + val coverMap = unique.map { coverResponse -> + val fileName = coverResponse.data.attributes.fileName + val mangaId = coverResponse.relationships.first { it.type.equals("manga", true) }.id + val thumbnailUrl = "${MdUtil.cdnUrl}/covers/$mangaId/$fileName" + mangaId to thumbnailUrl + }.toMap() + + return similarMangaParse(response, coverMap) } private fun similarMangaRequest(manga: MangaInfo): Request { @@ -29,12 +52,12 @@ class SimilarHandler( return GET(tempUrl, Headers.Builder().build(), CacheControl.FORCE_NETWORK) } - private fun similarMangaParse(response: Response): MangasPage { - val mangaList = response.parseAs().matches.map { + private fun similarMangaParse(response: SimilarMangaResponse, coverMap: Map): MangasPage { + val mangaList = response.matches.map { SManga.create().apply { url = MdUtil.buildMangaUrl(it.id) title = MdUtil.cleanString(it.title[lang] ?: it.title["en"]!!) - thumbnail_url = MdUtil.formThumbUrl(url) + thumbnail_url = coverMap[it.id] } } return MangasPage(mangaList, false) diff --git a/app/src/main/java/exh/md/handlers/serializers/MangaSerializer.kt b/app/src/main/java/exh/md/handlers/serializers/MangaSerializer.kt index 61d67fe89..ecdbf6671 100644 --- a/app/src/main/java/exh/md/handlers/serializers/MangaSerializer.kt +++ b/app/src/main/java/exh/md/handlers/serializers/MangaSerializer.kt @@ -98,6 +98,7 @@ data class CoverListResponse( @Serializable data class CoverResponse( val data: Cover, + val relationships: List ) @Serializable diff --git a/app/src/main/java/exh/md/utils/MdUtil.kt b/app/src/main/java/exh/md/utils/MdUtil.kt index 470d6b4ed..4e4d6fcdb 100644 --- a/app/src/main/java/exh/md/utils/MdUtil.kt +++ b/app/src/main/java/exh/md/utils/MdUtil.kt @@ -10,7 +10,10 @@ import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.all.MangaDex import exh.log.xLogD +import exh.log.xLogE import exh.md.handlers.serializers.AtHomeResponse +import exh.md.handlers.serializers.CoverListResponse +import exh.md.handlers.serializers.CoverResponse import exh.md.handlers.serializers.ListCallResponse import exh.md.handlers.serializers.LoginBodyToken import exh.md.handlers.serializers.MangaResponse @@ -24,6 +27,7 @@ import kotlinx.serialization.SerializationException import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import okhttp3.CacheControl import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient @@ -44,6 +48,7 @@ class MdUtil { const val apiUrl = "https://api.mangadex.org" const val imageUrlCacheNotFound = "https://cdn.statically.io/img/raw.githubusercontent.com/CarlosEsco/Neko/master/.github/manga_cover_not_found.png" const val atHomeUrl = "$apiUrl/at-home/server" + const val coverUrl = "$apiUrl/cover" const val chapterUrl = "$apiUrl/chapter/" const val chapterSuffix = "/chapter/" const val checkTokenUrl = "$apiUrl/auth/check" @@ -68,10 +73,10 @@ class MdUtil { }.build().toString() } - fun coverUrl(mangaId: String, coverId: String) = "$apiUrl/cover/?manga[]=$mangaId&ids[]=$coverId" + fun coverUrl(mangaId: String, coverId: String) = "$apiUrl/cover?manga[]=$mangaId&ids[]=$coverId" - const val similarCache = "https://raw.githubusercontent.com/goldbattle/MangadexRecomendations/master/output/api/" - const val similarCacheCdn = "https://cdn.statically.io/gh/goldbattle/MangadexRecomendations/master/output/api/" + const val similarCacheMapping = "https://api.similarmanga.com/mapping/mdex2search.csv" + const val similarCacheMangas = "https://api.similarmanga.com/manga/" const val similarBaseApi = "https://api.similarmanga.com/similar/" const val groupSearchUrl = "$baseUrl/groups/0/1/" @@ -96,6 +101,11 @@ class MdUtil { private const val scanlatorSeparator = " & " + const val contentRatingSafe = "safe" + const val contentRatingSuggestive = "suggestive" + const val contentRatingErotica = "erotica" + const val contentRatingPornographic = "pornographic" + val validOneShotFinalChapters = listOf("0", "1") val englishDescriptionTags = listOf( @@ -288,10 +298,10 @@ class MdUtil { return null } - fun atHomeUrlHostUrl(requestUrl: String, client: OkHttpClient): String { - val atHomeRequest = GET(requestUrl) + fun atHomeUrlHostUrl(requestUrl: String, client: OkHttpClient, cacheControl: CacheControl): String { + val atHomeRequest = GET(requestUrl, cache = cacheControl) val atHomeResponse = client.newCall(atHomeRequest).execute() - return atHomeResponse.parseAs(jsonParser).baseUrl + return jsonParser.decodeFromString(atHomeResponse.body!!.string()).baseUrl } val dateFormatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+SSS", Locale.US) @@ -300,15 +310,61 @@ class MdUtil { fun parseDate(dateAsString: String): Long = dateFormatter.parse(dateAsString)?.time ?: 0 - fun createMangaEntry(json: MangaResponse, lang: String, coverUrl: String): MangaInfo { - val key = buildMangaUrl(json.data.id) + fun createMangaEntry(json: MangaResponse, lang: String, coverUrl: String?): MangaInfo { return MangaInfo( - key = key, + key = buildMangaUrl(json.data.id), title = cleanString(json.data.attributes.title[lang] ?: json.data.attributes.title["en"]!!), - cover = coverUrl + cover = coverUrl.orEmpty() ) } + suspend fun getCoverUrl(dexId: String, coverId: String?, client: OkHttpClient): String { + coverId ?: return "" + val coverResponse = client.newCall(GET("$coverUrl/$coverId")) + .await().parseAs() + val fileName = coverResponse.data.attributes.fileName + return "$cdnUrl/covers/$dexId/$fileName" + } + + suspend fun getCoversFromMangaList(mangaResponseList: List, client: OkHttpClient): Map { + val idsAndCoverIds = mangaResponseList.mapNotNull { mangaResponse -> + val mangaId = mangaResponse.data.id + val coverId = mangaResponse.relationships.firstOrNull { relationship -> + relationship.type.equals("cover_art", true) + }?.id + if (coverId == null) { + null + } else { + Pair(mangaId, coverId) + } + }.toMap() + + return runCatching { + getBatchCoverUrls(idsAndCoverIds, client) + }.getOrNull()!! + } + + private suspend fun getBatchCoverUrls(ids: Map, client: OkHttpClient): Map { + try { + val url = coverUrl.toHttpUrl().newBuilder().apply { + ids.values.forEach { coverArtId -> + addQueryParameter("ids[]", coverArtId) + } + addQueryParameter("limit", ids.size.toString()) + }.build().toString() + val coverList = client.newCall(GET(url)).await().parseAs(jsonParser) + return coverList.results.map { coverResponse -> + val fileName = coverResponse.data.attributes.fileName + val mangaId = coverResponse.relationships.first { it.type.equals("manga", true) }.id + val thumbnailUrl = "$cdnUrl/covers/$mangaId/$fileName" + Pair(mangaId, thumbnailUrl) + }.toMap() + } catch (e: Exception) { + xLogE("Error getting covers", e) + throw e + } + } + fun getLoginBody(preferences: PreferencesHelper, mdList: MdList) = preferences.trackToken(mdList).get().nullIfBlank()?.let { try { jsonParser.decodeFromString(it)