From 29e43924901a0f7d322160afa2ad7d855287c19d Mon Sep 17 00:00:00 2001 From: Jobobby04 Date: Tue, 11 May 2021 13:06:55 -0400 Subject: [PATCH] Cleanup the mangadex list calls --- .../java/exh/md/handlers/FollowsHandler.kt | 29 +---- .../main/java/exh/md/handlers/MangaHandler.kt | 106 +++--------------- .../serializers/ListCallSerializer.kt | 11 ++ app/src/main/java/exh/md/utils/MdUtil.kt | 21 ++++ 4 files changed, 48 insertions(+), 119 deletions(-) create mode 100644 app/src/main/java/exh/md/handlers/serializers/ListCallSerializer.kt diff --git a/app/src/main/java/exh/md/handlers/FollowsHandler.kt b/app/src/main/java/exh/md/handlers/FollowsHandler.kt index 9d59532b3..e0059ccd1 100644 --- a/app/src/main/java/exh/md/handlers/FollowsHandler.kt +++ b/app/src/main/java/exh/md/handlers/FollowsHandler.kt @@ -19,6 +19,7 @@ import exh.md.handlers.serializers.MangaStatusResponse import exh.md.handlers.serializers.UpdateReadingStatus import exh.md.utils.FollowStatus import exh.md.utils.MdUtil +import exh.md.utils.mdListCall import exh.metadata.metadata.MangaDexSearchMetadata import exh.util.under import kotlinx.serialization.encodeToString @@ -197,34 +198,10 @@ class FollowsHandler( */ suspend fun fetchAllFollows(): List> { return withIOContext { - val response = client.newCall(followsListRequest(0)).await() - - if (response.code == 204) { - return@withIOContext emptyList() + val results = client.mdListCall { + followsListRequest(it) } - val mangaListResponse = response.parseAs(MdUtil.jsonParser) - val results = mangaListResponse.results.toMutableList() - - if (results.isEmpty()) { - return@withIOContext emptyList() - } - - var hasMoreResults = mangaListResponse.limit + mangaListResponse.offset under mangaListResponse.total - var lastOffset = mangaListResponse.offset - - while (hasMoreResults) { - val offset = lastOffset + mangaListResponse.limit - val newResponse = client.newCall(followsListRequest(offset)).await() - if (newResponse.code != 204) { - val newMangaListResponse = newResponse.parseAs(MdUtil.jsonParser) - results += newMangaListResponse.results - hasMoreResults = newMangaListResponse.limit + newMangaListResponse.offset under newMangaListResponse.total - lastOffset = newMangaListResponse.offset - } else { - hasMoreResults = false - } - } val statuses = results.chunked(100) .map { client.newCall(mangaStatusListRequest(results)).await().parseAs().statuses diff --git a/app/src/main/java/exh/md/handlers/MangaHandler.kt b/app/src/main/java/exh/md/handlers/MangaHandler.kt index afe645efe..d7ee76d88 100644 --- a/app/src/main/java/exh/md/handlers/MangaHandler.kt +++ b/app/src/main/java/exh/md/handlers/MangaHandler.kt @@ -3,7 +3,6 @@ package exh.md.handlers import eu.kanade.tachiyomi.data.database.models.Track import eu.kanade.tachiyomi.data.track.mdlist.MdList import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.network.await import eu.kanade.tachiyomi.network.parseAs import eu.kanade.tachiyomi.source.model.SChapter @@ -13,12 +12,11 @@ import eu.kanade.tachiyomi.source.model.toSChapter import eu.kanade.tachiyomi.source.model.toSManga import eu.kanade.tachiyomi.util.lang.runAsObservable import eu.kanade.tachiyomi.util.lang.withIOContext -import exh.md.handlers.serializers.ChapterListResponse import exh.md.handlers.serializers.ChapterResponse import exh.md.handlers.serializers.GroupListResponse import exh.md.utils.MdUtil +import exh.md.utils.mdListCall import exh.metadata.metadata.MangaDexSearchMetadata -import exh.util.under import kotlinx.coroutines.async import okhttp3.CacheControl import okhttp3.Headers @@ -66,95 +64,19 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l } fun fetchMangaDetailsObservable(manga: SManga, sourceId: Long): Observable { - return client.newCall(mangaRequest(manga.toMangaInfo())) - .asObservableSuccess() - .flatMap { response -> - runAsObservable({ - ApiMangaParser(client, lang).parseToManga(manga.toMangaInfo(), response, emptyList(), sourceId).toSManga() - }) - } + return runAsObservable({ + getMangaDetails(manga.toMangaInfo(), sourceId).toSManga() + }) } - fun fetchChapterListObservable(manga: SManga): Observable> { - return client.newCall(mangaFeedRequest(manga.toMangaInfo(), 0, lang)) - .asObservableSuccess() - .map { response -> - if (response.code == 204) { - return@map emptyList() - } - - val chapterListResponse = response.parseAs(MdUtil.jsonParser) - val results = chapterListResponse.results.toMutableList() - - var hasMoreResults = chapterListResponse.limit + chapterListResponse.offset under chapterListResponse.total - var lastOffset = chapterListResponse.offset - - while (hasMoreResults) { - val offset = lastOffset + chapterListResponse.limit - val newResponse = client.newCall(mangaFeedRequest(manga.toMangaInfo(), offset, lang)).execute() - if (newResponse.code != 204) { - val newChapterListResponse = newResponse - .parseAs(MdUtil.jsonParser) - results += newChapterListResponse.results - hasMoreResults = newChapterListResponse.limit + newChapterListResponse.offset under newChapterListResponse.total - lastOffset = newChapterListResponse.offset - } else { - hasMoreResults = false - } - } - val groupIds = - results.asSequence() - .map { chapter -> chapter.relationships } - .flatten() - .filter { it.type == "scanlation_group" } - .map { it.id } - .distinct() - .toList() - - val groupMap = runCatching { - groupIds.chunked(100).flatMapIndexed { index, ids -> - val groupResponse = client.newCall(groupIdRequest(ids, 100 * index)).execute() - if (groupResponse.code != 204) { - groupResponse - .parseAs(MdUtil.jsonParser) - .results - .map { group -> Pair(group.data.id, group.data.attributes.name) } - } else { - emptyList() - } - }.toMap() - }.getOrNull() ?: emptyMap() - - ApiMangaParser(client, lang).chapterListParse(results, groupMap).map { it.toSChapter() } - } - } + fun fetchChapterListObservable(manga: SManga): Observable> = runAsObservable({ + getChapterList(manga.toMangaInfo()).map { it.toSChapter() } + }) suspend fun getChapterList(manga: MangaInfo): List { return withIOContext { - val response = client.newCall(mangaFeedRequest(manga, 0, lang)).await() - - if (response.code == 204) { - return@withIOContext emptyList() - } - - val chapterListResponse = response.parseAs(MdUtil.jsonParser) - val results = chapterListResponse.results.toMutableList() - - var hasMoreResults = chapterListResponse.limit + chapterListResponse.offset under chapterListResponse.total - var lastOffset = chapterListResponse.offset - - while (hasMoreResults) { - val offset = lastOffset + chapterListResponse.limit - val newResponse = client.newCall(mangaFeedRequest(manga, offset, lang)).await() - if (newResponse.code != 204) { - val newChapterListResponse = newResponse - .parseAs(MdUtil.jsonParser) - results += newChapterListResponse.results - hasMoreResults = newChapterListResponse.limit + newChapterListResponse.offset under newChapterListResponse.total - lastOffset = newChapterListResponse.offset - } else { - hasMoreResults = false - } + val results = client.mdListCall { + mangaFeedRequest(manga, it, lang) } val groupMap = getGroupMap(results) @@ -165,12 +87,10 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l private suspend fun getGroupMap(results: List): Map { val groupIds = results.asSequence() - .map { chapter -> chapter.relationships } - .flatten() + .flatMap { chapter -> chapter.relationships } .filter { it.type == "scanlation_group" } .map { it.id } - .distinct() - .toList() + .toSet() return runCatching { groupIds.chunked(100).flatMapIndexed { index, ids -> @@ -178,12 +98,12 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l if (response.code != 204) { response .parseAs(MdUtil.jsonParser) - .results.map { group -> Pair(group.data.id, group.data.attributes.name) } + .results.map { group -> group.data.id to group.data.attributes.name } } else { emptyList() } }.toMap() - }.getOrNull() ?: emptyMap() + }.getOrNull().orEmpty() } suspend fun fetchRandomMangaId(): String { diff --git a/app/src/main/java/exh/md/handlers/serializers/ListCallSerializer.kt b/app/src/main/java/exh/md/handlers/serializers/ListCallSerializer.kt new file mode 100644 index 000000000..a3f0d8b31 --- /dev/null +++ b/app/src/main/java/exh/md/handlers/serializers/ListCallSerializer.kt @@ -0,0 +1,11 @@ +package exh.md.handlers.serializers + +import kotlinx.serialization.Serializable + +@Serializable +data class ListCallResponse( + val limit: Int, + val offset: Int, + val total: Int, + val results: List +) diff --git a/app/src/main/java/exh/md/utils/MdUtil.kt b/app/src/main/java/exh/md/utils/MdUtil.kt index 5cdb0a6ae..daed4c06b 100644 --- a/app/src/main/java/exh/md/utils/MdUtil.kt +++ b/app/src/main/java/exh/md/utils/MdUtil.kt @@ -3,6 +3,7 @@ package exh.md.utils import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.track.mdlist.MdList import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.await import eu.kanade.tachiyomi.network.parseAs import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.model.SChapter @@ -10,6 +11,7 @@ import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.all.MangaDex import exh.log.xLogD import exh.md.handlers.serializers.AtHomeResponse +import exh.md.handlers.serializers.ListCallResponse import exh.md.handlers.serializers.LoginBodyToken import exh.md.handlers.serializers.MangaResponse import exh.md.network.NoSessionException @@ -17,6 +19,7 @@ import exh.source.getMainSource import exh.util.floor import exh.util.nullIfBlank import exh.util.nullIfZero +import exh.util.under import kotlinx.serialization.SerializationException import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString @@ -24,6 +27,7 @@ import kotlinx.serialization.json.Json import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient +import okhttp3.Request import org.jsoup.parser.Parser import tachiyomi.source.model.MangaInfo import uy.kohesive.injekt.Injekt @@ -351,3 +355,20 @@ class MdUtil { } } } + +suspend inline fun OkHttpClient.mdListCall(request: (offset: Int) -> Request): List { + val results = mutableListOf() + var offset = 0 + + do { + val response = newCall(request(offset)).await() + if (response.code == 204) { + break + } + val mangaListResponse = response.parseAs>(MdUtil.jsonParser) + results += mangaListResponse.results + offset += mangaListResponse.limit + } while (offset under mangaListResponse.total) + + return results +}