Cleanup the mangadex list calls

app/src/main/java/exh/md/handlers/FollowsHandler.kt

@@ -19,6 +19,7 @@ import exh.md.handlers.serializers.MangaStatusResponse
 import exh.md.handlers.serializers.UpdateReadingStatus
 import exh.md.utils.FollowStatus
 import exh.md.utils.MdUtil
+import exh.md.utils.mdListCall
 import exh.metadata.metadata.MangaDexSearchMetadata
 import exh.util.under
 import kotlinx.serialization.encodeToString
@@ -197,34 +198,10 @@ class FollowsHandler(
      */
     suspend fun fetchAllFollows(): List<Pair<SManga, MangaDexSearchMetadata>> {
         return withIOContext {
-            val response = client.newCall(followsListRequest(0)).await()
+            val results = client.mdListCall<MangaResponse> {
-
+                followsListRequest(it)
-            if (response.code == 204) {
-                return@withIOContext emptyList()
             }
 
-            val mangaListResponse = response.parseAs<MangaListResponse>(MdUtil.jsonParser)
-            val results = mangaListResponse.results.toMutableList()
-
-            if (results.isEmpty()) {
-                return@withIOContext emptyList()
-            }
-
-            var hasMoreResults = mangaListResponse.limit + mangaListResponse.offset under mangaListResponse.total
-            var lastOffset = mangaListResponse.offset
-
-            while (hasMoreResults) {
-                val offset = lastOffset + mangaListResponse.limit
-                val newResponse = client.newCall(followsListRequest(offset)).await()
-                if (newResponse.code != 204) {
-                    val newMangaListResponse = newResponse.parseAs<MangaListResponse>(MdUtil.jsonParser)
-                    results += newMangaListResponse.results
-                    hasMoreResults = newMangaListResponse.limit + newMangaListResponse.offset under newMangaListResponse.total
-                    lastOffset = newMangaListResponse.offset
-                } else {
-                    hasMoreResults = false
-                }
-            }
             val statuses = results.chunked(100)
                 .map {
                     client.newCall(mangaStatusListRequest(results)).await().parseAs<MangaStatusListResponse>().statuses

app/src/main/java/exh/md/handlers/MangaHandler.kt

@@ -3,7 +3,6 @@ package exh.md.handlers
 import eu.kanade.tachiyomi.data.database.models.Track
 import eu.kanade.tachiyomi.data.track.mdlist.MdList
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.network.await
 import eu.kanade.tachiyomi.network.parseAs
 import eu.kanade.tachiyomi.source.model.SChapter
@@ -13,12 +12,11 @@ import eu.kanade.tachiyomi.source.model.toSChapter
 import eu.kanade.tachiyomi.source.model.toSManga
 import eu.kanade.tachiyomi.util.lang.runAsObservable
 import eu.kanade.tachiyomi.util.lang.withIOContext
-import exh.md.handlers.serializers.ChapterListResponse
 import exh.md.handlers.serializers.ChapterResponse
 import exh.md.handlers.serializers.GroupListResponse
 import exh.md.utils.MdUtil
+import exh.md.utils.mdListCall
 import exh.metadata.metadata.MangaDexSearchMetadata
-import exh.util.under
 import kotlinx.coroutines.async
 import okhttp3.CacheControl
 import okhttp3.Headers
@@ -66,95 +64,19 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l
     }
 
     fun fetchMangaDetailsObservable(manga: SManga, sourceId: Long): Observable<SManga> {
-        return client.newCall(mangaRequest(manga.toMangaInfo()))
+        return runAsObservable({
-            .asObservableSuccess()
+            getMangaDetails(manga.toMangaInfo(), sourceId).toSManga()
-            .flatMap { response ->
-                runAsObservable({
-                    ApiMangaParser(client, lang).parseToManga(manga.toMangaInfo(), response, emptyList(), sourceId).toSManga()
         })
     }
-    }
 
-    fun fetchChapterListObservable(manga: SManga): Observable<List<SChapter>> {
+    fun fetchChapterListObservable(manga: SManga): Observable<List<SChapter>> = runAsObservable({
-        return client.newCall(mangaFeedRequest(manga.toMangaInfo(), 0, lang))
+        getChapterList(manga.toMangaInfo()).map { it.toSChapter() }
-            .asObservableSuccess()
+    })
-            .map { response ->
-                if (response.code == 204) {
-                    return@map emptyList()
-                }
-
-                val chapterListResponse = response.parseAs<ChapterListResponse>(MdUtil.jsonParser)
-                val results = chapterListResponse.results.toMutableList()
-
-                var hasMoreResults = chapterListResponse.limit + chapterListResponse.offset under chapterListResponse.total
-                var lastOffset = chapterListResponse.offset
-
-                while (hasMoreResults) {
-                    val offset = lastOffset + chapterListResponse.limit
-                    val newResponse = client.newCall(mangaFeedRequest(manga.toMangaInfo(), offset, lang)).execute()
-                    if (newResponse.code != 204) {
-                        val newChapterListResponse = newResponse
-                            .parseAs<ChapterListResponse>(MdUtil.jsonParser)
-                        results += newChapterListResponse.results
-                        hasMoreResults = newChapterListResponse.limit + newChapterListResponse.offset under newChapterListResponse.total
-                        lastOffset = newChapterListResponse.offset
-                    } else {
-                        hasMoreResults = false
-                    }
-                }
-                val groupIds =
-                    results.asSequence()
-                        .map { chapter -> chapter.relationships }
-                        .flatten()
-                        .filter { it.type == "scanlation_group" }
-                        .map { it.id }
-                        .distinct()
-                        .toList()
-
-                val groupMap = runCatching {
-                    groupIds.chunked(100).flatMapIndexed { index, ids ->
-                        val groupResponse = client.newCall(groupIdRequest(ids, 100 * index)).execute()
-                        if (groupResponse.code != 204) {
-                            groupResponse
-                                .parseAs<GroupListResponse>(MdUtil.jsonParser)
-                                .results
-                                .map { group -> Pair(group.data.id, group.data.attributes.name) }
-                        } else {
-                            emptyList()
-                        }
-                    }.toMap()
-                }.getOrNull() ?: emptyMap()
-
-                ApiMangaParser(client, lang).chapterListParse(results, groupMap).map { it.toSChapter() }
-            }
-    }
 
     suspend fun getChapterList(manga: MangaInfo): List<ChapterInfo> {
         return withIOContext {
-            val response = client.newCall(mangaFeedRequest(manga, 0, lang)).await()
+            val results = client.mdListCall<ChapterResponse> {
-
+                mangaFeedRequest(manga, it, lang)
-            if (response.code == 204) {
-                return@withIOContext emptyList()
-            }
-
-            val chapterListResponse = response.parseAs<ChapterListResponse>(MdUtil.jsonParser)
-            val results = chapterListResponse.results.toMutableList()
-
-            var hasMoreResults = chapterListResponse.limit + chapterListResponse.offset under chapterListResponse.total
-            var lastOffset = chapterListResponse.offset
-
-            while (hasMoreResults) {
-                val offset = lastOffset + chapterListResponse.limit
-                val newResponse = client.newCall(mangaFeedRequest(manga, offset, lang)).await()
-                if (newResponse.code != 204) {
-                    val newChapterListResponse = newResponse
-                        .parseAs<ChapterListResponse>(MdUtil.jsonParser)
-                    results += newChapterListResponse.results
-                    hasMoreResults = newChapterListResponse.limit + newChapterListResponse.offset under newChapterListResponse.total
-                    lastOffset = newChapterListResponse.offset
-                } else {
-                    hasMoreResults = false
-                }
             }
 
             val groupMap = getGroupMap(results)
@@ -165,12 +87,10 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l
 
     private suspend fun getGroupMap(results: List<ChapterResponse>): Map<String, String> {
         val groupIds = results.asSequence()
-            .map { chapter -> chapter.relationships }
+            .flatMap { chapter -> chapter.relationships }
-            .flatten()
             .filter { it.type == "scanlation_group" }
             .map { it.id }
-            .distinct()
+            .toSet()
-            .toList()
 
         return runCatching {
             groupIds.chunked(100).flatMapIndexed { index, ids ->
@@ -178,12 +98,12 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l
                 if (response.code != 204) {
                     response
                         .parseAs<GroupListResponse>(MdUtil.jsonParser)
-                        .results.map { group -> Pair(group.data.id, group.data.attributes.name) }
+                        .results.map { group -> group.data.id to group.data.attributes.name }
                 } else {
                     emptyList()
                 }
             }.toMap()
-        }.getOrNull() ?: emptyMap()
+        }.getOrNull().orEmpty()
     }
 
     suspend fun fetchRandomMangaId(): String {

app/src/main/java/exh/md/handlers/serializers/ListCallSerializer.kt

@@ -0,0 +1,11 @@
+package exh.md.handlers.serializers
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class ListCallResponse<T>(
+    val limit: Int,
+    val offset: Int,
+    val total: Int,
+    val results: List<T>
+)

app/src/main/java/exh/md/utils/MdUtil.kt

@@ -3,6 +3,7 @@ package exh.md.utils
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.track.mdlist.MdList
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.await
 import eu.kanade.tachiyomi.network.parseAs
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.source.model.SChapter
@@ -10,6 +11,7 @@ import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.all.MangaDex
 import exh.log.xLogD
 import exh.md.handlers.serializers.AtHomeResponse
+import exh.md.handlers.serializers.ListCallResponse
 import exh.md.handlers.serializers.LoginBodyToken
 import exh.md.handlers.serializers.MangaResponse
 import exh.md.network.NoSessionException
@@ -17,6 +19,7 @@ import exh.source.getMainSource
 import exh.util.floor
 import exh.util.nullIfBlank
 import exh.util.nullIfZero
+import exh.util.under
 import kotlinx.serialization.SerializationException
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
@@ -24,6 +27,7 @@ import kotlinx.serialization.json.Json
 import okhttp3.Headers
 import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
+import okhttp3.Request
 import org.jsoup.parser.Parser
 import tachiyomi.source.model.MangaInfo
 import uy.kohesive.injekt.Injekt
@@ -351,3 +355,20 @@ class MdUtil {
         }
     }
 }
+
+suspend inline fun <reified T> OkHttpClient.mdListCall(request: (offset: Int) -> Request): List<T> {
+    val results = mutableListOf<T>()
+    var offset = 0
+
+    do {
+        val response = newCall(request(offset)).await()
+        if (response.code == 204) {
+            break
+        }
+        val mangaListResponse = response.parseAs<ListCallResponse<T>>(MdUtil.jsonParser)
+        results += mangaListResponse.results
+        offset += mangaListResponse.limit
+    } while (offset under mangaListResponse.total)
+
+    return results
+}