update to use includes to reduce api calls (#7603)

src/all/mangadex/build.gradle

@@ -6,7 +6,7 @@ ext {
     extName = 'MangaDex'
     pkgNameSuffix = 'all.mangadex'
     extClass = '.MangaDexFactory'
-    extVersionCode = 118
+    extVersionCode = 119
     libVersion = '1.2'
     containsNsfw = true
 }

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MDConstants.kt

@@ -10,11 +10,14 @@ object MDConstants {
         Regex("[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}")
 
     const val mangaLimit = 20
+    const val coverArt = "cover_art"
+    const val scanlator = "scanlation_group"
+    const val author = "author"
+    const val artist = "artist"
     const val cdnUrl = "https://uploads.mangadex.org"
 
     const val apiUrl = "https://api.mangadex.org"
     const val apiMangaUrl = "$apiUrl/manga"
-    const val apiCoverUrl = "$apiUrl/cover"
     const val atHomePostUrl = "https://api.mangadex.network/report"
     val whitespaceRegex = "\\s".toRegex()
 

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt

@@ -62,6 +62,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
             addQueryParameter("order[updatedAt]", "desc")
             addQueryParameter("limit", MDConstants.mangaLimit.toString())
             addQueryParameter("offset", helper.getMangaListOffset(page))
+            addQueryParameter("includes[]", MDConstants.coverArt)
             if (preferences.getBoolean(MDConstants.getContentRatingSafePrefKey(dexLang), false)) {
                 addQueryParameter("contentRating[]", "safe")
             }
@@ -105,26 +106,11 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
         val mangaListDto = helper.json.decodeFromString<MangaListDto>(response.body!!.string())
         val hasMoreResults = mangaListDto.limit + mangaListDto.offset < mangaListDto.total
 
-        val idsAndCoverIds = mangaListDto.results.mapNotNull { mangaDto ->
+        val mangaList = mangaListDto.results.map { mangaDto ->
-            val mangaId = mangaDto.data.id
+            val fileName = mangaDto.relationships.firstOrNull { relationshipDto ->
-            val coverId = mangaDto.relationships.firstOrNull { relationshipDto ->
+                relationshipDto.type.equals(MDConstants.coverArt, true)
-                relationshipDto.type.equals("cover_art", true)
+            }?.attributes?.fileName
-            }?.id
+            helper.createBasicManga(mangaDto, fileName)
-            if (coverId == null) {
-                null
-            } else {
-                Pair(mangaId, coverId)
-            }
-        }.toMap()
-
-        val results = runCatching {
-            helper.getBatchCoversUrl(idsAndCoverIds, client)
-        }.getOrNull()!!
-
-        val mangaList = mangaListDto.results.map {
-            helper.createBasicManga(it).apply {
-                thumbnail_url = results[url.substringAfter("/manga/")]
-            }
         }
 
         return MangasPage(mangaList, hasMoreResults)
@@ -141,6 +127,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
         if (query.startsWith(MDConstants.prefixIdSearch)) {
             val url = MDConstants.apiMangaUrl.toHttpUrlOrNull()!!.newBuilder()
                 .addQueryParameter("ids[]", query.removePrefix(MDConstants.prefixIdSearch))
+                .addQueryParameter("includes[]", MDConstants.coverArt)
             return GET(url.toString(), headers, CacheControl.FORCE_NETWORK)
         }
 
@@ -149,6 +136,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
         tempUrl.apply {
             addQueryParameter("limit", MDConstants.mangaLimit.toString())
             addQueryParameter("offset", (helper.getMangaListOffset(page)))
+            addQueryParameter("includes[]", MDConstants.coverArt)
             val actualQuery = query.replace(MDConstants.whitespaceRegex, " ")
             if (actualQuery.isNotBlank()) {
                 addQueryParameter("title", actualQuery)
@@ -181,16 +169,21 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
     /**
      * get manga details url throws exception if the url is the old format so people migrate
      */
-    fun apiMangaDetailsRequest(manga: SManga): Request {
+    private fun apiMangaDetailsRequest(manga: SManga): Request {
         if (!helper.containsUuid(manga.url.trim())) {
             throw Exception("Migrate this manga from MangaDex to MangaDex to update it")
         }
-        return GET("${MDConstants.apiUrl}${manga.url}", headers, CacheControl.FORCE_NETWORK)
+        val url = (MDConstants.apiUrl + manga.url).toHttpUrl().newBuilder().apply {
+            addQueryParameter("includes[]", MDConstants.coverArt)
+            addQueryParameter("includes[]", MDConstants.author)
+            addQueryParameter("includes[]", MDConstants.artist)
+        }.build().toString()
+        return GET(url, headers, CacheControl.FORCE_NETWORK)
     }
 
     override fun mangaDetailsParse(response: Response): SManga {
         val manga = helper.json.decodeFromString<MangaDto>(response.body!!.string())
-        return helper.createManga(manga, client, lang.substringBefore("-"))
+        return helper.createManga(manga, lang.substringBefore("-"))
     }
 
     // Chapter list section
@@ -247,11 +240,9 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
                 hasMoreResults = (limit + offset) < newChapterList.total
             }
 
-            val groupMap = helper.createGroupMap(chapterListResults.toList(), client)
-
             val now = Date().time
 
-            return chapterListResults.map { helper.createChapter(it, groupMap) }
+            return chapterListResults.map { helper.createChapter(it) }
                 .filter { it.date_upload <= now && "MangaPlus" != it.scanlator }
         } catch (e: Exception) {
             Log.e("MangaDex", "error parsing chapter list", e)

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt

@@ -2,11 +2,7 @@ package eu.kanade.tachiyomi.extension.all.mangadex
 
 import android.util.Log
 import eu.kanade.tachiyomi.extension.all.mangadex.dto.AtHomeDto
-import eu.kanade.tachiyomi.extension.all.mangadex.dto.AuthorListDto
 import eu.kanade.tachiyomi.extension.all.mangadex.dto.ChapterDto
-import eu.kanade.tachiyomi.extension.all.mangadex.dto.CoverDto
-import eu.kanade.tachiyomi.extension.all.mangadex.dto.CoverListDto
-import eu.kanade.tachiyomi.extension.all.mangadex.dto.GroupListDto
 import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDto
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.Page
@@ -16,7 +12,6 @@ import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
 import okhttp3.CacheControl
 import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import org.jsoup.parser.Parser
@@ -44,7 +39,7 @@ class MangaDexHelper() {
      * get chapters for manga (aka manga/$id/feed endpoint)
      */
     fun getChapterEndpoint(mangaId: String, offset: Int, langCode: String) =
-        "${MDConstants.apiMangaUrl}/$mangaId/feed?limit=500&offset=$offset&translatedLanguage[]=$langCode&order[volume]=desc&order[chapter]=desc"
+        "${MDConstants.apiMangaUrl}/$mangaId/feed?includes[]=${MDConstants.scanlator}&limit=500&offset=$offset&translatedLanguage[]=$langCode&order[volume]=desc&order[chapter]=desc"
 
     /**
      * Check if the manga url is a valid uuid
@@ -124,7 +119,7 @@ class MangaDexHelper() {
         tokenRequestUrl: String,
         client: OkHttpClient,
         headers: Headers,
-        cacheControl: CacheControl
+        cacheControl: CacheControl,
     ): String {
         if (cacheControl == CacheControl.FORCE_NETWORK) {
             tokenTracker[tokenRequestUrl] = Date().time
@@ -137,17 +132,20 @@ class MangaDexHelper() {
     /**
      * create an SManga from json element only basic elements
      */
-    fun createBasicManga(mangaDto: MangaDto): SManga {
+    fun createBasicManga(mangaDto: MangaDto, coverFileName: String?): SManga {
         return SManga.create().apply {
             url = "/manga/${mangaDto.data.id}"
             title = cleanString(mangaDto.data.attributes.title["en"] ?: "")
+            coverFileName?.let {
+                thumbnail_url = "${MDConstants.cdnUrl}/covers/${mangaDto.data.id}/$coverFileName"
+            }
         }
     }
 
     /**
      * Create an SManga from json element with all details
      */
-    fun createManga(mangaDto: MangaDto, client: OkHttpClient, lang: String): SManga {
+    fun createManga(mangaDto: MangaDto, lang: String): SManga {
         try {
             val data = mangaDto.data
             val attr = data.attributes
@@ -168,30 +166,17 @@ class MangaDexHelper() {
                 Locale(attr.originalLanguage ?: "").displayLanguage
             )
 
-            // get authors ignore if they error, artists are labelled as authors currently
+            val authors = mangaDto.relationships.filter { relationshipDto ->
-            val authorIds = mangaDto.relationships.filter { relationship ->
+                relationshipDto.type.equals(MDConstants.author, true)
-                relationship.type.equals("author", true)
+            }.mapNotNull { it.attributes!!.name }.distinct()
-            }.map { relationship -> relationship.id }
-                .distinct()
 
-            val artistIds = mangaDto.relationships.filter { relationship ->
+            val artists = mangaDto.relationships.filter { relationshipDto ->
-                relationship.type.equals("artist", true)
+                relationshipDto.type.equals(MDConstants.artist, true)
-            }.map { relationship -> relationship.id }
+            }.mapNotNull { it.attributes!!.name }.distinct()
-                .distinct()
 
-            val authorMap = runCatching {
+            val coverFileName = mangaDto.relationships.firstOrNull { relationshipDto ->
-                val ids = listOf(authorIds, artistIds).flatten().distinct()
+                relationshipDto.type.equals(MDConstants.coverArt, true)
-                    .joinToString("&ids[]=", "?ids[]=")
+            }?.attributes?.fileName
-                val response = client.newCall(GET("${MDConstants.apiUrl}/author$ids")).execute()
-                val authorListDto = json.decodeFromString<AuthorListDto>(response.body!!.string())
-                authorListDto.results.map { result ->
-                    result.data.id to cleanString(result.data.attributes.name)
-                }.toMap()
-            }.getOrNull() ?: emptyMap()
-
-            val coverId = mangaDto.relationships.filter { relationship ->
-                relationship.type.equals("cover_art", true)
-            }.map { relationship -> relationship.id }.firstOrNull()!!
 
             // get tag list
             val tags = mdFilters.getTags()
@@ -208,14 +193,11 @@ class MangaDexHelper() {
                 )
                 .filter { it.isNullOrBlank().not() }
 
-            return SManga.create().apply {
+            return createBasicManga(mangaDto, coverFileName).apply {
-                url = "/manga/${data.id}"
-                title = cleanString(attr.title["en"] ?: "")
                 description = cleanString(attr.description[lang] ?: attr.description["en"] ?: "")
-                author = authorIds.mapNotNull { authorMap[it] }.joinToString(", ")
+                author = authors.joinToString(", ")
-                artist = artistIds.mapNotNull { authorMap[it] }.joinToString(", ")
+                artist = artists.joinToString(", ")
                 status = getPublicationStatus(attr.status)
-                thumbnail_url = getCoverUrl(data.id, coverId, client)
                 genre = genreList.joinToString(", ")
             }
         } catch (e: Exception) {
@@ -224,52 +206,20 @@ class MangaDexHelper() {
         }
     }
 
-    /**
-     * This makes an api call per a unique group id found in the chapters hopefully Dex will eventually support
-     * batch ids
-     */
-    fun createGroupMap(
-        chapterListDto: List<ChapterDto>,
-        client: OkHttpClient
-    ): Map<String, String> {
-        val groupIds =
-            chapterListDto.map { chapterDto -> chapterDto.relationships }
-                .flatten()
-                .filter { relationshipDto -> relationshipDto.type.equals("scanlation_group", true) }
-                .map { relationshipDto -> relationshipDto.id }.distinct()
-
-        // ignore errors if request fails, there is no batch group search yet..
-        return runCatching {
-            groupIds.chunked(100).map { chunkIds ->
-                val ids = chunkIds.joinToString("&ids[]=", "?ids[]=")
-                val groupResponse =
-                    client.newCall(GET("${MDConstants.apiUrl}/group$ids")).execute()
-                // map results to pair id and name
-                json.decodeFromString<GroupListDto>(groupResponse.body!!.string())
-                    .results.map { result ->
-                        result.data.id to result.data.attributes.name
-                    }
-            }.flatten().toMap()
-        }.getOrNull() ?: emptyMap()
-    }
-
     /**
      * create the SChapter from json
      */
-    fun createChapter(chapterDto: ChapterDto, groupMap: Map<String, String>): SChapter {
+    fun createChapter(chapterDto: ChapterDto): SChapter {
         try {
             val data = chapterDto.data
             val attr = data.attributes
 
-            val scanlatorGroupIds =
+            val groups = chapterDto.relationships.filter { relationshipDto ->
-                chapterDto.relationships
-                    .filter { relationshipDto ->
                 relationshipDto.type.equals(
-                            "scanlation_group",
+                    MDConstants.scanlator,
                     true
                 )
-                    }
+            }.mapNotNull { it.attributes!!.name }
-                    .map { relationshipDto -> groupMap[relationshipDto.id] }
                 .joinToString(" & ")
 
             val chapterName = mutableListOf<String>()
@@ -306,41 +256,11 @@ class MangaDexHelper() {
                 url = "/chapter/${data.id}"
                 name = cleanString(chapterName.joinToString(" "))
                 date_upload = parseDate(attr.publishAt)
-                scanlator = scanlatorGroupIds
+                scanlator = groups
             }
         } catch (e: Exception) {
             Log.e("MangaDex", "error parsing chapter", e)
             throw(e)
         }
     }
-
-    private fun getCoverUrl(dexId: String, coverId: String, client: OkHttpClient): String {
-        val response =
-            client.newCall(GET("${MDConstants.apiCoverUrl}/$coverId"))
-                .execute()
-        val coverDto = json.decodeFromString<CoverDto>(response.body!!.string())
-        val fileName = coverDto.data.attributes.fileName
-        return "${MDConstants.cdnUrl}/covers/$dexId/$fileName"
-    }
-
-    fun getBatchCoversUrl(ids: Map<String, String>, client: OkHttpClient): Map<String, String> {
-
-        val url = MDConstants.apiCoverUrl.toHttpUrl().newBuilder().apply {
-            ids.values.forEach { coverArtId ->
-                addQueryParameter("ids[]", coverArtId)
-            }
-            addQueryParameter("limit", ids.size.toString())
-        }.build().toString()
-
-        val response = client.newCall(GET(url)).execute()
-        val coverListDto = json.decodeFromString<CoverListDto>(response.body!!.string())
-
-        return coverListDto.results.map { coverDto ->
-            val fileName = coverDto.data.attributes.fileName
-            val mangaId = coverDto.relationships
-                .first { relationshipDto -> relationshipDto.type.equals("manga", true) }
-                .id
-            mangaId to "${MDConstants.cdnUrl}/covers/$mangaId/$fileName"
-        }.toMap()
-    }
 }

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/ChapterDto.kt

@@ -1,4 +1,5 @@
 package eu.kanade.tachiyomi.extension.all.mangadex.dto
+
 import kotlinx.serialization.Serializable
 
 @Serializable
@@ -6,14 +7,14 @@ data class ChapterListDto(
     val limit: Int,
     val offset: Int,
     val total: Int,
-    val results: List<ChapterDto>
+    val results: List<ChapterDto>,
 )
 
 @Serializable
 data class ChapterDto(
     val result: String,
     val data: ChapterDataDto,
-    val relationships: List<RelationshipDto>
+    val relationships: List<RelationshipDto>,
 )
 
 @Serializable
@@ -34,28 +35,3 @@ data class ChapterAttributesDto(
     val dataSaver: List<String>,
     val hash: String,
 )
-
-@Serializable
-data class GroupListDto(
-    val limit: Int,
-    val offset: Int,
-    val total: Int,
-    val results: List<GroupDto>
-)
-
-@Serializable
-data class GroupDto(
-    val result: String,
-    val data: GroupDataDto,
-)
-
-@Serializable
-data class GroupDataDto(
-    val id: String,
-    val attributes: GroupAttributesDto,
-)
-
-@Serializable
-data class GroupAttributesDto(
-    val name: String,
-)

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/MangaDto.kt

@@ -21,13 +21,20 @@ data class MangaDto(
 data class RelationshipDto(
     val id: String,
     val type: String,
+    val attributes: IncludesAttributesDto? = null,
+)
+
+@Serializable
+data class IncludesAttributesDto(
+    val name: String? = null,
+    val fileName: String? = null,
 )
 
 @Serializable
 data class MangaDataDto(
     val id: String,
     val type: String,
-    val attributes: MangaAttributesDto
+    val attributes: MangaAttributesDto,
 )
 
 @Serializable
@@ -48,47 +55,5 @@ data class MangaAttributesDto(
 
 @Serializable
 data class TagDto(
-    val id: String
-)
-
-@Serializable
-data class AuthorListDto(
-    val results: List<AuthorDto>,
-)
-
-@Serializable
-data class AuthorDto(
-    val result: String,
-    val data: AuthorDataDto,
-)
-
-@Serializable
-data class AuthorDataDto(
     val id: String,
-    val attributes: AuthorAttributesDto,
-)
-
-@Serializable
-data class AuthorAttributesDto(
-    val name: String,
-)
-
-@Serializable
-data class CoverListDto(
-    val results: List<CoverDto>,
-)
-@Serializable
-data class CoverDto(
-    val data: CoverDataDto,
-    val relationships: List<RelationshipDto>
-)
-
-@Serializable
-data class CoverDataDto(
-    val attributes: CoverAttributesDto,
-)
-
-@Serializable
-data class CoverAttributesDto(
-    val fileName: String,
 )