Draff
/
Tachiyomi-Extensions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update to use includes to reduce api calls (#7603)

This commit is contained in:
Carlos 2021-06-10 21:11:54 -04:00 committed by GitHub
parent 675d44272a
commit 761e467896
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 62 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/all/mangadex/build.gradle
View File

@ -6,7 +6,7 @@ ext {
extName = 'MangaDex'
pkgNameSuffix = 'all.mangadex'
extClass = '.MangaDexFactory'
extVersionCode = 118
extVersionCode = 119
libVersion = '1.2'
containsNsfw = true
}

5
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MDConstants.kt
View File

@ -10,11 +10,14 @@ object MDConstants {
Regex("[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}")
const val mangaLimit = 20
const val coverArt = "cover_art"
const val scanlator = "scanlation_group"
const val author = "author"
const val artist = "artist"
const val cdnUrl = "https://uploads.mangadex.org"
const val apiUrl = "https://api.mangadex.org"
const val apiMangaUrl = "$apiUrl/manga"
const val apiCoverUrl = "$apiUrl/cover"
const val atHomePostUrl = "https://api.mangadex.network/report"
val whitespaceRegex = "\\s".toRegex()

45
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt
View File

@ -62,6 +62,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
addQueryParameter("order[updatedAt]", "desc")
addQueryParameter("limit", MDConstants.mangaLimit.toString())
addQueryParameter("offset", helper.getMangaListOffset(page))
addQueryParameter("includes[]", MDConstants.coverArt)
if (preferences.getBoolean(MDConstants.getContentRatingSafePrefKey(dexLang), false)) {
addQueryParameter("contentRating[]", "safe")
}
@ -105,26 +106,11 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
val mangaListDto = helper.json.decodeFromString<MangaListDto>(response.body!!.string())
val hasMoreResults = mangaListDto.limit + mangaListDto.offset < mangaListDto.total
val idsAndCoverIds = mangaListDto.results.mapNotNull { mangaDto ->
val mangaId = mangaDto.data.id
val coverId = mangaDto.relationships.firstOrNull { relationshipDto ->
relationshipDto.type.equals("cover_art", true)
}?.id
if (coverId == null) {
null
} else {
Pair(mangaId, coverId)
}
}.toMap()
val results = runCatching {
helper.getBatchCoversUrl(idsAndCoverIds, client)
}.getOrNull()!!
val mangaList = mangaListDto.results.map {
helper.createBasicManga(it).apply {
thumbnail_url = results[url.substringAfter("/manga/")]
}
val mangaList = mangaListDto.results.map { mangaDto ->
val fileName = mangaDto.relationships.firstOrNull { relationshipDto ->
relationshipDto.type.equals(MDConstants.coverArt, true)
}?.attributes?.fileName
helper.createBasicManga(mangaDto, fileName)
}
return MangasPage(mangaList, hasMoreResults)
@ -141,6 +127,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
if (query.startsWith(MDConstants.prefixIdSearch)) {
val url = MDConstants.apiMangaUrl.toHttpUrlOrNull()!!.newBuilder()
.addQueryParameter("ids[]", query.removePrefix(MDConstants.prefixIdSearch))
.addQueryParameter("includes[]", MDConstants.coverArt)
return GET(url.toString(), headers, CacheControl.FORCE_NETWORK)
}
@ -149,6 +136,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
tempUrl.apply {
addQueryParameter("limit", MDConstants.mangaLimit.toString())
addQueryParameter("offset", (helper.getMangaListOffset(page)))
addQueryParameter("includes[]", MDConstants.coverArt)
val actualQuery = query.replace(MDConstants.whitespaceRegex, " ")
if (actualQuery.isNotBlank()) {
addQueryParameter("title", actualQuery)
@ -174,23 +162,28 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
}
override fun mangaDetailsRequest(manga: SManga): Request {
//remove once redirect for /manga is fixed
// remove once redirect for /manga is fixed
return GET("${baseUrl}${manga.url.replace("manga", "title")}", headers)
}
/**
* get manga details url throws exception if the url is the old format so people migrate
*/
fun apiMangaDetailsRequest(manga: SManga): Request {
private fun apiMangaDetailsRequest(manga: SManga): Request {
if (!helper.containsUuid(manga.url.trim())) {
throw Exception("Migrate this manga from MangaDex to MangaDex to update it")
}
return GET("${MDConstants.apiUrl}${manga.url}", headers, CacheControl.FORCE_NETWORK)
val url = (MDConstants.apiUrl + manga.url).toHttpUrl().newBuilder().apply {
addQueryParameter("includes[]", MDConstants.coverArt)
addQueryParameter("includes[]", MDConstants.author)
addQueryParameter("includes[]", MDConstants.artist)
}.build().toString()
return GET(url, headers, CacheControl.FORCE_NETWORK)
}
override fun mangaDetailsParse(response: Response): SManga {
val manga = helper.json.decodeFromString<MangaDto>(response.body!!.string())
return helper.createManga(manga, client, lang.substringBefore("-"))
return helper.createManga(manga, lang.substringBefore("-"))
}
// Chapter list section
@ -247,11 +240,9 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
hasMoreResults = (limit + offset) < newChapterList.total
}
val groupMap = helper.createGroupMap(chapterListResults.toList(), client)
val now = Date().time
return chapterListResults.map { helper.createChapter(it, groupMap) }
return chapterListResults.map { helper.createChapter(it) }
.filter { it.date_upload <= now && "MangaPlus" != it.scanlator }
} catch (e: Exception) {
Log.e("MangaDex", "error parsing chapter list", e)

136
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt
View File

@ -2,11 +2,7 @@ package eu.kanade.tachiyomi.extension.all.mangadex
import android.util.Log
import eu.kanade.tachiyomi.extension.all.mangadex.dto.AtHomeDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.AuthorListDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.ChapterDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.CoverDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.CoverListDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.GroupListDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDto
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.Page
@ -16,7 +12,6 @@ import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import okhttp3.CacheControl
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import org.jsoup.parser.Parser
@ -44,7 +39,7 @@ class MangaDexHelper() {
* get chapters for manga (aka manga/$id/feed endpoint)
*/
fun getChapterEndpoint(mangaId: String, offset: Int, langCode: String) =
"${MDConstants.apiMangaUrl}/$mangaId/feed?limit=500&offset=$offset&translatedLanguage[]=$langCode&order[volume]=desc&order[chapter]=desc"
"${MDConstants.apiMangaUrl}/$mangaId/feed?includes[]=${MDConstants.scanlator}&limit=500&offset=$offset&translatedLanguage[]=$langCode&order[volume]=desc&order[chapter]=desc"
/**
* Check if the manga url is a valid uuid
@ -124,7 +119,7 @@ class MangaDexHelper() {
tokenRequestUrl: String,
client: OkHttpClient,
headers: Headers,
cacheControl: CacheControl
cacheControl: CacheControl,
): String {
if (cacheControl == CacheControl.FORCE_NETWORK) {
tokenTracker[tokenRequestUrl] = Date().time
@ -137,17 +132,20 @@ class MangaDexHelper() {
/**
* create an SManga from json element only basic elements
*/
fun createBasicManga(mangaDto: MangaDto): SManga {
fun createBasicManga(mangaDto: MangaDto, coverFileName: String?): SManga {
return SManga.create().apply {
url = "/manga/${mangaDto.data.id}"
title = cleanString(mangaDto.data.attributes.title["en"] ?: "")
coverFileName?.let {
thumbnail_url = "${MDConstants.cdnUrl}/covers/${mangaDto.data.id}/$coverFileName"
}
}
}
/**
* Create an SManga from json element with all details
*/
fun createManga(mangaDto: MangaDto, client: OkHttpClient, lang: String): SManga {
fun createManga(mangaDto: MangaDto, lang: String): SManga {
try {
val data = mangaDto.data
val attr = data.attributes
@ -168,30 +166,17 @@ class MangaDexHelper() {
Locale(attr.originalLanguage ?: "").displayLanguage
)
// get authors ignore if they error, artists are labelled as authors currently
val authorIds = mangaDto.relationships.filter { relationship ->
relationship.type.equals("author", true)
}.map { relationship -> relationship.id }
.distinct()
val authors = mangaDto.relationships.filter { relationshipDto ->
relationshipDto.type.equals(MDConstants.author, true)
}.mapNotNull { it.attributes!!.name }.distinct()
val artistIds = mangaDto.relationships.filter { relationship ->
relationship.type.equals("artist", true)
}.map { relationship -> relationship.id }
.distinct()
val artists = mangaDto.relationships.filter { relationshipDto ->
relationshipDto.type.equals(MDConstants.artist, true)
}.mapNotNull { it.attributes!!.name }.distinct()
val authorMap = runCatching {
val ids = listOf(authorIds, artistIds).flatten().distinct()
.joinToString("&ids[]=", "?ids[]=")
val response = client.newCall(GET("${MDConstants.apiUrl}/author$ids")).execute()
val authorListDto = json.decodeFromString<AuthorListDto>(response.body!!.string())
authorListDto.results.map { result ->
result.data.id to cleanString(result.data.attributes.name)
}.toMap()
}.getOrNull() ?: emptyMap()
val coverId = mangaDto.relationships.filter { relationship ->
relationship.type.equals("cover_art", true)
}.map { relationship -> relationship.id }.firstOrNull()!!
val coverFileName = mangaDto.relationships.firstOrNull { relationshipDto ->
relationshipDto.type.equals(MDConstants.coverArt, true)
}?.attributes?.fileName
// get tag list
val tags = mdFilters.getTags()
@ -208,14 +193,11 @@ class MangaDexHelper() {
)
.filter { it.isNullOrBlank().not() }
return SManga.create().apply {
url = "/manga/${data.id}"
title = cleanString(attr.title["en"] ?: "")
return createBasicManga(mangaDto, coverFileName).apply {
description = cleanString(attr.description[lang] ?: attr.description["en"] ?: "")
author = authorIds.mapNotNull { authorMap[it] }.joinToString(", ")
artist = artistIds.mapNotNull { authorMap[it] }.joinToString(", ")
author = authors.joinToString(", ")
artist = artists.joinToString(", ")
status = getPublicationStatus(attr.status)
thumbnail_url = getCoverUrl(data.id, coverId, client)
genre = genreList.joinToString(", ")
}
} catch (e: Exception) {
@ -224,53 +206,21 @@ class MangaDexHelper() {
}
}
/**
* This makes an api call per a unique group id found in the chapters hopefully Dex will eventually support
* batch ids
*/
fun createGroupMap(
chapterListDto: List<ChapterDto>,
client: OkHttpClient
): Map<String, String> {
val groupIds =
chapterListDto.map { chapterDto -> chapterDto.relationships }
.flatten()
.filter { relationshipDto -> relationshipDto.type.equals("scanlation_group", true) }
.map { relationshipDto -> relationshipDto.id }.distinct()
// ignore errors if request fails, there is no batch group search yet..
return runCatching {
groupIds.chunked(100).map { chunkIds ->
val ids = chunkIds.joinToString("&ids[]=", "?ids[]=")
val groupResponse =
client.newCall(GET("${MDConstants.apiUrl}/group$ids")).execute()
// map results to pair id and name
json.decodeFromString<GroupListDto>(groupResponse.body!!.string())
.results.map { result ->
result.data.id to result.data.attributes.name
}
}.flatten().toMap()
}.getOrNull() ?: emptyMap()
}
/**
* create the SChapter from json
*/
fun createChapter(chapterDto: ChapterDto, groupMap: Map<String, String>): SChapter {
fun createChapter(chapterDto: ChapterDto): SChapter {
try {
val data = chapterDto.data
val attr = data.attributes
val scanlatorGroupIds =
chapterDto.relationships
.filter { relationshipDto ->
relationshipDto.type.equals(
"scanlation_group",
true
)
}
.map { relationshipDto -> groupMap[relationshipDto.id] }
.joinToString(" & ")
val groups = chapterDto.relationships.filter { relationshipDto ->
relationshipDto.type.equals(
MDConstants.scanlator,
true
)
}.mapNotNull { it.attributes!!.name }
.joinToString(" & ")
val chapterName = mutableListOf<String>()
// Build chapter name
@ -306,41 +256,11 @@ class MangaDexHelper() {
url = "/chapter/${data.id}"
name = cleanString(chapterName.joinToString(" "))
date_upload = parseDate(attr.publishAt)
scanlator = scanlatorGroupIds
scanlator = groups
}
} catch (e: Exception) {
Log.e("MangaDex", "error parsing chapter", e)
throw(e)
}
}
private fun getCoverUrl(dexId: String, coverId: String, client: OkHttpClient): String {
val response =
client.newCall(GET("${MDConstants.apiCoverUrl}/$coverId"))
.execute()
val coverDto = json.decodeFromString<CoverDto>(response.body!!.string())
val fileName = coverDto.data.attributes.fileName
return "${MDConstants.cdnUrl}/covers/$dexId/$fileName"
}
fun getBatchCoversUrl(ids: Map<String, String>, client: OkHttpClient): Map<String, String> {
val url = MDConstants.apiCoverUrl.toHttpUrl().newBuilder().apply {
ids.values.forEach { coverArtId ->
addQueryParameter("ids[]", coverArtId)
}
addQueryParameter("limit", ids.size.toString())
}.build().toString()
val response = client.newCall(GET(url)).execute()
val coverListDto = json.decodeFromString<CoverListDto>(response.body!!.string())
return coverListDto.results.map { coverDto ->
val fileName = coverDto.data.attributes.fileName
val mangaId = coverDto.relationships
.first { relationshipDto -> relationshipDto.type.equals("manga", true) }
.id
mangaId to "${MDConstants.cdnUrl}/covers/$mangaId/$fileName"
}.toMap()
}
}

30
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/ChapterDto.kt
View File

@ -1,4 +1,5 @@
package eu.kanade.tachiyomi.extension.all.mangadex.dto
import kotlinx.serialization.Serializable
@Serializable
@ -6,14 +7,14 @@ data class ChapterListDto(
val limit: Int,
val offset: Int,
val total: Int,
val results: List<ChapterDto>
val results: List<ChapterDto>,
)
@Serializable
data class ChapterDto(
val result: String,
val data: ChapterDataDto,
val relationships: List<RelationshipDto>
val relationships: List<RelationshipDto>,
)
@Serializable
@ -34,28 +35,3 @@ data class ChapterAttributesDto(
val dataSaver: List<String>,
val hash: String,
)
@Serializable
data class GroupListDto(
val limit: Int,
val offset: Int,
val total: Int,
val results: List<GroupDto>
)
@Serializable
data class GroupDto(
val result: String,
val data: GroupDataDto,
)
@Serializable
data class GroupDataDto(
val id: String,
val attributes: GroupAttributesDto,
)
@Serializable
data class GroupAttributesDto(
val name: String,
)

51
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/MangaDto.kt
View File

@ -21,13 +21,20 @@ data class MangaDto(
data class RelationshipDto(
val id: String,
val type: String,
val attributes: IncludesAttributesDto? = null,
)
@Serializable
data class IncludesAttributesDto(
val name: String? = null,
val fileName: String? = null,
)
@Serializable
data class MangaDataDto(
val id: String,
val type: String,
val attributes: MangaAttributesDto
val attributes: MangaAttributesDto,
)
@Serializable
@ -48,47 +55,5 @@ data class MangaAttributesDto(
@Serializable
data class TagDto(
val id: String
)
@Serializable
data class AuthorListDto(
val results: List<AuthorDto>,
)
@Serializable
data class AuthorDto(
val result: String,
val data: AuthorDataDto,
)
@Serializable
data class AuthorDataDto(
val id: String,
val attributes: AuthorAttributesDto,
)
@Serializable
data class AuthorAttributesDto(
val name: String,
)
@Serializable
data class CoverListDto(
val results: List<CoverDto>,
)
@Serializable
data class CoverDto(
val data: CoverDataDto,
val relationships: List<RelationshipDto>
)
@Serializable
data class CoverDataDto(
val attributes: CoverAttributesDto,
)
@Serializable
data class CoverAttributesDto(
val fileName: String,
)