MangaDex: Switch to serialization, use lang descriptions if available, fix open in webview, fix publication status, possibly fix md@home retry (#7540)

* switch to kotlinx

* use baseurl in referer

* remove default sort cause we don't use at this time
update build.gradle

* Use correct field when parsing manga pub status

* Use current language if available in description map.

* potentially fix md@host refresh issue

* add default sort back since that's by number of follows

* use work around for webview credit @Nar1n

Co-authored-by: animusfracto <50589737+animusfracto@users.noreply.github.com>
Co-authored-by: Alessandro Jean <alessandrojean@gmail.com>
This commit is contained in:
Carlos 2021-06-08 20:58:56 -04:00 committed by GitHub
parent 6f39cbf575
commit 49c930d622
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 303 additions and 114 deletions

View File

@ -1,11 +1,12 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply plugin: 'kotlinx-serialization'
ext {
extName = 'MangaDex'
pkgNameSuffix = 'all.mangadex'
extClass = '.MangaDexFactory'
extVersionCode = 117
extVersionCode = 118
libVersion = '1.2'
containsNsfw = true
}

View File

@ -5,12 +5,10 @@ import android.content.SharedPreferences
import android.util.Log
import androidx.preference.PreferenceScreen
import androidx.preference.SwitchPreferenceCompat
import com.github.salomonbrys.kotson.array
import com.github.salomonbrys.kotson.get
import com.github.salomonbrys.kotson.int
import com.github.salomonbrys.kotson.obj
import com.github.salomonbrys.kotson.string
import com.google.gson.JsonParser
import eu.kanade.tachiyomi.extension.all.mangadex.dto.ChapterDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.ChapterListDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaListDto
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.ConfigurableSource
@ -20,6 +18,7 @@ import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import kotlinx.serialization.decodeFromString
import okhttp3.CacheControl
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
@ -35,7 +34,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
ConfigurableSource,
HttpSource() {
override val name = "MangaDex"
override val baseUrl = "https://www.mangadex.org"
override val baseUrl = "https://mangadex.org"
// after mvp comes out make current popular becomes latest (mvp doesnt have a browse page)
override val supportsLatest = false
@ -47,7 +46,7 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
private val helper = MangaDexHelper()
override fun headersBuilder() = Headers.Builder()
.add("Referer", "https://mangadex.org/")
.add("Referer", "$baseUrl/")
.add("User-Agent", "Tachiyomi " + System.getProperty("http.agent"))
override val client = network.client.newBuilder()
@ -103,25 +102,27 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
if (response.code == 204) {
return MangasPage(emptyList(), false)
}
val mangaListDto = helper.json.decodeFromString<MangaListDto>(response.body!!.string())
val hasMoreResults = mangaListDto.limit + mangaListDto.offset < mangaListDto.total
val mangaListResponse = JsonParser.parseString(response.body!!.string()).obj
val hasMoreResults =
(mangaListResponse["limit"].int + mangaListResponse["offset"].int) < mangaListResponse["total"].int
val idsAndCoverIds = mangaListResponse["results"].array.map { mangaJson ->
val mangaId = mangaJson["data"].obj["id"].string
val coverId = mangaJson["relationships"].array.filter { relationship ->
relationship["type"].string.equals("cover_art", true)
}.map { relationship -> relationship["id"].string }.first()
val idsAndCoverIds = mangaListDto.results.mapNotNull { mangaDto ->
val mangaId = mangaDto.data.id
val coverId = mangaDto.relationships.firstOrNull { relationshipDto ->
relationshipDto.type.equals("cover_art", true)
}?.id
if (coverId == null) {
null
} else {
Pair(mangaId, coverId)
}
}.toMap()
val results = runCatching {
helper.getBatchCoversUrl(idsAndCoverIds, client)
}.getOrNull()!!
val mangaList = mangaListResponse["results"].array.map {
helper.createBasicManga(it, client).apply {
val mangaList = mangaListDto.results.map {
helper.createBasicManga(it).apply {
thumbnail_url = results[url.substringAfter("/manga/")]
}
}
@ -173,7 +174,8 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
}
override fun mangaDetailsRequest(manga: SManga): Request {
return GET("${baseUrl}${manga.url}", headers)
//remove once redirect for /manga is fixed
return GET("${baseUrl}${manga.url.replace("manga", "title")}", headers)
}
/**
@ -187,8 +189,8 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
}
override fun mangaDetailsParse(response: Response): SManga {
val manga = JsonParser.parseString(response.body!!.string()).obj
return helper.createManga(manga, client)
val manga = helper.json.decodeFromString<MangaDto>(response.body!!.string())
return helper.createManga(manga, client, lang.substringBefore("-"))
}
// Chapter list section
@ -221,29 +223,28 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
return emptyList()
}
try {
val chapterListResponse = JsonParser.parseString(response.body!!.string()).obj
val chapterListResponse = helper.json.decodeFromString<ChapterListDto>(response.body!!.string())
val chapterListResults =
chapterListResponse["results"].array.map { it.obj }.toMutableList()
val chapterListResults = chapterListResponse.results.toMutableList()
val mangaId =
response.request.url.toString().substringBefore("/feed")
.substringAfter("${MDConstants.apiMangaUrl}/")
val limit = chapterListResponse["limit"].int
val limit = chapterListResponse.limit
var offset = chapterListResponse["offset"].int
var offset = chapterListResponse.offset
var hasMoreResults = (limit + offset) < chapterListResponse["total"].int
var hasMoreResults = (limit + offset) < chapterListResponse.total
// max results that can be returned is 500 so need to make more api calls if limit+offset > total chapters
while (hasMoreResults) {
offset += limit
val newResponse =
client.newCall(actualChapterListRequest(mangaId, offset)).execute()
val newChapterListJson = JsonParser.parseString(newResponse.body!!.string()).obj
chapterListResults.addAll(newChapterListJson["results"].array.map { it.obj })
hasMoreResults = (limit + offset) < newChapterListJson["total"].int
val newChapterList = helper.json.decodeFromString<ChapterListDto>(newResponse.body!!.string())
chapterListResults.addAll(newChapterList.results)
hasMoreResults = (limit + offset) < newChapterList.total
}
val groupMap = helper.createGroupMap(chapterListResults.toList(), client)
@ -272,14 +273,14 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
if (response.code == 204) {
return emptyList()
}
val chapterJson = JsonParser.parseString(response.body!!.string()).obj["data"]
val chapterDto = helper.json.decodeFromString<ChapterDto>(response.body!!.string()).data
val usingStandardHTTPS =
preferences.getBoolean(MDConstants.getStandardHttpsPreferenceKey(dexLang), false)
val atHomeRequestUrl = if (usingStandardHTTPS) {
"${MDConstants.apiUrl}/at-home/server/${chapterJson["id"].string}?forcePort443=true"
"${MDConstants.apiUrl}/at-home/server/${chapterDto.id}?forcePort443=true"
} else {
"${MDConstants.apiUrl}/at-home/server/${chapterJson["id"].string}"
"${MDConstants.apiUrl}/at-home/server/${chapterDto.id}"
}
val host =
@ -290,11 +291,12 @@ abstract class MangaDex(override val lang: String, val dexLang: String) :
// have to add the time, and url to the page because pages timeout within 30mins now
val now = Date().time
val hash = chapterJson["attributes"]["hash"].string
val hash = chapterDto.attributes.hash
val pageSuffix = if (usingDataSaver) {
chapterJson["attributes"]["dataSaver"].array.map { "/data-saver/$hash/${it.string}" }
chapterDto.attributes.dataSaver.map { "/data-saver/$hash/$it" }
} else {
chapterJson["attributes"]["data"].array.map { "/data/$hash/${it.string}" }
chapterDto.attributes.data.map { "/data/$hash/$it" }
}
return pageSuffix.mapIndexed { index, imgUrl ->

View File

@ -173,12 +173,12 @@ class MangaDexFilters {
Filter.Select<String>("Excluded tags mode", arrayOf("And", "Or"), 1)
val sortableList = listOf(
Pair("Default (Asc/Desc doesn't matter)", ""),
Pair("Number of follows", ""),
Pair("Created at", "createdAt"),
Pair("Updated at", "updatedAt"),
)
class SortFilter(sortables: Array<String>) : Filter.Sort("Sort", sortables, Selection(0, false))
class SortFilter(sortables: Array<String>) : Filter.Sort("Sort", sortables, Selection(1, false))
internal fun addFiltersToUrl(url: HttpUrl.Builder, filters: FilterList): String {
url.apply {

View File

@ -1,17 +1,19 @@
package eu.kanade.tachiyomi.extension.all.mangadex
import android.util.Log
import com.github.salomonbrys.kotson.array
import com.github.salomonbrys.kotson.get
import com.github.salomonbrys.kotson.nullString
import com.github.salomonbrys.kotson.obj
import com.github.salomonbrys.kotson.string
import com.google.gson.JsonElement
import com.google.gson.JsonParser
import eu.kanade.tachiyomi.extension.all.mangadex.dto.AtHomeDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.AuthorListDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.ChapterDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.CoverDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.CoverListDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.GroupListDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDto
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import okhttp3.CacheControl
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
@ -25,6 +27,14 @@ class MangaDexHelper() {
val mdFilters = MangaDexFilters()
val json = Json {
isLenient = true
ignoreUnknownKeys = true
allowSpecialFloatingPointValues = true
useArrayPolymorphism = true
prettyPrint = true
}
/**
* Gets the UUID from the url
*/
@ -85,6 +95,7 @@ class MangaDexHelper() {
// Check the token map to see if the md@home host is still valid
fun getValidImageUrlForPage(page: Page, headers: Headers, client: OkHttpClient): Request {
val data = page.url.split(",")
val mdAtHomeServerUrl =
when (Date().time - data[2].toLong() > MDConstants.mdAtHomeTokenLifespan) {
false -> data[0]
@ -96,7 +107,6 @@ class MangaDexHelper() {
?: 0
) > MDConstants.mdAtHomeTokenLifespan
) {
tokenTracker[tokenRequestUrl] = Date().time
CacheControl.FORCE_NETWORK
} else {
CacheControl.FORCE_CACHE
@ -116,37 +126,35 @@ class MangaDexHelper() {
headers: Headers,
cacheControl: CacheControl
): String {
if (cacheControl == CacheControl.FORCE_NETWORK) {
tokenTracker[tokenRequestUrl] = Date().time
}
val response =
client.newCall(GET(tokenRequestUrl, headers, cacheControl)).execute()
return JsonParser.parseString(response.body!!.string()).obj["baseUrl"].string
return json.decodeFromString<AtHomeDto>(response.body!!.string()).baseUrl
}
/**
* create an SManga from json element only basic elements
*/
fun createBasicManga(mangaJson: JsonElement, client: OkHttpClient): SManga {
val data = mangaJson["data"].obj
val dexId = data["id"].string
val attr = data["attributes"].obj
fun createBasicManga(mangaDto: MangaDto): SManga {
return SManga.create().apply {
url = "/manga/$dexId"
title = cleanString(attr["title"]["en"].string)
url = "/manga/${mangaDto.data.id}"
title = cleanString(mangaDto.data.attributes.title["en"] ?: "")
}
}
/**
* Create an SManga from json element with all details
*/
fun createManga(mangaJson: JsonElement, client: OkHttpClient): SManga {
fun createManga(mangaDto: MangaDto, client: OkHttpClient, lang: String): SManga {
try {
val data = mangaJson["data"].obj
val dexId = data["id"].string
val attr = data["attributes"].obj
val data = mangaDto.data
val attr = data.attributes
// things that will go with the genre tags but aren't actually genre
val tempContentRating = attr["contentRating"].nullString
val tempContentRating = attr.contentRating
val contentRating =
if (tempContentRating == null || tempContentRating.equals("safe", true)) {
null
@ -155,58 +163,59 @@ class MangaDexHelper() {
}
val nonGenres = listOf(
(attr["publicationDemographic"]?.nullString ?: "").capitalize(Locale.US),
(attr.publicationDemographic ?: "").capitalize(Locale.US),
contentRating,
Locale(attr["originalLanguage"].nullString ?: "").displayLanguage
Locale(attr.originalLanguage ?: "").displayLanguage
)
// get authors ignore if they error, artists are labelled as authors currently
val authorIds = mangaJson["relationships"].array.filter { relationship ->
relationship["type"].string.equals("author", true)
}.map { relationship -> relationship["id"].string }
val authorIds = mangaDto.relationships.filter { relationship ->
relationship.type.equals("author", true)
}.map { relationship -> relationship.id }
.distinct()
val artistIds = mangaJson["relationships"].array.filter { relationship ->
relationship["type"].string.equals("artist", true)
}.map { relationship -> relationship["id"].string }
val artistIds = mangaDto.relationships.filter { relationship ->
relationship.type.equals("artist", true)
}.map { relationship -> relationship.id }
.distinct()
val authorMap = runCatching {
val ids = listOf(authorIds, artistIds).flatten().distinct()
.joinToString("&ids[]=", "?ids[]=")
val response = client.newCall(GET("${MDConstants.apiUrl}/author$ids")).execute()
val json = JsonParser.parseString(response.body!!.string())
json.obj["results"].array.map { result ->
result["data"]["id"].string to
cleanString(result["data"]["attributes"]["name"].string)
val authorListDto = json.decodeFromString<AuthorListDto>(response.body!!.string())
authorListDto.results.map { result ->
result.data.id to cleanString(result.data.attributes.name)
}.toMap()
}.getOrNull() ?: emptyMap()
val coverId = mangaJson["relationships"].array.filter { relationship ->
relationship["type"].string.equals("cover_art", true)
}.map { relationship -> relationship["id"].string }.firstOrNull()!!
val coverId = mangaDto.relationships.filter { relationship ->
relationship.type.equals("cover_art", true)
}.map { relationship -> relationship.id }.firstOrNull()!!
// get tag list
val tags = mdFilters.getTags()
// map ids to tag names
val genreList = (
attr["tags"].array
.map { it["id"].string }
attr.tags
.map { it.id }
.map { dexId ->
tags.firstOrNull { it.id == dexId }
}.map { it?.name } +
}
.map { it?.name } +
nonGenres
)
.filter { it.isNullOrBlank().not() }
return SManga.create().apply {
url = "/manga/$dexId"
title = cleanString(attr["title"]["en"].string)
description = cleanString(attr["description"]["en"].string)
url = "/manga/${data.id}"
title = cleanString(attr.title["en"] ?: "")
description = cleanString(attr.description[lang] ?: attr.description["en"] ?: "")
author = authorIds.mapNotNull { authorMap[it] }.joinToString(", ")
artist = artistIds.mapNotNull { authorMap[it] }.joinToString(", ")
status = getPublicationStatus(attr["publicationDemographic"].nullString)
thumbnail_url = getCoverUrl(dexId, coverId, client)
status = getPublicationStatus(attr.status)
thumbnail_url = getCoverUrl(data.id, coverId, client)
genre = genreList.joinToString(", ")
}
} catch (e: Exception) {
@ -220,14 +229,14 @@ class MangaDexHelper() {
* batch ids
*/
fun createGroupMap(
chapterListResults: List<JsonElement>,
chapterListDto: List<ChapterDto>,
client: OkHttpClient
): Map<String, String> {
val groupIds =
chapterListResults.map { it["relationships"].array }
chapterListDto.map { chapterDto -> chapterDto.relationships }
.flatten()
.filter { it["type"].string == "scanlation_group" }
.map { it["id"].string }.distinct()
.filter { relationshipDto -> relationshipDto.type.equals("scanlation_group", true) }
.map { relationshipDto -> relationshipDto.id }.distinct()
// ignore errors if request fails, there is no batch group search yet..
return runCatching {
@ -236,11 +245,9 @@ class MangaDexHelper() {
val groupResponse =
client.newCall(GET("${MDConstants.apiUrl}/group$ids")).execute()
// map results to pair id and name
JsonParser.parseString(groupResponse.body!!.string())
.obj["results"].array.map { result ->
val id = result["data"]["id"].string
val name = result["data"]["attributes"]["name"].string
Pair(id, cleanString(name))
json.decodeFromString<GroupListDto>(groupResponse.body!!.string())
.results.map { result ->
result.data.id to result.data.attributes.name
}
}.flatten().toMap()
}.getOrNull() ?: emptyMap()
@ -249,31 +256,38 @@ class MangaDexHelper() {
/**
* create the SChapter from json
*/
fun createChapter(chapterJsonResponse: JsonElement, groupMap: Map<String, String>): SChapter {
fun createChapter(chapterDto: ChapterDto, groupMap: Map<String, String>): SChapter {
try {
val data = chapterJsonResponse["data"].obj
val data = chapterDto.data
val attr = data.attributes
val scanlatorGroupIds =
chapterJsonResponse["relationships"].array.filter { it["type"].string == "scanlation_group" }
.map { groupMap[it["id"].string] }
chapterDto.relationships
.filter { relationshipDto ->
relationshipDto.type.equals(
"scanlation_group",
true
)
}
.map { relationshipDto -> groupMap[relationshipDto.id] }
.joinToString(" & ")
val attr = data["attributes"]
val chapterName = mutableListOf<String>()
// Build chapter name
attr["volume"].nullString?.let {
attr.volume?.let {
if (it.isNotEmpty()) {
chapterName.add("Vol.$it")
}
}
attr["chapter"].nullString?.let {
attr.chapter?.let {
if (it.isNotEmpty()) {
chapterName.add("Ch.$it")
}
}
attr["title"].nullString?.let {
attr.title?.let {
if (it.isNotEmpty()) {
if (chapterName.isNotEmpty()) {
chapterName.add("-")
@ -289,9 +303,9 @@ class MangaDexHelper() {
// In future calculate [END] if non mvp api doesnt provide it
return SChapter.create().apply {
url = "/chapter/${data["id"].string}"
url = "/chapter/${data.id}"
name = cleanString(chapterName.joinToString(" "))
date_upload = parseDate(attr["publishAt"].string)
date_upload = parseDate(attr.publishAt)
scanlator = scanlatorGroupIds
}
} catch (e: Exception) {
@ -304,9 +318,8 @@ class MangaDexHelper() {
val response =
client.newCall(GET("${MDConstants.apiCoverUrl}/$coverId"))
.execute()
val coverJson = JsonParser.parseString(response.body!!.string()).obj
val fileName =
coverJson.obj["data"].obj["attributes"].obj["fileName"]?.nullString!!
val coverDto = json.decodeFromString<CoverDto>(response.body!!.string())
val fileName = coverDto.data.attributes.fileName
return "${MDConstants.cdnUrl}/covers/$dexId/$fileName"
}
@ -320,13 +333,14 @@ class MangaDexHelper() {
}.build().toString()
val response = client.newCall(GET(url)).execute()
val coverJson = JsonParser.parseString(response.body!!.string()).obj
val coverListDto = json.decodeFromString<CoverListDto>(response.body!!.string())
return coverJson.obj["results"].array.map { coverResult ->
val fileName = coverResult.obj["data"].obj["attributes"].obj["fileName"].string
val mangaId = coverResult.obj["relationships"].array.first { it["type"].string.equals("manga", true) }["id"].string
val url = "${MDConstants.cdnUrl}/covers/$mangaId/$fileName"
Pair(mangaId, url)
return coverListDto.results.map { coverDto ->
val fileName = coverDto.data.attributes.fileName
val mangaId = coverDto.relationships
.first { relationshipDto -> relationshipDto.type.equals("manga", true) }
.id
mangaId to "${MDConstants.cdnUrl}/covers/$mangaId/$fileName"
}.toMap()
}
}

View File

@ -0,0 +1,17 @@
package eu.kanade.tachiyomi.extension.all.mangadex.dto
import kotlinx.serialization.Serializable
@Serializable
data class AtHomeDto(
val baseUrl: String
)
@Serializable
data class ImageReportDto(
val url: String,
val success: Boolean,
val bytes: Int?,
val cached: Boolean,
val duration: Long,
)

View File

@ -0,0 +1,61 @@
package eu.kanade.tachiyomi.extension.all.mangadex.dto
import kotlinx.serialization.Serializable
@Serializable
data class ChapterListDto(
val limit: Int,
val offset: Int,
val total: Int,
val results: List<ChapterDto>
)
@Serializable
data class ChapterDto(
val result: String,
val data: ChapterDataDto,
val relationships: List<RelationshipDto>
)
@Serializable
data class ChapterDataDto(
val id: String,
val type: String,
val attributes: ChapterAttributesDto,
)
@Serializable
data class ChapterAttributesDto(
val title: String?,
val volume: String?,
val chapter: String?,
val translatedLanguage: String,
val publishAt: String,
val data: List<String>,
val dataSaver: List<String>,
val hash: String,
)
@Serializable
data class GroupListDto(
val limit: Int,
val offset: Int,
val total: Int,
val results: List<GroupDto>
)
@Serializable
data class GroupDto(
val result: String,
val data: GroupDataDto,
)
@Serializable
data class GroupDataDto(
val id: String,
val attributes: GroupAttributesDto,
)
@Serializable
data class GroupAttributesDto(
val name: String,
)

View File

@ -0,0 +1,94 @@
package eu.kanade.tachiyomi.extension.all.mangadex.dto
import kotlinx.serialization.Serializable
@Serializable
data class MangaListDto(
val limit: Int,
val offset: Int,
val total: Int,
val results: List<MangaDto>,
)
@Serializable
data class MangaDto(
val result: String,
val data: MangaDataDto,
val relationships: List<RelationshipDto>,
)
@Serializable
data class RelationshipDto(
val id: String,
val type: String,
)
@Serializable
data class MangaDataDto(
val id: String,
val type: String,
val attributes: MangaAttributesDto
)
@Serializable
data class MangaAttributesDto(
val title: Map<String, String>,
val altTitles: List<Map<String, String>>,
val description: Map<String, String>,
val links: Map<String, String>?,
val originalLanguage: String,
val lastVolume: String?,
val lastChapter: String?,
val contentRating: String?,
val publicationDemographic: String?,
val status: String?,
val year: Int?,
val tags: List<TagDto>,
)
@Serializable
data class TagDto(
val id: String
)
@Serializable
data class AuthorListDto(
val results: List<AuthorDto>,
)
@Serializable
data class AuthorDto(
val result: String,
val data: AuthorDataDto,
)
@Serializable
data class AuthorDataDto(
val id: String,
val attributes: AuthorAttributesDto,
)
@Serializable
data class AuthorAttributesDto(
val name: String,
)
@Serializable
data class CoverListDto(
val results: List<CoverDto>,
)
@Serializable
data class CoverDto(
val data: CoverDataDto,
val relationships: List<RelationshipDto>
)
@Serializable
data class CoverDataDto(
val attributes: CoverAttributesDto,
)
@Serializable
data class CoverAttributesDto(
val fileName: String,
)