Draff
/
Tachiyomi-Extensions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor the MangaDex code a bit. (#14902)

This commit is contained in:
Alessandro Jean 2023-01-11 16:59:30 -03:00 committed by GitHub
parent 8ad7f97c8f
commit ecdaaf98d0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 177 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/all/mangadex/build.gradle
View File

@ -6,7 +6,7 @@ ext {
extName = 'MangaDex'
pkgNameSuffix = 'all.mangadex'
extClass = '.MangaDexFactory'
extVersionCode = 176
extVersionCode = 177
isNsfw = true
}

89
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt
View File

@ -2,7 +2,6 @@ package eu.kanade.tachiyomi.extension.all.mangadex
import android.app.Application
import android.content.SharedPreferences
import android.util.Log
import androidx.preference.EditTextPreference
import androidx.preference.ListPreference
import androidx.preference.MultiSelectListPreference
@ -20,6 +19,7 @@ import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDataDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDto
import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaListDto
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservable
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.network.interceptor.rateLimit
import eu.kanade.tachiyomi.source.ConfigurableSource
@ -71,7 +71,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
preferences.sanitizeExistingUuidPrefs()
}
// POPULAR Manga Section
// Popular manga section
override fun popularMangaRequest(page: Int): Request {
val url = MDConstants.apiMangaUrl.toHttpUrl().newBuilder()
@ -112,7 +112,11 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
return MangasPage(mangaList, hasMoreResults)
}
// LATEST section API can't sort by date yet so not implemented
// Latest manga section
/**
* The API endpoint can't sort by date yet, so not implemented.
*/
override fun latestUpdatesParse(response: Response): MangasPage {
val chapterListDto = response.parseAs<ChapterListDto>()
val hasMoreResults = chapterListDto.limit + chapterListDto.offset < chapterListDto.total
@ -170,7 +174,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
return GET(url.build().toString(), headers, CacheControl.FORCE_NETWORK)
}
// SEARCH section
// Search manga section
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
return when {
@ -211,7 +215,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
private fun getMangaIdFromChapterId(id: String): Observable<String> {
return client.newCall(GET("${MDConstants.apiChapterUrl}/$id", headers))
.asObservableSuccess()
.asObservable()
.map { response ->
if (response.isSuccessful.not()) {
throw Exception(helper.intl.unableToProcessChapterRequest(response.code))
@ -317,6 +321,9 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
}
private fun searchMangaListParse(response: Response): List<SManga> {
// This check will be used as the source is doing additional requests to this
// that are not parsed by the asObservableSuccess() method. It should throw the
// HttpException from the app if it becomes available in a future version of extensions-lib.
if (response.isSuccessful.not()) {
throw Exception("HTTP error ${response.code}")
}
@ -360,7 +367,8 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
// Manga Details section
// Shenanigans to allow "open in webview" to show a webpage instead of JSON
// Workaround to allow "Open in WebView" to show a webpage instead of JSON.
// TODO: Replace with getMangaUrl when the repository is using extensions-lib 1.4
override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
return client.newCall(apiMangaDetailsRequest(manga))
.asObservableSuccess()
@ -370,7 +378,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
}
override fun mangaDetailsRequest(manga: SManga): Request {
// remove once redirect for /manga is fixed
// TODO: Remove once redirect for /manga is fixed.
val title = manga.title
val url = "${baseUrl}${manga.url.replace("manga", "title")}"
val shareUrl = "$url/" + helper.titleToSlug(title)
@ -378,7 +386,9 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
}
/**
* get manga details url throws exception if the url is the old format so people migrate
* Get the API endpoint URL for the entry details.
*
* @throws Exception if the url is the old format so people migrate
*/
private fun apiMangaDetailsRequest(manga: SManga): Request {
if (!helper.containsUuid(manga.url.trim())) {
@ -473,21 +483,24 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
}
// Chapter list section
/**
* get chapter list if manga url is old format throws exception
* Get the API endpoint URL for the first page of chapter list.
*
* @throws Exception if the url is the old format so people migrate
*/
override fun chapterListRequest(manga: SManga): Request {
if (!helper.containsUuid(manga.url)) {
throw Exception(helper.intl.migrateWarning)
}
return actualChapterListRequest(helper.getUUIDFromUrl(manga.url), 0)
return paginatedChapterListRequest(helper.getUUIDFromUrl(manga.url), 0)
}
/**
* Required because api is paged
* Required because the chapter list API endpoint is paginated.
*/
private fun actualChapterListRequest(mangaId: String, offset: Int): Request {
private fun paginatedChapterListRequest(mangaId: String, offset: Int): Request {
val url = helper.getChapterEndpoint(mangaId, offset, dexLang).toHttpUrl().newBuilder()
.addQueryParameter("contentRating[]", "safe")
.addQueryParameter("contentRating[]", "suggestive")
@ -504,37 +517,34 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
return emptyList()
}
try {
val chapterListResponse = response.parseAs<ChapterListDto>()
val chapterListResponse = response.parseAs<ChapterListDto>()
val chapterListResults = chapterListResponse.data.toMutableList()
val chapterListResults = chapterListResponse.data.toMutableList()
val mangaId = response.request.url.toString()
.substringBefore("/feed")
.substringAfter("${MDConstants.apiMangaUrl}/")
val mangaId = response.request.url.toString()
.substringBefore("/feed")
.substringAfter("${MDConstants.apiMangaUrl}/")
val limit = chapterListResponse.limit
val limit = chapterListResponse.limit
var offset = chapterListResponse.offset
var offset = chapterListResponse.offset
var hasMoreResults = (limit + offset) < chapterListResponse.total
var hasMoreResults = (limit + offset) < chapterListResponse.total
// Max results that can be returned is 500 so need to make more API
// calls if limit + offset > total chapters
while (hasMoreResults) {
offset += limit
val newRequest = actualChapterListRequest(mangaId, offset)
val newResponse = client.newCall(newRequest).execute()
val newChapterList = newResponse.parseAs<ChapterListDto>()
chapterListResults.addAll(newChapterList.data)
hasMoreResults = (limit + offset) < newChapterList.total
}
return chapterListResults.mapNotNull(helper::createChapter)
} catch (e: Exception) {
Log.e("MangaDex", "error parsing chapter list", e)
throw e
// Max results that can be returned is 500 so need to make more API
// calls if limit + offset > total chapters
while (hasMoreResults) {
offset += limit
val newRequest = paginatedChapterListRequest(mangaId, offset)
val newResponse = client.newCall(newRequest).execute()
val newChapterList = newResponse.parseAs<ChapterListDto>()
chapterListResults.addAll(newChapterList.data)
hasMoreResults = (limit + offset) < newChapterList.total
}
return chapterListResults
.filterNot { it.attributes!!.isInvalid }
.map(helper::createChapter)
}
override fun pageListRequest(chapter: SChapter): Request {
@ -557,7 +567,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
val atHomeDto = response.parseAs<AtHomeDto>()
val host = atHomeDto.baseUrl
// have to add the time, and url to the page because pages timeout within 30mins now
// Have to add the time, and url to the page because pages timeout within 30 minutes now.
val now = Date().time
val hash = atHomeDto.chapter.hash
@ -801,6 +811,11 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St
private val SharedPreferences.useDataSaver
get() = getBoolean(MDConstants.getDataSaverPreferenceKey(dexLang), false)
/**
* Previous versions of the extension allowed invalid UUID values to be stored in the
* preferences. This method clear invalid UUIDs in case the user have updated from
* a previous version with that behaviour.
*/
private fun SharedPreferences.sanitizeExistingUuidPrefs() {
if (getBoolean(MDConstants.getHasSanitizedUuidsPrefKey(dexLang), false)) {
return

12
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexFactory.kt
View File

@ -11,8 +11,8 @@ class MangaDexFactory : SourceFactory {
MangaDexBulgarian(),
MangaDexBurmese(),
MangaDexCatalan(),
MangaDexChineseSimp(),
MangaDexChineseTrad(),
MangaDexChineseSimplified(),
MangaDexChineseTraditional(),
MangaDexCzech(),
MangaDexDanish(),
MangaDexDutch(),
@ -42,7 +42,7 @@ class MangaDexFactory : SourceFactory {
MangaDexRomanian(),
MangaDexRussian(),
MangaDexSerboCroatian(),
MangaDexSpanishLTAM(),
MangaDexSpanishLatinAmerica(),
MangaDexSpanishSpain(),
MangaDexSwedish(),
MangaDexTamil(),
@ -58,8 +58,8 @@ class MangaDexBengali : MangaDex("bn", "bn")
class MangaDexBulgarian : MangaDex("bg", "bg")
class MangaDexBurmese : MangaDex("my", "my")
class MangaDexCatalan : MangaDex("ca", "ca")
class MangaDexChineseSimp : MangaDex("zh-Hans", "zh")
class MangaDexChineseTrad : MangaDex("zh-Hant", "zh-hk")
class MangaDexChineseSimplified : MangaDex("zh-Hans", "zh")
class MangaDexChineseTraditional : MangaDex("zh-Hant", "zh-hk")
class MangaDexCzech : MangaDex("cs", "cs")
class MangaDexDanish : MangaDex("da", "da")
class MangaDexDutch : MangaDex("nl", "nl")
@ -90,7 +90,7 @@ class MangaDexPortuguesePortugal : MangaDex("pt", "pt")
class MangaDexRomanian : MangaDex("ro", "ro")
class MangaDexRussian : MangaDex("ru", "ru")
class MangaDexSerboCroatian : MangaDex("sh", "sh")
class MangaDexSpanishLTAM : MangaDex("es-419", "es-la")
class MangaDexSpanishLatinAmerica : MangaDex("es-419", "es-la")
class MangaDexSpanishSpain : MangaDex("es", "es")
class MangaDexSwedish : MangaDex("sv", "sv")
class MangaDexTamil : MangaDex("ta", "ta")

228
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt
View File

@ -129,10 +129,11 @@ class MangaDexHelper(lang: String) {
/**
* Remove any HTML characters in description or chapter name to actual
* characters. For example &hearts; will show ♥
* characters. For example &hearts; will show ♥. It also removes
* Markdown syntax for links, italic and bold.
*/
private fun cleanString(string: String): String {
return Parser.unescapeEntities(string, false)
private fun String.removeEntitiesAndMarkdown(): String {
return Parser.unescapeEntities(this, false)
.substringBefore("---")
.replace(markdownLinksRegex, "$1")
.replace(markdownItalicBoldRegex, "$1")
@ -141,7 +142,7 @@ class MangaDexHelper(lang: String) {
}
/**
* Maps dex status to Tachi status.
* Maps MangaDex status to Tachiyomi status.
* Adapted from the MangaDex handler from TachiyomiSY.
*/
fun getPublicationStatus(attr: MangaAttributesDto, volumes: Map<String, AggregateVolume>): Int {
@ -173,7 +174,9 @@ class MangaDexHelper(lang: String) {
private fun parseDate(dateAsString: String): Long =
MDConstants.dateFormatter.parse(dateAsString)?.time ?: 0
// chapter url where we get the token, last request time
/**
* Chapter URL where we get the token, last request time.
*/
private val tokenTracker = hashMapOf<String, Long>()
companion object {
@ -190,7 +193,9 @@ class MangaDexHelper(lang: String) {
val trailingHyphenRegex = "-+$".toRegex()
}
// Check the token map to see if the md@home host is still valid
/**
* Check the token map to see if the MD@Home host is still valid.
*/
fun getValidImageUrlForPage(page: Page, headers: Headers, client: OkHttpClient): Request {
val data = page.url.split(",")
@ -199,12 +204,9 @@ class MangaDexHelper(lang: String) {
false -> data[0]
true -> {
val tokenRequestUrl = data[1]
val tokenLifespan = Date().time - (tokenTracker[tokenRequestUrl] ?: 0)
val cacheControl =
if (Date().time - (
tokenTracker[tokenRequestUrl]
?: 0
) > MDConstants.mdAtHomeTokenLifespan
) {
if (tokenLifespan > MDConstants.mdAtHomeTokenLifespan) {
CacheControl.FORCE_NETWORK
} else {
USE_CACHE
@ -212,11 +214,12 @@ class MangaDexHelper(lang: String) {
getMdAtHomeUrl(tokenRequestUrl, client, headers, cacheControl)
}
}
return GET(mdAtHomeServerUrl + page.imageUrl, headers)
}
/**
* get the md@home url
* Get the MD@Home URL.
*/
private fun getMdAtHomeUrl(
tokenRequestUrl: String,
@ -234,7 +237,7 @@ class MangaDexHelper(lang: String) {
return getMdAtHomeUrl(tokenRequestUrl, client, headers, CacheControl.FORCE_NETWORK)
}
return json.decodeFromString<AtHomeDto>(response.body!!.string()).baseUrl
return response.use { json.decodeFromString<AtHomeDto>(it.body!!.string()).baseUrl }
}
/**
@ -253,7 +256,7 @@ class MangaDexHelper(lang: String) {
}
/**
* create an SManga from json element only basic elements
* Create a [SManga] from the JSON element with only basic attributes filled.
*/
fun createBasicManga(
mangaDataDto: MangaDataDto,
@ -269,10 +272,10 @@ class MangaDexHelper(lang: String) {
?: mangaDataDto.attributes.altTitles
.find { (it[lang] ?: it["en"]) !== null }
?.values?.singleOrNull() // find something else from alt titles
title = cleanString(dirtyTitle ?: "")
title = (dirtyTitle ?: "").removeEntitiesAndMarkdown()
coverFileName?.let {
thumbnail_url = when (coverSuffix != null && coverSuffix != "") {
thumbnail_url = when (!coverSuffix.isNullOrEmpty()) {
true -> "${MDConstants.cdnUrl}/covers/${mangaDataDto.id}/$coverFileName$coverSuffix"
else -> "${MDConstants.cdnUrl}/covers/${mangaDataDto.id}/$coverFileName"
}
@ -281,7 +284,7 @@ class MangaDexHelper(lang: String) {
}
/**
* Create an SManga from json element with all details
* Create an [SManga] from the JSON element with all attributes filled.
*/
fun createManga(
mangaDataDto: MangaDataDto,
@ -290,128 +293,114 @@ class MangaDexHelper(lang: String) {
lang: String,
coverSuffix: String?
): SManga {
try {
val attr = mangaDataDto.attributes!!
val attr = mangaDataDto.attributes!!
// things that will go with the genre tags but aren't actually genre
val dexLocale = Locale.forLanguageTag(lang)
// Things that will go with the genre tags but aren't actually genre
val dexLocale = Locale.forLanguageTag(lang)
val nonGenres = listOfNotNull(
attr.publicationDemographic?.let { intl.publicationDemographic(it) },
attr.contentRating
.takeIf { it != ContentRatingDto.SAFE }
?.let { intl.contentRatingGenre(it) },
attr.originalLanguage
?.let { Locale.forLanguageTag(it) }
?.getDisplayName(dexLocale)
?.replaceFirstChar { it.uppercase(dexLocale) }
)
val nonGenres = listOfNotNull(
attr.publicationDemographic?.let { intl.publicationDemographic(it) },
attr.contentRating
.takeIf { it != ContentRatingDto.SAFE }
?.let { intl.contentRatingGenre(it) },
attr.originalLanguage
?.let { Locale.forLanguageTag(it) }
?.getDisplayName(dexLocale)
?.replaceFirstChar { it.uppercase(dexLocale) }
)
val authors = mangaDataDto.relationships
.filterIsInstance<AuthorDto>()
.mapNotNull { it.attributes?.name }
.distinct()
val authors = mangaDataDto.relationships
.filterIsInstance<AuthorDto>()
.mapNotNull { it.attributes?.name }
.distinct()
val artists = mangaDataDto.relationships
.filterIsInstance<ArtistDto>()
.mapNotNull { it.attributes?.name }
.distinct()
val artists = mangaDataDto.relationships
.filterIsInstance<ArtistDto>()
.mapNotNull { it.attributes?.name }
.distinct()
val coverFileName = firstVolumeCover ?: mangaDataDto.relationships
.filterIsInstance<CoverArtDto>()
.firstOrNull()
?.attributes?.fileName
val coverFileName = firstVolumeCover ?: mangaDataDto.relationships
.filterIsInstance<CoverArtDto>()
.firstOrNull()
?.attributes?.fileName
val tags = mdFilters.getTags(intl).associate { it.id to it.name }
val tags = mdFilters.getTags(intl).associate { it.id to it.name }
val genresMap = attr.tags
.groupBy({ it.attributes!!.group }) { tagDto -> tags[tagDto.id] }
.mapValues { it.value.filterNotNull().sortedWith(intl.collator) }
val genresMap = attr.tags
.groupBy({ it.attributes!!.group }) { tagDto -> tags[tagDto.id] }
.mapValues { it.value.filterNotNull().sortedWith(intl.collator) }
val genreList = MDConstants.tagGroupsOrder.flatMap { genresMap[it].orEmpty() } + nonGenres
val genreList = MDConstants.tagGroupsOrder.flatMap { genresMap[it].orEmpty() } + nonGenres
val desc = attr.description
val desc = attr.description
return createBasicManga(mangaDataDto, coverFileName, coverSuffix, lang).apply {
description = cleanString(desc[lang] ?: desc["en"] ?: "")
author = authors.joinToString(", ")
artist = artists.joinToString(", ")
status = getPublicationStatus(attr, chapters)
genre = genreList
.filter(String::isNotEmpty)
.joinToString(", ")
}
} catch (e: Exception) {
Log.e("MangaDex", "error parsing manga", e)
throw e
return createBasicManga(mangaDataDto, coverFileName, coverSuffix, lang).apply {
description = (desc[lang] ?: desc["en"] ?: "").removeEntitiesAndMarkdown()
author = authors.joinToString(", ")
artist = artists.joinToString(", ")
status = getPublicationStatus(attr, chapters)
genre = genreList
.filter(String::isNotEmpty)
.joinToString(", ")
}
}
/**
* create the SChapter from json
* Create the [SChapter] from the JSON element.
*/
fun createChapter(chapterDataDto: ChapterDataDto): SChapter? {
try {
val attr = chapterDataDto.attributes!!
fun createChapter(chapterDataDto: ChapterDataDto): SChapter {
val attr = chapterDataDto.attributes!!
val groups = chapterDataDto.relationships
.filterIsInstance<ScanlationGroupDto>()
.filterNot { it.id == MDConstants.legacyNoGroupId } // 'no group' left over from MDv3
.mapNotNull { it.attributes?.name }
.joinToString(" & ")
.ifEmpty {
// fall back to uploader name if no group
val users = chapterDataDto.relationships
.filterIsInstance<UserDto>()
.mapNotNull { it.attributes?.username }
if (users.isNotEmpty()) intl.uploadedBy(users) else ""
}
.ifEmpty { intl.noGroup } // "No Group" as final resort
val chapterName = mutableListOf<String>()
// Build chapter name
attr.volume?.let {
if (it.isNotEmpty()) {
chapterName.add("Vol.$it")
val groups = chapterDataDto.relationships
.filterIsInstance<ScanlationGroupDto>()
.filterNot { it.id == MDConstants.legacyNoGroupId } // 'no group' left over from MDv3
.mapNotNull { it.attributes?.name }
.joinToString(" & ")
.ifEmpty {
// Fallback to uploader name if no group is set.
val users = chapterDataDto.relationships
.filterIsInstance<UserDto>()
.mapNotNull { it.attributes?.username }
if (users.isNotEmpty()) intl.uploadedBy(users) else ""
}
.ifEmpty { intl.noGroup } // "No Group" as final resort
val chapterName = mutableListOf<String>()
// Build chapter name
attr.volume?.let {
if (it.isNotEmpty()) {
chapterName.add("Vol.$it")
}
}
attr.chapter?.let {
if (it.isNotEmpty()) {
chapterName.add("Ch.$it")
}
}
attr.title?.let {
if (it.isNotEmpty()) {
if (chapterName.isNotEmpty()) {
chapterName.add("-")
}
chapterName.add(it)
}
}
attr.chapter?.let {
if (it.isNotEmpty()) {
chapterName.add("Ch.$it")
}
}
// if volume, chapter and title is empty its a oneshot
if (chapterName.isEmpty()) {
chapterName.add("Oneshot")
}
attr.title?.let {
if (it.isNotEmpty()) {
if (chapterName.isNotEmpty()) {
chapterName.add("-")
}
chapterName.add(it)
}
}
// In future calculate [END] if non mvp api doesn't provide it
if (attr.externalUrl != null && attr.pages == 0) {
return null
}
// if volume, chapter and title is empty its a oneshot
if (chapterName.isEmpty()) {
chapterName.add("Oneshot")
}
// In future calculate [END] if non mvp api doesn't provide it
return SChapter.create().apply {
url = "/chapter/${chapterDataDto.id}"
name = cleanString(chapterName.joinToString(" "))
date_upload = parseDate(attr.publishAt)
scanlator = groups
}
} catch (e: Exception) {
Log.e("MangaDex", "error parsing chapter", e)
throw e
return SChapter.create().apply {
url = "/chapter/${chapterDataDto.id}"
name = chapterName.joinToString(" ").removeEntitiesAndMarkdown()
date_upload = parseDate(attr.publishAt)
scanlator = groups
}
}
@ -433,6 +422,9 @@ class MangaDexHelper(lang: String) {
* Adds a custom [TextWatcher] to the preference's [EditText] that show an
* error if the input value contains invalid UUIDs. If the validation fails,
* the Ok button is disabled to prevent the user from saving the value.
*
* This will likely need to be removed or revisited when the app migrates the
* extension preferences screen to Compose.
*/
fun setupEditTextUuidValidator(editText: EditText) {
editText.addTextChangedListener(object : TextWatcher {

9
src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/ChapterDto.kt
View File

@ -20,4 +20,11 @@ data class ChapterAttributesDto(
val pages: Int,
val publishAt: String,
val externalUrl: String?,
) : AttributesDto()
) : AttributesDto() {
/**
* Returns true if the chapter is from an external website and have no pages.
*/
val isInvalid: Boolean
get() = externalUrl != null && pages == 0
}