diff --git a/src/all/mangadex/build.gradle b/src/all/mangadex/build.gradle index 9142437e6..ccbb6e2d9 100644 --- a/src/all/mangadex/build.gradle +++ b/src/all/mangadex/build.gradle @@ -6,7 +6,7 @@ ext { extName = 'MangaDex' pkgNameSuffix = 'all.mangadex' extClass = '.MangaDexFactory' - extVersionCode = 176 + extVersionCode = 177 isNsfw = true } diff --git a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt index 82ba9eedd..9a5e8c80a 100644 --- a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt +++ b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt @@ -2,7 +2,6 @@ package eu.kanade.tachiyomi.extension.all.mangadex import android.app.Application import android.content.SharedPreferences -import android.util.Log import androidx.preference.EditTextPreference import androidx.preference.ListPreference import androidx.preference.MultiSelectListPreference @@ -20,6 +19,7 @@ import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDataDto import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaDto import eu.kanade.tachiyomi.extension.all.mangadex.dto.MangaListDto import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservable import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.network.interceptor.rateLimit import eu.kanade.tachiyomi.source.ConfigurableSource @@ -71,7 +71,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St preferences.sanitizeExistingUuidPrefs() } - // POPULAR Manga Section + // Popular manga section override fun popularMangaRequest(page: Int): Request { val url = MDConstants.apiMangaUrl.toHttpUrl().newBuilder() @@ -112,7 +112,11 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St return MangasPage(mangaList, hasMoreResults) } - // LATEST section API can't sort by date yet so not implemented + // Latest manga section + + /** + * The API endpoint can't sort by date yet, so not implemented. + */ override fun latestUpdatesParse(response: Response): MangasPage { val chapterListDto = response.parseAs() val hasMoreResults = chapterListDto.limit + chapterListDto.offset < chapterListDto.total @@ -170,7 +174,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St return GET(url.build().toString(), headers, CacheControl.FORCE_NETWORK) } - // SEARCH section + // Search manga section override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { return when { @@ -211,7 +215,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St private fun getMangaIdFromChapterId(id: String): Observable { return client.newCall(GET("${MDConstants.apiChapterUrl}/$id", headers)) - .asObservableSuccess() + .asObservable() .map { response -> if (response.isSuccessful.not()) { throw Exception(helper.intl.unableToProcessChapterRequest(response.code)) @@ -317,6 +321,9 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St } private fun searchMangaListParse(response: Response): List { + // This check will be used as the source is doing additional requests to this + // that are not parsed by the asObservableSuccess() method. It should throw the + // HttpException from the app if it becomes available in a future version of extensions-lib. if (response.isSuccessful.not()) { throw Exception("HTTP error ${response.code}") } @@ -360,7 +367,8 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St // Manga Details section - // Shenanigans to allow "open in webview" to show a webpage instead of JSON + // Workaround to allow "Open in WebView" to show a webpage instead of JSON. + // TODO: Replace with getMangaUrl when the repository is using extensions-lib 1.4 override fun fetchMangaDetails(manga: SManga): Observable { return client.newCall(apiMangaDetailsRequest(manga)) .asObservableSuccess() @@ -370,7 +378,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St } override fun mangaDetailsRequest(manga: SManga): Request { - // remove once redirect for /manga is fixed + // TODO: Remove once redirect for /manga is fixed. val title = manga.title val url = "${baseUrl}${manga.url.replace("manga", "title")}" val shareUrl = "$url/" + helper.titleToSlug(title) @@ -378,7 +386,9 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St } /** - * get manga details url throws exception if the url is the old format so people migrate + * Get the API endpoint URL for the entry details. + * + * @throws Exception if the url is the old format so people migrate */ private fun apiMangaDetailsRequest(manga: SManga): Request { if (!helper.containsUuid(manga.url.trim())) { @@ -473,21 +483,24 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St } // Chapter list section + /** - * get chapter list if manga url is old format throws exception + * Get the API endpoint URL for the first page of chapter list. + * + * @throws Exception if the url is the old format so people migrate */ override fun chapterListRequest(manga: SManga): Request { if (!helper.containsUuid(manga.url)) { throw Exception(helper.intl.migrateWarning) } - return actualChapterListRequest(helper.getUUIDFromUrl(manga.url), 0) + return paginatedChapterListRequest(helper.getUUIDFromUrl(manga.url), 0) } /** - * Required because api is paged + * Required because the chapter list API endpoint is paginated. */ - private fun actualChapterListRequest(mangaId: String, offset: Int): Request { + private fun paginatedChapterListRequest(mangaId: String, offset: Int): Request { val url = helper.getChapterEndpoint(mangaId, offset, dexLang).toHttpUrl().newBuilder() .addQueryParameter("contentRating[]", "safe") .addQueryParameter("contentRating[]", "suggestive") @@ -504,37 +517,34 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St return emptyList() } - try { - val chapterListResponse = response.parseAs() + val chapterListResponse = response.parseAs() - val chapterListResults = chapterListResponse.data.toMutableList() + val chapterListResults = chapterListResponse.data.toMutableList() - val mangaId = response.request.url.toString() - .substringBefore("/feed") - .substringAfter("${MDConstants.apiMangaUrl}/") + val mangaId = response.request.url.toString() + .substringBefore("/feed") + .substringAfter("${MDConstants.apiMangaUrl}/") - val limit = chapterListResponse.limit + val limit = chapterListResponse.limit - var offset = chapterListResponse.offset + var offset = chapterListResponse.offset - var hasMoreResults = (limit + offset) < chapterListResponse.total + var hasMoreResults = (limit + offset) < chapterListResponse.total - // Max results that can be returned is 500 so need to make more API - // calls if limit + offset > total chapters - while (hasMoreResults) { - offset += limit - val newRequest = actualChapterListRequest(mangaId, offset) - val newResponse = client.newCall(newRequest).execute() - val newChapterList = newResponse.parseAs() - chapterListResults.addAll(newChapterList.data) - hasMoreResults = (limit + offset) < newChapterList.total - } - - return chapterListResults.mapNotNull(helper::createChapter) - } catch (e: Exception) { - Log.e("MangaDex", "error parsing chapter list", e) - throw e + // Max results that can be returned is 500 so need to make more API + // calls if limit + offset > total chapters + while (hasMoreResults) { + offset += limit + val newRequest = paginatedChapterListRequest(mangaId, offset) + val newResponse = client.newCall(newRequest).execute() + val newChapterList = newResponse.parseAs() + chapterListResults.addAll(newChapterList.data) + hasMoreResults = (limit + offset) < newChapterList.total } + + return chapterListResults + .filterNot { it.attributes!!.isInvalid } + .map(helper::createChapter) } override fun pageListRequest(chapter: SChapter): Request { @@ -557,7 +567,7 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St val atHomeDto = response.parseAs() val host = atHomeDto.baseUrl - // have to add the time, and url to the page because pages timeout within 30mins now + // Have to add the time, and url to the page because pages timeout within 30 minutes now. val now = Date().time val hash = atHomeDto.chapter.hash @@ -801,6 +811,11 @@ abstract class MangaDex(final override val lang: String, private val dexLang: St private val SharedPreferences.useDataSaver get() = getBoolean(MDConstants.getDataSaverPreferenceKey(dexLang), false) + /** + * Previous versions of the extension allowed invalid UUID values to be stored in the + * preferences. This method clear invalid UUIDs in case the user have updated from + * a previous version with that behaviour. + */ private fun SharedPreferences.sanitizeExistingUuidPrefs() { if (getBoolean(MDConstants.getHasSanitizedUuidsPrefKey(dexLang), false)) { return diff --git a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexFactory.kt b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexFactory.kt index 6efe061ff..8860033d5 100644 --- a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexFactory.kt +++ b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexFactory.kt @@ -11,8 +11,8 @@ class MangaDexFactory : SourceFactory { MangaDexBulgarian(), MangaDexBurmese(), MangaDexCatalan(), - MangaDexChineseSimp(), - MangaDexChineseTrad(), + MangaDexChineseSimplified(), + MangaDexChineseTraditional(), MangaDexCzech(), MangaDexDanish(), MangaDexDutch(), @@ -42,7 +42,7 @@ class MangaDexFactory : SourceFactory { MangaDexRomanian(), MangaDexRussian(), MangaDexSerboCroatian(), - MangaDexSpanishLTAM(), + MangaDexSpanishLatinAmerica(), MangaDexSpanishSpain(), MangaDexSwedish(), MangaDexTamil(), @@ -58,8 +58,8 @@ class MangaDexBengali : MangaDex("bn", "bn") class MangaDexBulgarian : MangaDex("bg", "bg") class MangaDexBurmese : MangaDex("my", "my") class MangaDexCatalan : MangaDex("ca", "ca") -class MangaDexChineseSimp : MangaDex("zh-Hans", "zh") -class MangaDexChineseTrad : MangaDex("zh-Hant", "zh-hk") +class MangaDexChineseSimplified : MangaDex("zh-Hans", "zh") +class MangaDexChineseTraditional : MangaDex("zh-Hant", "zh-hk") class MangaDexCzech : MangaDex("cs", "cs") class MangaDexDanish : MangaDex("da", "da") class MangaDexDutch : MangaDex("nl", "nl") @@ -90,7 +90,7 @@ class MangaDexPortuguesePortugal : MangaDex("pt", "pt") class MangaDexRomanian : MangaDex("ro", "ro") class MangaDexRussian : MangaDex("ru", "ru") class MangaDexSerboCroatian : MangaDex("sh", "sh") -class MangaDexSpanishLTAM : MangaDex("es-419", "es-la") +class MangaDexSpanishLatinAmerica : MangaDex("es-419", "es-la") class MangaDexSpanishSpain : MangaDex("es", "es") class MangaDexSwedish : MangaDex("sv", "sv") class MangaDexTamil : MangaDex("ta", "ta") diff --git a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt index ae9aaf503..4fcf770e1 100644 --- a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt +++ b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt @@ -129,10 +129,11 @@ class MangaDexHelper(lang: String) { /** * Remove any HTML characters in description or chapter name to actual - * characters. For example ♥ will show ♥ + * characters. For example ♥ will show ♥. It also removes + * Markdown syntax for links, italic and bold. */ - private fun cleanString(string: String): String { - return Parser.unescapeEntities(string, false) + private fun String.removeEntitiesAndMarkdown(): String { + return Parser.unescapeEntities(this, false) .substringBefore("---") .replace(markdownLinksRegex, "$1") .replace(markdownItalicBoldRegex, "$1") @@ -141,7 +142,7 @@ class MangaDexHelper(lang: String) { } /** - * Maps dex status to Tachi status. + * Maps MangaDex status to Tachiyomi status. * Adapted from the MangaDex handler from TachiyomiSY. */ fun getPublicationStatus(attr: MangaAttributesDto, volumes: Map): Int { @@ -173,7 +174,9 @@ class MangaDexHelper(lang: String) { private fun parseDate(dateAsString: String): Long = MDConstants.dateFormatter.parse(dateAsString)?.time ?: 0 - // chapter url where we get the token, last request time + /** + * Chapter URL where we get the token, last request time. + */ private val tokenTracker = hashMapOf() companion object { @@ -190,7 +193,9 @@ class MangaDexHelper(lang: String) { val trailingHyphenRegex = "-+$".toRegex() } - // Check the token map to see if the md@home host is still valid + /** + * Check the token map to see if the MD@Home host is still valid. + */ fun getValidImageUrlForPage(page: Page, headers: Headers, client: OkHttpClient): Request { val data = page.url.split(",") @@ -199,12 +204,9 @@ class MangaDexHelper(lang: String) { false -> data[0] true -> { val tokenRequestUrl = data[1] + val tokenLifespan = Date().time - (tokenTracker[tokenRequestUrl] ?: 0) val cacheControl = - if (Date().time - ( - tokenTracker[tokenRequestUrl] - ?: 0 - ) > MDConstants.mdAtHomeTokenLifespan - ) { + if (tokenLifespan > MDConstants.mdAtHomeTokenLifespan) { CacheControl.FORCE_NETWORK } else { USE_CACHE @@ -212,11 +214,12 @@ class MangaDexHelper(lang: String) { getMdAtHomeUrl(tokenRequestUrl, client, headers, cacheControl) } } + return GET(mdAtHomeServerUrl + page.imageUrl, headers) } /** - * get the md@home url + * Get the MD@Home URL. */ private fun getMdAtHomeUrl( tokenRequestUrl: String, @@ -234,7 +237,7 @@ class MangaDexHelper(lang: String) { return getMdAtHomeUrl(tokenRequestUrl, client, headers, CacheControl.FORCE_NETWORK) } - return json.decodeFromString(response.body!!.string()).baseUrl + return response.use { json.decodeFromString(it.body!!.string()).baseUrl } } /** @@ -253,7 +256,7 @@ class MangaDexHelper(lang: String) { } /** - * create an SManga from json element only basic elements + * Create a [SManga] from the JSON element with only basic attributes filled. */ fun createBasicManga( mangaDataDto: MangaDataDto, @@ -269,10 +272,10 @@ class MangaDexHelper(lang: String) { ?: mangaDataDto.attributes.altTitles .find { (it[lang] ?: it["en"]) !== null } ?.values?.singleOrNull() // find something else from alt titles - title = cleanString(dirtyTitle ?: "") + title = (dirtyTitle ?: "").removeEntitiesAndMarkdown() coverFileName?.let { - thumbnail_url = when (coverSuffix != null && coverSuffix != "") { + thumbnail_url = when (!coverSuffix.isNullOrEmpty()) { true -> "${MDConstants.cdnUrl}/covers/${mangaDataDto.id}/$coverFileName$coverSuffix" else -> "${MDConstants.cdnUrl}/covers/${mangaDataDto.id}/$coverFileName" } @@ -281,7 +284,7 @@ class MangaDexHelper(lang: String) { } /** - * Create an SManga from json element with all details + * Create an [SManga] from the JSON element with all attributes filled. */ fun createManga( mangaDataDto: MangaDataDto, @@ -290,128 +293,114 @@ class MangaDexHelper(lang: String) { lang: String, coverSuffix: String? ): SManga { - try { - val attr = mangaDataDto.attributes!! + val attr = mangaDataDto.attributes!! - // things that will go with the genre tags but aren't actually genre - val dexLocale = Locale.forLanguageTag(lang) + // Things that will go with the genre tags but aren't actually genre + val dexLocale = Locale.forLanguageTag(lang) - val nonGenres = listOfNotNull( - attr.publicationDemographic?.let { intl.publicationDemographic(it) }, - attr.contentRating - .takeIf { it != ContentRatingDto.SAFE } - ?.let { intl.contentRatingGenre(it) }, - attr.originalLanguage - ?.let { Locale.forLanguageTag(it) } - ?.getDisplayName(dexLocale) - ?.replaceFirstChar { it.uppercase(dexLocale) } - ) + val nonGenres = listOfNotNull( + attr.publicationDemographic?.let { intl.publicationDemographic(it) }, + attr.contentRating + .takeIf { it != ContentRatingDto.SAFE } + ?.let { intl.contentRatingGenre(it) }, + attr.originalLanguage + ?.let { Locale.forLanguageTag(it) } + ?.getDisplayName(dexLocale) + ?.replaceFirstChar { it.uppercase(dexLocale) } + ) - val authors = mangaDataDto.relationships - .filterIsInstance() - .mapNotNull { it.attributes?.name } - .distinct() + val authors = mangaDataDto.relationships + .filterIsInstance() + .mapNotNull { it.attributes?.name } + .distinct() - val artists = mangaDataDto.relationships - .filterIsInstance() - .mapNotNull { it.attributes?.name } - .distinct() + val artists = mangaDataDto.relationships + .filterIsInstance() + .mapNotNull { it.attributes?.name } + .distinct() - val coverFileName = firstVolumeCover ?: mangaDataDto.relationships - .filterIsInstance() - .firstOrNull() - ?.attributes?.fileName + val coverFileName = firstVolumeCover ?: mangaDataDto.relationships + .filterIsInstance() + .firstOrNull() + ?.attributes?.fileName - val tags = mdFilters.getTags(intl).associate { it.id to it.name } + val tags = mdFilters.getTags(intl).associate { it.id to it.name } - val genresMap = attr.tags - .groupBy({ it.attributes!!.group }) { tagDto -> tags[tagDto.id] } - .mapValues { it.value.filterNotNull().sortedWith(intl.collator) } + val genresMap = attr.tags + .groupBy({ it.attributes!!.group }) { tagDto -> tags[tagDto.id] } + .mapValues { it.value.filterNotNull().sortedWith(intl.collator) } - val genreList = MDConstants.tagGroupsOrder.flatMap { genresMap[it].orEmpty() } + nonGenres + val genreList = MDConstants.tagGroupsOrder.flatMap { genresMap[it].orEmpty() } + nonGenres - val desc = attr.description + val desc = attr.description - return createBasicManga(mangaDataDto, coverFileName, coverSuffix, lang).apply { - description = cleanString(desc[lang] ?: desc["en"] ?: "") - author = authors.joinToString(", ") - artist = artists.joinToString(", ") - status = getPublicationStatus(attr, chapters) - genre = genreList - .filter(String::isNotEmpty) - .joinToString(", ") - } - } catch (e: Exception) { - Log.e("MangaDex", "error parsing manga", e) - throw e + return createBasicManga(mangaDataDto, coverFileName, coverSuffix, lang).apply { + description = (desc[lang] ?: desc["en"] ?: "").removeEntitiesAndMarkdown() + author = authors.joinToString(", ") + artist = artists.joinToString(", ") + status = getPublicationStatus(attr, chapters) + genre = genreList + .filter(String::isNotEmpty) + .joinToString(", ") } } /** - * create the SChapter from json + * Create the [SChapter] from the JSON element. */ - fun createChapter(chapterDataDto: ChapterDataDto): SChapter? { - try { - val attr = chapterDataDto.attributes!! + fun createChapter(chapterDataDto: ChapterDataDto): SChapter { + val attr = chapterDataDto.attributes!! - val groups = chapterDataDto.relationships - .filterIsInstance() - .filterNot { it.id == MDConstants.legacyNoGroupId } // 'no group' left over from MDv3 - .mapNotNull { it.attributes?.name } - .joinToString(" & ") - .ifEmpty { - // fall back to uploader name if no group - val users = chapterDataDto.relationships - .filterIsInstance() - .mapNotNull { it.attributes?.username } - if (users.isNotEmpty()) intl.uploadedBy(users) else "" - } - .ifEmpty { intl.noGroup } // "No Group" as final resort - - val chapterName = mutableListOf() - // Build chapter name - - attr.volume?.let { - if (it.isNotEmpty()) { - chapterName.add("Vol.$it") + val groups = chapterDataDto.relationships + .filterIsInstance() + .filterNot { it.id == MDConstants.legacyNoGroupId } // 'no group' left over from MDv3 + .mapNotNull { it.attributes?.name } + .joinToString(" & ") + .ifEmpty { + // Fallback to uploader name if no group is set. + val users = chapterDataDto.relationships + .filterIsInstance() + .mapNotNull { it.attributes?.username } + if (users.isNotEmpty()) intl.uploadedBy(users) else "" + } + .ifEmpty { intl.noGroup } // "No Group" as final resort + + val chapterName = mutableListOf() + // Build chapter name + + attr.volume?.let { + if (it.isNotEmpty()) { + chapterName.add("Vol.$it") + } + } + + attr.chapter?.let { + if (it.isNotEmpty()) { + chapterName.add("Ch.$it") + } + } + + attr.title?.let { + if (it.isNotEmpty()) { + if (chapterName.isNotEmpty()) { + chapterName.add("-") } + chapterName.add(it) } + } - attr.chapter?.let { - if (it.isNotEmpty()) { - chapterName.add("Ch.$it") - } - } + // if volume, chapter and title is empty its a oneshot + if (chapterName.isEmpty()) { + chapterName.add("Oneshot") + } - attr.title?.let { - if (it.isNotEmpty()) { - if (chapterName.isNotEmpty()) { - chapterName.add("-") - } - chapterName.add(it) - } - } + // In future calculate [END] if non mvp api doesn't provide it - if (attr.externalUrl != null && attr.pages == 0) { - return null - } - - // if volume, chapter and title is empty its a oneshot - if (chapterName.isEmpty()) { - chapterName.add("Oneshot") - } - - // In future calculate [END] if non mvp api doesn't provide it - - return SChapter.create().apply { - url = "/chapter/${chapterDataDto.id}" - name = cleanString(chapterName.joinToString(" ")) - date_upload = parseDate(attr.publishAt) - scanlator = groups - } - } catch (e: Exception) { - Log.e("MangaDex", "error parsing chapter", e) - throw e + return SChapter.create().apply { + url = "/chapter/${chapterDataDto.id}" + name = chapterName.joinToString(" ").removeEntitiesAndMarkdown() + date_upload = parseDate(attr.publishAt) + scanlator = groups } } @@ -433,6 +422,9 @@ class MangaDexHelper(lang: String) { * Adds a custom [TextWatcher] to the preference's [EditText] that show an * error if the input value contains invalid UUIDs. If the validation fails, * the Ok button is disabled to prevent the user from saving the value. + * + * This will likely need to be removed or revisited when the app migrates the + * extension preferences screen to Compose. */ fun setupEditTextUuidValidator(editText: EditText) { editText.addTextChangedListener(object : TextWatcher { diff --git a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/ChapterDto.kt b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/ChapterDto.kt index 9e9adeac5..36984ec53 100644 --- a/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/ChapterDto.kt +++ b/src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/dto/ChapterDto.kt @@ -20,4 +20,11 @@ data class ChapterAttributesDto( val pages: Int, val publishAt: String, val externalUrl: String?, -) : AttributesDto() +) : AttributesDto() { + + /** + * Returns true if the chapter is from an external website and have no pages. + */ + val isInvalid: Boolean + get() = externalUrl != null && pages == 0 +}