More Dex Updates (#6900)

* MangaDex: remove empty tag, fix genre typo

- remove empty tag caused by null/empty publicationDemographic/other from nonGenres if its null/empty
- fix typo genre uuid

* better method removing empty tag

Co-Authored-By: Carlos <2092019+CarlosEsco@users.noreply.github.com>

* revert pt-br changes

Changing it to pt-br instead of pt-BR changed the id of the source and made the Mangadex in Brazilian Portuguese to be separed from the other Brazilian extensions (that uses pt-BR as lang)

Co-Authored-By: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* Update build.gradle

* String updates to support standard HTTPS in MD@H node selection

* Code updates to only use nodes with standard HTTPS in MD@H node selection

* Update build.grade to indicate new version

* fix preferences and langs

* low md@home to refresh host sooner

* add content rating by default preferences(defaults to safe and suggestive)

* throw error for page list parsing instead of trying to continue

* clean up string

Co-authored-by: Riztard <riyanluqman@gmail.com>
Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>
Co-authored-by: fourquestionmarksokay <83817943+fourquestionmarksokay@users.noreply.github.com>

src/all/mangadex/build.gradle

@@ -5,7 +5,7 @@ ext {
     extName = 'MangaDex'
     pkgNameSuffix = 'all.mangadex'
     extClass = '.MangaDexFactory'
-    extVersionCode = 107
+    extVersionCode = 108
     libVersion = '1.2'
     containsNsfw = true
 }

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MDConstants.kt

@@ -17,6 +17,8 @@ object MDConstants {
 
     val tempCover = "https://i.imgur.com/6TrIues.jpg"
 
+    const val mdAtHomeTokenLifespan = 5 * 60 * 1000
+
     val dateFormatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss+SSS", Locale.US)
         .apply { timeZone = TimeZone.getTimeZone("UTC") }
 
@@ -26,5 +28,46 @@ object MDConstants {
     const val dataSaverPrefSummary = "Enables smaller more compressed images"
     const val dataSaverPref = "dataSaverV5"
 
-    const val mdAtHomeTokenLifespan = 10 * 60 * 1000
+    fun getDataSaverPreferenceKey(dexLang: String): String {
+        return "${dataSaverPref}_$dexLang"
+    }
+
+    const val standardHttpsPortTitle = "Use HTTPS port 443 only"
+    const val standardHttpsPortSummary =
+        "Enable to only request image servers that use port 443. This allows users with stricter firewall restrictions to access MangaDex images"
+    private const val standardHttpsPortPref = "usePort443"
+
+    fun getStandardHttpsPreferenceKey(dexLang: String): String {
+        return "${standardHttpsPortPref}_$dexLang"
+    }
+
+    const val showByDefaultPrefTitle = "Show only by default"
+
+    const val contentRatingSafePrefSummary = "Content Rating: Safe"
+    private const val contentRatingSafePref = "contentRatingSafe"
+
+    fun getContentRatingSafePrefKey(dexLang: String): String {
+        return "${contentRatingSafePref}_$dexLang"
+    }
+
+    const val contentRatingSuggestivePrefSummary = "Content Rating: Suggestive"
+    private const val contentRatingSuggestivePref = "contentRatingSuggestive"
+
+    fun getContentRatingSuggestivePrefKey(dexLang: String): String {
+        return "${contentRatingSuggestivePref}_$dexLang"
+    }
+
+    const val contentRatingEroticaPrefSummary = "Content Rating: Erotica"
+    private const val contentRatingEroticaPref = "contentRatingErotica"
+
+    fun getContentRatingEroticaPrefKey(dexLang: String): String {
+        return "${contentRatingEroticaPref}_$dexLang"
+    }
+
+    const val contentRatingPornographicPrefSummary = "Content Rating: Pornographic"
+    private const val contentRatingPornographicPref = "contentRatingPornographic"
+
+    fun getContentRatingPornographicPrefKey(dexLang: String): String {
+        return "${contentRatingPornographicPref}_$dexLang"
+    }
 }

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDex.kt

@@ -31,19 +31,20 @@ import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
 import java.util.Date
 
-abstract class MangaDex(override val lang: String) : ConfigurableSource, HttpSource() {
+abstract class MangaDex(override val lang: String, val dexLang: String) : ConfigurableSource,
+    HttpSource() {
     override val name = "MangaDex"
     override val baseUrl = "https://www.mangadex.org"
 
     // after mvp comes out make current popular becomes latest (mvp doesnt have a browse page)
     override val supportsLatest = false
 
-    private val helper = MangaDexHelper()
-
     private val preferences: SharedPreferences by lazy {
         Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
     }
 
+    private val helper = MangaDexHelper()
+
     override fun headersBuilder() = Headers.Builder().apply {
         add("User-Agent", "Tachiyomi " + System.getProperty("http.agent"))
     }
@@ -57,12 +58,35 @@ abstract class MangaDex(override val lang: String) : ConfigurableSource, HttpSou
     // POPULAR Manga Section
 
     override fun popularMangaRequest(page: Int): Request {
-        val url = MDConstants.apiMangaUrl.toHttpUrl().newBuilder()
+        val url = MDConstants.apiMangaUrl.toHttpUrl().newBuilder().apply {
-            .addQueryParameter("order[updatedAt]", "desc")
+            addQueryParameter("order[updatedAt]", "desc")
-            .addQueryParameter("limit", MDConstants.mangaLimit.toString())
+            addQueryParameter("limit", MDConstants.mangaLimit.toString())
-            .addQueryParameter("offset", helper.getMangaListOffset(page))
+            addQueryParameter("offset", helper.getMangaListOffset(page))
-            .build().toUrl().toString()
+            if (preferences.getBoolean(MDConstants.getContentRatingSafePrefKey(dexLang), false)) {
-
+                addQueryParameter("contentRating[]", "safe")
+            }
+            if (preferences.getBoolean(
+                    MDConstants.getContentRatingEroticaPrefKey(dexLang),
+                    false
+                )
+            ) {
+                addQueryParameter("contentRating[]", "suggestive")
+            }
+            if (preferences.getBoolean(
+                    MDConstants.getContentRatingSuggestivePrefKey(dexLang),
+                    false
+                )
+            ) {
+                addQueryParameter("contentRating[]", "erotica")
+            }
+            if (preferences.getBoolean(
+                    MDConstants.getContentRatingPornographicPrefKey(dexLang),
+                    false
+                )
+            ) {
+                addQueryParameter("contentRating[]", "pornographic")
+            }
+        }.build().toUrl().toString()
         return GET(
             url = url,
             headers = headers,
@@ -165,7 +189,7 @@ abstract class MangaDex(override val lang: String) : ConfigurableSource, HttpSou
      */
     private fun actualChapterListRequest(mangaId: String, offset: Int) =
         GET(
-            url = helper.getChapterEndpoint(mangaId, offset, lang),
+            url = helper.getChapterEndpoint(mangaId, offset, dexLang),
             headers = headers,
             cache = CacheControl.FORCE_NETWORK
         )
@@ -224,13 +248,27 @@ abstract class MangaDex(override val lang: String) : ConfigurableSource, HttpSou
     }
 
     override fun pageListParse(response: Response): List<Page> {
+        if (response.isSuccessful.not()) {
+            throw Exception("HTTP ${response.code}")
+        }
+        if (response.code == 204) {
+            return emptyList()
+        }
         val chapterJson = JsonParser.parseString(response.body!!.string()).obj["data"]
-        val atHomeRequestUrl = "${MDConstants.apiUrl}/at-home/server/${chapterJson["id"].string}"
+        val usingStandardHTTPS =
+            preferences.getBoolean(MDConstants.getStandardHttpsPreferenceKey(dexLang), false)
+
+        val atHomeRequestUrl = if (usingStandardHTTPS) {
+            "${MDConstants.apiUrl}/at-home/server/${chapterJson["id"].string}?ssl=true"
+        } else {
+            "${MDConstants.apiUrl}/at-home/server/${chapterJson["id"].string}"
+        }
 
         val host =
             helper.getMdAtHomeUrl(atHomeRequestUrl, client, headers, CacheControl.FORCE_NETWORK)
 
-        val usingDataSaver = preferences.getBoolean("${MDConstants.dataSaverPref}_$lang", false)
+        val usingDataSaver =
+            preferences.getBoolean(MDConstants.getDataSaverPreferenceKey(dexLang), false)
 
         // have to add the time, and url to the page because pages timeout within 30mins now
         val now = Date().time
@@ -257,19 +295,99 @@ abstract class MangaDex(override val lang: String) : ConfigurableSource, HttpSou
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
 
         val dataSaverPref = CheckBoxPreference(screen.context).apply {
-            key = "${MDConstants.dataSaverPref}_$lang"
+            key = MDConstants.getDataSaverPreferenceKey(dexLang)
             title = MDConstants.dataSaverPrefTitle
             summary = MDConstants.dataSaverPrefSummary
             setDefaultValue(false)
 
             setOnPreferenceChangeListener { _, newValue ->
                 val checkValue = newValue as Boolean
-                preferences.edit().putBoolean("${MDConstants.dataSaverPref}_$lang", checkValue)
+                preferences.edit()
+                    .putBoolean(MDConstants.getDataSaverPreferenceKey(dexLang), checkValue)
                     .commit()
             }
         }
+
+        val standardHTTPSPref = CheckBoxPreference(screen.context).apply {
+            key = MDConstants.getStandardHttpsPreferenceKey(dexLang)
+            title = MDConstants.standardHttpsPortTitle
+            summary = MDConstants.standardHttpsPortSummary
+            setDefaultValue(false)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit()
+                    .putBoolean(MDConstants.getStandardHttpsPreferenceKey(dexLang), checkValue)
+                    .commit()
+            }
+        }
+
+        val contentRatingSafePref = CheckBoxPreference(screen.context).apply {
+            key = MDConstants.getContentRatingSafePrefKey(dexLang)
+            title = MDConstants.showByDefaultPrefTitle
+            summary = MDConstants.contentRatingSafePrefSummary
+            setDefaultValue(true)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit()
+                    .putBoolean(MDConstants.getContentRatingSafePrefKey(dexLang), checkValue)
+                    .commit()
+            }
+        }
+
+        val contentRatingSuggestivePref = CheckBoxPreference(screen.context).apply {
+            key = MDConstants.getContentRatingSuggestivePrefKey(dexLang)
+            title = MDConstants.showByDefaultPrefTitle
+            summary = MDConstants.contentRatingSuggestivePrefSummary
+            setDefaultValue(true)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit()
+                    .putBoolean(MDConstants.getContentRatingSuggestivePrefKey(dexLang), checkValue)
+                    .commit()
+            }
+        }
+
+        val contentRatingEroticaPref = CheckBoxPreference(screen.context).apply {
+            key = MDConstants.getContentRatingEroticaPrefKey(dexLang)
+            title = MDConstants.showByDefaultPrefTitle
+            summary = MDConstants.contentRatingEroticaPrefSummary
+            setDefaultValue(false)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit()
+                    .putBoolean(MDConstants.getContentRatingEroticaPrefKey(dexLang), checkValue)
+                    .commit()
+            }
+        }
+
+        val contentRatingPornographicPref = CheckBoxPreference(screen.context).apply {
+            key = MDConstants.getContentRatingPornographicPrefKey(dexLang)
+            title = MDConstants.showByDefaultPrefTitle
+            summary = MDConstants.contentRatingPornographicPrefSummary
+            setDefaultValue(false)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit().putBoolean(
+                    MDConstants.getContentRatingPornographicPrefKey(dexLang),
+                    checkValue
+                )
+                    .commit()
+            }
+        }
+
         screen.addPreference(dataSaverPref)
+        screen.addPreference(standardHTTPSPref)
+        screen.addPreference(contentRatingSafePref)
+        screen.addPreference(contentRatingSuggestivePref)
+        screen.addPreference(contentRatingEroticaPref)
+        screen.addPreference(contentRatingPornographicPref)
     }
 
-    override fun getFilterList(): FilterList = helper.mdFilters.getMDFilterList()
+    override fun getFilterList(): FilterList =
+        helper.mdFilters.getMDFilterList(preferences, dexLang)
 }

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexFactory.kt

@@ -51,44 +51,45 @@ class MangaDexFactory : SourceFactory {
     )
 }
 
-class MangaDexEnglish : MangaDex("en")
+class MangaDexEnglish : MangaDex("en", "en")
-class MangaDexJapanese : MangaDex("ja")
+class MangaDexJapanese : MangaDex("ja", "ja")
-class MangaDexPolish : MangaDex("pl")
+class MangaDexPolish : MangaDex("pl", "pl")
-class MangaDexSerboCroatian : MangaDex("sh")
+class MangaDexSerboCroatian : MangaDex("sh", "sh")
-class MangaDexDutch : MangaDex("nl")
+class MangaDexDutch : MangaDex("nl", "nl")
-class MangaDexItalian : MangaDex("it")
+class MangaDexItalian : MangaDex("it", "it")
-class MangaDexRussian : MangaDex("ru")
+class MangaDexRussian : MangaDex("ru", "ru")
-class MangaDexGerman : MangaDex("de")
+class MangaDexGerman : MangaDex("de", "de")
-class MangaDexHungarian : MangaDex("hu")
+class MangaDexHungarian : MangaDex("hu", "hu")
-class MangaDexFrench : MangaDex("fr")
+class MangaDexFrench : MangaDex("fr", "fr")
-class MangaDexFinnish : MangaDex("fi")
+class MangaDexFinnish : MangaDex("fi", "fi")
-class MangaDexVietnamese : MangaDex("vi")
+class MangaDexVietnamese : MangaDex("vi", "vi")
-class MangaDexGreek : MangaDex("el")
+class MangaDexGreek : MangaDex("el", "el")
-class MangaDexBulgarian : MangaDex("bg")
+class MangaDexBulgarian : MangaDex("bg", "bg")
-class MangaDexSpanishSpain : MangaDex("es")
+class MangaDexSpanishSpain : MangaDex("es", "es")
-class MangaDexPortugueseBrazil : MangaDex("pt-br")
+class MangaDexPortugueseBrazil : MangaDex("pt-BR", "pt-br")
-class MangaDexPortuguesePortugal : MangaDex("pt")
+class MangaDexPortuguesePortugal : MangaDex("pt", "pt")
-class MangaDexSwedish : MangaDex("sv")
+class MangaDexSwedish : MangaDex("sv", "sv")
-class MangaDexArabic : MangaDex("ar")
+class MangaDexArabic : MangaDex("ar", "ar")
-class MangaDexDanish : MangaDex("da")
+class MangaDexDanish : MangaDex("da", "da")
-class MangaDexChineseSimp : MangaDex("zh")
+class MangaDexChineseSimp : MangaDex("zh-Hans", "zh")
-class MangaDexBengali : MangaDex("bn")
+class MangaDexBengali : MangaDex("bn", "bn")
-class MangaDexRomanian : MangaDex("ro")
+class MangaDexRomanian : MangaDex("ro", "ro")
-class MangaDexCzech : MangaDex("cs")
+class MangaDexCzech : MangaDex("cs", "cs")
-class MangaDexMongolian : MangaDex("mn")
+class MangaDexMongolian : MangaDex("mn", "mn")
-class MangaDexTurkish : MangaDex("tr")
+class MangaDexTurkish : MangaDex("tr", "tr")
-class MangaDexIndonesian : MangaDex("id")
+class MangaDexIndonesian : MangaDex("id", "id")
-class MangaDexKorean : MangaDex("ko")
+class MangaDexKorean : MangaDex("ko", "ko")
-class MangaDexSpanishLTAM : MangaDex("es-la")
+class MangaDexSpanishLTAM : MangaDex("es-419", "es-la")
-class MangaDexPersian : MangaDex("fa")
+class MangaDexPersian : MangaDex("fa", "fa")
-class MangaDexMalay : MangaDex("ms")
+class MangaDexMalay : MangaDex("ms", "ms")
-class MangaDexThai : MangaDex("th")
+class MangaDexThai : MangaDex("th", "th")
-class MangaDexCatalan : MangaDex("ca")
+class MangaDexCatalan : MangaDex("ca", "ca")
-class MangaDexFilipino : MangaDex("fi")
+class MangaDexFilipino : MangaDex("fil", "fi")
-class MangaDexChineseTrad : MangaDex("zh-hk")
+class MangaDexChineseTrad : MangaDex("zh-Hant", "zh-hk")
-class MangaDexUkrainian : MangaDex("uk")
+class MangaDexUkrainian : MangaDex("uk", "uk")
-class MangaDexBurmese : MangaDex("my")
+class MangaDexBurmese : MangaDex("my", "my")
-class MangaDexLithuanian : MangaDex("lt")
+class MangaDexLithuanian : MangaDex("lt", "lt")
-class MangaDexHebrew : MangaDex("he")
+class MangaDexHebrew : MangaDex("he", "he")
-class MangaDexHindi : MangaDex("hi")
+class MangaDexHindi : MangaDex("hi", "hi")
-class MangaDexNorwegian : MangaDex("no")
+class MangaDexNorwegian : MangaDex("no", "no")
+class Other : MangaDex("other", "NULL")

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexFilters.kt

@@ -1,5 +1,6 @@
 package eu.kanade.tachiyomi.extension.all.mangadex
 
+import android.content.SharedPreferences
 import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import okhttp3.HttpUrl
@@ -7,16 +8,43 @@ import java.util.Locale
 
 class MangaDexFilters {
 
-    internal fun getMDFilterList() = FilterList(
+    internal fun getMDFilterList(preferences: SharedPreferences, dexLang: String): FilterList {
-        OriginalLanguageList(getOriginalLanguage()),
+        val contentRatings = listOf(
-        ContentRatingList(getContentRating()),
+            ContentRating("Safe").apply {
-        DemographicList(getDemographics()),
+                state =
-        StatusList(getStatus()),
+                    preferences.getBoolean(MDConstants.getContentRatingSafePrefKey(dexLang), true)
-        SortFilter(sortableList.map { it.first }.toTypedArray()),
+            },
-        TagList(getTags()),
+            ContentRating("Suggestive").apply {
-        TagInclusionMode(),
+                state = preferences.getBoolean(
-        TagExclusionMode(),
+                    MDConstants.getContentRatingSuggestivePrefKey(dexLang),
-    )
+                    true
+                )
+            },
+            ContentRating("Erotica").apply {
+                state = preferences.getBoolean(
+                    MDConstants.getContentRatingEroticaPrefKey(dexLang),
+                    false
+                )
+            },
+            ContentRating("Pornographic").apply {
+                state = preferences.getBoolean(
+                    MDConstants.getContentRatingPornographicPrefKey(dexLang),
+                    false
+                )
+            },
+        )
+
+        return FilterList(
+            OriginalLanguageList(getOriginalLanguage()),
+            ContentRatingList(contentRatings),
+            DemographicList(getDemographics()),
+            StatusList(getStatus()),
+            SortFilter(sortableList.map { it.first }.toTypedArray()),
+            TagList(getTags()),
+            TagInclusionMode(),
+            TagExclusionMode(),
+        )
+    }
 
     private class Demographic(name: String) : Filter.CheckBox(name)
     private class DemographicList(demographics: List<Demographic>) :
@@ -45,13 +73,6 @@ class MangaDexFilters {
     private class ContentRatingList(contentRating: List<ContentRating>) :
         Filter.Group<ContentRating>("Content Rating", contentRating)
 
-    private fun getContentRating() = listOf(
-        ContentRating("Safe"),
-        ContentRating("Suggestive"),
-        ContentRating("Erotica"),
-        ContentRating("Pornographic")
-    )
-
     private class OriginalLanguage(name: String, val isoCode: String) : Filter.CheckBox(name)
     private class OriginalLanguageList(originalLanguage: List<OriginalLanguage>) :
         Filter.Group<OriginalLanguage>("Original language", originalLanguage)
@@ -108,7 +129,7 @@ class MangaDexFilters {
         Tag("c8cbe35b-1b2b-4a3f-9c37-db84c4514856", "Medical"),
         Tag("ac72833b-c4e9-4878-b9db-6c8a4a99444a", "Military"),
         Tag("dd1f77c5-dea9-4e2b-97ae-224af09caf99", "Monster Girls"),
-        Tag("t36fd93ea-e8b8-445e-b836-358f02b3d33d", "Monsters"),
+        Tag("36fd93ea-e8b8-445e-b836-358f02b3d33d", "Monsters"),
         Tag("f42fbf9e-188a-447b-9fdc-f19dc1e4d685", "Music"),
         Tag("ee968100-4191-4968-93d3-f82d72be7e46", "Mystery"),
         Tag("489dd859-9b61-4c37-af75-5b18e88daafc", "Ninja"),

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangaDexHelper.kt

@@ -147,7 +147,12 @@ class MangaDexHelper() {
             // things that will go with the genre tags but aren't actually genre
             val nonGenres = listOf(
                 (attr["publicationDemographic"]?.nullString ?: "").capitalize(Locale.US),
-                ("Content rating: " + (attr["contentRating"].nullString ?: "").capitalize(Locale.US)),
+                (
+                    "Content rating: " + (
+                        attr["contentRating"].nullString
+                            ?: ""
+                        ).capitalize(Locale.US)
+                    ),
                 Locale(attr["originalLanguage"].nullString ?: "").displayLanguage
             )
 
@@ -162,12 +167,13 @@ class MangaDexHelper() {
                 .distinct()
 
             val authorMap = runCatching {
-                val ids = listOf(authorIds, artistIds).flatten().distinct().joinToString("&ids[]=", "?ids[]=")
+                val ids = listOf(authorIds, artistIds).flatten().distinct()
+                    .joinToString("&ids[]=", "?ids[]=")
                 val response = client.newCall(GET("${MDConstants.apiUrl}/author$ids")).execute()
                 val json = JsonParser.parseString(response.body!!.string())
                 json.obj["results"].array.map { result ->
                     result["data"]["attributes"]["id"].string to
-                    cleanString(result["data"]["attributes"]["name"].string)
+                        cleanString(result["data"]["attributes"]["name"].string)
                 }.toMap()
             }.getOrNull() ?: emptyMap()
 
@@ -183,7 +189,7 @@ class MangaDexHelper() {
                     }.map { it?.name } +
                     nonGenres
                 )
-                .filterNotNull()
+                .filter { it.isNullOrBlank().not() }
 
             return SManga.create().apply {
                 url = "/manga/$dexId"