MangaRaw: rewrite parsing, add mirrors and split broken source (#13058)

* MangaRaw: rewrite parsing, add mirrors and split broken source

* Add (Broken) to Manga1001 extension name

* cleanup

* eliminate temporary property and optimize constructor performance

multisrc/overrides/mangaraw/manga1001/src/Manga1001.kt

@@ -1,5 +0,0 @@
-package eu.kanade.tachiyomi.extension.ja.manga1001
-
-import eu.kanade.tachiyomi.multisrc.mangaraw.MangaRaw
-
-class Manga1001 : MangaRaw("Manga1001", "https://manga1001.top")

multisrc/overrides/mangaraw/manga9co/src/Manga9co.kt

@@ -1,41 +0,0 @@
-package eu.kanade.tachiyomi.extension.ja.manga9co
-
-import eu.kanade.tachiyomi.multisrc.mangaraw.MangaRaw
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.SChapter
-import eu.kanade.tachiyomi.source.model.SManga
-import okhttp3.Request
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
-
-class Manga9co : MangaRaw("Manga9co", "https://manga9.co/") {
-
-    override fun popularMangaRequest(page: Int): Request = GET("$baseUrl/top/?page=$page", headers)
-
-    override fun popularMangaSelector() = ".col-sm-4.my-2"
-
-    override fun popularMangaNextPageSelector() = "nextpostslink"
-
-    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/$page", headers)
-
-    override fun latestUpdatesSelector() = popularMangaSelector()
-
-    override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
-
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
-        GET("$baseUrl/?s=$query&page=$page", headers)
-
-    override fun mangaDetailsParse(document: Document) = SManga.create().apply {
-        description = document.select("strong").last().text().trim()
-    }
-
-    override fun chapterListSelector() = ".list-scoll a"
-
-    override fun chapterFromElement(element: Element) = SChapter.create().apply {
-        url = element.attr("href").replace(baseUrl, "")
-        name = element.text().trim()
-    }
-
-    override val imageSelector = ".card-wrap > img"
-}

multisrc/overrides/mangaraw/manga9co/src/MangaRaw.kt

@@ -0,0 +1,64 @@
+package eu.kanade.tachiyomi.extension.ja.manga9co
+
+import android.app.Application
+import androidx.preference.ListPreference
+import androidx.preference.PreferenceScreen
+import eu.kanade.tachiyomi.multisrc.mangaraw.MangaRawTheme
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.ConfigurableSource
+import eu.kanade.tachiyomi.source.model.FilterList
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import org.jsoup.select.Evaluator
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+
+class MangaRaw : MangaRawTheme("MangaRaw", ""), ConfigurableSource {
+    // See https://github.com/tachiyomiorg/tachiyomi-extensions/commits/master/src/ja/mangaraw
+    override val versionId = 2
+    override val id = 4572869149806246133
+
+    override val supportsLatest = true
+    override val baseUrl: String
+    private val selectors: Selectors
+
+    init {
+        val mirrors = MIRRORS
+        val mirrorIndex = Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
+            .getString(MIRROR_PREF, "0")!!.toInt().coerceAtMost(mirrors.size - 1)
+        baseUrl = "https://" + mirrors[mirrorIndex]
+        selectors = getSelectors(mirrorIndex)
+    }
+
+    override fun String.sanitizeTitle() = substringBeforeLast('(').trimEnd()
+
+    override fun popularMangaRequest(page: Int) = GET("$baseUrl/top/?page=$page", headers)
+    override fun popularMangaSelector() = selectors.listMangaSelector
+    override fun popularMangaNextPageSelector() = ".nextpostslink"
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
+        GET("$baseUrl/?s=$query&page=$page", headers)
+
+    override fun Document.getSanitizedDetails(): Element =
+        selectFirst(selectors.detailsSelector).apply {
+            val recommendClass = selectors.recommendClass
+            children().find { it.hasClass(recommendClass) }?.remove()
+            selectFirst(Evaluator.Class("list-scoll")).remove()
+        }
+
+    override fun chapterListSelector() = ".list-scoll a"
+    override fun String.sanitizeChapter() = substring(lastIndexOf('【') + 1, length - 1)
+
+    override fun pageSelector() = Evaluator.Class("card-wrap")
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        ListPreference(screen.context).apply {
+            key = MIRROR_PREF
+            title = "Mirror"
+            summary = "Requires app restart to take effect\nSelected: %s"
+            entries = MIRRORS
+            entryValues = MIRRORS.indices.map { it.toString() }.toTypedArray()
+            setDefaultValue("0")
+        }.let { screen.addPreference(it) }
+    }
+}

multisrc/overrides/mangaraw/manga9co/src/MangaRawConstants.kt

@@ -0,0 +1,23 @@
+package eu.kanade.tachiyomi.extension.ja.manga9co
+
+internal const val MIRROR_PREF = "MIRROR"
+internal val MIRRORS get() = arrayOf("manga9.co", "mangaraw.co", "mangaraw.lol", "mangarawjp.com")
+
+internal fun getSelectors(mirrorIndex: Int) = when (mirrorIndex) {
+    0, 1, 2 -> Selectors(
+        listMangaSelector = ".card",
+        detailsSelector = "div:has(> main)",
+        recommendClass = "container"
+    )
+    else -> Selectors(
+        listMangaSelector = ".post-list:not(.last-hidden) > .item",
+        detailsSelector = "#post-data",
+        recommendClass = "post-list"
+    )
+}
+
+internal class Selectors(
+    val listMangaSelector: String,
+    val detailsSelector: String,
+    val recommendClass: String,
+)

multisrc/overrides/mangaraw/syosetu/src/SyoSetu.kt

@@ -1,9 +1,36 @@
 package eu.kanade.tachiyomi.extension.ja.syosetu
 
-import eu.kanade.tachiyomi.multisrc.mangaraw.MangaRaw
+import eu.kanade.tachiyomi.multisrc.mangaraw.MangaRawTheme
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.model.FilterList
 import okhttp3.Request
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import org.jsoup.select.Evaluator
 
-class SyoSetu : MangaRaw("SyoSetu", "https://syosetu.top") {
+class SyoSetu : MangaRawTheme("SyoSetu", "https://syosetu.top") {
     // syosetu.top doesn't have a popular manga page redirect to latest manga request
     override fun popularMangaRequest(page: Int): Request = latestUpdatesRequest(page)
+
+    override val supportsLatest = false
+
+    override fun String.sanitizeTitle() =
+        substring(0, lastIndexOf("RAW", ignoreCase = true))
+            .trimEnd('(', ' ', ',')
+
+    override fun popularMangaSelector() = "article"
+    override fun popularMangaNextPageSelector() = ".next.page-numbers"
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
+        GET("$baseUrl/page/$page?s=$query")
+
+    override fun Document.getSanitizedDetails(): Element =
+        selectFirst(Evaluator.Tag("article")).selectFirst(Evaluator.Class("content-wrap-inner")).apply {
+            selectFirst(Evaluator.Class("chaplist")).remove()
+        }
+
+    override fun chapterListSelector() = ".chaplist a"
+    override fun String.sanitizeChapter() = substringAfterLast(" - ")
+
+    override fun pageSelector() = Evaluator.Tag("figure")
 }

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangaraw/MangaRawGenerator.kt

@@ -4,16 +4,15 @@ import generator.ThemeSourceData.SingleLang
 import generator.ThemeSourceGenerator
 
 class MangaRawGenerator : ThemeSourceGenerator {
-    override val themeClass = "MangaRaw"
+    override val themeClass = "MangaRawTheme"
 
     override val themePkg = "mangaraw"
 
-    override val baseVersionCode: Int = 1
+    override val baseVersionCode = 2
 
     override val sources = listOf(
-        SingleLang("Manga1001", "https://manga1001.top", "ja", overrideVersionCode = 1),
         SingleLang("SyoSetu", "https://syosetu.top", "ja", overrideVersionCode = 1),
-        SingleLang("Manga9co", "https://manga9.co", "ja", overrideVersionCode = 1),
+        SingleLang("MangaRaw", "https://manga9.co", "ja", pkgName = "manga9co", overrideVersionCode = 1),
     )
 
     companion object {

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangaraw/MangaRawTheme.kt

@@ -0,0 +1,75 @@
+package eu.kanade.tachiyomi.multisrc.mangaraw
+
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.ParsedHttpSource
+import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
+import org.jsoup.select.Evaluator
+
+abstract class MangaRawTheme(
+    override val name: String,
+    override val baseUrl: String,
+    override val lang: String = "ja"
+) : ParsedHttpSource() {
+
+    override fun headersBuilder() = super.headersBuilder().add("Referer", baseUrl)
+
+    protected abstract fun String.sanitizeTitle(): String
+
+    override fun popularMangaFromElement(element: Element) = SManga.create().apply {
+        url = element.selectFirst(Evaluator.Tag("a")).attr("href").removePrefix(baseUrl)
+        title = element.selectFirst(Evaluator.Tag("h3")).text().sanitizeTitle()
+        thumbnail_url = element.selectFirst(Evaluator.Tag("img"))?.absUrl("data-src")
+    }
+
+    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/$page", headers)
+    override fun latestUpdatesSelector() = popularMangaSelector()
+    override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
+    override fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element)
+
+    override fun searchMangaSelector() = popularMangaSelector()
+    override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
+    override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
+
+    /** Other recommended manga must be removed. Make sure the last `<p>` is description. */
+    protected abstract fun Document.getSanitizedDetails(): Element
+
+    override fun mangaDetailsParse(document: Document) = SManga.create().apply {
+        val root = document.getSanitizedDetails()
+        thumbnail_url = root.selectFirst(Evaluator.Tag("img"))?.run {
+            absUrl("data-src").ifEmpty { absUrl("src") }
+        }
+        description = root.select(Evaluator.Tag("p")).lastOrNull { it.text().isNotEmpty() }
+            ?.run {
+                select(Evaluator.Tag("br")).prepend("\\n")
+                text().replace("\\n ", "\n")
+            }
+        genre = root.select(Evaluator.AttributeWithValueContaining("rel", "tag"))
+            .flatMapTo(mutableSetOf()) { element ->
+                val text = element.ownText()
+                if (text.all { it.code < 128 }) return@flatMapTo listOf(text)
+                text.split(' ', '.', '・', '。')
+            }.joinToString()
+    }
+
+    protected abstract fun String.sanitizeChapter(): String
+
+    override fun chapterFromElement(element: Element) = SChapter.create().apply {
+        url = element.attr("href").removePrefix(baseUrl)
+        name = element.text().sanitizeChapter()
+    }
+
+    protected abstract fun pageSelector(): Evaluator
+
+    override fun pageListParse(document: Document): List<Page> {
+        val imgSelector = Evaluator.Tag("img")
+        return document.select(pageSelector()).mapIndexed { index, element ->
+            Page(index, imageUrl = element.selectFirst(imgSelector).attr("data-src"))
+        }
+    }
+
+    override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("Not used.")
+}

src/ja/manga1001/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest package="eu.kanade.tachiyomi.extension" />

src/ja/manga1001/build.gradle

@@ -0,0 +1,11 @@
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+
+ext {
+    extName = 'Manga1001 (Broken)'
+    pkgNameSuffix = 'ja.manga1001'
+    extClass = '.Manga1001'
+    extVersionCode = 2
+}
+
+apply from: "$rootDir/common.gradle"

src/ja/manga1001/src/eu/kanade/tachiyomi/extension/ja/manga1001/Manga1001.kt

@@ -1,4 +1,4 @@
-package eu.kanade.tachiyomi.multisrc.mangaraw
+package eu.kanade.tachiyomi.extension.ja.manga1001
 
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -13,11 +13,11 @@ import org.jsoup.nodes.Element
 
 /**
  * Common parsers of mangaraw sites, follow manga1001.top by default.
+ * FIXME: manga1001.top changed its theme
  */
-abstract class MangaRaw(
+class Manga1001 : ParsedHttpSource() {
-    override val name: String,
+    override val name = "Manga1001"
-    override val baseUrl: String,
+    override val baseUrl = "https://manga1001.top"
-) : ParsedHttpSource() {
 
     protected open val imageSelector = ".wp-block-image > img"