MangaDemon: Update to new site structure (#4857)

* fix

* bruh

src/en/mangademon/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'Manga Demon'
     extClass = '.MangaDemon'
-    extVersionCode = 12
+    extVersionCode = 13
     isNsfw = false
 }
 

src/en/mangademon/src/eu/kanade/tachiyomi/extension/en/mangademon/MangaDemon.kt

@@ -9,129 +9,62 @@ import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
-import eu.kanade.tachiyomi.util.asJsoup
 import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.Interceptor
 import okhttp3.Request
 import okhttp3.Response
-import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import rx.Observable
-import java.io.IOException
+import java.text.ParseException
-import java.net.URLEncoder
 import java.text.SimpleDateFormat
 import java.util.Locale
 
 class MangaDemon : ParsedHttpSource() {
 
+    override val versionId = 2
+
     override val lang = "en"
     override val supportsLatest = true
     override val name = "Manga Demon"
-    override val baseUrl = "https://mgdemon.org"
+    override val baseUrl = "https://demonicscans.org"
 
     override val client = network.cloudflareClient.newBuilder()
         .rateLimit(1)
-        .addInterceptor { chain ->
-            val request = chain.request()
-            val headers = request.headers.newBuilder()
-                .removeAll("Accept-Encoding")
-                .build()
-            chain.proceed(request.newBuilder().headers(headers).build())
-        }
-        .addInterceptor(::dynamicUrlInterceptor)
         .build()
 
-    // Cache suffix
-    private var dynamicUrlSuffix = ""
-    private var dynamicUrlSuffixUpdated: Long = 0
-    private val dynamicUrlSuffixValidity: Long = 10 * 60 // 10 minutes
-
-    private fun dynamicUrlInterceptor(chain: Interceptor.Chain): Response {
-        val request = chain.request()
-        val timeNow = System.currentTimeMillis() / 1000
-
-        // Check if request requires an up-to-date suffix
-        if (request.url.pathSegments[0] == "manga") {
-            // Force update suffix if required
-            if (timeNow - dynamicUrlSuffixUpdated > dynamicUrlSuffixValidity) {
-                client.newCall(GET(baseUrl)).execute()
-                if (timeNow - dynamicUrlSuffixUpdated > dynamicUrlSuffixValidity) {
-                    throw IOException("Failed to update dynamic url suffix")
-                }
-            }
-
-            val newPath = request.url
-                .encodedPath
-                .replaceAfterLast("-", dynamicUrlSuffix)
-
-            val newUrl = request.url.newBuilder()
-                .encodedPath(newPath)
-                .build()
-
-            val newRequest = request.newBuilder()
-                .url(newUrl)
-                .build()
-
-            return chain.proceed(newRequest)
-        }
-
-        // Always update suffix
-        val response = chain.proceed(request)
-        val document = Jsoup.parse(
-            response.peekBody(Long.MAX_VALUE).string(),
-            request.url.toString(),
-        )
-
-        val links = document.select("a[href^='/manga/']")
-
-        // Get the most popular suffix after last `-`
-        val suffix = links.map { it.attr("href").substringAfterLast("-") }
-            .groupBy { it }
-            .maxByOrNull { it.value.size }
-            ?.key
-
-        if (suffix != null) {
-            dynamicUrlSuffix = suffix
-            dynamicUrlSuffixUpdated = timeNow
-        }
-
-        return response
-    }
-
     override fun headersBuilder() = super.headersBuilder()
-        .add("Referer", baseUrl)
+        .add("Referer", "$baseUrl/")
 
-    // latest
+    override fun popularMangaRequest(page: Int): Request {
-    override fun latestUpdatesRequest(page: Int): Request {
+        return GET("$baseUrl/advanced.php?list=$page&status=all&orderby=VIEWS%20DESC", headers)
-        return GET("$baseUrl/updates.php?list=$page", headers)
     }
 
-    override fun latestUpdatesNextPageSelector() = ".pagination a:contains(Next)"
+    override fun popularMangaNextPageSelector() = "div.pagination > ul > a > li:contains(Next)"
 
-    override fun latestUpdatesSelector() = "div.leftside"
+    override fun popularMangaSelector() = "div#advanced-content > div.advanced-element"
+
+    override fun popularMangaFromElement(element: Element) = SManga.create().apply {
+        setUrlWithoutDomain(element.selectFirst("a")!!.attr("abs:href"))
+        title = element.selectFirst("h1")!!.ownText()
+        thumbnail_url = element.selectFirst("img")?.attr("abs:src")
+    }
+
+    override fun latestUpdatesRequest(page: Int): Request {
+        return GET("$baseUrl/lastupdates.php?list=$page", headers)
+    }
+
+    override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
+
+    override fun latestUpdatesSelector() = "div#updates-container > div.updates-element"
 
     override fun latestUpdatesFromElement(element: Element) = SManga.create().apply {
-        element.select("a").apply {
+        with(element.selectFirst("div.updates-element-info")!!) {
-            title = attr("title")
+            setUrlWithoutDomain(selectFirst("a")!!.attr("abs:href"))
-            val url = URLEncoder.encode(attr("href"), "UTF-8")
+            title = selectFirst("a")!!.ownText()
-            setUrlWithoutDomain(url)
         }
-        thumbnail_url = element.select("img").attr("abs:src")
+        thumbnail_url = element.selectFirst("div.thumb img")!!.attr("abs:src")
     }
 
-    // Popular
-    override fun popularMangaRequest(page: Int): Request {
-        return GET("$baseUrl/browse.php?list=$page", headers)
-    }
-
-    override fun popularMangaNextPageSelector() = latestUpdatesNextPageSelector()
-
-    override fun popularMangaSelector() = latestUpdatesSelector()
-
-    override fun popularMangaFromElement(element: Element) = latestUpdatesFromElement(element)
-
-    // Search
     override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
         return if (query.isNotEmpty()) {
             super.fetchSearchManga(page, query, filters)
@@ -142,8 +75,25 @@ class MangaDemon : ParsedHttpSource() {
         }
     }
 
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        val url = "$baseUrl/search.php".toHttpUrl().newBuilder()
+            .addQueryParameter("manga", query)
+            .build()
+        return GET(url, headers)
+    }
+
+    override fun searchMangaSelector() = "body > a[href]"
+
+    override fun searchMangaNextPageSelector() = null
+
+    override fun searchMangaFromElement(element: Element) = SManga.create().apply {
+        setUrlWithoutDomain(element.attr("abs:href"))
+        title = element.selectFirst("div.seach-right > div")!!.ownText()
+        thumbnail_url = element.selectFirst("img")!!.attr("abs:src")
+    }
+
     private fun filterSearchRequest(page: Int, filters: FilterList): Request {
-        val url = "$baseUrl/browse.php".toHttpUrl().newBuilder().apply {
+        val url = "$baseUrl/advanced.php".toHttpUrl().newBuilder().apply {
             addQueryParameter("list", page.toString())
             filters.forEach { filter ->
                 when (filter) {
@@ -170,36 +120,14 @@ class MangaDemon : ParsedHttpSource() {
 
     override fun getFilterList() = getFilters()
 
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+    override fun mangaDetailsParse(document: Document) = SManga.create().apply {
-        val url = "$baseUrl/search.php".toHttpUrl().newBuilder()
+        with(document.selectFirst("div#manga-info-container")!!) {
-            .addQueryParameter("manga", query)
+            title = selectFirst("h1.big-fat-titles")!!.ownText()
-            .build()
+            thumbnail_url = selectFirst("div#manga-page img")!!.attr("abs:src")
-        return GET(url, headers)
+            genre = select("div.genres-list > li").joinToString { it.text() }
-    }
+            description = selectFirst("div#manga-info-rightColumn > div > div.white-font")!!.text()
-
+            author = select("div#manga-info-stats > div:has(> li:eq(0):contains(Author)) > li:eq(1)").text()
-    override fun searchMangaSelector() = "a.boxsizing"
+            status = parseStatus(select("div#manga-info-stats > div:has(> li:eq(0):contains(Status)) > li:eq(1)").text())
-
-    override fun searchMangaFromElement(element: Element) = SManga.create().apply {
-        title = element.text()
-        val url = URLEncoder.encode(element.attr("href"), "UTF-8")
-        setUrlWithoutDomain(url)
-        val urlSorter = title.replace(":", "%20")
-        thumbnail_url = ("https://readermc.org/images/thumbnails/$urlSorter.webp")
-    }
-
-    override fun searchMangaNextPageSelector() = null
-
-    // Manga details
-    override fun mangaDetailsParse(document: Document): SManga {
-        val infoElement = document.select("article")
-
-        return SManga.create().apply {
-            title = infoElement.select("h1.novel-title").text()
-            author = infoElement.select("div.author > [itemprop=author]").text()
-            status = parseStatus(infoElement.select("span:has(small:containsOwn(Status))").text())
-            genre = infoElement.select("a.property-item").joinToString { it.text() }
-            description = infoElement.select("p.description").text()
-            thumbnail_url = infoElement.select("img#thumbonail").attr("src")
         }
     }
 
@@ -210,68 +138,31 @@ class MangaDemon : ParsedHttpSource() {
         else -> SManga.UNKNOWN
     }
 
-    override fun chapterListSelector() = "ul.chapter-list li"
+    override fun chapterListSelector() = "div#chapters-list a.chplinks"
 
-    // Get Chapters
+    override fun chapterFromElement(element: Element) = SChapter.create().apply {
-    override fun chapterFromElement(element: Element): SChapter {
+        setUrlWithoutDomain(element.attr("abs:href"))
-        return SChapter.create().apply {
+        name = element.ownText()
-            element.select("a").let { urlElement ->
+        date_upload = parseDate(element.selectFirst("span")?.ownText())
-                val url = URLEncoder.encode(urlElement.attr("href"), "UTF-8")
-                setUrlWithoutDomain(url)
-                name = element.select("strong.chapter-title").text()
-            }
-            val date = element.select("time.chapter-update").text()
-            date_upload = parseDate(date)
-        }
     }
 
-    private fun parseDate(dateStr: String): Long {
+    private fun parseDate(dateStr: String?): Long {
-        return runCatching { DATE_FORMATTER.parse(dateStr)?.time }
+        return try {
-            .getOrNull() ?: 0L
+            dateStr?.let { DATE_FORMATTER.parse(it)?.time } ?: 0
-    }
+        } catch (_: ParseException) {
-
+            0L
-    companion object {
-        private val DATE_FORMATTER by lazy {
-            SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH)
         }
-
-        private val loadMoreEndpointRegex by lazy { Regex("""GET[^/]+([^=]+)""") }
     }
 
     override fun pageListParse(document: Document): List<Page> {
-        val baseImages = document.select("img.imgholder")
+        return document.select("img.imgholder").mapIndexed { i, element ->
-            .map { it.attr("abs:src") }
+            Page(i, "", element.attr("abs:src"))
-            .toMutableList()
-
-        baseImages.addAll(loadMoreImages(document))
-
-        return baseImages.mapIndexed { i, img -> Page(i, "", img) }
-    }
-
-    private fun loadMoreImages(document: Document): List<String> {
-        val buttonHtml = document.selectFirst("img.imgholder ~ button")
-            ?.attr("onclick")?.replace("\"", "\'")
-            ?: return emptyList()
-
-        val id = buttonHtml.substringAfter("\'").substringBefore("\'").trim()
-        val funcName = buttonHtml.substringBefore("(").trim()
-
-        val endpoint = document.selectFirst("script:containsData($funcName)")
-            ?.data()
-            ?.let { loadMoreEndpointRegex.find(it)?.groupValues?.get(1) }
-            ?: return emptyList()
-
-        val response = client.newCall(GET("$baseUrl$endpoint=$id", headers)).execute()
-
-        if (!response.isSuccessful) {
-            response.close()
-            return emptyList()
         }
-
-        return response.use { it.asJsoup() }
-            .select("img")
-            .map { it.attr("abs:src") }
     }
 
     override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException()
+
+    companion object {
+        private val DATE_FORMATTER = SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH)
+    }
 }

src/en/mangademon/src/eu/kanade/tachiyomi/extension/en/mangademon/MangaDemonFilters.kt

@@ -45,7 +45,6 @@ class GenreFilter : Filter.Group<CheckBoxFilter>(
 
     companion object {
         private val genres = listOf(
-            Pair("All", "all"),
             Pair("Action", "1"),
             Pair("Adventure", "2"),
             Pair("Comedy", "3"),