Madara: Rework search filters and some improvements (#1361)

Signed-off-by: Rama Bondan Prakoso <ramanarubp@gmail.com>

src/all/madara/build.gradle

@@ -5,7 +5,7 @@ ext {
     appName = 'Tachiyomi: Madara'
     pkgNameSuffix = "all.madara"
     extClass = '.MadaraFactory'
-    extVersionCode = 7
+    extVersionCode = 8
     libVersion = '1.2'
 }
 

src/all/madara/src/eu/kanade/tachiyomi/extension/all/madara/Madara.kt

@@ -1,18 +1,18 @@
 package eu.kanade.tachiyomi.extension.all.madara
 
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.network.asObservable
 import eu.kanade.tachiyomi.source.model.*
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import okhttp3.*
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
 import java.text.ParseException
 import java.text.SimpleDateFormat
 import java.util.*
-import eu.kanade.tachiyomi.source.model.FilterList
+import java.util.concurrent.TimeUnit
-import okhttp3.CacheControl
+import org.jsoup.nodes.Document
-import okhttp3.FormBody
+import org.jsoup.nodes.Element
-import okhttp3.Request
+import rx.Observable
-import eu.kanade.tachiyomi.network.POST
 
 open class Madara(
         override val name: String,
@@ -21,6 +21,11 @@ open class Madara(
         private val dateFormat: SimpleDateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US)
 ) : ParsedHttpSource() {
 
+    override val client: OkHttpClient = network.cloudflareClient.newBuilder()
+        .connectTimeout(10, TimeUnit.SECONDS)
+        .readTimeout(30, TimeUnit.SECONDS)
+        .build()
+
     override val supportsLatest = true
 
     // Popular Manga
@@ -62,11 +67,11 @@ open class Madara(
         return POST("$baseUrl/wp-admin/admin-ajax.php", headers, form.build(), CacheControl.FORCE_NETWORK)
     }
 
-    override fun popularMangaNextPageSelector(): String? = searchMangaNextPageSelector()
+    override fun popularMangaNextPageSelector(): String? = "body:not(:has(.no-posts))"
 
     // Latest Updates
 
-    override fun latestUpdatesSelector() = "div.page-item-detail"
+    override fun latestUpdatesSelector() = popularMangaSelector()
 
     override fun latestUpdatesFromElement(element: Element): SManga {
         // Even if it's different from the popular manga's list, the relevant classes are the same
@@ -91,7 +96,7 @@ open class Madara(
         return POST("$baseUrl/wp-admin/admin-ajax.php", headers, form.build(), CacheControl.FORCE_NETWORK)
     }
 
-    override fun latestUpdatesNextPageSelector(): String? = searchMangaNextPageSelector()
+    override fun latestUpdatesNextPageSelector(): String? = popularMangaNextPageSelector()
 
     override fun latestUpdatesParse(response: Response): MangasPage {
         val mp = super.latestUpdatesParse(response)
@@ -99,24 +104,98 @@ open class Madara(
         return MangasPage(mangas, mp.hasNextPage)
     }
 
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        return client.newCall(searchMangaRequest(page, query, filters))
+            .asObservable().doOnNext { response ->
+                if(!response.isSuccessful) {
+                    response.close()
+                    // Error message for exceeding last page
+                    if (response.code() == 404)
+                        error("Already on the Last Page!")
+                    else throw Exception("HTTP error ${response.code()}")
+                }
+            }
+            .map { response ->
+                searchMangaParse(response)
+            }
+    }
+
     // Search Manga
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val form = FormBody.Builder().apply {
+        val url = HttpUrl.parse("$baseUrl/page/$page/")!!.newBuilder()
-            add("action", "madara_load_more")
+        url.addQueryParameter("s", query)
-            add("page", (page-1).toString())
+        url.addQueryParameter("post_type", "wp-manga")
-            add("template", "madara-core/content/content-search")
+        filters.forEach { filter ->
-            add("vars[s]", query)
+            when (filter) {
-            add("vars[orderby]", "")
+                is AuthorFilter -> {
-            add("vars[paged]", (page-1).toString())
+                    if(filter.state.isNotBlank()) {
-            add("vars[template]", "search")
+                        url.addQueryParameter("author", filter.state)
-            add("vars[post_type]", "wp-manga")
+                    }
-            add("vars[post_status]", "publish")
+                }
-            add("vars[manga_archives_item_layout]", "default")
+                is ArtistFilter -> {
+                    if(filter.state.isNotBlank()) {
+                        url.addQueryParameter("artist", filter.state)
+                    }
+                }
+                is YearFilter -> {
+                    if(filter.state.isNotBlank()) {
+                        url.addQueryParameter("release", filter.state)
+                    }
+                }
+                is StatusFilter -> {
+                    filter.state.forEach {
+                        if (it.state) {
+                            url.addQueryParameter("status[]", it.id)
+                        }
+                    }
+                }
+                is OrderByFilter -> {
+                    if(filter.state != 0) {
+                        url.addQueryParameter("m_orderby", filter.toUriPart())
+                    }
+                }
+            }
         }
-        return POST("$baseUrl/wp-admin/admin-ajax.php", headers, form.build(), CacheControl.FORCE_NETWORK)
+        return GET(url.build().toString(), headers)
     }
 
+    private class AuthorFilter : Filter.Text("Author")
+    private class ArtistFilter : Filter.Text("Artist")
+    private class YearFilter : Filter.Text("Year of Released")
+    private class StatusFilter(status: List<Tag>) : Filter.Group<Tag>("Status", status)
+    private class OrderByFilter : UriPartFilter("Order By", arrayOf(
+            Pair("<select>", ""),
+            Pair("Latest", "latest"),
+            Pair("A-Z", "alphabet"),
+            Pair("Rating", "rating"),
+            Pair("Trending", "trending"),
+            Pair("Most Views", "views"),
+            Pair("New", "new-manga")
+    ))
+
+    override fun getFilterList() = FilterList(
+            AuthorFilter(),
+            ArtistFilter(),
+            YearFilter(),
+            StatusFilter(getStatusList()),
+            OrderByFilter()
+    )
+
+    private fun getStatusList() = listOf(
+            Tag("end" , "Completed"),
+            Tag("on-going" , "Ongoing"),
+            Tag("canceled" , "Canceled"),
+            Tag("on-hold" , "On Hold")
+    )
+
+    private open class UriPartFilter(displayName: String, val vals: Array<Pair<String, String>>) :
+        Filter.Select<String>(displayName, vals.map { it.first }.toTypedArray()) {
+        fun toUriPart() = vals[state].second
+    }
+
+    open class Tag(val id: String, name: String) : Filter.CheckBox(name)
+
     override fun searchMangaSelector() = "div.c-tabs-item__content"
 
     override fun searchMangaFromElement(element: Element): SManga {
@@ -135,7 +214,7 @@ open class Madara(
         return manga
     }
 
-    override fun searchMangaNextPageSelector() = "body:not(:has(.no-posts))"
+    override fun searchMangaNextPageSelector() = "div.nav-previous"
 
     // Manga Details Parse
 
@@ -173,21 +252,27 @@ open class Madara(
                     else -> SManga.UNKNOWN
                 }
             }
+            val genres = mutableListOf<String>()
+            select("div.genres-content a").forEach { element ->
+                val genre = element.text()
+                genres.add(genre)
+            }
+            manga.genre = genres.joinToString(", ")
         }
 
         return manga
     }
 
-    override fun chapterListSelector() = "div.listing-chapters_wrap li.wp-manga-chapter"
+    override fun chapterListSelector() = "li.wp-manga-chapter"
 
     override fun chapterFromElement(element: Element): SChapter {
         val chapter = SChapter.create()
 
         with(element) {
             select("a").first()?.let { urlElement ->
-                chapter.setUrlWithoutDomain(urlElement.attr("abs:href").let {
+                chapter.url = urlElement.attr("abs:href").let {
                     it.substringBefore("?style=paged") + if(!it.endsWith("?style=list")) "?style=list" else ""
-                })
+                }
                 chapter.name = urlElement.text()
             }
 
@@ -195,7 +280,7 @@ open class Madara(
             if (select("img").attr("alt").isNotBlank()) {
                 chapter.date_upload = parseRelativeDate(select("img").attr("alt")) ?: 0
             } else {
-            // For a chapter date that's text
+                // For a chapter date that's text
                 select("span.chapter-release-date i").first()?.let {
                     chapter.date_upload = parseChapterDate(it.text()) ?: 0
                 }
@@ -257,6 +342,13 @@ open class Madara(
         }
     }
 
+    override fun pageListRequest(chapter: SChapter): Request {
+        if (chapter.url.startsWith("http")) {
+            return GET(chapter.url, headers)
+        }
+        return super.pageListRequest(chapter)
+    }
+
     open val pageListParseSelector = "div.page-break"
 
     override fun pageListParse(document: Document): List<Page> {

src/all/madara/src/eu/kanade/tachiyomi/extension/all/madara/MadaraFactory.kt

@@ -1,16 +1,16 @@
 package eu.kanade.tachiyomi.extension.all.madara
 
 
+import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.SourceFactory
-import java.text.SimpleDateFormat
+import eu.kanade.tachiyomi.source.model.FilterList
-import java.util.*
+import eu.kanade.tachiyomi.source.model.SChapter
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.*
 import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.Request
+import java.text.SimpleDateFormat
+import java.util.Locale
 import okhttp3.Response
-import org.jsoup.nodes.Element
+import okhttp3.Request
 
 class MadaraFactory : SourceFactory {
     override fun createSources(): List<Source> = listOf(
@@ -45,14 +45,22 @@ class MadaraFactory : SourceFactory {
     )
 }
 
-class Mangasushi : Madara("Mangasushi", "https://mangasushi.net", "en")
+class Mangasushi : Madara("Mangasushi", "https://mangasushi.net", "en") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
 class NinjaScans : Madara("NinjaScans", "https://ninjascans.com", "en")
-class ReadManhua : Madara("ReadManhua", "https://readmanhua.net", "en", dateFormat = SimpleDateFormat("dd MMM yy", Locale.US))
+class ReadManhua : Madara("ReadManhua", "https://readmanhua.net", "en",
+    dateFormat = SimpleDateFormat("dd MMM yy", Locale.US)) {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
 class ZeroScans : Madara("ZeroScans", "https://zeroscans.com", "en")
 class IsekaiScanCom : Madara("IsekaiScan.com", "http://isekaiscan.com/", "en")
 class HappyTeaScans : Madara("Happy Tea Scans", "https://happyteascans.com/", "en")
-class JustForFun : Madara("Just For Fun", "https://just-for-fun.ru/", "ru", dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US))
+class JustForFun : Madara("Just For Fun", "https://just-for-fun.ru/", "ru",
-class AoCTranslations : Madara("Agent of Change Translations", "https://aoc.moe/", "en") {
+    dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US)) {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
+class AoCTranslations : Madara("Agent of Change Translations", "https://aoc.moe/", "en"){
     override fun chapterListParse(response: Response): List<SChapter> {
         val chapters = mutableListOf<SChapter>()
         val document = response.asJsoup()
@@ -60,13 +68,12 @@ class AoCTranslations : Madara("Agent of Change Translations", "https://aoc.moe/
         // For when it's a normal chapter list
         if (document.select(chapterListSelector()).hasText()) {
             document.select(chapterListSelector())
-                .filter { it.select("a").attr("href").contains(baseUrl)}
                 .map { chapters.add(chapterFromElement(it)) }
         } else {
-        // For their "fancy" volume/chapter lists
+            // For their "fancy" volume/chapter lists
             document.select("div.wpb_wrapper:contains(volume) a")
                 .filter { it.attr("href").contains(baseUrl) && !it.attr("href").contains("imgur")}
-                .map {
+                .map { it ->
                     val chapter = SChapter.create()
                     if (it.attr("href").contains("volume")) {
                         val volume = it.attr("href").substringAfter("volume-").substringBefore("/")
@@ -84,79 +91,66 @@ class AoCTranslations : Madara("Agent of Change Translations", "https://aoc.moe/
         return chapters.reversed()
     }
 }
-class Kanjiku : Madara("Kanjiku", "https://kanjiku.net/", "de", dateFormat = SimpleDateFormat("dd. MMM yyyy", Locale.GERMAN))
+class Kanjiku : Madara("Kanjiku", "https://kanjiku.net/", "de",
-class KomikGo : Madara("KomikGo", "https://komikgo.com/", "id")
+    dateFormat = SimpleDateFormat("dd. MMM yyyy", Locale.GERMAN))
+class KomikGo : Madara("KomikGo", "https://komikgo.com/", "id") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
 class LuxyScans : Madara("Luxy Scans", "https://luxyscans.com/", "en")
 class TritiniaScans : Madara("Tritinia Scans", "http://tritiniascans.ml/", "en") {
     override fun popularMangaRequest(page: Int): Request = GET("$baseUrl/?m_orderby=views", headers)
     override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/?m_orderby=latest", headers)
     override fun latestUpdatesNextPageSelector(): String? = null
     override fun popularMangaNextPageSelector(): String? = null
-
-    // Source's search seems broken (doesn't return results)
-    private var searchQuery = ""
-
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        searchQuery = query.toLowerCase()
-        return popularMangaRequest(1)
-    }
-
-    override fun searchMangaParse(response: Response): MangasPage {
-        val searchMatches = mutableListOf<SManga>()
-        val document = response.asJsoup()
-
-        document.select(searchMangaSelector())
-            .filter { it.text().toLowerCase().contains(searchQuery) }
-            .map { searchMatches.add(searchMangaFromElement(it)) }
-
-        return MangasPage(searchMatches, false)
-    }
-
-    override fun searchMangaSelector() = popularMangaSelector()
-
-    override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
-
-    override fun searchMangaNextPageSelector() = "Not needed"
-
 }
-class TsubakiNoScan : Madara("Tsubaki No Scan", "https://tsubakinoscan.com/", "fr", dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US))
+
-class YokaiJump : Madara("Yokai Jump", "https://yokaijump.fr/", "fr", dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US))
+class TsubakiNoScan : Madara("Tsubaki No Scan", "https://tsubakinoscan.com/",
-class ZManga : Madara("ZManga", "https://zmanga.org/", "es")
+    "fr", dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US))
-class MangazukiMe : Madara("Mangazuki.me", "https://mangazuki.me/", "en"){
+class YokaiJump : Madara("Yokai Jump", "https://yokaijump.fr/", "fr",
-    override fun chapterListParse(response: Response): List<SChapter> {
+    dateFormat = SimpleDateFormat("dd/MM/yy", Locale.US)) {
-        val chapters = mutableListOf<SChapter>()
+        override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
-        response.asJsoup().select(chapterListSelector())
-            .filter { it.select("a").attr("href").contains(baseUrl) }
-            .map { chapters.add(chapterFromElement(it)) }
-        return chapters
-    }
 }
+class ZManga : Madara("ZManga", "https://zmanga.org/", "es") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
+class MangazukiMe : Madara("Mangazuki.me", "https://mangazuki.me/", "en")
 class MangazukiOnline : Madara("Mangazuki.online", "https://www.mangazuki.online/", "en") {
-    override fun chapterListSelector() = "li.wp-manga-chapter:has(a)"
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
-    override fun chapterListParse(response: Response): List<SChapter> {
+}
-        val chapters = mutableListOf<SChapter>()
+class MangazukiClubJP : Madara("Mangazuki.club", "https://mangazuki.club/", "ja") {
-        response.asJsoup().select(chapterListSelector())
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
-            .filter { it.select("a").attr("href").contains(baseUrl) }
+}
-            .map { chapters.add(chapterFromElement(it)) }
+class MangazukiClubKO : Madara("Mangazuki.club", "https://mangazuki.club/", "ko") {
-        return chapters
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
-    }
 }
-class MangazukiClubJP : Madara("Mangazuki.club", "https://mangazuki.club/", "ja")
-class MangazukiClubKO : Madara("Mangazuki.club", "https://mangazuki.club/", "ko")
 class FirstKissManga : Madara("1st Kiss", "https://1stkissmanga.com/", "en") {
     override val pageListParseSelector = "div.reading-content img"
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
+class Mangalike : Madara("Mangalike", "https://mangalike.net/", "en") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
+class MangaSY : Madara("Manga SY", "https://www.mangasy.com/", "en") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
 }
-class Mangalike : Madara("Mangalike", "https://mangalike.net/", "en")
-class MangaSY : Madara("Manga SY", "https://www.mangasy.com/", "en")
 class ManwhaClub : Madara("Manwha Club", "https://manhwa.club/", "en")
 class WuxiaWorld : Madara("WuxiaWorld", "https://wuxiaworld.site/", "en") {
     override fun popularMangaRequest(page: Int): Request = GET("$baseUrl/tag/webcomics/page/$page/?m_orderby=views", headers)
     override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/tag/webcomics/page/$page/?m_orderby=latest", headers)
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = GET("$baseUrl/?s=$query&post_type=wp-manga&genre[]=webcomics", headers)
+    override fun popularMangaSelector() = "div.page-item-detail"
-    override fun latestUpdatesNextPageSelector(): String? = "div.nav-previous"
+    override fun latestUpdatesSelector() = "div.page-item-detail"
-    override fun popularMangaNextPageSelector(): String? = "div.nav-previous"
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = throw Exception("Not used")
+    override fun getFilterList() = FilterList()
+}
+class YoManga : Madara("Yo Manga", "https://yomanga.info/", "en") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
+class ManyToon : Madara("ManyToon", "https://manytoon.com/", "en") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
+class ChibiManga : Madara("Chibi Manga", "http://www.cmreader.info/", "en") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
+}
+class ZinManga : Madara("Zin Translator", "https://zinmanga.com/", "en") {
+    override fun searchMangaNextPageSelector() = "nav.navigation-ajax"
 }
-class YoManga : Madara("Yo Manga", "https://yomanga.info/", "en")
-class ManyToon : Madara("ManyToon", "https://manytoon.com/", "en")
-class ChibiManga : Madara("Chibi Manga", "http://www.cmreader.info/", "en")
-class ZinManga : Madara("Zin Translator", "https://zinmanga.com/", "en")