MangaBox - refactor search, source list changes (#3264)

src/all/mangabox/build.gradle

@@ -5,7 +5,7 @@ ext {
     appName = 'Tachiyomi: MangaBox (Mangakakalot and others)'
     pkgNameSuffix = 'all.mangabox'
     extClass = '.MangaBoxFactory'
-    extVersionCode = 21
+    extVersionCode = 22
     libVersion = '1.2'
 }
 

src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBox.kt

@@ -59,9 +59,9 @@ abstract class MangaBox(
         return GET("$baseUrl/$latestUrlPath$page", headers)
     }
 
-    override fun popularMangaFromElement(element: Element): SManga {
+    protected fun mangaFromElement(element: Element, urlSelector: String = "h3 a"): SManga {
         return SManga.create().apply {
-            element.select("h3 a").first().let {
+            element.select(urlSelector).first().let {
                 url = it.attr("abs:href").substringAfter(baseUrl) // intentionally not using setUrlWithoutDomain
                 title = it.text()
             }
@@ -69,29 +69,46 @@ abstract class MangaBox(
         }
     }
 
-    override fun latestUpdatesFromElement(element: Element): SManga = popularMangaFromElement(element)
+    override fun popularMangaFromElement(element: Element): SManga = mangaFromElement(element)
+
+    override fun latestUpdatesFromElement(element: Element): SManga = mangaFromElement(element)
 
     override fun popularMangaNextPageSelector() = "div.group_page, div.group-page a:not([href]) + a:not(:contains(Last))"
 
     override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        return if (query.isNotBlank()) {
+        return if (query.isNotBlank() && getAdvancedGenreFilters().isEmpty()) {
             GET("$baseUrl/$simpleQueryPath${normalizeSearchQuery(query)}?page=$page", headers)
         } else {
-            val url = HttpUrl.parse("$baseUrl/manga_list")!!.newBuilder()
+            val url = HttpUrl.parse(baseUrl)!!.newBuilder()
-            url.addQueryParameter("page", page.toString())
+            if (getAdvancedGenreFilters().isNotEmpty()) {
-
+                url.addPathSegment("advanced_search")
-            filters.forEach { filter ->
+                url.addQueryParameter("page", page.toString())
-                when (filter) {
+                url.addQueryParameter("keyw", normalizeSearchQuery(query))
-                    is SortFilter -> {
+                var genreInclude = ""
-                        url.addQueryParameter("type", filter.toUriPart())
+                var genreExclude = ""
+                filters.forEach { filter ->
+                    when (filter) {
+                        is KeywordFilter -> filter.toUriPart()?.let { url.addQueryParameter("keyt", it) }
+                        is SortFilter -> url.addQueryParameter("orby", filter.toUriPart())
+                        is StatusFilter -> url.addQueryParameter("sts", filter.toUriPart())
+                        is AdvGenreFilter -> {
+                            filter.state.forEach { if (it.isIncluded()) genreInclude += "_${it.id}" }
+                            filter.state.forEach { if (it.isExcluded()) genreExclude += "_${it.id}" }
+                        }
                     }
-                    is StatusFilter -> {
+                }
-                        url.addQueryParameter("state", filter.toUriPart())
+                url.addQueryParameter("g_i", genreInclude)
-                    }
+                url.addQueryParameter("g_e", genreExclude)
-                    is GenreFilter -> {
+            } else {
-                        url.addQueryParameter("category", filter.toUriPart())
+                url.addPathSegment("manga_list")
+                url.addQueryParameter("page", page.toString())
+                filters.forEach { filter ->
+                    when (filter) {
+                        is SortFilter -> url.addQueryParameter("type", filter.toUriPart())
+                        is StatusFilter -> url.addQueryParameter("state", filter.toUriPart())
+                        is GenreFilter -> url.addQueryParameter("category", filter.toUriPart())
                     }
                 }
             }
@@ -101,7 +118,7 @@ abstract class MangaBox(
 
     override fun searchMangaSelector() = ".panel_story_list .story_item"
 
-    override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
+    override fun searchMangaFromElement(element: Element) = mangaFromElement(element)
 
     override fun searchMangaNextPageSelector() = "a.page_select + a:not(.page_last), a.page-select + a:not(.page-last)"
 
@@ -165,6 +182,13 @@ abstract class MangaBox(
 
     override fun chapterListSelector() = "div.chapter-list div.row, ul.row-content-chapter li"
 
+    protected open val alternateChapterDateSelector = String()
+
+    private fun Element.selectDateFromElement(): Element {
+        val defaultChapterDateSelector = "span"
+        return this.select(defaultChapterDateSelector).lastOrNull() ?: this.select(alternateChapterDateSelector).last()
+    }
+
     override fun chapterFromElement(element: Element): SChapter {
         return SChapter.create().apply {
             element.select("a").let {
@@ -172,11 +196,11 @@ abstract class MangaBox(
                 name = it.text()
                 scanlator = HttpUrl.parse(it.attr("abs:href"))!!.host() // show where chapters are actually from
             }
-            date_upload = parseChapterDate(element.select("span").last().text(), element.ownerDocument().location()) ?: 0
+            date_upload = parseChapterDate(element.selectDateFromElement().text(), scanlator!!) ?: 0
         }
     }
 
-    private fun parseChapterDate(date: String, url: String): Long? {
+    private fun parseChapterDate(date: String, host: String): Long? {
         return if ("ago" in date) {
             val value = date.split(' ')[0].toIntOrNull()
             val cal = Calendar.getInstance()
@@ -188,7 +212,7 @@ abstract class MangaBox(
             }?.timeInMillis
         } else {
             try {
-                if (url.contains("manganelo")) {
+                if (host.contains("manganelo", ignoreCase = true)) {
                     // Nelo's date format
                     SimpleDateFormat("MMM dd,yy", Locale.ENGLISH).parse(date)
                 } else {
@@ -245,32 +269,54 @@ abstract class MangaBox(
         return str
     }
 
-    override fun getFilterList() = FilterList(
+    override fun getFilterList() = if (getAdvancedGenreFilters().isNotEmpty()) {
+        FilterList(
+            KeywordFilter(getKeywordFilters()),
+            SortFilter(getSortFilters()),
+            StatusFilter(getStatusFilters()),
+            AdvGenreFilter(getAdvancedGenreFilters())
+        )
+    } else {
+        FilterList(
             Filter.Header("NOTE: Ignored if using text search!"),
             Filter.Separator(),
-            SortFilter(),
+            SortFilter(getSortFilters()),
-            StatusFilter(getStatusPairs()),
+            StatusFilter(getStatusFilters()),
-            GenreFilter(getGenrePairs())
+            GenreFilter(getGenreFilters())
+        )
+    }
+
+    private class KeywordFilter(vals: Array<Pair<String?, String>>) : UriPartFilter("Keyword search ", vals)
+    private class SortFilter(vals: Array<Pair<String?, String>>) : UriPartFilter("Order by", vals)
+    private class StatusFilter(vals: Array<Pair<String?, String>>) : UriPartFilter("Status", vals)
+    private class GenreFilter(vals: Array<Pair<String?, String>>) : UriPartFilter("Category", vals)
+
+    // For advanced search, specifically tri-state genres
+    private class AdvGenreFilter(vals: List<AdvGenre>) : Filter.Group<AdvGenre>("Category", vals)
+    class AdvGenre(val id: String?, name: String) : Filter.TriState(name)
+
+    // keyt query parameter
+    private fun getKeywordFilters(): Array<Pair<String?, String>> = arrayOf(
+        Pair(null, "Everything"),
+        Pair("title", "Title"),
+        Pair("alternative", "Alt title"),
+        Pair("author", "Author")
     )
 
-    private class SortFilter : UriPartFilter("Sort", arrayOf(
+    private fun getSortFilters(): Array<Pair<String?, String>> = arrayOf(
-            Pair("latest", "Latest"),
+        Pair("latest", "Latest"),
-            Pair("newest", "Newest"),
+        Pair("newest", "Newest"),
-            Pair("topview", "Top read")
+        Pair("topview", "Top read")
-    ))
+    )
 
-    open class StatusFilter(statusPairs: Array<Pair<String, String>>) : UriPartFilter("Status", statusPairs)
+    open fun getStatusFilters(): Array<Pair<String?, String>> = arrayOf(
-
-    open fun getStatusPairs() = arrayOf(
         Pair("all", "ALL"),
         Pair("completed", "Completed"),
         Pair("ongoing", "Ongoing"),
         Pair("drop", "Dropped")
     )
 
-    private class GenreFilter(genrePairs: Array<Pair<String, String>>) : UriPartFilter("Category", genrePairs)
+    open fun getGenreFilters(): Array<Pair<String?, String>> = arrayOf(
-
-    open fun getGenrePairs(): Array<Pair<String, String>> = arrayOf(
         Pair("all", "ALL"),
         Pair("2", "Action"),
         Pair("3", "Adult"),
@@ -314,7 +360,10 @@ abstract class MangaBox(
         Pair("42", "Yuri")
     )
 
-    open class UriPartFilter(displayName: String, private val vals: Array<Pair<String, String>>) :
+    // To be overridden if using tri-state genres
+    protected open fun getAdvancedGenreFilters(): List<AdvGenre> = emptyList()
+
+    open class UriPartFilter(displayName: String, private val vals: Array<Pair<String?, String>>) :
         Filter.Select<String>(displayName, vals.map { it.second }.toTypedArray()) {
         fun toUriPart() = vals[state].first
     }

src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBoxFactory.kt

@@ -3,117 +3,41 @@ package eu.kanade.tachiyomi.extension.all.mangabox
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.Source
 import eu.kanade.tachiyomi.source.SourceFactory
-import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.util.asJsoup
 import java.text.SimpleDateFormat
 import java.util.Locale
 import okhttp3.Headers
 import okhttp3.Request
 import okhttp3.Response
+import org.jsoup.nodes.Element
 
 class MangaBoxFactory : SourceFactory {
     override fun createSources(): List<Source> = listOf(
         Mangakakalot(),
         Manganelo(),
         Mangabat(),
-        MangaOnl(),
+        OtherMangakakalot(),
-        OtherMangakakalot()
+        Mangairo()
     )
 }
 
-/**
- * Base MangaBox class allows for genre search using query parameters in URLs
- * MangaBoxPathedGenres class extends base class, genre search only uses path segments in URLs
- */
-
-abstract class MangaBoxPathedGenres(
-    name: String,
-    baseUrl: String,
-    lang: String,
-    dateformat: SimpleDateFormat = SimpleDateFormat("MMM-dd-yy", Locale.ENGLISH)
-) : MangaBox(name, baseUrl, lang, dateformat) {
-    override fun getFilterList() = FilterList(
-        Filter.Header("NOTE: Ignored if using text search!"),
-        Filter.Separator(),
-        GenreFilter(getGenrePairs())
-    )
-    class GenreFilter(genrePairs: Array<Pair<String, String>>) : UriPartFilter("Category", genrePairs)
-    // Pair("path_segment/", "display name")
-    abstract override fun getGenrePairs(): Array<Pair<String, String>>
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        return if (query.isNotBlank()) {
-            GET("$baseUrl/$simpleQueryPath${normalizeSearchQuery(query)}?page=$page", headers)
-        } else {
-            var url = "$baseUrl/"
-
-            filters.forEach { filter ->
-                when (filter) {
-                    is GenreFilter -> {
-                        url += filter.toUriPart()
-                    }
-                }
-            }
-            GET(url + page, headers)
-        }
-    }
-}
-
 class Mangakakalot : MangaBox("Mangakakalot", "https://mangakakalot.com", "en") {
     override fun headersBuilder(): Headers.Builder = super.headersBuilder().set("Referer", "https://manganelo.com") // for covers
     override fun searchMangaSelector() = "${super.searchMangaSelector()}, div.list-truyen-item-wrap"
 }
 
-class Manganelo : MangaBoxPathedGenres("Manganelo", "https://manganelo.com", "en") {
+class Manganelo : MangaBox("Manganelo", "https://manganelo.com", "en") {
     // Nelo's date format is part of the base class
     override fun popularMangaRequest(page: Int): Request = GET("$baseUrl/genre-all/$page?type=topview", headers)
     override fun popularMangaSelector() = "div.content-genres-item"
     override val latestUrlPath = "genre-all/"
     override fun searchMangaSelector() = "div.search-story-item, div.content-genres-item"
-    override fun getGenrePairs() = arrayOf(
+    override fun getAdvancedGenreFilters(): List<AdvGenre> = getGenreFilters()
-        Pair("genre-all/", "All"),
+        .drop(1)
-        Pair("genre-2/", "Action"),
+        .map { AdvGenre(it.first, it.second) }
-        Pair("genre-3/", "Adult"),
-        Pair("genre-4/", "Adventure"),
-        Pair("genre-6/", "Comedy"),
-        Pair("genre-7/", "Cooking"),
-        Pair("genre-9/", "Doujinshi"),
-        Pair("genre-10/", "Drama"),
-        Pair("genre-11/", "Ecchi"),
-        Pair("genre-12/", "Fantasy"),
-        Pair("genre-13/", "Gender bender"),
-        Pair("genre-14/", "Harem"),
-        Pair("genre-15/", "Historical"),
-        Pair("genre-16/", "Horror"),
-        Pair("genre-45/", "Isekai"),
-        Pair("genre-17/", "Josei"),
-        Pair("genre-44/", "Manhua"),
-        Pair("genre-43/", "Manhwa"),
-        Pair("genre-19/", "Martial arts"),
-        Pair("genre-20/", "Mature"),
-        Pair("genre-21/", "Mecha"),
-        Pair("genre-22/", "Medical"),
-        Pair("genre-24/", "Mystery"),
-        Pair("genre-25/", "One shot"),
-        Pair("genre-26/", "Psychological"),
-        Pair("genre-27/", "Romance"),
-        Pair("genre-28/", "School life"),
-        Pair("genre-29/", "Sci fi"),
-        Pair("genre-30/", "Seinen"),
-        Pair("genre-31/", "Shoujo"),
-        Pair("genre-32/", "Shoujo ai"),
-        Pair("genre-33/", "Shounen"),
-        Pair("genre-34/", "Shounen ai"),
-        Pair("genre-35/", "Slice of life"),
-        Pair("genre-36/", "Smut"),
-        Pair("genre-37/", "Sports"),
-        Pair("genre-38/", "Supernatural"),
-        Pair("genre-39/", "Tragedy"),
-        Pair("genre-40/", "Webtoons"),
-        Pair("genre-41/", "Yaoi"),
-        Pair("genre-42/", "Yuri")
-    )
 }
 
 class Mangabat : MangaBox("Mangabat", "https://mangabat.com", "en", SimpleDateFormat("MMM dd,yy", Locale.ENGLISH)) {
@@ -121,79 +45,28 @@ class Mangabat : MangaBox("Mangabat", "https://mangabat.com", "en", SimpleDateFo
     override fun popularMangaSelector() = "div.list-story-item"
     override val latestUrlPath = "manga-list-all/"
     override fun searchMangaSelector() = "div.list-story-item"
-}
+    override fun getAdvancedGenreFilters(): List<AdvGenre> = getGenreFilters()
-
+        .drop(1)
-class MangaOnl : MangaBoxPathedGenres("MangaOnl", "https://mangaonl.com", "en") {
+        .map { AdvGenre(it.first, it.second) }
-    override val popularUrlPath = "story-list-ty-topview-st-all-ca-all-"
-    override val latestUrlPath = "story-list-ty-latest-st-all-ca-all-"
-    override fun popularMangaSelector() = "div.story_item"
-    override val mangaDetailsMainSelector = "div.panel_story_info, ${super.mangaDetailsMainSelector}" // Some manga link to Nelo
-    override val thumbnailSelector = "img.story_avatar, ${super.thumbnailSelector}"
-    override val descriptionSelector = "div.panel_story_info_description, ${super.descriptionSelector}"
-    override fun chapterListSelector() = "div.chapter_list_title + ul li, ${super.chapterListSelector()}"
-    override val pageListSelector = "div.container_readchapter img, ${super.pageListSelector}"
-    override fun getGenrePairs() = arrayOf(
-        Pair("story-list-ty-latest-st-all-ca-all-", "ALL"),
-        Pair("story-list-ty-latest-st-all-ca-2-", "Action"),
-        Pair("story-list-ty-latest-st-all-ca-3-", "Adult"),
-        Pair("story-list-ty-latest-st-all-ca-4-", "Adventure"),
-        Pair("story-list-ty-latest-st-all-ca-6-", "Comedy"),
-        Pair("story-list-ty-latest-st-all-ca-7-", "Cooking"),
-        Pair("story-list-ty-latest-st-all-ca-9-", "Doujinshi"),
-        Pair("story-list-ty-latest-st-all-ca-10-", "Drama"),
-        Pair("story-list-ty-latest-st-all-ca-11-", "Ecchi"),
-        Pair("story-list-ty-latest-st-all-ca-12-", "Fantasy"),
-        Pair("story-list-ty-latest-st-all-ca-13-", "Gender bender"),
-        Pair("story-list-ty-latest-st-all-ca-14-", "Harem"),
-        Pair("story-list-ty-latest-st-all-ca-15-", "Historical"),
-        Pair("story-list-ty-latest-st-all-ca-16-", "Horror"),
-        Pair("story-list-ty-latest-st-all-ca-45-", "Isekai"),
-        Pair("story-list-ty-latest-st-all-ca-17-", "Josei"),
-        Pair("story-list-ty-latest-st-all-ca-43-", "Manhwa"),
-        Pair("story-list-ty-latest-st-all-ca-44-", "Manhua"),
-        Pair("story-list-ty-latest-st-all-ca-19-", "Martial arts"),
-        Pair("story-list-ty-latest-st-all-ca-20-", "Mature"),
-        Pair("story-list-ty-latest-st-all-ca-21-", "Mecha"),
-        Pair("story-list-ty-latest-st-all-ca-22-", "Medical"),
-        Pair("story-list-ty-latest-st-all-ca-24-", "Mystery"),
-        Pair("story-list-ty-latest-st-all-ca-25-", "One shot"),
-        Pair("story-list-ty-latest-st-all-ca-26-", "Psychological"),
-        Pair("story-list-ty-latest-st-all-ca-27-", "Romance"),
-        Pair("story-list-ty-latest-st-all-ca-28-", "School life"),
-        Pair("story-list-ty-latest-st-all-ca-29-", "Sci fi"),
-        Pair("story-list-ty-latest-st-all-ca-30-", "Seinen"),
-        Pair("story-list-ty-latest-st-all-ca-31-", "Shoujo"),
-        Pair("story-list-ty-latest-st-all-ca-32-", "Shoujo ai"),
-        Pair("story-list-ty-latest-st-all-ca-33-", "Shounen"),
-        Pair("story-list-ty-latest-st-all-ca-34-", "Shounen ai"),
-        Pair("story-list-ty-latest-st-all-ca-35-", "Slice of life"),
-        Pair("story-list-ty-latest-st-all-ca-36-", "Smut"),
-        Pair("story-list-ty-latest-st-all-ca-37-", "Sports"),
-        Pair("story-list-ty-latest-st-all-ca-38-", "Supernatural"),
-        Pair("story-list-ty-latest-st-all-ca-39-", "Tragedy"),
-        Pair("story-list-ty-latest-st-all-ca-40-", "Webtoons"),
-        Pair("story-list-ty-latest-st-all-ca-41-", "Yaoi"),
-        Pair("story-list-ty-latest-st-all-ca-42-", "Yuri")
-    )
 }
 
 class OtherMangakakalot : MangaBox("Mangakakalots (unoriginal)", "https://mangakakalots.com", "en") {
     override fun searchMangaSelector(): String = "${super.searchMangaSelector()}, div.list-truyen-item-wrap"
     override fun searchMangaParse(response: Response): MangasPage {
         val document = response.asJsoup()
-        val mangas = document.select(searchMangaSelector()).map { searchMangaFromElement(it) }
+        val mangas = document.select(searchMangaSelector()).map { mangaFromElement(it) }
         val hasNextPage = !response.request().url().toString()
             .contains(document.select(searchMangaNextPageSelector()).attr("href"))
 
         return MangasPage(mangas, hasNextPage)
     }
     override fun searchMangaNextPageSelector() = "div.group_page a:last-of-type"
-    override fun getStatusPairs() = arrayOf(
+    override fun getStatusFilters(): Array<Pair<String?, String>> = arrayOf(
         Pair("all", "ALL"),
         Pair("Completed", "Completed"),
         Pair("Ongoing", "Ongoing")
     )
-    override fun getGenrePairs() = arrayOf(
+    override fun getGenreFilters(): Array<Pair<String?, String>> = arrayOf(
         Pair("all", "ALL"),
         Pair("Action", "Action"),
         Pair("Adult", "Adult"),
@@ -237,3 +110,23 @@ class OtherMangakakalot : MangaBox("Mangakakalots (unoriginal)", "https://mangak
         Pair("Yuri", "Yuri")
     )
 }
+
+class Mangairo : MangaBox("Mangairo", "https://m.mangairo.com", "en", SimpleDateFormat("MMM-dd-yy", Locale.ENGLISH)) {
+    override val popularUrlPath = "manga-list/type-topview/ctg-all/state-all/page-"
+    override fun popularMangaSelector() = "div.story-item"
+    override val latestUrlPath = "manga-list/type-latest/ctg-all/state-all/page-"
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        return GET("$baseUrl/$simpleQueryPath${normalizeSearchQuery(query)}?page=$page", headers)
+    }
+    override fun searchMangaSelector() = "div.story-item"
+    override fun searchMangaFromElement(element: Element): SManga = mangaFromElement(element, "h2 a")
+    override fun searchMangaNextPageSelector() = "div.group-page a.select + a:not(.go-p-end)"
+    override val mangaDetailsMainSelector = "${super.mangaDetailsMainSelector}, div.story_content"
+    override val thumbnailSelector = "${super.thumbnailSelector}, div.story_info_left img"
+    override val descriptionSelector = "${super.descriptionSelector}, div#story_discription p"
+    override fun chapterListSelector() = "${super.chapterListSelector()}, div#chapter_list li"
+    override val alternateChapterDateSelector = "p"
+    override val pageListSelector = "${super.pageListSelector}, div.panel-read-story img"
+    // will have to write a separate searchMangaRequest to get filters working for this source
+    override fun getFilterList() = FilterList()
+}