From 6b0b72dd837d9aa96a7fe17f030b377f57330cd3 Mon Sep 17 00:00:00 2001 From: Mike <51273546+SnakeDoc83@users.noreply.github.com> Date: Wed, 20 May 2020 22:45:17 -0400 Subject: [PATCH] MangaBox - refactor search, source list changes (#3264) --- src/all/mangabox/build.gradle | 2 +- .../extension/all/mangabox/MangaBox.kt | 121 ++++++++---- .../extension/all/mangabox/MangaBoxFactory.kt | 175 ++++-------------- 3 files changed, 120 insertions(+), 178 deletions(-) diff --git a/src/all/mangabox/build.gradle b/src/all/mangabox/build.gradle index 1603dc243..1a2b415f5 100644 --- a/src/all/mangabox/build.gradle +++ b/src/all/mangabox/build.gradle @@ -5,7 +5,7 @@ ext { appName = 'Tachiyomi: MangaBox (Mangakakalot and others)' pkgNameSuffix = 'all.mangabox' extClass = '.MangaBoxFactory' - extVersionCode = 21 + extVersionCode = 22 libVersion = '1.2' } diff --git a/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBox.kt b/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBox.kt index bd4fb6242..324d60ad3 100644 --- a/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBox.kt +++ b/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBox.kt @@ -59,9 +59,9 @@ abstract class MangaBox( return GET("$baseUrl/$latestUrlPath$page", headers) } - override fun popularMangaFromElement(element: Element): SManga { + protected fun mangaFromElement(element: Element, urlSelector: String = "h3 a"): SManga { return SManga.create().apply { - element.select("h3 a").first().let { + element.select(urlSelector).first().let { url = it.attr("abs:href").substringAfter(baseUrl) // intentionally not using setUrlWithoutDomain title = it.text() } @@ -69,29 +69,46 @@ abstract class MangaBox( } } - override fun latestUpdatesFromElement(element: Element): SManga = popularMangaFromElement(element) + override fun popularMangaFromElement(element: Element): SManga = mangaFromElement(element) + + override fun latestUpdatesFromElement(element: Element): SManga = mangaFromElement(element) override fun popularMangaNextPageSelector() = "div.group_page, div.group-page a:not([href]) + a:not(:contains(Last))" override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector() override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - return if (query.isNotBlank()) { + return if (query.isNotBlank() && getAdvancedGenreFilters().isEmpty()) { GET("$baseUrl/$simpleQueryPath${normalizeSearchQuery(query)}?page=$page", headers) } else { - val url = HttpUrl.parse("$baseUrl/manga_list")!!.newBuilder() - url.addQueryParameter("page", page.toString()) - - filters.forEach { filter -> - when (filter) { - is SortFilter -> { - url.addQueryParameter("type", filter.toUriPart()) + val url = HttpUrl.parse(baseUrl)!!.newBuilder() + if (getAdvancedGenreFilters().isNotEmpty()) { + url.addPathSegment("advanced_search") + url.addQueryParameter("page", page.toString()) + url.addQueryParameter("keyw", normalizeSearchQuery(query)) + var genreInclude = "" + var genreExclude = "" + filters.forEach { filter -> + when (filter) { + is KeywordFilter -> filter.toUriPart()?.let { url.addQueryParameter("keyt", it) } + is SortFilter -> url.addQueryParameter("orby", filter.toUriPart()) + is StatusFilter -> url.addQueryParameter("sts", filter.toUriPart()) + is AdvGenreFilter -> { + filter.state.forEach { if (it.isIncluded()) genreInclude += "_${it.id}" } + filter.state.forEach { if (it.isExcluded()) genreExclude += "_${it.id}" } + } } - is StatusFilter -> { - url.addQueryParameter("state", filter.toUriPart()) - } - is GenreFilter -> { - url.addQueryParameter("category", filter.toUriPart()) + } + url.addQueryParameter("g_i", genreInclude) + url.addQueryParameter("g_e", genreExclude) + } else { + url.addPathSegment("manga_list") + url.addQueryParameter("page", page.toString()) + filters.forEach { filter -> + when (filter) { + is SortFilter -> url.addQueryParameter("type", filter.toUriPart()) + is StatusFilter -> url.addQueryParameter("state", filter.toUriPart()) + is GenreFilter -> url.addQueryParameter("category", filter.toUriPart()) } } } @@ -101,7 +118,7 @@ abstract class MangaBox( override fun searchMangaSelector() = ".panel_story_list .story_item" - override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element) + override fun searchMangaFromElement(element: Element) = mangaFromElement(element) override fun searchMangaNextPageSelector() = "a.page_select + a:not(.page_last), a.page-select + a:not(.page-last)" @@ -165,6 +182,13 @@ abstract class MangaBox( override fun chapterListSelector() = "div.chapter-list div.row, ul.row-content-chapter li" + protected open val alternateChapterDateSelector = String() + + private fun Element.selectDateFromElement(): Element { + val defaultChapterDateSelector = "span" + return this.select(defaultChapterDateSelector).lastOrNull() ?: this.select(alternateChapterDateSelector).last() + } + override fun chapterFromElement(element: Element): SChapter { return SChapter.create().apply { element.select("a").let { @@ -172,11 +196,11 @@ abstract class MangaBox( name = it.text() scanlator = HttpUrl.parse(it.attr("abs:href"))!!.host() // show where chapters are actually from } - date_upload = parseChapterDate(element.select("span").last().text(), element.ownerDocument().location()) ?: 0 + date_upload = parseChapterDate(element.selectDateFromElement().text(), scanlator!!) ?: 0 } } - private fun parseChapterDate(date: String, url: String): Long? { + private fun parseChapterDate(date: String, host: String): Long? { return if ("ago" in date) { val value = date.split(' ')[0].toIntOrNull() val cal = Calendar.getInstance() @@ -188,7 +212,7 @@ abstract class MangaBox( }?.timeInMillis } else { try { - if (url.contains("manganelo")) { + if (host.contains("manganelo", ignoreCase = true)) { // Nelo's date format SimpleDateFormat("MMM dd,yy", Locale.ENGLISH).parse(date) } else { @@ -245,32 +269,54 @@ abstract class MangaBox( return str } - override fun getFilterList() = FilterList( + override fun getFilterList() = if (getAdvancedGenreFilters().isNotEmpty()) { + FilterList( + KeywordFilter(getKeywordFilters()), + SortFilter(getSortFilters()), + StatusFilter(getStatusFilters()), + AdvGenreFilter(getAdvancedGenreFilters()) + ) + } else { + FilterList( Filter.Header("NOTE: Ignored if using text search!"), Filter.Separator(), - SortFilter(), - StatusFilter(getStatusPairs()), - GenreFilter(getGenrePairs()) + SortFilter(getSortFilters()), + StatusFilter(getStatusFilters()), + GenreFilter(getGenreFilters()) + ) + } + + private class KeywordFilter(vals: Array>) : UriPartFilter("Keyword search ", vals) + private class SortFilter(vals: Array>) : UriPartFilter("Order by", vals) + private class StatusFilter(vals: Array>) : UriPartFilter("Status", vals) + private class GenreFilter(vals: Array>) : UriPartFilter("Category", vals) + + // For advanced search, specifically tri-state genres + private class AdvGenreFilter(vals: List) : Filter.Group("Category", vals) + class AdvGenre(val id: String?, name: String) : Filter.TriState(name) + + // keyt query parameter + private fun getKeywordFilters(): Array> = arrayOf( + Pair(null, "Everything"), + Pair("title", "Title"), + Pair("alternative", "Alt title"), + Pair("author", "Author") ) - private class SortFilter : UriPartFilter("Sort", arrayOf( - Pair("latest", "Latest"), - Pair("newest", "Newest"), - Pair("topview", "Top read") - )) + private fun getSortFilters(): Array> = arrayOf( + Pair("latest", "Latest"), + Pair("newest", "Newest"), + Pair("topview", "Top read") + ) - open class StatusFilter(statusPairs: Array>) : UriPartFilter("Status", statusPairs) - - open fun getStatusPairs() = arrayOf( + open fun getStatusFilters(): Array> = arrayOf( Pair("all", "ALL"), Pair("completed", "Completed"), Pair("ongoing", "Ongoing"), Pair("drop", "Dropped") ) - private class GenreFilter(genrePairs: Array>) : UriPartFilter("Category", genrePairs) - - open fun getGenrePairs(): Array> = arrayOf( + open fun getGenreFilters(): Array> = arrayOf( Pair("all", "ALL"), Pair("2", "Action"), Pair("3", "Adult"), @@ -314,7 +360,10 @@ abstract class MangaBox( Pair("42", "Yuri") ) - open class UriPartFilter(displayName: String, private val vals: Array>) : + // To be overridden if using tri-state genres + protected open fun getAdvancedGenreFilters(): List = emptyList() + + open class UriPartFilter(displayName: String, private val vals: Array>) : Filter.Select(displayName, vals.map { it.second }.toTypedArray()) { fun toUriPart() = vals[state].first } diff --git a/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBoxFactory.kt b/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBoxFactory.kt index 9f88037dc..e3cc2ed15 100644 --- a/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBoxFactory.kt +++ b/src/all/mangabox/src/eu/kanade/tachiyomi/extension/all/mangabox/MangaBoxFactory.kt @@ -3,117 +3,41 @@ package eu.kanade.tachiyomi.extension.all.mangabox import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceFactory -import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage +import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.util.asJsoup import java.text.SimpleDateFormat import java.util.Locale import okhttp3.Headers import okhttp3.Request import okhttp3.Response +import org.jsoup.nodes.Element class MangaBoxFactory : SourceFactory { override fun createSources(): List = listOf( Mangakakalot(), Manganelo(), Mangabat(), - MangaOnl(), - OtherMangakakalot() + OtherMangakakalot(), + Mangairo() ) } -/** - * Base MangaBox class allows for genre search using query parameters in URLs - * MangaBoxPathedGenres class extends base class, genre search only uses path segments in URLs - */ - -abstract class MangaBoxPathedGenres( - name: String, - baseUrl: String, - lang: String, - dateformat: SimpleDateFormat = SimpleDateFormat("MMM-dd-yy", Locale.ENGLISH) -) : MangaBox(name, baseUrl, lang, dateformat) { - override fun getFilterList() = FilterList( - Filter.Header("NOTE: Ignored if using text search!"), - Filter.Separator(), - GenreFilter(getGenrePairs()) - ) - class GenreFilter(genrePairs: Array>) : UriPartFilter("Category", genrePairs) - // Pair("path_segment/", "display name") - abstract override fun getGenrePairs(): Array> - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - return if (query.isNotBlank()) { - GET("$baseUrl/$simpleQueryPath${normalizeSearchQuery(query)}?page=$page", headers) - } else { - var url = "$baseUrl/" - - filters.forEach { filter -> - when (filter) { - is GenreFilter -> { - url += filter.toUriPart() - } - } - } - GET(url + page, headers) - } - } -} - class Mangakakalot : MangaBox("Mangakakalot", "https://mangakakalot.com", "en") { override fun headersBuilder(): Headers.Builder = super.headersBuilder().set("Referer", "https://manganelo.com") // for covers override fun searchMangaSelector() = "${super.searchMangaSelector()}, div.list-truyen-item-wrap" } -class Manganelo : MangaBoxPathedGenres("Manganelo", "https://manganelo.com", "en") { +class Manganelo : MangaBox("Manganelo", "https://manganelo.com", "en") { // Nelo's date format is part of the base class override fun popularMangaRequest(page: Int): Request = GET("$baseUrl/genre-all/$page?type=topview", headers) override fun popularMangaSelector() = "div.content-genres-item" override val latestUrlPath = "genre-all/" override fun searchMangaSelector() = "div.search-story-item, div.content-genres-item" - override fun getGenrePairs() = arrayOf( - Pair("genre-all/", "All"), - Pair("genre-2/", "Action"), - Pair("genre-3/", "Adult"), - Pair("genre-4/", "Adventure"), - Pair("genre-6/", "Comedy"), - Pair("genre-7/", "Cooking"), - Pair("genre-9/", "Doujinshi"), - Pair("genre-10/", "Drama"), - Pair("genre-11/", "Ecchi"), - Pair("genre-12/", "Fantasy"), - Pair("genre-13/", "Gender bender"), - Pair("genre-14/", "Harem"), - Pair("genre-15/", "Historical"), - Pair("genre-16/", "Horror"), - Pair("genre-45/", "Isekai"), - Pair("genre-17/", "Josei"), - Pair("genre-44/", "Manhua"), - Pair("genre-43/", "Manhwa"), - Pair("genre-19/", "Martial arts"), - Pair("genre-20/", "Mature"), - Pair("genre-21/", "Mecha"), - Pair("genre-22/", "Medical"), - Pair("genre-24/", "Mystery"), - Pair("genre-25/", "One shot"), - Pair("genre-26/", "Psychological"), - Pair("genre-27/", "Romance"), - Pair("genre-28/", "School life"), - Pair("genre-29/", "Sci fi"), - Pair("genre-30/", "Seinen"), - Pair("genre-31/", "Shoujo"), - Pair("genre-32/", "Shoujo ai"), - Pair("genre-33/", "Shounen"), - Pair("genre-34/", "Shounen ai"), - Pair("genre-35/", "Slice of life"), - Pair("genre-36/", "Smut"), - Pair("genre-37/", "Sports"), - Pair("genre-38/", "Supernatural"), - Pair("genre-39/", "Tragedy"), - Pair("genre-40/", "Webtoons"), - Pair("genre-41/", "Yaoi"), - Pair("genre-42/", "Yuri") - ) + override fun getAdvancedGenreFilters(): List = getGenreFilters() + .drop(1) + .map { AdvGenre(it.first, it.second) } } class Mangabat : MangaBox("Mangabat", "https://mangabat.com", "en", SimpleDateFormat("MMM dd,yy", Locale.ENGLISH)) { @@ -121,79 +45,28 @@ class Mangabat : MangaBox("Mangabat", "https://mangabat.com", "en", SimpleDateFo override fun popularMangaSelector() = "div.list-story-item" override val latestUrlPath = "manga-list-all/" override fun searchMangaSelector() = "div.list-story-item" -} - -class MangaOnl : MangaBoxPathedGenres("MangaOnl", "https://mangaonl.com", "en") { - override val popularUrlPath = "story-list-ty-topview-st-all-ca-all-" - override val latestUrlPath = "story-list-ty-latest-st-all-ca-all-" - override fun popularMangaSelector() = "div.story_item" - override val mangaDetailsMainSelector = "div.panel_story_info, ${super.mangaDetailsMainSelector}" // Some manga link to Nelo - override val thumbnailSelector = "img.story_avatar, ${super.thumbnailSelector}" - override val descriptionSelector = "div.panel_story_info_description, ${super.descriptionSelector}" - override fun chapterListSelector() = "div.chapter_list_title + ul li, ${super.chapterListSelector()}" - override val pageListSelector = "div.container_readchapter img, ${super.pageListSelector}" - override fun getGenrePairs() = arrayOf( - Pair("story-list-ty-latest-st-all-ca-all-", "ALL"), - Pair("story-list-ty-latest-st-all-ca-2-", "Action"), - Pair("story-list-ty-latest-st-all-ca-3-", "Adult"), - Pair("story-list-ty-latest-st-all-ca-4-", "Adventure"), - Pair("story-list-ty-latest-st-all-ca-6-", "Comedy"), - Pair("story-list-ty-latest-st-all-ca-7-", "Cooking"), - Pair("story-list-ty-latest-st-all-ca-9-", "Doujinshi"), - Pair("story-list-ty-latest-st-all-ca-10-", "Drama"), - Pair("story-list-ty-latest-st-all-ca-11-", "Ecchi"), - Pair("story-list-ty-latest-st-all-ca-12-", "Fantasy"), - Pair("story-list-ty-latest-st-all-ca-13-", "Gender bender"), - Pair("story-list-ty-latest-st-all-ca-14-", "Harem"), - Pair("story-list-ty-latest-st-all-ca-15-", "Historical"), - Pair("story-list-ty-latest-st-all-ca-16-", "Horror"), - Pair("story-list-ty-latest-st-all-ca-45-", "Isekai"), - Pair("story-list-ty-latest-st-all-ca-17-", "Josei"), - Pair("story-list-ty-latest-st-all-ca-43-", "Manhwa"), - Pair("story-list-ty-latest-st-all-ca-44-", "Manhua"), - Pair("story-list-ty-latest-st-all-ca-19-", "Martial arts"), - Pair("story-list-ty-latest-st-all-ca-20-", "Mature"), - Pair("story-list-ty-latest-st-all-ca-21-", "Mecha"), - Pair("story-list-ty-latest-st-all-ca-22-", "Medical"), - Pair("story-list-ty-latest-st-all-ca-24-", "Mystery"), - Pair("story-list-ty-latest-st-all-ca-25-", "One shot"), - Pair("story-list-ty-latest-st-all-ca-26-", "Psychological"), - Pair("story-list-ty-latest-st-all-ca-27-", "Romance"), - Pair("story-list-ty-latest-st-all-ca-28-", "School life"), - Pair("story-list-ty-latest-st-all-ca-29-", "Sci fi"), - Pair("story-list-ty-latest-st-all-ca-30-", "Seinen"), - Pair("story-list-ty-latest-st-all-ca-31-", "Shoujo"), - Pair("story-list-ty-latest-st-all-ca-32-", "Shoujo ai"), - Pair("story-list-ty-latest-st-all-ca-33-", "Shounen"), - Pair("story-list-ty-latest-st-all-ca-34-", "Shounen ai"), - Pair("story-list-ty-latest-st-all-ca-35-", "Slice of life"), - Pair("story-list-ty-latest-st-all-ca-36-", "Smut"), - Pair("story-list-ty-latest-st-all-ca-37-", "Sports"), - Pair("story-list-ty-latest-st-all-ca-38-", "Supernatural"), - Pair("story-list-ty-latest-st-all-ca-39-", "Tragedy"), - Pair("story-list-ty-latest-st-all-ca-40-", "Webtoons"), - Pair("story-list-ty-latest-st-all-ca-41-", "Yaoi"), - Pair("story-list-ty-latest-st-all-ca-42-", "Yuri") - ) + override fun getAdvancedGenreFilters(): List = getGenreFilters() + .drop(1) + .map { AdvGenre(it.first, it.second) } } class OtherMangakakalot : MangaBox("Mangakakalots (unoriginal)", "https://mangakakalots.com", "en") { override fun searchMangaSelector(): String = "${super.searchMangaSelector()}, div.list-truyen-item-wrap" override fun searchMangaParse(response: Response): MangasPage { val document = response.asJsoup() - val mangas = document.select(searchMangaSelector()).map { searchMangaFromElement(it) } + val mangas = document.select(searchMangaSelector()).map { mangaFromElement(it) } val hasNextPage = !response.request().url().toString() .contains(document.select(searchMangaNextPageSelector()).attr("href")) return MangasPage(mangas, hasNextPage) } override fun searchMangaNextPageSelector() = "div.group_page a:last-of-type" - override fun getStatusPairs() = arrayOf( + override fun getStatusFilters(): Array> = arrayOf( Pair("all", "ALL"), Pair("Completed", "Completed"), Pair("Ongoing", "Ongoing") ) - override fun getGenrePairs() = arrayOf( + override fun getGenreFilters(): Array> = arrayOf( Pair("all", "ALL"), Pair("Action", "Action"), Pair("Adult", "Adult"), @@ -237,3 +110,23 @@ class OtherMangakakalot : MangaBox("Mangakakalots (unoriginal)", "https://mangak Pair("Yuri", "Yuri") ) } + +class Mangairo : MangaBox("Mangairo", "https://m.mangairo.com", "en", SimpleDateFormat("MMM-dd-yy", Locale.ENGLISH)) { + override val popularUrlPath = "manga-list/type-topview/ctg-all/state-all/page-" + override fun popularMangaSelector() = "div.story-item" + override val latestUrlPath = "manga-list/type-latest/ctg-all/state-all/page-" + override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { + return GET("$baseUrl/$simpleQueryPath${normalizeSearchQuery(query)}?page=$page", headers) + } + override fun searchMangaSelector() = "div.story-item" + override fun searchMangaFromElement(element: Element): SManga = mangaFromElement(element, "h2 a") + override fun searchMangaNextPageSelector() = "div.group-page a.select + a:not(.go-p-end)" + override val mangaDetailsMainSelector = "${super.mangaDetailsMainSelector}, div.story_content" + override val thumbnailSelector = "${super.thumbnailSelector}, div.story_info_left img" + override val descriptionSelector = "${super.descriptionSelector}, div#story_discription p" + override fun chapterListSelector() = "${super.chapterListSelector()}, div#chapter_list li" + override val alternateChapterDateSelector = "p" + override val pageListSelector = "${super.pageListSelector}, div.panel-read-story img" + // will have to write a separate searchMangaRequest to get filters working for this source + override fun getFilterList() = FilterList() +}