From f14439bcad00ae6ed7ddb704f61de5642c086f13 Mon Sep 17 00:00:00 2001 From: Mike <51273546+SnakeDoc83@users.noreply.github.com> Date: Wed, 1 Jul 2020 04:49:50 -0400 Subject: [PATCH] Mangasee - update to use Manga Life code (#3662) --- src/en/mangasee/build.gradle | 2 +- .../extension/en/mangasee/Mangasee.kt | 427 +++++++++++------- 2 files changed, 263 insertions(+), 166 deletions(-) diff --git a/src/en/mangasee/build.gradle b/src/en/mangasee/build.gradle index e6f4fe2df..060993560 100644 --- a/src/en/mangasee/build.gradle +++ b/src/en/mangasee/build.gradle @@ -5,7 +5,7 @@ ext { extName = 'Mangasee' pkgNameSuffix = 'en.mangasee' extClass = '.Mangasee' - extVersionCode = 7 + extVersionCode = 8 libVersion = '1.2' } diff --git a/src/en/mangasee/src/eu/kanade/tachiyomi/extension/en/mangasee/Mangasee.kt b/src/en/mangasee/src/eu/kanade/tachiyomi/extension/en/mangasee/Mangasee.kt index 8ccc61e59..504e8e9d5 100644 --- a/src/en/mangasee/src/eu/kanade/tachiyomi/extension/en/mangasee/Mangasee.kt +++ b/src/en/mangasee/src/eu/kanade/tachiyomi/extension/en/mangasee/Mangasee.kt @@ -1,72 +1,174 @@ package eu.kanade.tachiyomi.extension.en.mangasee -import eu.kanade.tachiyomi.network.POST +import com.github.salomonbrys.kotson.fromJson +import com.github.salomonbrys.kotson.get +import com.github.salomonbrys.kotson.nullString +import com.github.salomonbrys.kotson.string +import com.google.gson.GsonBuilder +import com.google.gson.JsonArray +import com.google.gson.JsonElement +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.ParsedHttpSource +import eu.kanade.tachiyomi.source.online.HttpSource +import eu.kanade.tachiyomi.util.asJsoup import java.text.SimpleDateFormat import java.util.Locale -import java.util.regex.Pattern -import okhttp3.FormBody -import okhttp3.HttpUrl +import java.util.concurrent.TimeUnit +import okhttp3.Headers +import okhttp3.OkHttpClient import okhttp3.Request -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element +import okhttp3.Response +import rx.Observable -class Mangasee : ParsedHttpSource() { +/** + * Exact same code as Manga Life except for better chapter names thanks to Regex + * Probably should make this a multi-source extension, but decided that that's a problem for a different day + */ + +class Mangasee : HttpSource() { override val id: Long = 9 override val name = "Mangasee" - override val baseUrl = "https://mangaseeonline.us" + override val baseUrl = "https://mangasee123.com" override val lang = "en" override val supportsLatest = true - private val recentUpdatesPattern = Pattern.compile("(.*?)\\s(\\d+\\.?\\d*)\\s?(Completed)?") + override val client: OkHttpClient = network.cloudflareClient.newBuilder() + .connectTimeout(1, TimeUnit.MINUTES) + .readTimeout(1, TimeUnit.MINUTES) + .writeTimeout(1, TimeUnit.MINUTES) + .build() - override fun popularMangaSelector() = "div.requested > div.row" + override fun headersBuilder(): Headers.Builder = Headers.Builder() + .add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/77.0") + + private val gson = GsonBuilder().setLenient().create() + + private lateinit var directory: List + + // Popular + + override fun fetchPopularManga(page: Int): Observable { + return if (page == 1) { + client.newCall(popularMangaRequest(page)) + .asObservableSuccess() + .map { response -> + popularMangaParse(response) + } + } else { + Observable.just(parseDirectory(page)) + } + } override fun popularMangaRequest(page: Int): Request { - val (body, requestUrl) = convertQueryToPost(page, "$baseUrl/search/request.php?sortBy=popularity&sortOrder=descending") - return POST(requestUrl, headers, body.build()) + return GET("$baseUrl/search/", headers) } - override fun popularMangaFromElement(element: Element): SManga { - val manga = SManga.create() - element.select("a.resultLink").first().let { - manga.setUrlWithoutDomain(it.attr("href")) - manga.title = it.text() + // don't use ";" for substringBefore() ! + private fun directoryFromResponse(response: Response): String { + return response.asJsoup().select("script:containsData(MainFunction)").first().data() + .substringAfter("vm.Directory = ").substringBefore("vm.GetIntValue").trim() + .replace(";", " ") + } + + override fun popularMangaParse(response: Response): MangasPage { + directory = gson.fromJson(directoryFromResponse(response)) + .sortedByDescending { it["v"].string } + return parseDirectory(1) + } + + private fun parseDirectory(page: Int): MangasPage { + val mangas = mutableListOf() + val endRange = ((page * 24) - 1).let { if (it <= directory.lastIndex) it else directory.lastIndex } + + for (i in (((page - 1) * 24)..endRange)) { + mangas.add(SManga.create().apply { + title = directory[i]["s"].string + url = "/manga/${directory[i]["i"].string}" + thumbnail_url = "https://static.mangaboss.net/cover/${directory[i]["i"].string}.jpg" + }) } - manga.thumbnail_url = element.select("img").attr("abs:src") - return manga + return MangasPage(mangas, endRange < directory.lastIndex) } - override fun popularMangaNextPageSelector() = "button.requestMore" + // Latest - override fun searchMangaSelector() = "div.requested > div.row" + override fun fetchLatestUpdates(page: Int): Observable { + return if (page == 1) { + client.newCall(latestUpdatesRequest(page)) + .asObservableSuccess() + .map { response -> + latestUpdatesParse(response) + } + } else { + Observable.just(parseDirectory(page)) + } + } + + override fun latestUpdatesRequest(page: Int): Request = popularMangaRequest(1) + + override fun latestUpdatesParse(response: Response): MangasPage { + directory = gson.fromJson(directoryFromResponse(response)) + .sortedByDescending { it["lt"].string } + return parseDirectory(1) + } + + // Search + + override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { + return if (page == 1) { + client.newCall(searchMangaRequest(page, query, filters)) + .asObservableSuccess() + .map { response -> + searchMangaParse(response, query, filters) + } + } else { + Observable.just(parseDirectory(page)) + } + } + + override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = popularMangaRequest(1) + + private fun searchMangaParse(response: Response, query: String, filters: FilterList): MangasPage { + directory = gson.fromJson(directoryFromResponse(response)) + .filter { it["s"].string.contains(query, ignoreCase = true) } - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = HttpUrl.parse("$baseUrl/search/request.php")!!.newBuilder() - if (query.isNotEmpty()) url.addQueryParameter("keyword", query) val genres = mutableListOf() val genresNo = mutableListOf() + var sortBy: String for (filter in if (filters.isEmpty()) getFilterList() else filters) { when (filter) { is Sort -> { - if (filter.state?.index != 0) - url.addQueryParameter("sortBy", if (filter.state?.index == 1) "dateUpdated" else "popularity") - if (filter.state?.ascending != true) - url.addQueryParameter("sortOrder", "descending") + sortBy = when (filter.state?.index) { + 1 -> "ls" + 2 -> "v" + else -> "s" + } + directory = if (filter.state?.ascending != true) { + directory.sortedByDescending { it[sortBy].string } + } else { + directory.sortedByDescending { it[sortBy].string }.reversed() + } } - is SelectField -> if (filter.state != 0) url.addQueryParameter(filter.key, filter.values[filter.state]) - is TextField -> if (filter.state.isNotEmpty()) url.addQueryParameter(filter.key, filter.state) + is SelectField -> if (filter.state != 0) directory = when (filter.name) { + "Scan Status" -> directory.filter { it["ss"].string.contains(filter.values[filter.state], ignoreCase = true) } + "Publish Status" -> directory.filter { it["ps"].string.contains(filter.values[filter.state], ignoreCase = true) } + "Type" -> directory.filter { it["t"].string.contains(filter.values[filter.state], ignoreCase = true) } + "Translation" -> directory.filter { it["o"].string.contains("yes", ignoreCase = true) } + else -> directory + } + is YearField -> if (filter.state.isNotEmpty()) directory = directory.filter { it["y"].string.contains(filter.state) } + is AuthorField -> if (filter.state.isNotEmpty()) directory = directory.filter { e -> e["a"].asJsonArray.any { it.string.contains(filter.state, ignoreCase = true) } } is GenreList -> filter.state.forEach { genre -> when (genre.state) { Filter.TriState.STATE_INCLUDE -> genres.add(genre.name) @@ -75,167 +177,162 @@ class Mangasee : ParsedHttpSource() { } } } - if (genres.isNotEmpty()) url.addQueryParameter("genre", genres.joinToString(",")) - if (genresNo.isNotEmpty()) url.addQueryParameter("genreNo", genresNo.joinToString(",")) + if (genres.isNotEmpty()) genres.map { genre -> directory = directory.filter { e -> e["g"].asJsonArray.any { it.string.contains(genre, ignoreCase = true) } } } + if (genresNo.isNotEmpty()) genresNo.map { genre -> directory = directory.filterNot { e -> e["g"].asJsonArray.any { it.string.contains(genre, ignoreCase = true) } } } - val (body, requestUrl) = convertQueryToPost(page, url.toString()) - return POST(requestUrl, headers, body.build()) + return parseDirectory(1) } - private fun convertQueryToPost(page: Int, urlString: String): Pair { - val url = HttpUrl.parse(urlString)!! - val body = FormBody.Builder().add("page", page.toString()) - for (i in 0 until url.querySize()) { - body.add(url.queryParameterName(i), url.queryParameterValue(i)) + override fun searchMangaParse(response: Response): MangasPage = throw UnsupportedOperationException("Not used") + + // Details + + override fun mangaDetailsParse(response: Response): SManga { + return response.asJsoup().select("div.BoxBody > div.row").let { info -> + SManga.create().apply { + title = info.select("h1").text() + author = info.select("li.list-group-item:has(span:contains(Author)) a").first()?.text() + genre = info.select("li.list-group-item:has(span:contains(Genre)) a").joinToString { it.text() } + status = info.select("li.list-group-item:has(span:contains(Status)) a:contains(publish)").text().toStatus() + description = info.select("div.Content").text() + thumbnail_url = info.select("img").attr("abs:src") + } } - val requestUrl = url.scheme() + "://" + url.host() + url.encodedPath() - return Pair(body, requestUrl) } - override fun searchMangaFromElement(element: Element): SManga { - val manga = SManga.create() - element.select("a.resultLink").first().let { - manga.setUrlWithoutDomain(it.attr("href")) - manga.title = it.text() - } - manga.thumbnail_url = element.select("img").attr("abs:src") - return manga - } - - override fun searchMangaNextPageSelector() = "button.requestMore" - - override fun mangaDetailsParse(document: Document): SManga { - val detailElement = document.select("div.well > div.row").first() - - val manga = SManga.create() - manga.author = detailElement.select("a[href^=/search/?author=]").first()?.text() - manga.genre = detailElement.select("span.details > div.row > div:has(b:contains(Genre(s))) > a").joinToString { it.text() } - manga.description = detailElement.select("strong:contains(Description:) + div").first()?.text() - manga.status = detailElement.select("a[href^=/search/?status=]").first()?.text().orEmpty().let { parseStatus(it) } - manga.thumbnail_url = detailElement.select("div > img").first()?.absUrl("src") - return manga - } - - private fun parseStatus(status: String) = when { - status.contains("Ongoing (Scan)") -> SManga.ONGOING - status.contains("Complete (Scan)") -> SManga.COMPLETED + private fun String.toStatus() = when { + this.contains("Ongoing", ignoreCase = true) -> SManga.ONGOING + this.contains("Complete", ignoreCase = true) -> SManga.COMPLETED else -> SManga.UNKNOWN } - override fun chapterListSelector() = "div.chapter-list > a" + // Chapters - Mind special cases like decimal chapters (e.g. One Punch Man) and manga with seasons (e.g. The Gamer) - override fun chapterFromElement(element: Element): SChapter { - val urlElement = element.select("a").first() + private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.getDefault()) - val chapter = SChapter.create() - chapter.setUrlWithoutDomain(urlElement.attr("href")) - chapter.name = element.select("span.chapterLabel").firstOrNull()?.text() ?: "" - chapter.date_upload = element.select("time").first()?.attr("datetime")?.let { parseChapterDate(it) } ?: 0 - return chapter + private fun chapterURLEncode(e: String): String { + var index = "" + val t = e.substring(0, 1).toInt() + if (1 != t) { index = "-index-$t" } + val n = e.substring(1, e.length - 1) + var suffix = "" + val path = e.substring(e.length - 1).toInt() + if (0 != path) { suffix = ".$path" } + return "-chapter-$n$index$suffix.html" } - private fun parseChapterDate(dateAsString: String): Long { - return SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US).parse(dateAsString).time - } + private val chapterImageRegex = Regex("""^0+""") - override fun pageListParse(document: Document): List { - val pageArr = document.select("script:containsData(PageArr={)").first().data() - .substringAfter("PageArr=").substringBefore(";") - return JSONObject(pageArr).let { jsonObject -> - jsonObject.keys() - .asSequence() - .toList() - .filter { it.toIntOrNull() is Int } - .mapIndexed { i, key -> Page(i, "", jsonObject.getString(key)) } + private fun chapterImage(e: String): String { + val a = e.substring(1, e.length - 1).replace(chapterImageRegex, "") + val b = e.substring(e.length - 1).toInt() + return if (b == 0) { + a + } else { + "$a.$b" } } - override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not used") + override fun chapterListParse(response: Response): List { + val vmChapters = response.asJsoup().select("script:containsData(MainFunction)").first().data() + .substringAfter("vm.Chapters = ").substringBefore(";") - override fun latestUpdatesNextPageSelector() = "button.requestMore" - - override fun latestUpdatesSelector(): String = "a.latestSeries" - - override fun latestUpdatesRequest(page: Int): Request { - val url = "$baseUrl/home/latest.request.php" - val (body, requestUrl) = convertQueryToPost(page, url) - return POST(requestUrl, headers, body.build()) - } - - override fun latestUpdatesFromElement(element: Element): SManga { - val manga = SManga.create() - element.select("a.latestSeries").first().let { - val chapterUrl = it.attr("href") - val indexOfMangaUrl = chapterUrl.indexOf("-chapter-") - val indexOfLastPath = chapterUrl.lastIndexOf("/") - val mangaUrl = chapterUrl.substring(indexOfLastPath, indexOfMangaUrl) - val defaultText = it.select("p.clamp2").text() - val m = recentUpdatesPattern.matcher(defaultText) - val title = if (m.matches()) m.group(1) else defaultText - manga.setUrlWithoutDomain("/manga$mangaUrl") - manga.title = title + return gson.fromJson(vmChapters).map { json -> + val indexChapter = json["Chapter"].string + SChapter.create().apply { + name = json["ChapterName"].nullString.let { if (it.isNullOrEmpty()) "${json["Type"].string} ${chapterImage(indexChapter)}" else it } + url = "/read-online/" + response.request().url().toString().substringAfter("/manga/") + chapterURLEncode(indexChapter) + date_upload = try { + dateFormat.parse(json["Date"].string.substringBefore(" "))?.time ?: 0 + } catch (_: Exception) { + 0L + } + } } - manga.thumbnail_url = element.select("img").attr("abs:src") - return manga } + // Pages + + override fun pageListParse(response: Response): List { + val document = response.asJsoup() + val script = document.select("script:containsData(MainFunction)").first().data() + val curChapter = gson.fromJson(script.substringAfter("vm.CurChapter = ").substringBefore(";")) + + val pageTotal = curChapter["Page"].string.toInt() + + val host = "https://" + script.substringAfter("vm.CurPathName = \"").substringBefore("\"") + val titleURI = script.substringAfter("vm.IndexName = \"").substringBefore("\"") + val seasonURI = curChapter["Directory"].string + .let { if (it.isEmpty()) "" else "$it/" } + val path = "$host/manga/$titleURI/$seasonURI" + + val chNum = chapterImage(curChapter["Chapter"].string) + + return IntRange(1, pageTotal).mapIndexed { i, _ -> + val imageNum = (i + 1).toString().let { "000$it" }.let { it.substring(it.length - 3) } + Page(i, "", "$path$chNum-$imageNum.png") + } + } + + override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException("Not used") + + // Filters + private class Sort : Filter.Sort("Sort", arrayOf("Alphabetically", "Date updated", "Popularity"), Selection(2, false)) private class Genre(name: String) : Filter.TriState(name) - private class TextField(name: String, val key: String) : Filter.Text(name) - private class SelectField(name: String, val key: String, values: Array, state: Int = 0) : Filter.Select(name, values, state) + private class YearField : Filter.Text("Years") + private class AuthorField : Filter.Text("Author") + private class SelectField(name: String, values: Array, state: Int = 0) : Filter.Select(name, values, state) private class GenreList(genres: List) : Filter.Group("Genres", genres) override fun getFilterList() = FilterList( - TextField("Years", "year"), - TextField("Author", "author"), - SelectField("Scan Status", "status", arrayOf("Any", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing")), - SelectField("Publish Status", "pstatus", arrayOf("Any", "Cancelled", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing", "Unfinished")), - SelectField("Type", "type", arrayOf("Any", "Doujinshi", "Manga", "Manhua", "Manhwa", "OEL", "One-shot")), - Sort(), - GenreList(getGenreList()) + YearField(), + AuthorField(), + SelectField("Scan Status", arrayOf("Any", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing")), + SelectField("Publish Status", arrayOf("Any", "Cancelled", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing", "Unfinished")), + SelectField("Type", arrayOf("Any", "Doujinshi", "Manga", "Manhua", "Manhwa", "OEL", "One-shot")), + SelectField("Translation", arrayOf("Any", "Official Only")), + Sort(), + GenreList(getGenreList()) ) - // [...document.querySelectorAll(".genres .list-group-item")].map(el => `Genre("${el.getAttribute('value')}")`).join(',\n') - // https://mangaseeonline.us/search/ + // copied over from Manga Life private fun getGenreList() = listOf( - Genre("Action"), - Genre("Adult"), - Genre("Adventure"), - Genre("Comedy"), - Genre("Doujinshi"), - Genre("Drama"), - Genre("Ecchi"), - Genre("Fantasy"), - Genre("Gender Bender"), - Genre("Harem"), - Genre("Hentai"), - Genre("Historical"), - Genre("Horror"), - Genre("Isekai"), - Genre("Josei"), - Genre("Lolicon"), - Genre("Martial Arts"), - Genre("Mature"), - Genre("Mecha"), - Genre("Mystery"), - Genre("Psychological"), - Genre("Romance"), - Genre("School Life"), - Genre("Sci-fi"), - Genre("Seinen"), - Genre("Seinen Supernatural"), - Genre("Shotacon"), - Genre("Shoujo"), - Genre("Shoujo Ai"), - Genre("Shounen"), - Genre("Shounen Ai"), - Genre("Slice of Life"), - Genre("Smut"), - Genre("Sport"), - Genre("Sports"), - Genre("Supernatural"), - Genre("Tragedy"), - Genre("Yaoi"), - Genre("Yuri") + Genre("Action"), + Genre("Adult"), + Genre("Adventure"), + Genre("Comedy"), + Genre("Doujinshi"), + Genre("Drama"), + Genre("Ecchi"), + Genre("Fantasy"), + Genre("Gender Bender"), + Genre("Harem"), + Genre("Hentai"), + Genre("Historical"), + Genre("Horror"), + Genre("Josei"), + Genre("Lolicon"), + Genre("Martial Arts"), + Genre("Mature"), + Genre("Mecha"), + Genre("Mystery"), + Genre("Psychological"), + Genre("Romance"), + Genre("School Life"), + Genre("Sci-fi"), + Genre("Seinen"), + Genre("Shotacon"), + Genre("Shoujo"), + Genre("Shoujo Ai"), + Genre("Shounen"), + Genre("Shounen Ai"), + Genre("Slice of Life"), + Genre("Smut"), + Genre("Sports"), + Genre("Supernatural"), + Genre("Tragedy"), + Genre("Yaoi"), + Genre("Yuri") ) }