MangaLife update (#2021)

* MangaLife update

* Filters

* chapterListParse Refactor

* Fix Chapter Name

* Refactor imageNum

* Refactor pageListParse - chNum

Co-authored-by: happywillow0 <45346080+happywillow0@users.noreply.github.com>
This commit is contained in:
Mike 2020-01-09 19:53:07 -05:00 committed by arkon
parent eaa687aa0b
commit f1403b6439
2 changed files with 226 additions and 140 deletions

View File

@ -5,8 +5,13 @@ ext {
appName = 'Tachiyomi: MangaLife' appName = 'Tachiyomi: MangaLife'
pkgNameSuffix = 'en.mangalife' pkgNameSuffix = 'en.mangalife'
extClass = '.MangaLife' extClass = '.MangaLife'
extVersionCode = 1 extVersionCode = 2
libVersion = '1.2' libVersion = '1.2'
} }
dependencies {
compileOnly 'com.google.code.gson:gson:2.8.2'
compileOnly 'com.github.salomonbrys.kotson:kotson:2.5.0'
}
apply from: "$rootDir/common.gradle" apply from: "$rootDir/common.gradle"

View File

@ -1,71 +1,165 @@
package eu.kanade.tachiyomi.extension.en.mangalife package eu.kanade.tachiyomi.extension.en.mangalife
import eu.kanade.tachiyomi.network.POST import com.github.salomonbrys.kotson.fromJson
import com.github.salomonbrys.kotson.get
import com.github.salomonbrys.kotson.string
import com.google.gson.GsonBuilder
import com.google.gson.JsonArray
import com.google.gson.JsonElement
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.* import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.ParsedHttpSource import eu.kanade.tachiyomi.source.online.HttpSource
import okhttp3.FormBody import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.Headers import okhttp3.Headers
import okhttp3.HttpUrl import okhttp3.OkHttpClient
import okhttp3.Request import okhttp3.Request
import org.jsoup.nodes.Document import okhttp3.Response
import org.jsoup.nodes.Element import rx.Observable
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.regex.Pattern import java.util.Locale
import java.util.concurrent.TimeUnit
class MangaLife : ParsedHttpSource() { /**
* Source responds to requests with their full database as a JsonArray, then sorts/filters it client-side
* We'll take the database on first requests, then do what we want with it
*/
class MangaLife : HttpSource() {
override val name = "MangaLife" override val name = "MangaLife"
override val baseUrl = "https://mangalife.us" override val baseUrl = "https://manga4life.com"
override val lang = "en" override val lang = "en"
override val supportsLatest = true override val supportsLatest = true
private val recentUpdatesPattern = Pattern.compile("(.*?)\\s(\\d+\\.?\\d*)\\s?(Completed)?") override val client: OkHttpClient = network.cloudflareClient.newBuilder()
.connectTimeout(1, TimeUnit.MINUTES)
.readTimeout(1, TimeUnit.MINUTES)
.writeTimeout(1, TimeUnit.MINUTES)
.build()
private val indexPattern = Pattern.compile("-index-(.*?)-") override fun headersBuilder(): Headers.Builder = Headers.Builder()
.add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0")
private val catalogHeaders = Headers.Builder().apply { private val gson = GsonBuilder().setLenient().create()
add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)")
add("Host", "mangalife.us")
}.build()
override fun popularMangaSelector() = "div.requested > div.row" private lateinit var directory: List<JsonElement>
// Popular
override fun fetchPopularManga(page: Int): Observable<MangasPage> {
return if (page == 1) {
client.newCall(popularMangaRequest(page))
.asObservableSuccess()
.map { response ->
popularMangaParse(response)
}
} else {
Observable.just(parseDirectory(page))
}
}
override fun popularMangaRequest(page: Int): Request { override fun popularMangaRequest(page: Int): Request {
val (body, requestUrl) = convertQueryToPost(page, "$baseUrl/search/request.php?sortBy=popularity&sortOrder=descending") return GET("$baseUrl/search/", headers)
return POST(requestUrl, catalogHeaders, body.build())
} }
override fun popularMangaFromElement(element: Element): SManga { // don't use ";" for substringBefore() !
val manga = SManga.create() private fun directoryFromResponse(response: Response): String {
element.select("a.resultLink").first().let { return response.asJsoup().select("script:containsData(MainFunction)").first().data()
manga.setUrlWithoutDomain(it.attr("href")) .substringAfter("vm.Directory = ").substringBefore("vm.GetIntValue").trim()
manga.title = it.text() .replace(";", " ")
}
return manga
} }
override fun popularMangaNextPageSelector() = "button.requestMore" override fun popularMangaParse(response: Response): MangasPage {
directory = gson.fromJson<JsonArray>(directoryFromResponse(response))
.sortedByDescending { it["v"].string }
return parseDirectory(1)
}
override fun searchMangaSelector() = "div.requested > div.row" private fun parseDirectory(page: Int): MangasPage {
val mangas = mutableListOf<SManga>()
val endRange = ((page * 24) - 1).let { if (it <= directory.lastIndex) it else directory.lastIndex }
for (i in (((page - 1) * 24) .. endRange)){
mangas.add(SManga.create().apply {
title = directory[i]["s"].string
url = "/manga/${directory[i]["i"].string}"
thumbnail_url = "https://static.mangaboss.net/cover/${directory[i]["i"].string}.jpg"
})
}
return MangasPage(mangas, endRange < directory.lastIndex)
}
// Latest
override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
return if (page == 1) {
client.newCall(latestUpdatesRequest(page))
.asObservableSuccess()
.map { response ->
latestUpdatesParse(response)
}
} else {
Observable.just(parseDirectory(page))
}
}
override fun latestUpdatesRequest(page: Int): Request = popularMangaRequest(1)
override fun latestUpdatesParse(response: Response): MangasPage {
directory = gson.fromJson<JsonArray>(directoryFromResponse(response))
.sortedByDescending { it["lt"].string }
return parseDirectory(1)
}
// Search
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
return if (page == 1) {
client.newCall(searchMangaRequest(page, query, filters))
.asObservableSuccess()
.map { response ->
searchMangaParse(response, query, filters)
}
} else {
Observable.just(parseDirectory(page))
}
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = popularMangaRequest(1)
private fun searchMangaParse(response: Response, query: String, filters: FilterList): MangasPage {
directory = gson.fromJson<JsonArray>(directoryFromResponse(response))
.filter { it["s"].string.contains(query, ignoreCase = true) }
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = HttpUrl.parse("$baseUrl/search/request.php")!!.newBuilder()
if (!query.isEmpty()) url.addQueryParameter("keyword", query)
val genres = mutableListOf<String>() val genres = mutableListOf<String>()
val genresNo = mutableListOf<String>() val genresNo = mutableListOf<String>()
var sortBy: String
for (filter in if (filters.isEmpty()) getFilterList() else filters) { for (filter in if (filters.isEmpty()) getFilterList() else filters) {
when (filter) { when (filter) {
is Sort -> { is Sort -> {
if (filter.state?.index != 0) sortBy = when (filter.state?.index) {
url.addQueryParameter("sortBy", if (filter.state?.index == 1) "dateUpdated" else "popularity") 1 -> "ls"
if (filter.state?.ascending != true) 2 -> "v"
url.addQueryParameter("sortOrder", "descending") else -> "s"
} }
is SelectField -> if (filter.state != 0) url.addQueryParameter(filter.key, filter.values[filter.state]) directory = if (filter.state?.ascending != true) {
is TextField -> if (!filter.state.isEmpty()) url.addQueryParameter(filter.key, filter.state) directory.sortedByDescending { it[sortBy].string }
} else {
directory.sortedByDescending { it[sortBy].string }.reversed()
}
}
is SelectField -> if (filter.state != 0) directory = when (filter.name) {
"Scan Status" -> directory.filter { it["ss"].string.contains(filter.values[filter.state], ignoreCase = true) }
"Publish Status" -> directory.filter { it["ps"].string.contains(filter.values[filter.state], ignoreCase = true) }
"Type" -> directory.filter { it["t"].string.contains(filter.values[filter.state], ignoreCase = true) }
else -> directory
}
is YearField -> if (filter.state.isNotEmpty()) directory = directory.filter { it["y"].string.contains(filter.state) }
is AuthorField -> if (filter.state.isNotEmpty()) directory = directory.filter { e -> e["a"].asJsonArray.any { it.string.contains(filter.state, ignoreCase = true) } }
is GenreList -> filter.state.forEach { genre -> is GenreList -> filter.state.forEach { genre ->
when (genre.state) { when (genre.state) {
Filter.TriState.STATE_INCLUDE -> genres.add(genre.name) Filter.TriState.STATE_INCLUDE -> genres.add(genre.name)
@ -74,131 +168,118 @@ class MangaLife : ParsedHttpSource() {
} }
} }
} }
if (genres.isNotEmpty()) url.addQueryParameter("genre", genres.joinToString(",")) if (genres.isNotEmpty()) genres.map { genre -> directory = directory.filter { e -> e["g"].asJsonArray.any { it.string.contains(genre, ignoreCase = true) } } }
if (genresNo.isNotEmpty()) url.addQueryParameter("genreNo", genresNo.joinToString(",")) if (genresNo.isNotEmpty()) genresNo.map { genre -> directory = directory.filterNot { e -> e["g"].asJsonArray.any { it.string.contains(genre, ignoreCase = true) } } }
val (body, requestUrl) = convertQueryToPost(page, url.toString()) return parseDirectory(1)
return POST(requestUrl, catalogHeaders, body.build())
} }
private fun convertQueryToPost(page: Int, url: String): Pair<FormBody.Builder, String> { override fun searchMangaParse(response: Response): MangasPage = throw UnsupportedOperationException("Not used")
val url = HttpUrl.parse(url)!!
val body = FormBody.Builder().add("page", page.toString()) // Details
for (i in 0 until url.querySize()) {
body.add(url.queryParameterName(i), url.queryParameterValue(i)) override fun mangaDetailsParse(response: Response): SManga {
return response.asJsoup().select("div.BoxBody > div.row").let { info ->
SManga.create().apply {
title = info.select("h1").text()
author = info.select("li.list-group-item:has(span:contains(Author)) a").first()?.text()
genre = info.select("li.list-group-item:has(span:contains(Genre)) a").joinToString { it.text() }
status = info.select("li.list-group-item:has(span:contains(Status)) a:contains(publish)").text().toStatus()
description = info.select("div.Content").text()
thumbnail_url = info.select("img").attr("abs:src")
}
} }
val requestUrl = url.scheme() + "://" + url.host() + url.encodedPath()
return Pair(body, requestUrl)
} }
override fun searchMangaFromElement(element: Element): SManga { private fun String.toStatus() = when {
val manga = SManga.create() this.contains("Ongoing", ignoreCase = true) -> SManga.ONGOING
element.select("a.resultLink").first().let { this.contains("Complete", ignoreCase = true) -> SManga.COMPLETED
manga.setUrlWithoutDomain(it.attr("href"))
manga.title = it.text()
}
return manga
}
override fun searchMangaNextPageSelector() = "button.requestMore"
override fun mangaDetailsParse(document: Document): SManga {
val detailElement = document.select("div.well > div.row").first()
val manga = SManga.create()
manga.author = detailElement.select("a[href^=/search/?author=]").first()?.text()
manga.genre = detailElement.select("span.details > div.row > div:has(b:contains(Genre(s))) > a").map { it.text() }.joinToString()
manga.description = detailElement.select("strong:contains(Description:) + div").first()?.text()
manga.status = detailElement.select("a[href^=/search/?status=]").first()?.text().orEmpty().let { parseStatus(it) }
manga.thumbnail_url = detailElement.select("div > img").first()?.absUrl("src")
return manga
}
private fun parseStatus(status: String) = when {
status.contains("Ongoing (Scan)") -> SManga.ONGOING
status.contains("Complete (Scan)") -> SManga.COMPLETED
else -> SManga.UNKNOWN else -> SManga.UNKNOWN
} }
override fun chapterListSelector() = "div.chapter-list > a" // Chapters - Mind special cases like decimal chapters (e.g. One Punch Man) and manga with seasons (e.g. The Gamer)
override fun chapterFromElement(element: Element): SChapter { private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.getDefault())
val urlElement = element.select("a").first()
val chapter = SChapter.create() private fun chapterURLEncode(e: String ):String {
chapter.setUrlWithoutDomain(urlElement.attr("href"))
chapter.name = element.select("span.chapterLabel").first().text()?.let { it } ?: ""
chapter.date_upload = element.select("time").first()?.attr("datetime")?.let { parseChapterDate(it) } ?: 0
return chapter
}
private fun parseChapterDate(dateAsString: String): Long {
return SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").parse(dateAsString).time
}
override fun pageListParse(document: Document): List<Page> {
val fullUrl = document.baseUri()
val url = fullUrl.substringBeforeLast('/')
val pages = mutableListOf<Page>()
val series = document.select("input.IndexName").first().attr("value")
val chapter = document.select("span.CurChapter").first().text()
var index = "" var index = ""
val t = e.substring(0,1).toInt()
val m = indexPattern.matcher(fullUrl) if (1 != t) { index = "-index-$t" }
if (m.find()) { val n = e.substring(1,e.length-1)
val indexNumber = m.group(1) var suffix = ""
index = "-index-$indexNumber" val path = e.substring(e.length-1).toInt()
if (0 != path) {suffix = ".$path"}
return "-chapter-$n$index$suffix.html"
} }
document.select("div.ContainerNav").first().select("select.PageSelect > option").forEach { private fun chapterImage(e: String): String {
pages.add(Page(pages.size, "$url/$series-chapter-$chapter$index-page-${pages.size + 1}.html")) val a = e.substring(1,e.length-1)
val b = e.substring(e.length-1).toInt()
return if (b == 0) {
a
} else {
"$a.$b"
} }
pages.getOrNull(0)?.imageUrl = imageUrlParse(document)
return pages
} }
override fun imageUrlParse(document: Document): String = document.select("img.CurImage").attr("src") override fun chapterListParse(response: Response): List<SChapter> {
val vmChapters = response.asJsoup().select("script:containsData(MainFunction)").first().data()
.substringAfter("vm.Chapters = ").substringBefore(";")
override fun latestUpdatesNextPageSelector() = "button.requestMore" return gson.fromJson<JsonArray>(vmChapters).map{ json ->
val indexChapter = json["Chapter"].string
override fun latestUpdatesSelector(): String = "a.latestSeries" SChapter.create().apply {
name = json["ChapterName"].string.let { if (it.isNotEmpty()) it else "${json["Type"].string} ${chapterImage(indexChapter)}" }
override fun latestUpdatesRequest(page: Int): Request { url = "/read-online/" + response.request().url().toString().substringAfter("/manga/") + chapterURLEncode(indexChapter)
val url = "$baseUrl/home/latest.request.php" date_upload = try {
val (body, requestUrl) = convertQueryToPost(page, url) dateFormat.parse(json["Date"].string.substringBefore(" ")).time
return POST(requestUrl, catalogHeaders, body.build()) } catch (_: Exception) {
0L
}
}
}
} }
override fun latestUpdatesFromElement(element: Element): SManga { // Pages
val manga = SManga.create()
element.select("a.latestSeries").first().let { override fun pageListParse(response: Response): List<Page> {
val chapterUrl = it.attr("href") val document = response.asJsoup()
val indexOfMangaUrl = chapterUrl.indexOf("-chapter-") val script = document.select("script:containsData(MainFunction)").first().data()
val indexOfLastPath = chapterUrl.lastIndexOf("/") val curChapter = gson.fromJson<JsonElement>(script.substringAfter("vm.CurChapter = ").substringBefore(";"))
val mangaUrl = chapterUrl.substring(indexOfLastPath, indexOfMangaUrl)
val defaultText = it.select("p.clamp2").text() val pageTotal = curChapter["Page"].string.toInt()
val m = recentUpdatesPattern.matcher(defaultText)
val title = if (m.matches()) m.group(1) else defaultText val host = "https://" + script.substringAfter("vm.CurPathName = \"").substringBefore("\"")
manga.setUrlWithoutDomain("/manga$mangaUrl") val titleURI = script.substringAfter("vm.IndexName = \"").substringBefore("\"")
manga.title = title val seasonURI = curChapter["Directory"].string
.let { if (it.isEmpty()) "" else "$it/" }
val path = "$host/manga/$titleURI/$seasonURI"
var chNum = chapterImage(curChapter["Chapter"].string)
return IntRange(1, pageTotal).mapIndexed { i, _ ->
var imageNum = (i + 1).toString().let { "000$it" }.let { it.substring(it.length-3) }
Page(i, "", path + "$chNum-$imageNum.png")
} }
return manga
} }
private class Sort : Filter.Sort("Sort", arrayOf("Alphabetically", "Date updated", "Popularity"), Filter.Sort.Selection(2, false)) override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException("Not used")
// Filters
private class Sort : Filter.Sort("Sort", arrayOf("Alphabetically", "Date updated", "Popularity"), Selection(2, false))
private class Genre(name: String) : Filter.TriState(name) private class Genre(name: String) : Filter.TriState(name)
private class TextField(name: String, val key: String) : Filter.Text(name) private class YearField : Filter.Text("Years")
private class SelectField(name: String, val key: String, values: Array<String>, state: Int = 0) : Filter.Select<String>(name, values, state) private class AuthorField : Filter.Text("Author")
private class SelectField(name: String, values: Array<String>, state: Int = 0) : Filter.Select<String>(name, values, state)
private class GenreList(genres: List<Genre>) : Filter.Group<Genre>("Genres", genres) private class GenreList(genres: List<Genre>) : Filter.Group<Genre>("Genres", genres)
override fun getFilterList() = FilterList( override fun getFilterList() = FilterList(
TextField("Years", "year"), YearField(),
TextField("Author", "author"), AuthorField(),
SelectField("Scan Status", "status", arrayOf("Any", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing")), SelectField("Scan Status", arrayOf("Any", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing")),
SelectField("Publish Status", "pstatus", arrayOf("Any", "Cancelled", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing", "Unfinished")), SelectField("Publish Status", arrayOf("Any", "Cancelled", "Complete", "Discontinued", "Hiatus", "Incomplete", "Ongoing", "Unfinished")),
SelectField("Type", "type", arrayOf("Any", "Doujinshi", "Manga", "Manhua", "Manhwa", "OEL", "One-shot")), SelectField("Type", arrayOf("Any", "Doujinshi", "Manga", "Manhua", "Manhwa", "OEL", "One-shot")),
Sort(), Sort(),
GenreList(getGenreList()) GenreList(getGenreList())
) )