FMReader - new factory extension (#1561)

* LHReader - new factory extension

* progress

* progress 2

* Update LHReader.kt

* Rename, last changes before review
This commit is contained in:
Mike 2019-10-09 18:58:55 -04:00 committed by arkon
parent 3b17675878
commit a27ea45025
9 changed files with 646 additions and 0 deletions

View File

@ -0,0 +1,12 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
ext {
appName = 'Tachiyomi: FMReader (multiple aggregators)'
pkgNameSuffix = 'all.fmreader'
extClass = '.FMReaderFactory'
extVersionCode = 1
libVersion = '1.2'
}
apply from: "$rootDir/common.gradle"

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 72 KiB

View File

@ -0,0 +1,392 @@
package eu.kanade.tachiyomi.extension.all.fmreader
// For sites based on the Flat-Manga CMS
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.*
import java.util.*
abstract class FMReader (
override val name: String,
override val baseUrl: String,
override val lang: String
) : ParsedHttpSource() {
override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient
override fun headersBuilder() = Headers.Builder().apply {
add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64) Gecko/20100101 Firefox/69.0")
add("Referer", baseUrl)
}
open val requestPath = "manga-list.html"
override fun popularMangaRequest(page: Int): Request =
GET("$baseUrl/$requestPath?listType=pagination&page=$page&sort=views&sort_type=DESC", headers)
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = HttpUrl.parse("$baseUrl/$requestPath?")!!.newBuilder()
.addQueryParameter("name", query)
.addQueryParameter("page", page.toString())
(if (filters.isEmpty()) getFilterList() else filters).forEach { filter ->
when (filter) {
is Status -> {
val status = arrayOf("", "1", "2")[filter.state]
url.addQueryParameter("m_status", status)
}
is TextField -> url.addQueryParameter(filter.key, filter.state)
is GenreList -> {
var genre = String()
var ungenre = String()
filter.state.forEach {
if (it.isIncluded()) genre += ",${it.name}"
if (it.isExcluded()) ungenre += ",${it.name}"
}
url.addQueryParameter("genre", genre)
url.addQueryParameter("ungenre", ungenre)
}
is SortBy -> {
url.addQueryParameter("sort", when {
filter.state?.index == 0 -> "name"
filter.state?.index == 1 -> "views"
else -> "last_update"
})
if (filter.state?.ascending == true)
url.addQueryParameter("sort_type", "ASC")
}
}
}
return GET(url.toString(), headers)
}
override fun latestUpdatesRequest(page: Int): Request =
GET("$baseUrl/$requestPath?listType=pagination&page=$page&sort=last_update&sort_type=DESC")
// for sources that don't have the "page x of y" element
fun defaultMangaParse(response: Response): MangasPage = super.popularMangaParse(response)
override fun popularMangaParse(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = mutableListOf<SManga>()
var hasNextPage = true
document.select(popularMangaSelector()).map{ mangas.add(popularMangaFromElement(it)) }
// check if there's a next page
document.select(popularMangaNextPageSelector()).first().text().split(" ").let {
val currentPage = it[1]
val lastPage = it[3]
if (currentPage == lastPage) hasNextPage = false
}
return MangasPage(mangas, hasNextPage)
}
override fun latestUpdatesParse(response: Response) = popularMangaParse(response)
override fun searchMangaParse(response: Response) = popularMangaParse(response)
override fun popularMangaSelector() = "div.media"
override fun latestUpdatesSelector() = popularMangaSelector()
override fun searchMangaSelector() = popularMangaSelector()
override fun popularMangaFromElement(element: Element): SManga {
val manga = SManga.create()
element.select("h3 a").let {
manga.setUrlWithoutDomain(it.attr("abs:href"))
manga.title = it.text()
}
manga.thumbnail_url = element.select("img").let{
if (it.hasAttr("src")) {
it.attr("abs:src")
} else {
it.attr("abs:data-original")
}
}
return manga
}
override fun latestUpdatesFromElement(element: Element): SManga =
popularMangaFromElement(element)
override fun searchMangaFromElement(element: Element): SManga = popularMangaFromElement(element)
// selects an element with text "x of y pages", must be first element if multiple elements are selected
override fun popularMangaNextPageSelector() = "div.col-lg-9 button.btn-info"
override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
override fun mangaDetailsParse(document: Document): SManga {
val manga = SManga.create()
val infoElement = document.select("div.row").first()
manga.author = infoElement.select("li a.btn-info").text()
manga.genre = infoElement.select("li a.btn-danger").joinToString { it.text() }
manga.status = parseStatus(infoElement.select("li a.btn-success").first().text().toLowerCase())
manga.description = document.select("div.row ~ div.row p").text().trim()
manga.thumbnail_url = infoElement.select("img.thumbnail").attr("abs:src")
return manga
}
// languages: en, vi, tr
fun parseStatus(element: String): Int = when (element) {
"completed", "complete", "incomplete", "đã hoàn thành", "tamamlandı" -> SManga.COMPLETED
"ongoing", "on going", "updating", "chưa hoàn thành", "đang cập nhật", "devam ediyor" -> SManga.ONGOING
else -> SManga.UNKNOWN
}
override fun chapterListSelector() = "div#list-chapters p, table.table tr"
open val chapterUrlSelector = "a"
open val chapterTimeSelector = "time"
override fun chapterFromElement(element: Element): SChapter {
val chapter = SChapter.create()
element.select(chapterUrlSelector).first().let{
chapter.setUrlWithoutDomain(it.attr("abs:href"))
chapter.name = it.text()
}
chapter.date_upload = element.select(chapterTimeSelector).let{ if(it.hasText()) parseChapterDate(it.text()) else 0 }
return chapter
}
// gets the number from "1 day ago"
open val dateValueIndex = 0
// gets the unit of time (day, week hour) from "1 day ago"
open val dateWordIndex = 1
private fun parseChapterDate(date: String): Long {
val value = date.split(' ')[dateValueIndex].toInt()
val dateWord = date.split(' ')[dateWordIndex].let{
if (it.contains("(")) {
it.substringBefore("(")
} else {
it.substringBefore("s")
}
}
// languages: en, vi, es, tr
return when (dateWord) {
"min", "minute", "phút", "minuto", "dakika" -> Calendar.getInstance().apply {
add(Calendar.MINUTE, value * -1)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
"hour", "giờ", "hora", "saat" -> Calendar.getInstance().apply {
add(Calendar.HOUR_OF_DAY, value * -1)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
"day", "ngày", "día", "gün" -> Calendar.getInstance().apply {
add(Calendar.DATE, value * -1)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
"week", "tuần", "semana", "hafta" -> Calendar.getInstance().apply {
add(Calendar.DATE, value * 7 * -1)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
"month", "tháng", "mes", "ay" -> Calendar.getInstance().apply {
add(Calendar.MONTH, value * -1)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
"year", "năm", "año", "yıl" -> Calendar.getInstance().apply {
add(Calendar.YEAR, value * -1)
set(Calendar.SECOND, 0)
set(Calendar.MILLISECOND, 0)
}.timeInMillis
else -> {
return 0
}
}
}
override fun pageListParse(document: Document): List<Page> {
val pages = mutableListOf<Page>()
document.select("img.chapter-img").forEachIndexed { i, img ->
pages.add(Page(i, "", img.attr("abs:data-src").let{ if (it.isNotEmpty()) it else img.attr("abs:src") }))
}
return pages
}
override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not used")
private class TextField(name: String, val key: String) : Filter.Text(name)
private class Status : Filter.Select<String>("Status", arrayOf("Any", "Completed", "Ongoing"))
private class GenreList(genres: List<Genre>) : Filter.Group<Genre>("Genre", genres)
class Genre(name: String, val id: String = name.replace(' ', '+')) : Filter.TriState(name)
private class SortBy : Filter.Sort("Sorted By", arrayOf("A-Z", "Most vỉews", "Last updated"), Selection(1, false))
// TODO: Country (leftover from original LHTranslation)
override fun getFilterList() = FilterList(
TextField("Author", "author"),
TextField("Group", "group"),
Status(),
SortBy(),
GenreList(getGenreList())
)
// [...document.querySelectorAll("div.panel-body a")].map((el,i) => `Genre("${el.innerText.trim()}")`).join(',\n')
// on https://lhtranslation.net/search
open fun getGenreList() = listOf(
Genre("Action"),
Genre("18+"),
Genre("Adult"),
Genre("Anime"),
Genre("Comedy"),
Genre("Comic"),
Genre("Doujinshi"),
Genre("Drama"),
Genre("Ecchi"),
Genre("Fantasy"),
Genre("Gender Bender"),
Genre("Harem"),
Genre("Historical"),
Genre("Horror"),
Genre("Josei"),
Genre("Live action"),
Genre("Manhua"),
Genre("Manhwa"),
Genre("Martial Art"),
Genre("Mature"),
Genre("Mecha"),
Genre("Mystery"),
Genre("One shot"),
Genre("Psychological"),
Genre("Romance"),
Genre("School Life"),
Genre("Sci-fi"),
Genre("Seinen"),
Genre("Shoujo"),
Genre("Shojou Ai"),
Genre("Shounen"),
Genre("Shounen Ai"),
Genre("Slice of Life"),
Genre("Smut"),
Genre("Sports"),
Genre("Supernatural"),
Genre("Tragedy"),
Genre("Adventure"),
Genre("Yaoi")
)
// from manhwa18.com/search, removed a few that didn't return results/wouldn't be terribly useful
fun getAdultGenreList() = listOf(
Genre("18"),
Genre("Action"),
Genre("Adult"),
Genre("Adventure"),
Genre("Anime"),
Genre("Comedy"),
Genre("Comic"),
Genre("Doujinshi"),
Genre("Drama"),
Genre("Ecchi"),
Genre("Fantasy"),
Genre("Gender Bender"),
Genre("Harem"),
Genre("Historical"),
Genre("Horror"),
Genre("Josei"),
Genre("Live action"),
Genre("Magic"),
Genre("Manhua"),
Genre("Manhwa"),
Genre("Martial Arts"),
Genre("Mature"),
Genre("Mecha"),
Genre("Mystery"),
Genre("Oneshot"),
Genre("Psychological"),
Genre("Romance"),
Genre("School Life"),
Genre("Sci-fi"),
Genre("Seinen"),
Genre("Shoujo"),
Genre("Shoujo Ai"),
Genre("Shounen"),
Genre("Shounen Ai"),
Genre("Slice of life"),
Genre("Smut"),
Genre("Soft Yaoi"),
Genre("Soft Yuri"),
Genre("Sports"),
Genre("Supernatural"),
Genre("Tragedy"),
Genre("VnComic"),
Genre("Webtoon")
)
// taken from readcomiconline.org/search
fun getComicsGenreList() = listOf(
Genre("Action"),
Genre("Adventure"),
Genre("Anthology"),
Genre("Anthropomorphic"),
Genre("Biography"),
Genre("Children"),
Genre("Comedy"),
Genre("Crime"),
Genre("Drama"),
Genre("Family"),
Genre("Fantasy"),
Genre("Fighting"),
Genre("GraphicNovels"),
Genre("Historical"),
Genre("Horror"),
Genre("LeadingLadies"),
Genre("LGBTQ"),
Genre("Literature"),
Genre("Manga"),
Genre("MartialArts"),
Genre("Mature"),
Genre("Military"),
Genre("Mystery"),
Genre("Mythology"),
Genre("Personal"),
Genre("Political"),
Genre("Post-Apocalyptic"),
Genre("Psychological"),
Genre("Pulp"),
Genre("Religious"),
Genre("Robots"),
Genre("Romance"),
Genre("Schoollife"),
Genre("Sci-Fi"),
Genre("Sliceoflife"),
Genre("Sport"),
Genre("Spy"),
Genre("Superhero"),
Genre("Supernatural"),
Genre("Suspense"),
Genre("Thriller"),
Genre("Vampires"),
Genre("VideoGames"),
Genre("War"),
Genre("Western"),
Genre("Zombies")
)
}

View File

@ -0,0 +1,242 @@
package eu.kanade.tachiyomi.extension.all.fmreader
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.SourceFactory
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.*
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import rx.Observable
class FMReaderFactory : SourceFactory {
override fun createSources(): List<Source> = listOf(
LHTranslation(),
MangaHato(),
ManhwaScan(),
MangaTiki(),
MangaBone(),
YoloManga(),
MangaLeer(),
AiLoveManga(),
ReadComicOnlineOrg(),
MangaWeek(),
HanaScan(),
RawLH(),
Manhwa18(),
TruyenTranhLH(),
EighteenLHPlus(),
MangaTR(),
Comicastle()
)
}
/** For future sources: when testing and popularMangaRequest() returns a Jsoup error instead of results
* most likely the fix is to override popularMangaNextPageSelector() */
class LHTranslation : FMReader("LHTranslation", "https://lhtranslation.net", "en")
class MangaHato : FMReader("MangaHato", "https://mangahato.com", "ja")
class ManhwaScan : FMReader("ManhwaScan", "https://manhwascan.com", "en")
class MangaTiki : FMReader("MangaTiki", "https://mangatiki.com", "ja")
class MangaBone : FMReader("MangaBone", "https://mangabone.com", "en")
class YoloManga : FMReader("Yolo Manga", "https://yolomanga.ca", "es") {
override fun chapterListSelector() = "div#tab-chapper ~ div#tab-chapper table tr"
}
class MangaLeer : FMReader("MangaLeer", "https://mangaleer.com", "es") {
override val dateValueIndex = 1
override val dateWordIndex = 2
}
class AiLoveManga : FMReader("AiLoveManga", "https://ailovemanga.com", "vi") {
override val requestPath = "danh-sach-truyen.html"
// TODO: could add a genre search (different URL paths for genres)
override fun getFilterList() = FilterList()
// I don't know why, but I have to override searchMangaRequest to make it work for this source
override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = GET("$baseUrl/$requestPath?name=$query&page=$page")
override fun chapterListSelector() = "div#tab-chapper table tr"
override fun mangaDetailsParse(document: Document): SManga {
val manga = SManga.create()
val infoElement = document.select("div.container:has(img)").first()
manga.author = infoElement.select("a.btn-info").first().text()
manga.artist = infoElement.select("a.btn-info + a").text()
manga.genre = infoElement.select("a.btn-danger").joinToString { it.text() }
manga.status = parseStatus(infoElement.select("a.btn-success").text().toLowerCase())
manga.description = document.select("div.col-sm-8 p").text().trim()
manga.thumbnail_url = infoElement.select("img").attr("abs:src")
return manga
}
}
class ReadComicOnlineOrg : FMReader("ReadComicOnline.org", "https://readcomiconline.org", "en") {
override val client: OkHttpClient = network.cloudflareClient.newBuilder()
.addInterceptor { requestIntercept(it) }
.build()
private fun requestIntercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
return if (response.headers("set-cookie").isNotEmpty()) {
val body = FormBody.Builder()
.add("dqh_firewall", "%2F")
.build()
val cookie = mutableListOf<String>()
response.headers("set-cookie").map{ cookie.add(it.substringBefore(" ")) }
headers.newBuilder().add("Cookie", cookie.joinToString { " " }).build()
client.newCall(POST(request.url().toString(), headers, body)).execute()
} else {
response
}
}
override val requestPath = "comic-list.html"
override fun pageListParse(document: Document): List<Page> {
val pages = mutableListOf<Page>()
document.select("div#divImage > select:first-of-type option").forEachIndexed{ i, imgPage ->
pages.add(Page(i, imgPage.attr("value"), ""))
}
return pages.dropLast(1) // last page is a comments page
}
override fun imageUrlRequest(page: Page): Request = GET(baseUrl + page.url, headers)
override fun imageUrlParse(document: Document): String = document.select("img.chapter-img").attr("abs:src").trim()
override fun getGenreList() = getComicsGenreList()
}
class MangaWeek : FMReader("MangaWeek", "https://mangaweek.com", "en")
class HanaScan : FMReader("HanaScan (RawQQ)", "http://rawqq.com", "ja") {
override fun popularMangaNextPageSelector() = "div.col-md-8 button"
}
class RawLH : FMReader("RawLH", "https://lhscan.net", "ja") {
override fun popularMangaNextPageSelector() = "div.col-md-8 button"
}
class Manhwa18 : FMReader("Manhwa18", "https://manhwa18.com", "en") {
override fun getGenreList() = getAdultGenreList()
}
class TruyenTranhLH : FMReader("TruyenTranhLH", "https://truyentranhlh.net", "vi") {
override val requestPath = "danh-sach-truyen.html"
}
class EighteenLHPlus : FMReader("18LHPlus", "https://18lhplus.com", "en") {
override fun getGenreList() = getAdultGenreList()
}
class MangaTR : FMReader("Manga-TR", "https://manga-tr.com", "tr") {
override fun popularMangaNextPageSelector() = "div.btn-group:not(div.btn-block) button.btn-info"
// TODO: genre search possible but a bit of a pain
override fun getFilterList() = FilterList()
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = GET("$baseUrl/arama.html?icerik=$query", headers)
override fun searchMangaParse(response: Response): MangasPage {
val mangas = mutableListOf<SManga>()
response.asJsoup().select("div.row a[data-toggle]")
.filterNot { it.siblingElements().text().contains("Novel") }
.map { mangas.add(searchMangaFromElement(it)) }
return MangasPage(mangas, false)
}
override fun searchMangaFromElement(element: Element): SManga {
val manga = SManga.create()
manga.setUrlWithoutDomain(element.attr("abs:href"))
manga.title = element.text()
return manga
}
override fun mangaDetailsParse(document: Document): SManga {
val manga = SManga.create()
val infoElement = document.select("div#tab1").first()
manga.author = infoElement.select("table + table tr + tr td a").first()?.text()
manga.artist = infoElement.select("table + table tr + tr td + td a").first()?.text()
manga.genre = infoElement.select("div#tab1 table + table tr + tr td + td + td").text()
manga.status = parseStatus(infoElement.select("div#tab1 table tr + tr td a").first().text().toLowerCase())
manga.description = infoElement.select("div.well").text().trim()
manga.thumbnail_url = document.select("img.thumbnail").attr("abs:src")
return manga
}
override fun chapterListSelector() = "tr.table-bordered"
override val chapterUrlSelector = "td[align=left] > a"
override val chapterTimeSelector = "td[align=right]"
private val chapterListHeaders = headers.newBuilder().add("X-Requested-With", "XMLHttpRequest").build()
override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
return if (manga.status != SManga.LICENSED) {
val requestUrl = "$baseUrl/cek/fetch_pages_manga.php?manga_cek=${manga.url.substringAfter("manga-").substringBefore(".")}"
client.newCall(GET(requestUrl, chapterListHeaders))
.asObservableSuccess()
.map { response ->
chapterListParse(response, requestUrl)
}
} else {
Observable.error(Exception("Licensed - No chapters to show"))
}
}
private fun chapterListParse(response: Response, requestUrl: String): List<SChapter> {
val chapters = mutableListOf<SChapter>()
var document = response.asJsoup()
var moreChapters = true
var nextPage = 2
// chapters are paginated
while (moreChapters) {
document.select(chapterListSelector()).map{ chapters.add(chapterFromElement(it)) }
if (document.select("a[data-page=$nextPage]").isNotEmpty()) {
val body = FormBody.Builder()
.add("page", nextPage.toString())
.build()
document = client.newCall(POST(requestUrl, chapterListHeaders, body)).execute().asJsoup()
nextPage++
} else {
moreChapters = false
}
}
return chapters
}
override fun pageListRequest(chapter: SChapter): Request = GET("$baseUrl/${chapter.url.substringAfter("cek/")}", headers)
override fun pageListParse(document: Document): List<Page> {
val pages = mutableListOf<Page>()
document.select("div.chapter-content select:first-of-type option").forEachIndexed{ i, imgPage ->
pages.add(Page(i, "$baseUrl/${imgPage.attr("value")}", ""))
}
return pages.dropLast(1) // last page is a comments page
}
override fun imageUrlParse(document: Document): String = document.select("img.chapter-img").attr("abs:src").trim()
}
class Comicastle : FMReader("Comicastle", "https://www.comicastle.org", "en") {
override val requestPath = "comic-dir"
// this source doesn't have the "page x of y" element
override fun popularMangaNextPageSelector() = "li:contains(»)"
override fun popularMangaParse(response: Response) = defaultMangaParse(response)
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = GET("$baseUrl/comic-dir?q=$query", headers)
override fun searchMangaParse(response: Response): MangasPage = defaultMangaParse(response)
override fun getFilterList() = FilterList()
override fun mangaDetailsParse(document: Document): SManga {
val manga = SManga.create()
val infoElement = document.select("div.col-md-9").first()
manga.author = infoElement.select("tr + tr td a").first().text()
manga.artist = infoElement.select("tr + tr td + td a").text()
manga.genre = infoElement.select("tr + tr td + td + td").text()
manga.description = infoElement.select("p").text().trim()
manga.thumbnail_url = document.select("img.manga-cover").attr("abs:src")
return manga
}
override fun chapterListSelector() = "div.col-md-9 table:last-of-type tr"
override fun chapterListParse(response: Response): List<SChapter> = super.chapterListParse(response).reversed()
override fun pageListParse(document: Document): List<Page> {
val pages = mutableListOf<Page>()
document.select("div.text-center select option").forEachIndexed{ i, imgPage ->
pages.add(Page(i, imgPage.attr("value"), ""))
}
return pages
}
override fun imageUrlParse(document: Document): String = document.select("img.chapter-img").attr("abs:src").trim()
override fun getGenreList() = getComicsGenreList()
}