Remove MP sources due to cat-and-mouse game (#8679)

* Remove MP sources due to cat-and-mouse game.

* Add sources to the autocloser.
This commit is contained in:
Alessandro Jean 2021-08-20 16:11:55 -03:00 committed by GitHub
parent 42f573c8ce
commit 60037bfe4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1 additions and 566 deletions

View File

@ -32,7 +32,7 @@ jobs:
},
{
"type": "both",
"regex": ".*(mangago|mangafox|hq\\s*dragon|manga\\s*host|supermangas|superhentais|union\\s*mangas|yes\\s*mangas|manhuascan|heroscan|manhwahot).*",
"regex": ".*(mangago|mangafox|hq\\s*dragon|manga\\s*host|supermangas|superhentais|union\\s*mangas|yes\\s*mangas|manhuascan|heroscan|manhwahot|leitor\\.?net|manga\\s*livre).*",
"ignoreCase": true,
"message": "{match} will not be added back as it is too difficult to maintain. Read #3475 for more information"
},

View File

@ -1,4 +0,0 @@
dependencies {
implementation project(':lib-ratelimit')
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 68 KiB

View File

@ -1,47 +0,0 @@
package eu.kanade.tachiyomi.extension.pt.leitornet
import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor
import eu.kanade.tachiyomi.multisrc.mangasproject.MangasProject
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.SChapter
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import java.util.concurrent.TimeUnit
class LeitorNet : MangasProject("Leitor.net", "https://leitor.net", "pt-BR") {
// Use the old generated id when the source did have the name "mangásPROJECT" and
// did have mangas in their catalogue. Now they "only have webtoons" and
// became a different website, but they still use the same structure.
// Existing mangas and other titles in the library still work.
override val id: Long = 2225174659569980836
override val client: OkHttpClient = super.client.newBuilder()
.addInterceptor(RateLimitInterceptor(1, 3, TimeUnit.SECONDS))
.build()
override val licensedCheck = true
/**
* Temporary fix to bypass Cloudflare.
*/
override fun pageListRequest(chapter: SChapter): Request {
val newHeaders = super.pageListRequest(chapter).headers.newBuilder()
.set("Referer", "https://mangalivre.net/home")
.build()
val newChapterUrl = chapter.url
.replace("/manga/", "/ler/")
.replace("/(\\d+)/capitulo-".toRegex(), "/online/$1/capitulo-")
return GET("https://mangalivre.net$newChapterUrl", newHeaders)
}
override fun getChapterUrl(response: Response): String {
return super.getChapterUrl(response)
.replace("https://mangalivre.net", baseUrl)
.replace("/ler/", "/manga/")
.replace("/online/", "/")
}
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 69 KiB

View File

@ -1,76 +0,0 @@
package eu.kanade.tachiyomi.extension.pt.mangalivre
import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor
import eu.kanade.tachiyomi.multisrc.mangasproject.MangasProject
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import java.util.concurrent.TimeUnit
class MangaLivre : MangasProject("Mangá Livre", "https://mangalivre.net", "pt-BR") {
// Hardcode the id because the language wasn't specific.
override val id: Long = 4762777556012432014
override val client: OkHttpClient = super.client.newBuilder()
.addInterceptor(RateLimitInterceptor(1, 3, TimeUnit.SECONDS))
.build()
override fun popularMangaRequest(page: Int): Request {
val originalRequestUrl = super.popularMangaRequest(page).url.toString()
return GET(originalRequestUrl + DEFAULT_TYPE, sourceHeaders)
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
if (query.isNotEmpty()) {
return super.searchMangaRequest(page, query, filters)
}
val popularRequestUrl = super.popularMangaRequest(page).url.toString()
val type = filters.filterIsInstance<TypeFilter>()
.firstOrNull()?.selected?.value ?: DEFAULT_TYPE
return GET(popularRequestUrl + type, sourceHeaders)
}
override fun searchMangaParse(response: Response): MangasPage {
if (response.request.url.pathSegments.contains("search")) {
return super.searchMangaParse(response)
}
return popularMangaParse(response)
}
private fun getContentTypes(): List<ContentType> = listOf(
ContentType("Mangás", "manga"),
ContentType("Manhuas", "manhua"),
ContentType("Webtoons", "webtoon"),
ContentType("Novels", "novel"),
ContentType("Todos", "")
)
private data class ContentType(val name: String, val value: String) {
override fun toString() = name
}
private class TypeFilter(contentTypes: List<ContentType>) :
Filter.Select<ContentType>("Tipo de conteúdo", contentTypes.toTypedArray()) {
val selected: ContentType
get() = values[state]
}
override fun getFilterList(): FilterList = FilterList(
Filter.Header(FILTER_WARNING),
TypeFilter(getContentTypes())
)
companion object {
private const val FILTER_WARNING = "O filtro abaixo é ignorado durante a busca!"
private const val DEFAULT_TYPE = "manga"
}
}

View File

@ -1,349 +0,0 @@
package eu.kanade.tachiyomi.multisrc.mangasproject
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.decodeFromJsonElement
import okhttp3.FormBody
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.text.ParseException
import java.text.SimpleDateFormat
import java.util.Locale
import java.util.concurrent.TimeUnit
abstract class MangasProject(
override val name: String,
override val baseUrl: String,
override val lang: String
) : HttpSource() {
override val supportsLatest = true
// Sometimes the site is slow.
override val client: OkHttpClient = network.cloudflareClient.newBuilder()
.connectTimeout(1, TimeUnit.MINUTES)
.readTimeout(1, TimeUnit.MINUTES)
.writeTimeout(1, TimeUnit.MINUTES)
.build()
override fun headersBuilder(): Headers.Builder = Headers.Builder()
.add("Referer", baseUrl)
// Use internal headers to allow "Open in WebView" to work.
private fun sourceHeadersBuilder(): Headers.Builder = headersBuilder()
.add("Accept", ACCEPT_JSON)
.add("X-Requested-With", "XMLHttpRequest")
protected val sourceHeaders: Headers by lazy { sourceHeadersBuilder().build() }
private val json: Json by injectLazy()
protected open val licensedCheck: Boolean = false
override fun popularMangaRequest(page: Int): Request {
return GET("$baseUrl/home/most_read?page=$page&type=", sourceHeaders)
}
override fun popularMangaParse(response: Response): MangasPage {
val result = response.parseAs<MangasProjectMostReadDto>()
val popularMangas = result.mostRead.map(::popularMangaFromObject)
val hasNextPage = response.request.url.queryParameter("page")!!.toInt() < 10
return MangasPage(popularMangas, hasNextPage)
}
private fun popularMangaFromObject(serie: MangasProjectSerieDto) = SManga.create().apply {
title = serie.serieName
thumbnail_url = serie.cover
url = serie.link
}
override fun latestUpdatesRequest(page: Int): Request {
return GET("$baseUrl/home/releases?page=$page&type=", sourceHeaders)
}
override fun latestUpdatesParse(response: Response): MangasPage {
val result = response.parseAs<MangasProjectReleasesDto>()
val latestMangas = result.releases.map(::latestMangaFromObject)
val hasNextPage = response.request.url.queryParameter("page")!!.toInt() < 5
return MangasPage(latestMangas, hasNextPage)
}
private fun latestMangaFromObject(serie: MangasProjectSerieDto) = SManga.create().apply {
title = serie.name
thumbnail_url = serie.image
url = serie.link
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val form = FormBody.Builder()
.add("search", query)
.build()
val newHeaders = sourceHeadersBuilder()
.add("Content-Length", form.contentLength().toString())
.add("Content-Type", form.contentType().toString())
.build()
return POST("$baseUrl/lib/search/series.json", newHeaders, form)
}
override fun searchMangaParse(response: Response): MangasPage {
val result = response.parseAs<MangasProjectSearchDto>()
// If "series" have boolean false value, then it doesn't have results.
if (result.series is JsonPrimitive)
return MangasPage(emptyList(), false)
val searchMangas = json.decodeFromJsonElement<List<MangasProjectSerieDto>>(result.series)
.map(::searchMangaFromObject)
return MangasPage(searchMangas, false)
}
private fun searchMangaFromObject(serie: MangasProjectSerieDto) = SManga.create().apply {
title = serie.name
thumbnail_url = serie.cover
url = serie.link
}
override fun mangaDetailsParse(response: Response): SManga {
val document = response.asJsoup()
val seriesData = document.select("#series-data")
val isCompleted = seriesData.select("span.series-author i.complete-series").first() != null
// Check if the manga was removed by the publisher.
val seriesBlocked = document.select("div.series-blocked-img:has(img[src$=blocked.svg])").first()
val seriesAuthors = document.select("div#series-data span.series-author").text()
.substringAfter("Completo")
.substringBefore("+")
.split("&")
.groupBy(
{ it.contains("(Arte)") },
{
it.replace(" (Arte)", "")
.trim()
.split(", ")
.reversed()
.joinToString(" ")
}
)
return SManga.create().apply {
thumbnail_url = seriesData.select("div.series-img > div.cover > img").attr("src")
description = seriesData.select("span.series-desc > span").text()
status = parseStatus(seriesBlocked, isCompleted)
author = seriesAuthors[false]?.joinToString(", ") ?: author
artist = seriesAuthors[true]?.joinToString(", ") ?: author
genre = seriesData.select("div#series-data ul.tags li")
.joinToString { it.text() }
}
}
private fun parseStatus(seriesBlocked: Element?, isCompleted: Boolean) = when {
seriesBlocked != null && licensedCheck -> SManga.LICENSED
isCompleted -> SManga.COMPLETED
else -> SManga.ONGOING
}
override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
if (manga.status != SManga.LICENSED)
return super.fetchChapterList(manga)
return Observable.error(Exception(MANGA_REMOVED))
}
private fun chapterListRequestPaginated(mangaUrl: String, id: String, page: Int): Request {
val newHeaders = sourceHeadersBuilder()
.set("Referer", baseUrl + mangaUrl)
.build()
return GET("$baseUrl/series/chapters_list.json?page=$page&id_serie=$id", newHeaders)
}
override fun chapterListParse(response: Response): List<SChapter> {
val document = response.asJsoup()
val licensedMessage = document.select("div.series-blocked-img:has(img[src$=blocked.svg])").firstOrNull()
if (licensedMessage != null && licensedCheck) {
// If the manga is licensed and has been removed from the source,
// the extension will not fetch the chapters, even if they are returned
// by the API. This is just to mimic the website behavior.
throw Exception(MANGA_REMOVED)
}
val mangaUrl = response.request.url.toString().replace(baseUrl, "")
val mangaId = mangaUrl.substringAfterLast("/")
var page = 1
var chapterListRequest = chapterListRequestPaginated(mangaUrl, mangaId, page)
var chapterListResult = client.newCall(chapterListRequest).execute()
.parseAs<MangasProjectChapterListDto>()
if (chapterListResult.chapters is JsonPrimitive)
return emptyList()
val chapters = json.decodeFromJsonElement<List<MangasProjectChapterDto>>(chapterListResult.chapters)
.flatMap(::chaptersFromObject)
.toMutableList()
// If the result has less than the default per page, return right away
// to prevent extra API calls to get the chapters that does not exist.
if (chapters.size < 30) {
return chapters
}
// Otherwise, call the next pages of the API endpoint.
chapterListRequest = chapterListRequestPaginated(mangaUrl, mangaId, ++page)
chapterListResult = client.newCall(chapterListRequest).execute().parseAs()
while (chapterListResult.chapters is JsonArray) {
chapters += json.decodeFromJsonElement<List<MangasProjectChapterDto>>(chapterListResult.chapters)
.flatMap(::chaptersFromObject)
.toMutableList()
chapterListRequest = chapterListRequestPaginated(mangaUrl, mangaId, ++page)
chapterListResult = client.newCall(chapterListRequest).execute().parseAs()
}
return chapters
}
private fun chaptersFromObject(chapter: MangasProjectChapterDto): List<SChapter> {
return chapter.releases.values.map { release ->
SChapter.create().apply {
name = "Cap. ${chapter.number}" +
(if (chapter.name.isEmpty()) "" else " - ${chapter.name}")
date_upload = chapter.dateCreated.substringBefore("T").toDate()
scanlator = release.scanlators
.mapNotNull { scan -> scan.name.ifEmpty { null } }
.sorted()
.joinToString()
url = release.link
chapter_number = chapter.number.toFloatOrNull() ?: -1f
}
}
}
override fun pageListRequest(chapter: SChapter): Request {
val newHeaders = headersBuilder()
.add("Accept", ACCEPT)
.add("Accept-Language", ACCEPT_LANGUAGE)
.set("Referer", "$baseUrl/home")
.build()
return GET(baseUrl + chapter.url, newHeaders)
}
private fun pageListApiRequest(chapterUrl: String, token: String): Request {
val newHeaders = sourceHeadersBuilder()
.set("Referer", chapterUrl)
.build()
val id = chapterUrl
.substringBeforeLast("/")
.substringAfterLast("/")
return GET("$baseUrl/leitor/pages/$id.json?key=$token", newHeaders)
}
override fun pageListParse(response: Response): List<Page> {
val document = response.asJsoup()
val readerToken = getReaderToken(document) ?: throw Exception(TOKEN_NOT_FOUND)
val chapterUrl = getChapterUrl(response)
val apiRequest = pageListApiRequest(chapterUrl, readerToken)
val apiResponse = client.newCall(apiRequest).execute()
.parseAs<MangasProjectReaderDto>()
return apiResponse.images
.filter { it.startsWith("http") }
.mapIndexed { i, imageUrl -> Page(i, chapterUrl, imageUrl) }
}
open fun getChapterUrl(response: Response): String {
return response.request.url.toString()
}
protected open fun getReaderToken(document: Document): String? {
return document.select("script[src*=\"reader.\"]").firstOrNull()
?.attr("abs:src")
?.toHttpUrlOrNull()
?.queryParameter("token")
}
override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl!!)
override fun imageUrlParse(response: Response): String = ""
override fun imageRequest(page: Page): Request {
val newHeaders = headersBuilder()
.removeAll("Referer")
.set("Accept", ACCEPT_IMAGE)
.build()
return GET(page.imageUrl!!, newHeaders)
}
private inline fun <reified T> Response.parseAs(): T {
val responseBody = body?.string().orEmpty()
val errorResult = json.decodeFromString<MangasProjectErrorDto>(responseBody)
if (errorResult.message.isNullOrEmpty().not()) {
throw Exception(errorResult.message)
}
return json.decodeFromString(responseBody)
}
private fun String.toDate(): Long {
return try {
DATE_FORMATTER.parse(this)?.time ?: 0L
} catch (e: ParseException) {
0L
}
}
companion object {
private const val ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9," +
"image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
private const val ACCEPT_JSON = "application/json, text/javascript, */*; q=0.01"
private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
private const val ACCEPT_LANGUAGE = "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7,es;q=0.6,gl;q=0.5"
private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) }
private const val MANGA_REMOVED = "Mangá licenciado e removido pela fonte."
private const val TOKEN_NOT_FOUND = "Não foi possível obter o token de leitura."
}
}

View File

@ -1,64 +0,0 @@
package eu.kanade.tachiyomi.multisrc.mangasproject
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonElement
@Serializable
data class MangasProjectErrorDto(
val code: Int? = null,
val message: String? = null
)
@Serializable
data class MangasProjectMostReadDto(
@SerialName("most_read") val mostRead: List<MangasProjectSerieDto> = emptyList()
)
@Serializable
data class MangasProjectReleasesDto(
val releases: List<MangasProjectSerieDto> = emptyList()
)
@Serializable
data class MangasProjectSearchDto(
val series: JsonElement
)
@Serializable
data class MangasProjectSerieDto(
val cover: String = "",
val image: String = "",
val link: String,
val name: String = "",
@SerialName("serie_name") val serieName: String = ""
)
@Serializable
data class MangasProjectChapterListDto(
val chapters: JsonElement
)
@Serializable
data class MangasProjectChapterDto(
@SerialName("date_created") val dateCreated: String,
@SerialName("chapter_name") val name: String,
val number: String,
val releases: Map<String, MangasProjectChapterReleaseDto> = emptyMap()
)
@Serializable
data class MangasProjectChapterReleaseDto(
val link: String,
val scanlators: List<MangasProjectScanlatorDto> = emptyList()
)
@Serializable
data class MangasProjectScanlatorDto(
val name: String
)
@Serializable
data class MangasProjectReaderDto(
val images: List<String> = emptyList()
)

View File

@ -1,25 +0,0 @@
package eu.kanade.tachiyomi.multisrc.mangasproject
import generator.ThemeSourceData.SingleLang
import generator.ThemeSourceGenerator
class MangasProjectGenerator : ThemeSourceGenerator {
override val themePkg = "mangasproject"
override val themeClass = "MangasProject"
override val baseVersionCode: Int = 10
override val sources = listOf(
SingleLang("Leitor.net", "https://leitor.net", "pt-BR", className = "LeitorNet", isNsfw = true),
SingleLang("Mangá Livre", "https://mangalivre.net", "pt-BR", className = "MangaLivre", isNsfw = true)
)
companion object {
@JvmStatic
fun main(args: Array<String>) {
MangasProjectGenerator().createAll()
}
}
}