Remove MP sources due to cat-and-mouse game (#8679)

* Remove MP sources due to cat-and-mouse game.

* Add sources to the autocloser.

.github/workflows/issue_closer.yml

@@ -32,7 +32,7 @@ jobs:
               },
               {
                 "type": "both",
-                "regex": ".*(mangago|mangafox|hq\\s*dragon|manga\\s*host|supermangas|superhentais|union\\s*mangas|yes\\s*mangas|manhuascan|heroscan|manhwahot).*",
+                "regex": ".*(mangago|mangafox|hq\\s*dragon|manga\\s*host|supermangas|superhentais|union\\s*mangas|yes\\s*mangas|manhuascan|heroscan|manhwahot|leitor\\.?net|manga\\s*livre).*",
                 "ignoreCase": true,
                 "message": "{match} will not be added back as it is too difficult to maintain. Read #3475 for more information"
               },

multisrc/overrides/mangasproject/default/additional.gradle.kts

@@ -1,4 +0,0 @@
-
-dependencies {
-    implementation project(':lib-ratelimit')
-}

multisrc/overrides/mangasproject/leitornet/src/LeitorNet.kt

@@ -1,47 +0,0 @@
-package eu.kanade.tachiyomi.extension.pt.leitornet
-
-import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor
-import eu.kanade.tachiyomi.multisrc.mangasproject.MangasProject
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.SChapter
-import okhttp3.OkHttpClient
-import okhttp3.Request
-import okhttp3.Response
-import java.util.concurrent.TimeUnit
-
-class LeitorNet : MangasProject("Leitor.net", "https://leitor.net", "pt-BR") {
-
-    // Use the old generated id when the source did have the name "mangásPROJECT" and
-    // did have mangas in their catalogue. Now they "only have webtoons" and
-    // became a different website, but they still use the same structure.
-    // Existing mangas and other titles in the library still work.
-    override val id: Long = 2225174659569980836
-
-    override val client: OkHttpClient = super.client.newBuilder()
-        .addInterceptor(RateLimitInterceptor(1, 3, TimeUnit.SECONDS))
-        .build()
-
-    override val licensedCheck = true
-
-    /**
-     * Temporary fix to bypass Cloudflare.
-     */
-    override fun pageListRequest(chapter: SChapter): Request {
-        val newHeaders = super.pageListRequest(chapter).headers.newBuilder()
-            .set("Referer", "https://mangalivre.net/home")
-            .build()
-
-        val newChapterUrl = chapter.url
-            .replace("/manga/", "/ler/")
-            .replace("/(\\d+)/capitulo-".toRegex(), "/online/$1/capitulo-")
-
-        return GET("https://mangalivre.net$newChapterUrl", newHeaders)
-    }
-
-    override fun getChapterUrl(response: Response): String {
-        return super.getChapterUrl(response)
-            .replace("https://mangalivre.net", baseUrl)
-            .replace("/ler/", "/manga/")
-            .replace("/online/", "/")
-    }
-}

multisrc/overrides/mangasproject/mangalivre/src/MangaLivre.kt

@@ -1,76 +0,0 @@
-package eu.kanade.tachiyomi.extension.pt.mangalivre
-
-import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor
-import eu.kanade.tachiyomi.multisrc.mangasproject.MangasProject
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.source.model.Filter
-import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.MangasPage
-import okhttp3.OkHttpClient
-import okhttp3.Request
-import okhttp3.Response
-import java.util.concurrent.TimeUnit
-
-class MangaLivre : MangasProject("Mangá Livre", "https://mangalivre.net", "pt-BR") {
-
-    // Hardcode the id because the language wasn't specific.
-    override val id: Long = 4762777556012432014
-
-    override val client: OkHttpClient = super.client.newBuilder()
-        .addInterceptor(RateLimitInterceptor(1, 3, TimeUnit.SECONDS))
-        .build()
-
-    override fun popularMangaRequest(page: Int): Request {
-        val originalRequestUrl = super.popularMangaRequest(page).url.toString()
-        return GET(originalRequestUrl + DEFAULT_TYPE, sourceHeaders)
-    }
-
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        if (query.isNotEmpty()) {
-            return super.searchMangaRequest(page, query, filters)
-        }
-
-        val popularRequestUrl = super.popularMangaRequest(page).url.toString()
-        val type = filters.filterIsInstance<TypeFilter>()
-            .firstOrNull()?.selected?.value ?: DEFAULT_TYPE
-
-        return GET(popularRequestUrl + type, sourceHeaders)
-    }
-
-    override fun searchMangaParse(response: Response): MangasPage {
-        if (response.request.url.pathSegments.contains("search")) {
-            return super.searchMangaParse(response)
-        }
-
-        return popularMangaParse(response)
-    }
-
-    private fun getContentTypes(): List<ContentType> = listOf(
-        ContentType("Mangás", "manga"),
-        ContentType("Manhuas", "manhua"),
-        ContentType("Webtoons", "webtoon"),
-        ContentType("Novels", "novel"),
-        ContentType("Todos", "")
-    )
-
-    private data class ContentType(val name: String, val value: String) {
-        override fun toString() = name
-    }
-
-    private class TypeFilter(contentTypes: List<ContentType>) :
-        Filter.Select<ContentType>("Tipo de conteúdo", contentTypes.toTypedArray()) {
-
-        val selected: ContentType
-            get() = values[state]
-    }
-
-    override fun getFilterList(): FilterList = FilterList(
-        Filter.Header(FILTER_WARNING),
-        TypeFilter(getContentTypes())
-    )
-
-    companion object {
-        private const val FILTER_WARNING = "O filtro abaixo é ignorado durante a busca!"
-        private const val DEFAULT_TYPE = "manga"
-    }
-}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangasproject/MangasProject.kt

@@ -1,349 +0,0 @@
-package eu.kanade.tachiyomi.multisrc.mangasproject
-
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.MangasPage
-import eu.kanade.tachiyomi.source.model.Page
-import eu.kanade.tachiyomi.source.model.SChapter
-import eu.kanade.tachiyomi.source.model.SManga
-import eu.kanade.tachiyomi.source.online.HttpSource
-import eu.kanade.tachiyomi.util.asJsoup
-import kotlinx.serialization.decodeFromString
-import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.JsonArray
-import kotlinx.serialization.json.JsonPrimitive
-import kotlinx.serialization.json.decodeFromJsonElement
-import okhttp3.FormBody
-import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
-import okhttp3.OkHttpClient
-import okhttp3.Request
-import okhttp3.Response
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
-import rx.Observable
-import uy.kohesive.injekt.injectLazy
-import java.text.ParseException
-import java.text.SimpleDateFormat
-import java.util.Locale
-import java.util.concurrent.TimeUnit
-
-abstract class MangasProject(
-    override val name: String,
-    override val baseUrl: String,
-    override val lang: String
-) : HttpSource() {
-
-    override val supportsLatest = true
-
-    // Sometimes the site is slow.
-    override val client: OkHttpClient = network.cloudflareClient.newBuilder()
-        .connectTimeout(1, TimeUnit.MINUTES)
-        .readTimeout(1, TimeUnit.MINUTES)
-        .writeTimeout(1, TimeUnit.MINUTES)
-        .build()
-
-    override fun headersBuilder(): Headers.Builder = Headers.Builder()
-        .add("Referer", baseUrl)
-
-    // Use internal headers to allow "Open in WebView" to work.
-    private fun sourceHeadersBuilder(): Headers.Builder = headersBuilder()
-        .add("Accept", ACCEPT_JSON)
-        .add("X-Requested-With", "XMLHttpRequest")
-
-    protected val sourceHeaders: Headers by lazy { sourceHeadersBuilder().build() }
-
-    private val json: Json by injectLazy()
-
-    protected open val licensedCheck: Boolean = false
-
-    override fun popularMangaRequest(page: Int): Request {
-        return GET("$baseUrl/home/most_read?page=$page&type=", sourceHeaders)
-    }
-
-    override fun popularMangaParse(response: Response): MangasPage {
-        val result = response.parseAs<MangasProjectMostReadDto>()
-
-        val popularMangas = result.mostRead.map(::popularMangaFromObject)
-
-        val hasNextPage = response.request.url.queryParameter("page")!!.toInt() < 10
-
-        return MangasPage(popularMangas, hasNextPage)
-    }
-
-    private fun popularMangaFromObject(serie: MangasProjectSerieDto) = SManga.create().apply {
-        title = serie.serieName
-        thumbnail_url = serie.cover
-        url = serie.link
-    }
-
-    override fun latestUpdatesRequest(page: Int): Request {
-        return GET("$baseUrl/home/releases?page=$page&type=", sourceHeaders)
-    }
-
-    override fun latestUpdatesParse(response: Response): MangasPage {
-        val result = response.parseAs<MangasProjectReleasesDto>()
-
-        val latestMangas = result.releases.map(::latestMangaFromObject)
-
-        val hasNextPage = response.request.url.queryParameter("page")!!.toInt() < 5
-
-        return MangasPage(latestMangas, hasNextPage)
-    }
-
-    private fun latestMangaFromObject(serie: MangasProjectSerieDto) = SManga.create().apply {
-        title = serie.name
-        thumbnail_url = serie.image
-        url = serie.link
-    }
-
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val form = FormBody.Builder()
-            .add("search", query)
-            .build()
-
-        val newHeaders = sourceHeadersBuilder()
-            .add("Content-Length", form.contentLength().toString())
-            .add("Content-Type", form.contentType().toString())
-            .build()
-
-        return POST("$baseUrl/lib/search/series.json", newHeaders, form)
-    }
-
-    override fun searchMangaParse(response: Response): MangasPage {
-        val result = response.parseAs<MangasProjectSearchDto>()
-
-        // If "series" have boolean false value, then it doesn't have results.
-        if (result.series is JsonPrimitive)
-            return MangasPage(emptyList(), false)
-
-        val searchMangas = json.decodeFromJsonElement<List<MangasProjectSerieDto>>(result.series)
-            .map(::searchMangaFromObject)
-
-        return MangasPage(searchMangas, false)
-    }
-
-    private fun searchMangaFromObject(serie: MangasProjectSerieDto) = SManga.create().apply {
-        title = serie.name
-        thumbnail_url = serie.cover
-        url = serie.link
-    }
-
-    override fun mangaDetailsParse(response: Response): SManga {
-        val document = response.asJsoup()
-
-        val seriesData = document.select("#series-data")
-
-        val isCompleted = seriesData.select("span.series-author i.complete-series").first() != null
-
-        // Check if the manga was removed by the publisher.
-        val seriesBlocked = document.select("div.series-blocked-img:has(img[src$=blocked.svg])").first()
-
-        val seriesAuthors = document.select("div#series-data span.series-author").text()
-            .substringAfter("Completo")
-            .substringBefore("+")
-            .split("&")
-            .groupBy(
-                { it.contains("(Arte)") },
-                {
-                    it.replace(" (Arte)", "")
-                        .trim()
-                        .split(", ")
-                        .reversed()
-                        .joinToString(" ")
-                }
-            )
-
-        return SManga.create().apply {
-            thumbnail_url = seriesData.select("div.series-img > div.cover > img").attr("src")
-            description = seriesData.select("span.series-desc > span").text()
-
-            status = parseStatus(seriesBlocked, isCompleted)
-            author = seriesAuthors[false]?.joinToString(", ") ?: author
-            artist = seriesAuthors[true]?.joinToString(", ") ?: author
-            genre = seriesData.select("div#series-data ul.tags li")
-                .joinToString { it.text() }
-        }
-    }
-
-    private fun parseStatus(seriesBlocked: Element?, isCompleted: Boolean) = when {
-        seriesBlocked != null && licensedCheck -> SManga.LICENSED
-        isCompleted -> SManga.COMPLETED
-        else -> SManga.ONGOING
-    }
-
-    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
-        if (manga.status != SManga.LICENSED)
-            return super.fetchChapterList(manga)
-
-        return Observable.error(Exception(MANGA_REMOVED))
-    }
-
-    private fun chapterListRequestPaginated(mangaUrl: String, id: String, page: Int): Request {
-        val newHeaders = sourceHeadersBuilder()
-            .set("Referer", baseUrl + mangaUrl)
-            .build()
-
-        return GET("$baseUrl/series/chapters_list.json?page=$page&id_serie=$id", newHeaders)
-    }
-
-    override fun chapterListParse(response: Response): List<SChapter> {
-        val document = response.asJsoup()
-        val licensedMessage = document.select("div.series-blocked-img:has(img[src$=blocked.svg])").firstOrNull()
-
-        if (licensedMessage != null && licensedCheck) {
-            // If the manga is licensed and has been removed from the source,
-            // the extension will not fetch the chapters, even if they are returned
-            // by the API. This is just to mimic the website behavior.
-            throw Exception(MANGA_REMOVED)
-        }
-
-        val mangaUrl = response.request.url.toString().replace(baseUrl, "")
-        val mangaId = mangaUrl.substringAfterLast("/")
-        var page = 1
-
-        var chapterListRequest = chapterListRequestPaginated(mangaUrl, mangaId, page)
-        var chapterListResult = client.newCall(chapterListRequest).execute()
-            .parseAs<MangasProjectChapterListDto>()
-
-        if (chapterListResult.chapters is JsonPrimitive)
-            return emptyList()
-
-        val chapters = json.decodeFromJsonElement<List<MangasProjectChapterDto>>(chapterListResult.chapters)
-            .flatMap(::chaptersFromObject)
-            .toMutableList()
-
-        // If the result has less than the default per page, return right away
-        // to prevent extra API calls to get the chapters that does not exist.
-        if (chapters.size < 30) {
-            return chapters
-        }
-
-        // Otherwise, call the next pages of the API endpoint.
-        chapterListRequest = chapterListRequestPaginated(mangaUrl, mangaId, ++page)
-        chapterListResult = client.newCall(chapterListRequest).execute().parseAs()
-
-        while (chapterListResult.chapters is JsonArray) {
-            chapters += json.decodeFromJsonElement<List<MangasProjectChapterDto>>(chapterListResult.chapters)
-                .flatMap(::chaptersFromObject)
-                .toMutableList()
-
-            chapterListRequest = chapterListRequestPaginated(mangaUrl, mangaId, ++page)
-            chapterListResult = client.newCall(chapterListRequest).execute().parseAs()
-        }
-
-        return chapters
-    }
-
-    private fun chaptersFromObject(chapter: MangasProjectChapterDto): List<SChapter> {
-        return chapter.releases.values.map { release ->
-            SChapter.create().apply {
-                name = "Cap. ${chapter.number}" +
-                    (if (chapter.name.isEmpty()) "" else " - ${chapter.name}")
-                date_upload = chapter.dateCreated.substringBefore("T").toDate()
-                scanlator = release.scanlators
-                    .mapNotNull { scan -> scan.name.ifEmpty { null } }
-                    .sorted()
-                    .joinToString()
-                url = release.link
-                chapter_number = chapter.number.toFloatOrNull() ?: -1f
-            }
-        }
-    }
-
-    override fun pageListRequest(chapter: SChapter): Request {
-        val newHeaders = headersBuilder()
-            .add("Accept", ACCEPT)
-            .add("Accept-Language", ACCEPT_LANGUAGE)
-            .set("Referer", "$baseUrl/home")
-            .build()
-
-        return GET(baseUrl + chapter.url, newHeaders)
-    }
-
-    private fun pageListApiRequest(chapterUrl: String, token: String): Request {
-        val newHeaders = sourceHeadersBuilder()
-            .set("Referer", chapterUrl)
-            .build()
-
-        val id = chapterUrl
-            .substringBeforeLast("/")
-            .substringAfterLast("/")
-
-        return GET("$baseUrl/leitor/pages/$id.json?key=$token", newHeaders)
-    }
-
-    override fun pageListParse(response: Response): List<Page> {
-        val document = response.asJsoup()
-        val readerToken = getReaderToken(document) ?: throw Exception(TOKEN_NOT_FOUND)
-        val chapterUrl = getChapterUrl(response)
-
-        val apiRequest = pageListApiRequest(chapterUrl, readerToken)
-        val apiResponse = client.newCall(apiRequest).execute()
-            .parseAs<MangasProjectReaderDto>()
-
-        return apiResponse.images
-            .filter { it.startsWith("http") }
-            .mapIndexed { i, imageUrl -> Page(i, chapterUrl, imageUrl) }
-    }
-
-    open fun getChapterUrl(response: Response): String {
-        return response.request.url.toString()
-    }
-
-    protected open fun getReaderToken(document: Document): String? {
-        return document.select("script[src*=\"reader.\"]").firstOrNull()
-            ?.attr("abs:src")
-            ?.toHttpUrlOrNull()
-            ?.queryParameter("token")
-    }
-
-    override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl!!)
-
-    override fun imageUrlParse(response: Response): String = ""
-
-    override fun imageRequest(page: Page): Request {
-        val newHeaders = headersBuilder()
-            .removeAll("Referer")
-            .set("Accept", ACCEPT_IMAGE)
-            .build()
-
-        return GET(page.imageUrl!!, newHeaders)
-    }
-
-    private inline fun <reified T> Response.parseAs(): T {
-        val responseBody = body?.string().orEmpty()
-
-        val errorResult = json.decodeFromString<MangasProjectErrorDto>(responseBody)
-
-        if (errorResult.message.isNullOrEmpty().not()) {
-            throw Exception(errorResult.message)
-        }
-
-        return json.decodeFromString(responseBody)
-    }
-
-    private fun String.toDate(): Long {
-        return try {
-            DATE_FORMATTER.parse(this)?.time ?: 0L
-        } catch (e: ParseException) {
-            0L
-        }
-    }
-
-    companion object {
-        private const val ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9," +
-            "image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"
-        private const val ACCEPT_JSON = "application/json, text/javascript, */*; q=0.01"
-        private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
-        private const val ACCEPT_LANGUAGE = "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7,es;q=0.6,gl;q=0.5"
-        private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
-            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
-
-        private val DATE_FORMATTER by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) }
-
-        private const val MANGA_REMOVED = "Mangá licenciado e removido pela fonte."
-        private const val TOKEN_NOT_FOUND = "Não foi possível obter o token de leitura."
-    }
-}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangasproject/MangasProjectDto.kt

@@ -1,64 +0,0 @@
-package eu.kanade.tachiyomi.multisrc.mangasproject
-
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.JsonElement
-
-@Serializable
-data class MangasProjectErrorDto(
-    val code: Int? = null,
-    val message: String? = null
-)
-
-@Serializable
-data class MangasProjectMostReadDto(
-    @SerialName("most_read") val mostRead: List<MangasProjectSerieDto> = emptyList()
-)
-
-@Serializable
-data class MangasProjectReleasesDto(
-    val releases: List<MangasProjectSerieDto> = emptyList()
-)
-
-@Serializable
-data class MangasProjectSearchDto(
-    val series: JsonElement
-)
-
-@Serializable
-data class MangasProjectSerieDto(
-    val cover: String = "",
-    val image: String = "",
-    val link: String,
-    val name: String = "",
-    @SerialName("serie_name") val serieName: String = ""
-)
-
-@Serializable
-data class MangasProjectChapterListDto(
-    val chapters: JsonElement
-)
-
-@Serializable
-data class MangasProjectChapterDto(
-    @SerialName("date_created") val dateCreated: String,
-    @SerialName("chapter_name") val name: String,
-    val number: String,
-    val releases: Map<String, MangasProjectChapterReleaseDto> = emptyMap()
-)
-
-@Serializable
-data class MangasProjectChapterReleaseDto(
-    val link: String,
-    val scanlators: List<MangasProjectScanlatorDto> = emptyList()
-)
-
-@Serializable
-data class MangasProjectScanlatorDto(
-    val name: String
-)
-
-@Serializable
-data class MangasProjectReaderDto(
-    val images: List<String> = emptyList()
-)

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangasproject/MangasProjectGenerator.kt

@@ -1,25 +0,0 @@
-package eu.kanade.tachiyomi.multisrc.mangasproject
-
-import generator.ThemeSourceData.SingleLang
-import generator.ThemeSourceGenerator
-
-class MangasProjectGenerator : ThemeSourceGenerator {
-
-    override val themePkg = "mangasproject"
-
-    override val themeClass = "MangasProject"
-
-    override val baseVersionCode: Int = 10
-
-    override val sources = listOf(
-        SingleLang("Leitor.net", "https://leitor.net", "pt-BR", className = "LeitorNet", isNsfw = true),
-        SingleLang("Mangá Livre", "https://mangalivre.net", "pt-BR", className = "MangaLivre", isNsfw = true)
-    )
-
-    companion object {
-        @JvmStatic
-        fun main(args: Array<String>) {
-            MangasProjectGenerator().createAll()
-        }
-    }
-}