HeanCms: Re-add fetchAllTitles (#17590)

* Re-add fetchAllTitles

* fetchAllTitles in pagelistParse

* Lint

* Change allTitlesRequest parameters

* Rename val

* Remove unnecesary permSlug in chapters
This commit is contained in:
bapeey 2023-08-19 09:56:33 -05:00 committed by GitHub
parent dce77b7290
commit 31d420cd50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 246 additions and 186 deletions

View File

@ -1,21 +1,10 @@
package eu.kanade.tachiyomi.extension.pt.reaperscans package eu.kanade.tachiyomi.extension.pt.reaperscans
import eu.kanade.tachiyomi.multisrc.heancms.Genre import eu.kanade.tachiyomi.multisrc.heancms.Genre
import eu.kanade.tachiyomi.multisrc.heancms.GenreFilter
import eu.kanade.tachiyomi.multisrc.heancms.HeanCms import eu.kanade.tachiyomi.multisrc.heancms.HeanCms
import eu.kanade.tachiyomi.multisrc.heancms.HeanCmsSeriesDto
import eu.kanade.tachiyomi.multisrc.heancms.SortByFilter
import eu.kanade.tachiyomi.multisrc.heancms.StatusFilter
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.interceptor.rateLimitHost import eu.kanade.tachiyomi.network.interceptor.rateLimitHost
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.TimeZone import java.util.TimeZone
@ -32,87 +21,15 @@ class ReaperScans : HeanCms(
// Site changed from Madara to HeanCms. // Site changed from Madara to HeanCms.
override val versionId = 2 override val versionId = 2
override val fetchAllTitles = true
override val useNewQueryEndpoint = true
override val coverPath: String = "" override val coverPath: String = ""
override val dateFormat: SimpleDateFormat = super.dateFormat.apply { override val dateFormat: SimpleDateFormat = super.dateFormat.apply {
timeZone = TimeZone.getTimeZone("GMT+01:00") timeZone = TimeZone.getTimeZone("GMT+01:00")
} }
override fun popularMangaRequest(page: Int): Request {
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", "")
.addQueryParameter("series_status", "All")
.addQueryParameter("order", "desc")
.addQueryParameter("orderBy", "total_views")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", "[]")
return GET(url.build(), headers)
}
override fun latestUpdatesRequest(page: Int): Request {
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", "")
.addQueryParameter("series_status", "All")
.addQueryParameter("order", "desc")
.addQueryParameter("orderBy", "latest")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", "[]")
return GET(url.build(), headers)
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val sortByFilter = filters.firstInstanceOrNull<SortByFilter>()
val statusFilter = filters.firstInstanceOrNull<StatusFilter>()
val tagIds = filters.firstInstanceOrNull<GenreFilter>()?.state.orEmpty()
.filter(Genre::state)
.map(Genre::id)
.joinToString(",", prefix = "[", postfix = "]")
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", query)
.addQueryParameter("series_status", statusFilter?.selected?.value ?: "All")
.addQueryParameter("order", if (sortByFilter?.state?.ascending == true) "asc" else "desc")
.addQueryParameter("orderBy", sortByFilter?.selected ?: "total_views")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", tagIds)
return GET(url.build(), headers)
}
override fun chapterListParse(response: Response): List<SChapter> {
val result = response.parseAs<HeanCmsSeriesDto>()
val currentTimestamp = System.currentTimeMillis()
return result.seasons.orEmpty()
.flatMap { it.chapters.orEmpty() }
.filterNot { it.price == 1 }
.map { it.toSChapter(result.slug, dateFormat) }
.filter { it.date_upload <= currentTimestamp }
}
override fun pageListRequest(chapter: SChapter) = GET(baseUrl + chapter.url, headers)
override fun pageListParse(response: Response): List<Page> {
val document = response.asJsoup()
val images = document.selectFirst("div.min-h-screen > div.container > p.items-center")
return images?.select("img").orEmpty().mapIndexed { i, img ->
val imageUrl = if (img.hasClass("lazy")) img.absUrl("data-src") else img.absUrl("src")
Page(i, "", imageUrl)
}
}
override fun getGenreList(): List<Genre> = listOf( override fun getGenreList(): List<Genre> = listOf(
Genre("Artes Marciais", 2), Genre("Artes Marciais", 2),
Genre("Aventura", 10), Genre("Aventura", 10),

View File

@ -1,20 +1,9 @@
package eu.kanade.tachiyomi.extension.es.yugenmangas package eu.kanade.tachiyomi.extension.es.yugenmangas
import eu.kanade.tachiyomi.multisrc.heancms.Genre import eu.kanade.tachiyomi.multisrc.heancms.Genre
import eu.kanade.tachiyomi.multisrc.heancms.GenreFilter
import eu.kanade.tachiyomi.multisrc.heancms.HeanCms import eu.kanade.tachiyomi.multisrc.heancms.HeanCms
import eu.kanade.tachiyomi.multisrc.heancms.HeanCmsSeriesDto
import eu.kanade.tachiyomi.multisrc.heancms.SortByFilter
import eu.kanade.tachiyomi.multisrc.heancms.StatusFilter
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.interceptor.rateLimitHost import eu.kanade.tachiyomi.network.interceptor.rateLimitHost
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Request
import okhttp3.Response
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.TimeZone import java.util.TimeZone
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
@ -30,6 +19,9 @@ class YugenMangas :
// Site changed from Madara to HeanCms. // Site changed from Madara to HeanCms.
override val versionId = 2 override val versionId = 2
override val fetchAllTitles = true
override val useNewQueryEndpoint = true
override val client = super.client.newBuilder() override val client = super.client.newBuilder()
.connectTimeout(60, TimeUnit.SECONDS) .connectTimeout(60, TimeUnit.SECONDS)
.readTimeout(90, TimeUnit.SECONDS) .readTimeout(90, TimeUnit.SECONDS)
@ -42,81 +34,6 @@ class YugenMangas :
timeZone = TimeZone.getTimeZone("UTC") timeZone = TimeZone.getTimeZone("UTC")
} }
override fun popularMangaRequest(page: Int): Request {
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", "")
.addQueryParameter("series_status", "All")
.addQueryParameter("order", "desc")
.addQueryParameter("orderBy", "total_views")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", "[]")
return GET(url.build(), headers)
}
override fun latestUpdatesRequest(page: Int): Request {
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", "")
.addQueryParameter("series_status", "All")
.addQueryParameter("order", "desc")
.addQueryParameter("orderBy", "latest")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", "[]")
return GET(url.build(), headers)
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val sortByFilter = filters.firstInstanceOrNull<SortByFilter>()
val statusFilter = filters.firstInstanceOrNull<StatusFilter>()
val tagIds = filters.firstInstanceOrNull<GenreFilter>()?.state.orEmpty()
.filter(Genre::state)
.map(Genre::id)
.joinToString(",", prefix = "[", postfix = "]")
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", query)
.addQueryParameter("series_status", statusFilter?.selected?.value ?: "All")
.addQueryParameter("order", if (sortByFilter?.state?.ascending == true) "asc" else "desc")
.addQueryParameter("orderBy", sortByFilter?.selected ?: "total_views")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", tagIds)
return GET(url.build(), headers)
}
override fun chapterListParse(response: Response): List<SChapter> {
val result = response.parseAs<HeanCmsSeriesDto>()
val currentTimestamp = System.currentTimeMillis()
return result.seasons.orEmpty()
.flatMap { it.chapters.orEmpty() }
.filterNot { it.price == 1 }
.map { it.toSChapter(result.slug, dateFormat) }
.filter { it.date_upload <= currentTimestamp }
}
override fun pageListRequest(chapter: SChapter) = GET(baseUrl + chapter.url, headers)
override fun pageListParse(response: Response): List<Page> {
val document = response.asJsoup()
val images = document.selectFirst("div.min-h-screen > div.container > p.items-center")
return images?.select("img").orEmpty().mapIndexed { i, img ->
val imageUrl = if (img.hasClass("lazy")) img.absUrl("data-src") else img.absUrl("src")
Page(i, "", imageUrl)
}
}
override fun getGenreList(): List<Genre> = listOf( override fun getGenreList(): List<Genre> = listOf(
Genre("+18", 1), Genre("+18", 1),
Genre("Acción", 36), Genre("Acción", 36),

View File

@ -9,10 +9,12 @@ import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.decodeFromString import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
import okhttp3.Headers import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient import okhttp3.OkHttpClient
import okhttp3.Request import okhttp3.Request
@ -33,6 +35,12 @@ abstract class HeanCms(
override val client: OkHttpClient = network.cloudflareClient override val client: OkHttpClient = network.cloudflareClient
protected open val fetchAllTitles = false
protected open val useNewQueryEndpoint = false
private var seriesSlugMap: Map<String, HeanCmsTitle>? = null
/** /**
* Custom Json instance to make usage of `encodeDefaults`, * Custom Json instance to make usage of `encodeDefaults`,
* which is not enabled on the injected instance of the app. * which is not enabled on the injected instance of the app.
@ -54,6 +62,10 @@ abstract class HeanCms(
.add("Referer", "$baseUrl/") .add("Referer", "$baseUrl/")
override fun popularMangaRequest(page: Int): Request { override fun popularMangaRequest(page: Int): Request {
if (useNewQueryEndpoint) {
return newEndpointPopularMangaRequest(page)
}
val payloadObj = HeanCmsQuerySearchPayloadDto( val payloadObj = HeanCmsQuerySearchPayloadDto(
page = page, page = page,
order = "desc", order = "desc",
@ -72,23 +84,45 @@ abstract class HeanCms(
return POST("$apiUrl/series/querysearch", apiHeaders, payload) return POST("$apiUrl/series/querysearch", apiHeaders, payload)
} }
protected fun newEndpointPopularMangaRequest(page: Int): Request {
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", "")
.addQueryParameter("series_status", "All")
.addQueryParameter("order", "desc")
.addQueryParameter("orderBy", "total_views")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", "[]")
return GET(url.build(), headers)
}
override fun popularMangaParse(response: Response): MangasPage { override fun popularMangaParse(response: Response): MangasPage {
val json = response.body.string() val json = response.body.string()
if (json.startsWith("{")) { if (json.startsWith("{")) {
val result = json.parseAs<HeanCmsQuerySearchDto>() val result = json.parseAs<HeanCmsQuerySearchDto>()
val mangaList = result.data.map { it.toSManga(apiUrl, coverPath) } val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
fetchAllTitles()
return MangasPage(mangaList, result.meta?.hasNextPage ?: false) return MangasPage(mangaList, result.meta?.hasNextPage ?: false)
} }
val mangaList = json.parseAs<List<HeanCmsSeriesDto>>() val mangaList = json.parseAs<List<HeanCmsSeriesDto>>()
.map { it.toSManga(apiUrl, coverPath) } .map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
fetchAllTitles()
return MangasPage(mangaList, hasNextPage = false) return MangasPage(mangaList, hasNextPage = false)
} }
override fun latestUpdatesRequest(page: Int): Request { override fun latestUpdatesRequest(page: Int): Request {
if (useNewQueryEndpoint) {
return newEndpointLatestUpdatesRequest(page)
}
val payloadObj = HeanCmsQuerySearchPayloadDto( val payloadObj = HeanCmsQuerySearchPayloadDto(
page = page, page = page,
order = "desc", order = "desc",
@ -107,6 +141,20 @@ abstract class HeanCms(
return POST("$apiUrl/series/querysearch", apiHeaders, payload) return POST("$apiUrl/series/querysearch", apiHeaders, payload)
} }
protected fun newEndpointLatestUpdatesRequest(page: Int): Request {
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", "")
.addQueryParameter("series_status", "All")
.addQueryParameter("order", "desc")
.addQueryParameter("orderBy", "latest")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", "[]")
return GET(url.build(), headers)
}
override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response) override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response)
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> { override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
@ -121,6 +169,10 @@ abstract class HeanCms(
} }
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
if (useNewQueryEndpoint) {
return newEndpointSearchMangaRequest(page, query, filters)
}
if (query.isNotBlank()) { if (query.isNotBlank()) {
val searchPayloadObj = HeanCmsSearchPayloadDto(query) val searchPayloadObj = HeanCmsSearchPayloadDto(query)
val searchPayload = json.encodeToString(searchPayloadObj) val searchPayload = json.encodeToString(searchPayloadObj)
@ -158,16 +210,40 @@ abstract class HeanCms(
return POST("$apiUrl/series/querysearch", apiHeaders, payload) return POST("$apiUrl/series/querysearch", apiHeaders, payload)
} }
protected fun newEndpointSearchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val sortByFilter = filters.firstInstanceOrNull<SortByFilter>()
val statusFilter = filters.firstInstanceOrNull<StatusFilter>()
val tagIds = filters.firstInstanceOrNull<GenreFilter>()?.state.orEmpty()
.filter(Genre::state)
.map(Genre::id)
.joinToString(",", prefix = "[", postfix = "]")
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("query_string", query)
.addQueryParameter("series_status", statusFilter?.selected?.value ?: "All")
.addQueryParameter("order", if (sortByFilter?.state?.ascending == true) "asc" else "desc")
.addQueryParameter("orderBy", sortByFilter?.selected ?: "total_views")
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", "12")
.addQueryParameter("tags_ids", tagIds)
return GET(url.build(), headers)
}
override fun searchMangaParse(response: Response): MangasPage { override fun searchMangaParse(response: Response): MangasPage {
val json = response.body.string() val json = response.body.string()
if (response.request.url.pathSegments.last() == "search") { if (response.request.url.pathSegments.last() == "search") {
fetchAllTitles()
val result = json.parseAs<List<HeanCmsSearchDto>>() val result = json.parseAs<List<HeanCmsSearchDto>>()
val mangaList = result val mangaList = result
.filter { it.type == "Comic" } .filter { it.type == "Comic" }
.map { .map {
it.slug = it.slug.replace(TIMESTAMP_REGEX, "") it.slug = it.slug.replace(TIMESTAMP_REGEX, "")
it.toSManga(apiUrl, coverPath) it.toSManga(apiUrl, coverPath, seriesSlugMap.orEmpty(), fetchAllTitles)
} }
return MangasPage(mangaList, false) return MangasPage(mangaList, false)
@ -175,28 +251,51 @@ abstract class HeanCms(
if (json.startsWith("{")) { if (json.startsWith("{")) {
val result = json.parseAs<HeanCmsQuerySearchDto>() val result = json.parseAs<HeanCmsQuerySearchDto>()
val mangaList = result.data.map { it.toSManga(apiUrl, coverPath) } val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
fetchAllTitles()
return MangasPage(mangaList, result.meta?.hasNextPage ?: false) return MangasPage(mangaList, result.meta?.hasNextPage ?: false)
} }
val mangaList = json.parseAs<List<HeanCmsSeriesDto>>() val mangaList = json.parseAs<List<HeanCmsSeriesDto>>()
.map { it.toSManga(apiUrl, coverPath) } .map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
fetchAllTitles()
return MangasPage(mangaList, hasNextPage = false) return MangasPage(mangaList, hasNextPage = false)
} }
override fun getMangaUrl(manga: SManga) = baseUrl + manga.url override fun getMangaUrl(manga: SManga): String {
override fun mangaDetailsRequest(manga: SManga): Request {
val seriesSlug = manga.url val seriesSlug = manga.url
.substringAfterLast("/") .substringAfterLast("/")
.toPermSlugIfNeeded()
val currentSlug = seriesSlugMap?.get(seriesSlug)?.slug ?: seriesSlug
return "$baseUrl/series/$currentSlug"
}
override fun mangaDetailsRequest(manga: SManga): Request {
if (fetchAllTitles && manga.url.contains(TIMESTAMP_REGEX)) {
throw Exception(intl.urlChangedError(name))
}
val seriesSlug = manga.url
.substringAfterLast("/")
.toPermSlugIfNeeded()
fetchAllTitles()
val seriesDetails = seriesSlugMap?.get(seriesSlug)
val currentSlug = seriesDetails?.slug ?: seriesSlug
val currentStatus = seriesDetails?.status ?: manga.status
val apiHeaders = headersBuilder() val apiHeaders = headersBuilder()
.add("Accept", ACCEPT_JSON) .add("Accept", ACCEPT_JSON)
.build() .build()
return GET("$apiUrl/series/$seriesSlug#${manga.status}", apiHeaders) return GET("$apiUrl/series/$currentSlug#$currentStatus", apiHeaders)
} }
override fun mangaDetailsParse(response: Response): SManga { override fun mangaDetailsParse(response: Response): SManga {
@ -204,7 +303,7 @@ abstract class HeanCms(
val result = runCatching { response.parseAs<HeanCmsSeriesDto>() } val result = runCatching { response.parseAs<HeanCmsSeriesDto>() }
val seriesDetails = result.getOrNull()?.toSManga(apiUrl, coverPath) val seriesDetails = result.getOrNull()?.toSManga(apiUrl, coverPath, fetchAllTitles)
?: throw Exception(intl.urlChangedError(name)) ?: throw Exception(intl.urlChangedError(name))
return seriesDetails.apply { return seriesDetails.apply {
@ -220,6 +319,14 @@ abstract class HeanCms(
val currentTimestamp = System.currentTimeMillis() val currentTimestamp = System.currentTimeMillis()
if (useNewQueryEndpoint) {
return result.seasons.orEmpty()
.flatMap { it.chapters.orEmpty() }
.filterNot { it.price == 1 }
.map { it.toSChapter(result.slug, dateFormat) }
.filter { it.date_upload <= currentTimestamp }
}
return result.chapters.orEmpty() return result.chapters.orEmpty()
.filterNot { it.price == 1 } .filterNot { it.price == 1 }
.map { it.toSChapter(result.slug, dateFormat) } .map { it.toSChapter(result.slug, dateFormat) }
@ -230,6 +337,10 @@ abstract class HeanCms(
override fun getChapterUrl(chapter: SChapter): String = baseUrl + chapter.url override fun getChapterUrl(chapter: SChapter): String = baseUrl + chapter.url
override fun pageListRequest(chapter: SChapter): Request { override fun pageListRequest(chapter: SChapter): Request {
if (useNewQueryEndpoint) {
return GET(baseUrl + chapter.url, headers)
}
val chapterId = chapter.url.substringAfterLast("#") val chapterId = chapter.url.substringAfterLast("#")
val apiHeaders = headersBuilder() val apiHeaders = headersBuilder()
@ -240,6 +351,17 @@ abstract class HeanCms(
} }
override fun pageListParse(response: Response): List<Page> { override fun pageListParse(response: Response): List<Page> {
if (useNewQueryEndpoint) {
val document = response.asJsoup()
val images = document.selectFirst("div.min-h-screen > div.container > p.items-center")
return images?.select("img").orEmpty().mapIndexed { i, img ->
val imageUrl = if (img.hasClass("lazy")) img.absUrl("data-src") else img.absUrl("src")
Page(i, "", imageUrl)
}
}
return response.parseAs<HeanCmsReaderDto>().content?.images.orEmpty() return response.parseAs<HeanCmsReaderDto>().content?.images.orEmpty()
.filterNot { imageUrl -> .filterNot { imageUrl ->
// Their image server returns HTTP 403 for hidden files that starts // Their image server returns HTTP 403 for hidden files that starts
@ -266,6 +388,94 @@ abstract class HeanCms(
return GET(page.imageUrl!!, imageHeaders) return GET(page.imageUrl!!, imageHeaders)
} }
protected open fun fetchAllTitles() {
if (!seriesSlugMap.isNullOrEmpty() || !fetchAllTitles) {
return
}
val result = runCatching {
var hasNextPage = true
var page = 1
val tempMap = mutableMapOf<String, HeanCmsTitle>()
while (hasNextPage) {
val response = client.newCall(allTitlesRequest(page)).execute()
val json = response.body.string()
if (json.startsWith("{")) {
val result = json.parseAs<HeanCmsQuerySearchDto>()
tempMap.putAll(parseAllTitles(result.data))
hasNextPage = result.meta?.hasNextPage ?: false
page++
} else {
val result = json.parseAs<List<HeanCmsSeriesDto>>()
tempMap.putAll(parseAllTitles(result))
hasNextPage = false
}
}
tempMap.toMap()
}
seriesSlugMap = result.getOrNull()
}
protected open fun allTitlesRequest(page: Int): Request {
if (useNewQueryEndpoint) {
val url = "$apiUrl/query".toHttpUrl().newBuilder()
.addQueryParameter("series_type", "Comic")
.addQueryParameter("page", page.toString())
.addQueryParameter("perPage", PER_PAGE_MANGA_TITLES.toString())
return GET(url.build(), headers)
}
val payloadObj = HeanCmsQuerySearchPayloadDto(
page = page,
order = "desc",
orderBy = "total_views",
type = "Comic",
)
val payload = json.encodeToString(payloadObj).toRequestBody(JSON_MEDIA_TYPE)
val apiHeaders = headersBuilder()
.add("Accept", ACCEPT_JSON)
.add("Content-Type", payload.contentType().toString())
.build()
return POST("$apiUrl/series/querysearch", apiHeaders, payload)
}
protected open fun parseAllTitles(result: List<HeanCmsSeriesDto>): Map<String, HeanCmsTitle> {
return result
.filter { it.type == "Comic" }
.associateBy(
keySelector = { it.slug.replace(TIMESTAMP_REGEX, "") },
valueTransform = {
HeanCmsTitle(
slug = it.slug,
thumbnailFileName = it.thumbnail,
status = it.status?.toStatus() ?: SManga.UNKNOWN,
)
},
)
}
/**
* Used to store the current slugs for sources that change it periodically and for the
* search that doesn't return the thumbnail URLs.
*/
data class HeanCmsTitle(val slug: String, val thumbnailFileName: String, val status: Int)
private fun String.toPermSlugIfNeeded(): String {
return if (fetchAllTitles) {
this.replace(TIMESTAMP_REGEX, "")
} else {
this
}
}
protected open fun getStatusList(): List<Status> = listOf( protected open fun getStatusList(): List<Status> = listOf(
Status(intl.statusAll, "All"), Status(intl.statusAll, "All"),
Status(intl.statusOngoing, "Ongoing"), Status(intl.statusOngoing, "Ongoing"),
@ -312,6 +522,8 @@ abstract class HeanCms(
val TIMESTAMP_REGEX = """-\d{13}$""".toRegex() val TIMESTAMP_REGEX = """-\d{13}$""".toRegex()
private const val PER_PAGE_MANGA_TITLES = 10000
const val SEARCH_PREFIX = "slug:" const val SEARCH_PREFIX = "slug:"
} }
} }

View File

@ -35,10 +35,15 @@ data class HeanCmsSearchDto(
fun toSManga( fun toSManga(
apiUrl: String, apiUrl: String,
coverPath: String, coverPath: String,
slugMap: Map<String, HeanCms.HeanCmsTitle>,
fetchAllTiles: Boolean,
): SManga = SManga.create().apply { ): SManga = SManga.create().apply {
val slugOnly = slug.toPermSlugIfNeeded(fetchAllTiles)
val thumbnailFileName = slugMap[slugOnly]?.thumbnailFileName
title = this@HeanCmsSearchDto.title title = this@HeanCmsSearchDto.title
thumbnail_url = thumbnail?.toAbsoluteThumbnailUrl(apiUrl, coverPath) thumbnail_url = thumbnail?.toAbsoluteThumbnailUrl(apiUrl, coverPath)
url = "/series/$slug" ?: thumbnailFileName?.toAbsoluteThumbnailUrl(apiUrl, coverPath)
url = "/series/$slugOnly"
} }
} }
@ -61,8 +66,10 @@ data class HeanCmsSeriesDto(
fun toSManga( fun toSManga(
apiUrl: String, apiUrl: String,
coverPath: String, coverPath: String,
fetchAllTiles: Boolean,
): SManga = SManga.create().apply { ): SManga = SManga.create().apply {
val descriptionBody = this@HeanCmsSeriesDto.description?.let(Jsoup::parseBodyFragment) val descriptionBody = this@HeanCmsSeriesDto.description?.let(Jsoup::parseBodyFragment)
val slugOnly = slug.toPermSlugIfNeeded(fetchAllTiles)
title = this@HeanCmsSeriesDto.title title = this@HeanCmsSeriesDto.title
author = this@HeanCmsSeriesDto.author?.trim() author = this@HeanCmsSeriesDto.author?.trim()
@ -76,7 +83,7 @@ data class HeanCmsSeriesDto(
thumbnail_url = thumbnail.ifEmpty { null } thumbnail_url = thumbnail.ifEmpty { null }
?.toAbsoluteThumbnailUrl(apiUrl, coverPath) ?.toAbsoluteThumbnailUrl(apiUrl, coverPath)
status = this@HeanCmsSeriesDto.status?.toStatus() ?: SManga.UNKNOWN status = this@HeanCmsSeriesDto.status?.toStatus() ?: SManga.UNKNOWN
url = "/series/$slug" url = "/series/$slugOnly"
} }
} }
@ -98,7 +105,6 @@ data class HeanCmsChapterDto(
@SerialName("created_at") val createdAt: String, @SerialName("created_at") val createdAt: String,
val price: Int? = null, val price: Int? = null,
) { ) {
fun toSChapter(seriesSlug: String, dateFormat: SimpleDateFormat): SChapter = SChapter.create().apply { fun toSChapter(seriesSlug: String, dateFormat: SimpleDateFormat): SChapter = SChapter.create().apply {
name = this@HeanCmsChapterDto.name.trim() name = this@HeanCmsChapterDto.name.trim()
date_upload = runCatching { dateFormat.parse(createdAt)?.time } date_upload = runCatching { dateFormat.parse(createdAt)?.time }
@ -134,6 +140,14 @@ private fun String.toAbsoluteThumbnailUrl(apiUrl: String, coverPath: String): St
return if (startsWith("https://")) this else "$apiUrl/$coverPath$this" return if (startsWith("https://")) this else "$apiUrl/$coverPath$this"
} }
private fun String.toPermSlugIfNeeded(fetchAllTitles: Boolean): String {
return if (fetchAllTitles) {
this.replace(HeanCms.TIMESTAMP_REGEX, "")
} else {
this
}
}
fun String.toStatus(): Int = when (this) { fun String.toStatus(): Int = when (this) {
"Ongoing" -> SManga.ONGOING "Ongoing" -> SManga.ONGOING
"Hiatus" -> SManga.ON_HIATUS "Hiatus" -> SManga.ON_HIATUS

View File

@ -9,7 +9,7 @@ class HeanCmsGenerator : ThemeSourceGenerator {
override val themeClass = "HeanCms" override val themeClass = "HeanCms"
override val baseVersionCode: Int = 16 override val baseVersionCode: Int = 17
override val sources = listOf( override val sources = listOf(
SingleLang("Glorious Scan", "https://gloriousscan.com", "pt-BR", overrideVersionCode = 17), SingleLang("Glorious Scan", "https://gloriousscan.com", "pt-BR", overrideVersionCode = 17),