Draff/Tachiyomi-Extensions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HeanCMS: Add fetchStrategy (#17320)

Browse Source 
* Add fetchStrategy

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>

* Remove YugenMangas companion object

* Lol

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* Update multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt

Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* Fixes

* get slug directly instead convert to SManga

* change new result function

* remove unnecessary var

* unused imports

* Exact search NOT RETURN LATEST TIMESTAMP O.o

* parse response directly

Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>

* change search query

---------

Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>
Co-authored-by: Alessandro Jean <14254807+alessandrojean@users.noreply.github.com>
This commit is contained in:
Rolando Lecca 2023-07-31 12:56:44 -05:00 committed by GitHub
parent 31ae3fc43a
commit f9834fcca0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 123 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
multisrc/overrides/heancms/reaperscans/src/ReaperScans.kt
View File

@ -21,7 +21,7 @@ class ReaperScans : HeanCms(
// Site changed from Madara to HeanCms.
override val versionId = 2
override val fetchAllTitles = true
override val fetchAllTitlesStrategy = FetchAllStrategy.SEARCH_ALL
override val coverPath: String = ""

47
multisrc/overrides/heancms/yugenmangas/src/YugenMangas.kt
View File

@ -1,16 +1,8 @@
package eu.kanade.tachiyomi.extension.es.yugenmangas
import android.app.Application
import android.content.SharedPreferences
import android.widget.Toast
import androidx.preference.PreferenceScreen
import androidx.preference.SwitchPreferenceCompat
import eu.kanade.tachiyomi.multisrc.heancms.Genre
import eu.kanade.tachiyomi.multisrc.heancms.HeanCms
import eu.kanade.tachiyomi.network.interceptor.rateLimit
import eu.kanade.tachiyomi.source.ConfigurableSource
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.text.SimpleDateFormat
import java.util.TimeZone
import java.util.concurrent.TimeUnit
@ -21,17 +13,12 @@ class YugenMangas :
"https://yugenmangas.net",
"es",
"https://api.yugenmangas.net",
),
ConfigurableSource {
private val preferences: SharedPreferences by lazy {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
}
) {
// Site changed from Madara to HeanCms.
override val versionId = 2
override val fetchAllTitles = getFetchAllSeriesPref()
override val fetchAllTitlesStrategy = FetchAllStrategy.SEARCH_EACH
override val client = super.client.newBuilder()
.connectTimeout(60, TimeUnit.SECONDS)
@ -45,26 +32,6 @@ class YugenMangas :
timeZone = TimeZone.getTimeZone("UTC")
}
private fun getFetchAllSeriesPref(): Boolean {
return preferences.getBoolean(FETCH_ALL_SERIES_PREF, FETCH_ALL_SERIES_DEFAULT_VALUE)
}
override fun setupPreferenceScreen(screen: PreferenceScreen) {
val fetchAllSeriesPreference = SwitchPreferenceCompat(screen.context).apply {
key = FETCH_ALL_SERIES_PREF
title = FETCH_ALL_SERIES_TITLE
summary = FETCH_ALL_SERIES_SUMMARY
setDefaultValue(FETCH_ALL_SERIES_DEFAULT_VALUE)
setOnPreferenceChangeListener { _, newValue ->
Toast.makeText(screen.context, "Reinicia la app para aplicar los cambios.", Toast.LENGTH_LONG).show()
true
}
}
screen.addPreference(fetchAllSeriesPreference)
}
override fun getGenreList(): List<Genre> = listOf(
Genre("+18", 1),
Genre("Acción", 36),
@ -115,14 +82,4 @@ class YugenMangas :
Genre("Yaoi", 43),
Genre("Yuri", 44),
)
companion object {
private const val FETCH_ALL_SERIES_PREF = "fetchAllSeriesPref"
private const val FETCH_ALL_SERIES_TITLE = "Buscar todas las series"
private const val FETCH_ALL_SERIES_SUMMARY = "Busca las URLs actuales de las series. " +
"Habilitar esta opción evita la necesidad de migrar, " +
"pero puede llevar tiempo dependiendo del número total de series. \n" +
"Tendrá que migrar cada vez que cambie esta opción."
private const val FETCH_ALL_SERIES_DEFAULT_VALUE = true
}
}

124
multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt
View File

@ -1,5 +1,7 @@
package eu.kanade.tachiyomi.multisrc.heancms
import android.app.Application
import android.content.SharedPreferences
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.source.model.Filter
@ -19,6 +21,8 @@ import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import rx.Observable
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.text.SimpleDateFormat
import java.util.Locale
@ -29,6 +33,10 @@ abstract class HeanCms(
protected val apiUrl: String = baseUrl.replace("://", "://api."),
) : HttpSource() {
private val preferences: SharedPreferences by lazy {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
}
override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient
@ -45,7 +53,7 @@ abstract class HeanCms(
protected val intl by lazy { HeanCmsIntl(lang) }
protected open val fetchAllTitles: Boolean = false
protected open val fetchAllTitlesStrategy = FetchAllStrategy.NONE
protected open val coverPath: String = "cover/"
@ -81,7 +89,7 @@ abstract class HeanCms(
if (json.startsWith("{")) {
val result = json.parseAs<HeanCmsQuerySearchDto>()
val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) }
fetchAllTitles()
@ -89,7 +97,7 @@ abstract class HeanCms(
}
val mangaList = json.parseAs<List<HeanCmsSeriesDto>>()
.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
.map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) }
fetchAllTitles()
@ -179,14 +187,14 @@ abstract class HeanCms(
val result = json.parseAs<List<HeanCmsSearchDto>>()
val mangaList = result
.filter { it.type == "Comic" }
.map { it.toSManga(apiUrl, coverPath, seriesSlugMap.orEmpty(), fetchAllTitles) }
.map { it.toSManga(apiUrl, coverPath, seriesSlugMap.orEmpty(), fetchAllTitlesStrategy) }
return MangasPage(mangaList, false)
}
if (json.startsWith("{")) {
val result = json.parseAs<HeanCmsQuerySearchDto>()
val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
val mangaList = result.data.map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) }
fetchAllTitles()
@ -194,7 +202,7 @@ abstract class HeanCms(
}
val mangaList = json.parseAs<List<HeanCmsSeriesDto>>()
.map { it.toSManga(apiUrl, coverPath, fetchAllTitles) }
.map { it.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy) }
fetchAllTitles()
@ -206,12 +214,38 @@ abstract class HeanCms(
.substringAfterLast("/")
.replace(TIMESTAMP_REGEX, "")
val currentSlug = seriesSlugMap?.get(seriesSlug)?.slug ?: manga.url.substringAfterLast("/")
val currentSlug = if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) {
preferences.slugMap[seriesSlug] ?: manga.url.substringAfterLast("/")
} else {
seriesSlugMap?.get(seriesSlug)?.slug ?: manga.url.substringAfterLast("/")
}
return "$baseUrl/series/$currentSlug"
}
override fun mangaDetailsRequest(manga: SManga): Request {
if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) {
val searchQuery = manga.title.trim()
.replaceAfterLast(" ", "").trim()
.let {
if (it.length > 2) it.dropLast(1).trim() else it
}
val searchPayloadObj = HeanCmsSearchPayloadDto(searchQuery)
val searchPayload = json.encodeToString(searchPayloadObj)
.toRequestBody(JSON_MEDIA_TYPE)
val apiHeaders = headersBuilder()
.add("Accept", ACCEPT_JSON)
.add("Content-Type", searchPayload.contentType().toString())
.build()
val mangaSlug = manga.url
.substringAfterLast("/")
.replace(TIMESTAMP_REGEX, "")
return POST("$apiUrl/series/search#$mangaSlug", apiHeaders, searchPayload)
}
val seriesSlug = manga.url
.substringAfterLast("/")
.replace(TIMESTAMP_REGEX, "")
@ -230,30 +264,72 @@ abstract class HeanCms(
}
override fun mangaDetailsParse(response: Response): SManga {
val result = runCatching { response.parseAs<HeanCmsSeriesDto>() }
val seriesDetails = result.getOrNull()?.toSManga(apiUrl, coverPath, fetchAllTitles)
val mangaStatus = response.request.url.fragment?.toIntOrNull() ?: SManga.UNKNOWN
val result = runCatching {
if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) {
val originalSlug = response.request.url.fragment!!
val searchResult = response.parseAs<List<HeanCmsSearchDto>>()
searchResultToSeries(originalSlug, searchResult)
} else {
response.parseAs<HeanCmsSeriesDto>()
}
}
val seriesDetails = result.getOrNull()?.toSManga(apiUrl, coverPath, fetchAllTitlesStrategy)
?: throw Exception(intl.urlChangedError(name))
return seriesDetails.apply {
status = status.takeUnless { it == SManga.UNKNOWN }
?: response.request.url.fragment?.toIntOrNull() ?: SManga.UNKNOWN
?: mangaStatus
}
}
override fun chapterListRequest(manga: SManga): Request = mangaDetailsRequest(manga)
override fun chapterListParse(response: Response): List<SChapter> {
val result = response.parseAs<HeanCmsSeriesDto>()
val seriesSlug = response.request.url.pathSegments.last()
val result = if (fetchAllTitlesStrategy == FetchAllStrategy.SEARCH_EACH) {
val originalSlug = response.request.url.fragment!!
val searchResult = response.parseAs<List<HeanCmsSearchDto>>()
searchResultToSeries(originalSlug, searchResult)
} else {
response.parseAs<HeanCmsSeriesDto>()
}
val currentTimestamp = System.currentTimeMillis()
return result.chapters.orEmpty()
.filterNot { it.price == 1 }
.map { it.toSChapter(seriesSlug, dateFormat) }
.map { it.toSChapter(result.slug, dateFormat) }
.filter { it.date_upload <= currentTimestamp }
.reversed()
}
private fun searchResultToSeries(originalSlug: String, searchResult: List<HeanCmsSearchDto>): HeanCmsSeriesDto {
val mangaSlug = searchResult
.filter { it.type == "Comic" }
.map { it.slug }
.find { it.replace(TIMESTAMP_REGEX, "") == originalSlug }
?: throw Exception(intl.urlChangedError(name))
val apiHeaders = headersBuilder()
.add("Accept", ACCEPT_JSON)
.build()
val detailsRequest = GET("$apiUrl/series/$mangaSlug", apiHeaders)
val result = client.newCall(detailsRequest).execute()
.parseAs<HeanCmsSeriesDto>()
val permSlug = result.slug
.substringAfterLast("/")
.replace(TIMESTAMP_REGEX, "")
preferences.slugMap = preferences.slugMap.toMutableMap()
.also { it[permSlug] = result.slug.substringAfterLast("/") }
return result
}
override fun getChapterUrl(chapter: SChapter): String = baseUrl + chapter.url
override fun pageListRequest(chapter: SChapter): Request {
@ -309,7 +385,7 @@ abstract class HeanCms(
protected open fun getGenreList(): List<Genre> = emptyList()
protected open fun fetchAllTitles() {
if (!seriesSlugMap.isNullOrEmpty() || !fetchAllTitles) {
if (!seriesSlugMap.isNullOrEmpty() || fetchAllTitlesStrategy != FetchAllStrategy.SEARCH_ALL) {
return
}
@ -358,6 +434,18 @@ abstract class HeanCms(
return POST("$apiUrl/series/querysearch", apiHeaders, payload)
}
protected var SharedPreferences.slugMap: MutableMap<String, String>
get() {
val jsonMap = getString(PREF_URL_MAP, "{}")!!
val slugMap = runCatching { json.decodeFromString<Map<String, String>>(jsonMap) }
return slugMap.getOrNull()?.toMutableMap() ?: mutableMapOf()
}
set(newSlugMap) {
edit()
.putString(PREF_URL_MAP, json.encodeToString(newSlugMap))
.commit()
}
protected open fun parseAllTitles(result: List<HeanCmsSeriesDto>): Map<String, HeanCmsTitle> {
return result
.filter { it.type == "Comic" }
@ -401,6 +489,12 @@ abstract class HeanCms(
*/
data class HeanCmsTitle(val slug: String, val thumbnailFileName: String, val status: Int)
enum class FetchAllStrategy {
NONE,
SEARCH_EACH,
SEARCH_ALL,
}
companion object {
private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
private const val ACCEPT_JSON = "application/json, text/plain, */*"
@ -410,5 +504,7 @@ abstract class HeanCms(
val TIMESTAMP_REGEX = "-\\d+$".toRegex()
const val SEARCH_PREFIX = "slug:"
private const val PREF_URL_MAP = "pref_url_map"
}
}

13
multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt
View File

@ -1,5 +1,6 @@
package eu.kanade.tachiyomi.multisrc.heancms
import eu.kanade.tachiyomi.multisrc.heancms.HeanCms.FetchAllStrategy
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import kotlinx.serialization.SerialName
@ -36,9 +37,9 @@ data class HeanCmsSearchDto(
apiUrl: String,
coverPath: String,
slugMap: Map<String, HeanCms.HeanCmsTitle>,
usePermSlug: Boolean,
fetchStrategy: FetchAllStrategy = FetchAllStrategy.NONE,
): SManga = SManga.create().apply {
val slug = if (!usePermSlug) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "")
val slug = if (fetchStrategy == FetchAllStrategy.NONE) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "")
val thumbnailFileName = slugMap[slug]?.thumbnailFileName
title = this@HeanCmsSearchDto.title
@ -63,9 +64,13 @@ data class HeanCmsSeriesDto(
val chapters: List<HeanCmsChapterDto>? = emptyList(),
) {
fun toSManga(apiUrl: String, coverPath: String, usePermSlug: Boolean): SManga = SManga.create().apply {
fun toSManga(
apiUrl: String,
coverPath: String,
fetchStrategy: FetchAllStrategy = FetchAllStrategy.NONE,
): SManga = SManga.create().apply {
val descriptionBody = this@HeanCmsSeriesDto.description?.let(Jsoup::parseBodyFragment)
val slug = if (!usePermSlug) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "")
val slug = if (fetchStrategy == FetchAllStrategy.NONE) slug else slug.replace(HeanCms.TIMESTAMP_REGEX, "")
title = this@HeanCmsSeriesDto.title
author = this@HeanCmsSeriesDto.author?.trim()

2
multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsGenerator.kt
View File

@ -9,7 +9,7 @@ class HeanCmsGenerator : ThemeSourceGenerator {
override val themeClass = "HeanCms"
override val baseVersionCode: Int = 14
override val baseVersionCode: Int = 15
override val sources = listOf(
SingleLang("Glorious Scan", "https://gloriousscan.com", "pt-BR", overrideVersionCode = 17),