From 4554a0c71793006e054f0a1e71a6d20fd6de31b2 Mon Sep 17 00:00:00 2001
From: Wackery <32279125+webmsgr@users.noreply.github.com>
Date: Fri, 2 Aug 2024 03:33:54 -0700
Subject: [PATCH] Reaper Scans: Update to new website (#4359)
* Update reaper scans to new website
* Reaper Scans: Fix requested changes
* Reaper Scans: remove unused property.
* Reaper Scans: Fix no pages found error
they broke it already???
* Reaper Scans: Fix extra images in chapters.
* Reaper Scans: Replace scraping chapter pages with api
* Reaper Scans: Fix lint
hopefully
* HeanCms: Add cdnUrl option
for sources who host thumbnails/pages outside of the api domain
* Reaper Scans: it was a multisrc this whole time
pain.
* Reaper Scans: Remove id override
* Reaper Scans: Correct version number
---
.../tachiyomi/multisrc/heancms/HeanCms.kt | 8 +-
.../tachiyomi/multisrc/heancms/HeanCmsDto.kt | 8 +-
src/en/reaperscans/AndroidManifest.xml | 22 --
src/en/reaperscans/build.gradle | 4 +-
.../extension/en/reaperscans/ReaperScans.kt | 363 +-----------------
.../en/reaperscans/ReaperScansDto.kt | 21 -
.../en/reaperscans/ReaperScansUrlActivity.kt | 34 --
7 files changed, 19 insertions(+), 441 deletions(-)
delete mode 100644 src/en/reaperscans/AndroidManifest.xml
delete mode 100644 src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt
delete mode 100644 src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansUrlActivity.kt
diff --git a/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt b/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt
index 6909288cd..d22e92437 100644
--- a/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt
+++ b/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCms.kt
@@ -72,6 +72,8 @@ abstract class HeanCms(
protected open val coverPath: String = ""
+ protected open val cdnUrl = apiUrl
+
protected open val mangaSubDirectory: String = "series"
protected open val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ", Locale.US)
@@ -242,7 +244,7 @@ abstract class HeanCms(
val seriesResult = result.getOrNull()
?: throw Exception(intl.format("url_changed_error", name, name))
- val seriesDetails = seriesResult.toSManga(apiUrl, coverPath, mangaSubDirectory)
+ val seriesDetails = seriesResult.toSManga(cdnUrl, coverPath, mangaSubDirectory)
return seriesDetails.apply {
status = status.takeUnless { it == SManga.UNKNOWN }
@@ -345,8 +347,8 @@ abstract class HeanCms(
}
}
- private fun String.toAbsoluteUrl(): String {
- return if (startsWith("https://") || startsWith("http://")) this else "$apiUrl/$coverPath$this"
+ protected open fun String.toAbsoluteUrl(): String {
+ return if (startsWith("https://") || startsWith("http://")) this else "$cdnUrl/$coverPath$this"
}
override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl!!)
diff --git a/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt b/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt
index 06d3f38df..e74df068e 100644
--- a/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt
+++ b/lib-multisrc/heancms/src/eu/kanade/tachiyomi/multisrc/heancms/HeanCmsDto.kt
@@ -63,7 +63,7 @@ class HeanCmsSeriesDto(
) {
fun toSManga(
- apiUrl: String,
+ cdnUrl: String,
coverPath: String,
mangaSubDirectory: String,
): SManga = SManga.create().apply {
@@ -79,7 +79,7 @@ class HeanCmsSeriesDto(
.sortedBy(HeanCmsTagDto::name)
.joinToString { it.name }
thumbnail_url = thumbnail.ifEmpty { null }
- ?.toAbsoluteThumbnailUrl(apiUrl, coverPath)
+ ?.toAbsoluteThumbnailUrl(cdnUrl, coverPath)
status = this@HeanCmsSeriesDto.status?.toStatus() ?: SManga.UNKNOWN
url = "/$mangaSubDirectory/$slug#$id"
}
@@ -161,8 +161,8 @@ class HeanCmsGenreDto(
val name: String,
)
-private fun String.toAbsoluteThumbnailUrl(apiUrl: String, coverPath: String): String {
- return if (startsWith("https://") || startsWith("http://")) this else "$apiUrl/$coverPath$this"
+private fun String.toAbsoluteThumbnailUrl(cdnUrl: String, coverPath: String): String {
+ return if (startsWith("https://") || startsWith("http://")) this else "$cdnUrl/$coverPath$this"
}
fun String.toStatus(): Int = when (this) {
diff --git a/src/en/reaperscans/AndroidManifest.xml b/src/en/reaperscans/AndroidManifest.xml
deleted file mode 100644
index 3905aa593..000000000
--- a/src/en/reaperscans/AndroidManifest.xml
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<manifest package="eu.kanade.tachiyomi.extension" xmlns:android="http://schemas.android.com/apk/res/android">
- <application>
- <activity
- android:name=".en.reaperscans.ReaperScansUrlActivity"
- android:excludeFromRecents="true"
- android:exported="true"
- android:theme="@android:style/Theme.NoDisplay">
- <intent-filter>
- <action android:name="android.intent.action.VIEW" />
-
- <category android:name="android.intent.category.DEFAULT" />
- <category android:name="android.intent.category.BROWSABLE" />
-
- <data
- android:host="reaperscans.com"
- android:pathPattern="/comics/..*"
- android:scheme="https" />
- </intent-filter>
- </activity>
- </application>
-</manifest>
diff --git a/src/en/reaperscans/build.gradle b/src/en/reaperscans/build.gradle
index 481395bcc..8939e1561 100644
--- a/src/en/reaperscans/build.gradle
+++ b/src/en/reaperscans/build.gradle
@@ -1,7 +1,9 @@
ext {
extName = 'Reaper Scans'
extClass = '.ReaperScans'
- extVersionCode = 49
+ themePkg = 'heancms'
+ baseUrl = 'https://reaperscans.com'
+ overrideVersionCode = 25
}
apply from: "$rootDir/common.gradle"
diff --git a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt
index 88db88acf..cacca9a16 100644
--- a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt
+++ b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt
@@ -1,363 +1,14 @@
package eu.kanade.tachiyomi.extension.en.reaperscans
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.network.interceptor.rateLimit
-import eu.kanade.tachiyomi.source.model.FilterList
-import eu.kanade.tachiyomi.source.model.MangasPage
-import eu.kanade.tachiyomi.source.model.Page
-import eu.kanade.tachiyomi.source.model.SChapter
-import eu.kanade.tachiyomi.source.model.SManga
-import eu.kanade.tachiyomi.source.online.ParsedHttpSource
-import eu.kanade.tachiyomi.util.asJsoup
-import kotlinx.serialization.decodeFromString
-import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.add
-import kotlinx.serialization.json.addJsonObject
-import kotlinx.serialization.json.buildJsonObject
-import kotlinx.serialization.json.contentOrNull
-import kotlinx.serialization.json.jsonPrimitive
-import kotlinx.serialization.json.put
-import kotlinx.serialization.json.putJsonArray
-import kotlinx.serialization.json.putJsonObject
-import okhttp3.Headers
-import okhttp3.MediaType.Companion.toMediaType
-import okhttp3.OkHttpClient
-import okhttp3.Request
-import okhttp3.RequestBody.Companion.toRequestBody
-import okhttp3.Response
-import org.jsoup.Jsoup
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
-import org.jsoup.select.Elements
-import rx.Observable
-import uy.kohesive.injekt.injectLazy
-import java.util.Calendar
-import java.util.concurrent.TimeUnit
-import kotlin.random.Random
+import eu.kanade.tachiyomi.multisrc.heancms.HeanCms
-class ReaperScans : ParsedHttpSource() {
+class ReaperScans : HeanCms("Reaper Scans", "https://reaperscans.com", "en") {
- override val name = "Reaper Scans"
+ override val versionId = 2
- override val baseUrl = "https://reaperscans.com"
+ override val useNewChapterEndpoint = true
+ override val useNewQueryEndpoint = true
+ override val enableLogin = true
- override val lang = "en"
-
- override val id = 5177220001642863679
-
- override val supportsLatest = true
-
- private val json: Json by injectLazy()
-
- override val client: OkHttpClient = network.cloudflareClient.newBuilder()
- .rateLimit(1, 2, TimeUnit.SECONDS)
- .addInterceptor { chain ->
- val request = chain.request()
- val headers = request.headers.newBuilder()
- .removeAll("X-Requested-With")
- .build()
- chain.proceed(request.newBuilder().headers(headers).build())
- }
- .build()
-
- override fun headersBuilder(): Headers.Builder = super.headersBuilder()
- .set("X-Requested-With", randomString((1..20).random())) // For WebView, removed in interceptor
-
- // Popular
- override fun popularMangaRequest(page: Int) = GET("$baseUrl/comics?page=$page", headers)
-
- override fun popularMangaNextPageSelector(): String = "button[wire:click*=nextPage]"
-
- override fun popularMangaSelector(): String = "li"
-
- override fun popularMangaFromElement(element: Element): SManga {
- return SManga.create().apply {
- element.select("a.text-white").let {
- title = it.text()
- setUrlWithoutDomain(it.attr("href"))
- }
- thumbnail_url = element.select("img").imgAttr()
- }
- }
-
- // Latest
- override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/latest/comics?page=$page", headers)
-
- override fun latestUpdatesNextPageSelector(): String = "button[wire:click*=nextPage]"
-
- override fun latestUpdatesSelector(): String = ".grid > div"
-
- override fun latestUpdatesFromElement(element: Element): SManga {
- return SManga.create().apply {
- element.select("p > a").let {
- title = it.text().trim()
- setUrlWithoutDomain(it.attr("href"))
- }
- thumbnail_url = element.select("img").imgAttr()
- }
- }
-
- // Search
- override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
- val response = client.newCall(GET(baseUrl)).execute()
- val soup = response.asJsoup()
-
- val csrfToken = soup.selectFirst("meta[name=csrf-token]")?.attr("content")
-
- val livewareData = soup.selectFirst("div[wire:initial-data*=comics]")
- ?.attr("wire:initial-data")
- ?.parseJson<LiveWireDataDto>()
-
- if (csrfToken == null) error("Couldn't find csrf-token")
- if (livewareData == null) error("Couldn't find LiveWireData")
-
- val routeName = livewareData.fingerprint["name"]?.jsonPrimitive?.contentOrNull
- ?: error("Couldn't find routeName")
-
- // Javascript: (Math.random() + 1).toString(36).substring(8)
- val generateId = { -> "1.${Random.nextLong().toString(36)}".substring(10) } // Not exactly the same, but results in a 3-5 character string
- val payload = buildJsonObject {
- put("fingerprint", livewareData.fingerprint)
- put("serverMemo", livewareData.serverMemo)
- putJsonArray("updates") {
- addJsonObject {
- put("type", "syncInput")
- putJsonObject("payload") {
- put("id", generateId())
- put("name", "query")
- put("value", query)
- }
- }
- }
- }.toString().toRequestBody(JSON_MEDIA_TYPE)
-
- val headers = Headers.Builder()
- .add("x-csrf-token", csrfToken)
- .add("x-livewire", "true")
- .build()
-
- return POST("$baseUrl/livewire/message/$routeName", headers, payload)
- }
-
- override fun searchMangaSelector(): String = "a[href*=/comics/]"
-
- override fun searchMangaParse(response: Response): MangasPage {
- val html = response.parseJson<LiveWireResponseDto>().effects.html
- val mangas = Jsoup.parse(html, baseUrl).select(searchMangaSelector()).map { element ->
- searchMangaFromElement(element)
- }
- return MangasPage(mangas, false)
- }
-
- override fun searchMangaFromElement(element: Element): SManga {
- return SManga.create().apply {
- setUrlWithoutDomain(element.attr("href"))
- element.select("img").first()?.let {
- thumbnail_url = it.imgAttr()
- }
- title = element.select("p").first()!!.text()
- }
- }
-
- override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
- if (query.startsWith(PREFIX_ID_SEARCH)) {
- val realUrl = "/comics/" + query.removePrefix(PREFIX_ID_SEARCH)
- val manga = SManga.create().apply {
- url = realUrl
- }
- return fetchMangaDetails(manga).map {
- MangasPage(listOf(it.apply { url = realUrl }), false)
- }
- }
- return super.fetchSearchManga(page, query, filters)
- }
-
- // Details
- override fun mangaDetailsParse(document: Document): SManga {
- return SManga.create().apply {
- thumbnail_url = document.select("div > img").first()!!.imgAttr()
- title = document.select("h1").first()!!.text()
-
- status = when (document.select("dt:contains(Release Status)").next().first()!!.text()) {
- "On hold" -> SManga.ON_HIATUS
- "Complete" -> SManga.COMPLETED
- "Ongoing" -> SManga.ONGOING
- "Dropped" -> SManga.CANCELLED
- else -> SManga.UNKNOWN
- }
-
- genre = mutableListOf<String>().apply {
- when (document.select("dt:contains(Source Language)").next().first()!!.text()) {
- "Korean" -> "Manhwa"
- "Chinese" -> "Manhua"
- "Japanese" -> "Manga"
- else -> null
- }?.let { add(it) }
- }.takeIf { it.isNotEmpty() }?.joinToString(",")
-
- description = document.select("section > div:nth-child(1) > div > p").first()!!.text()
- }
- }
-
- // Chapters
- private fun chapterListNextPageSelector(): String = "button[wire:click*=nextPage]"
-
- override fun chapterListSelector() = "div[wire:id] > div > ul[role=list] > li"
-
- override fun chapterListParse(response: Response): List<SChapter> {
- val document = response.asJsoup()
- val chapters = mutableListOf<SChapter>()
- document.select(chapterListSelector()).forEach { chapters.add(chapterFromElement(it)) }
- var hasNextPage = document.selectFirst(chapterListNextPageSelector()) != null
-
- if (!hasNextPage) {
- return chapters
- }
-
- val csrfToken = document.selectFirst("meta[name=csrf-token]")?.attr("content")
- ?: error("Couldn't find csrf-token")
-
- val livewareData = document.selectFirst("div[wire:initial-data*=Models\\\\Comic]")
- ?.attr("wire:initial-data")
- ?.parseJson<LiveWireDataDto>()
- ?: error("Couldn't find LiveWireData")
-
- val routeName = livewareData.fingerprint["name"]?.jsonPrimitive?.contentOrNull
- ?: error("Couldn't find routeName")
-
- val fingerprint = livewareData.fingerprint
- var serverMemo = livewareData.serverMemo
-
- var pageToQuery = 2
-
- // Javascript: (Math.random() + 1).toString(36).substring(8)
- val generateId = { "1.${Random.nextLong().toString(36)}".substring(10) } // Not exactly the same, but results in a 3-5 character string
- while (hasNextPage) {
- val payload = buildJsonObject {
- put("fingerprint", fingerprint)
- put("serverMemo", serverMemo)
- putJsonArray("updates") {
- addJsonObject {
- put("type", "callMethod")
- putJsonObject("payload") {
- put("id", generateId())
- put("method", "gotoPage")
- putJsonArray("params") {
- add(pageToQuery)
- add("page")
- }
- }
- }
- }
- }.toString().toRequestBody(JSON_MEDIA_TYPE)
-
- val headers = Headers.Builder()
- .add("x-csrf-token", csrfToken)
- .add("x-livewire", "true")
- .build()
-
- val request = POST("$baseUrl/livewire/message/$routeName", headers, payload)
-
- val responseData = client.newCall(request).execute().parseJson<LiveWireResponseDto>()
-
- // response contains state that we need to preserve
- serverMemo = serverMemo.mergeLeft(responseData.serverMemo)
- val chaptersHtml = Jsoup.parse(responseData.effects.html, baseUrl)
- chaptersHtml.select(chapterListSelector()).forEach { chapters.add(chapterFromElement(it)) }
- hasNextPage = chaptersHtml.selectFirst(chapterListNextPageSelector()) != null
- pageToQuery++
- }
-
- return chapters
- }
-
- override fun chapterFromElement(element: Element): SChapter {
- return SChapter.create().apply {
- element.selectFirst("a")?.let { urlElement ->
- setUrlWithoutDomain(urlElement.attr("href"))
- urlElement.select("p").let {
- name = it.getOrNull(0)?.text() ?: ""
- date_upload = it.getOrNull(1)?.text()?.parseRelativeDate() ?: 0
- }
- }
- }
- }
-
- // Page
- override fun pageListParse(document: Document): List<Page> {
- document.select("noscript").remove()
- return document.select("img.max-w-full").mapIndexed { index, element ->
- Page(index, imageUrl = element.imgAttr())
- }
- }
-
- // Helpers
- private inline fun <reified T> Response.parseJson(): T = use {
- it.body.string().parseJson()
- }
-
- private inline fun <reified T> String.parseJson(): T = json.decodeFromString(this)
-
- /**
- * Recursively merges j2 onto j1 in place
- * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's
- *
- */
- private fun JsonObject.mergeLeft(j2: JsonObject): JsonObject = buildJsonObject {
- val j1 = this@mergeLeft
- j1.entries.forEach { (key, value) -> put(key, value) }
- j2.entries.forEach { (key, value) ->
- val j1Value = j1[key]
- when {
- j1Value !is JsonObject -> put(key, value)
- value is JsonObject -> put(key, j1Value.mergeLeft(value))
- }
- }
- }
-
- /**
- * Parses dates in this form: 21 hours ago
- * Taken from multisrc/madara/Madara.kt
- */
- private fun String.parseRelativeDate(): Long {
- val number = Regex("""(\d+)""").find(this)?.value?.toIntOrNull() ?: return 0
- val cal = Calendar.getInstance()
-
- return when {
- contains("day") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
- contains("hour") -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
- contains("minute") -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
- contains("second") -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
- contains("week") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis
- contains("month") -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
- contains("year") -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
- else -> 0
- }
- }
-
- private fun Element.imgAttr(): String = when {
- hasAttr("data-lazy-src") -> attr("abs:data-lazy-src")
- hasAttr("data-src") -> attr("abs:data-src")
- hasAttr("data-cfsrc") -> attr("abs:data-cfsrc")
- else -> attr("abs:src")
- }
-
- private fun Elements.imgAttr(): String = this.first()!!.imgAttr()
-
- private fun randomString(length: Int): String {
- val charPool = ('a'..'z') + ('A'..'Z')
- return List(length) { charPool.random() }.joinToString("")
- }
-
- // Unused
- override fun searchMangaNextPageSelector() = throw UnsupportedOperationException()
-
- override fun imageUrlParse(document: Document) = throw UnsupportedOperationException()
-
- companion object {
- private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType()
- const val PREFIX_ID_SEARCH = "id:"
- }
+ override val cdnUrl = "https://media.reaperscans.com/file/4SRBHm"
}
diff --git a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt
deleted file mode 100644
index 62246fb66..000000000
--- a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt
+++ /dev/null
@@ -1,21 +0,0 @@
-package eu.kanade.tachiyomi.extension.en.reaperscans
-
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.JsonObject
-
-@Serializable
-data class LiveWireResponseDto(
- val effects: LiveWireEffectsDto,
- val serverMemo: JsonObject,
-)
-
-@Serializable
-data class LiveWireEffectsDto(
- val html: String,
-)
-
-@Serializable
-data class LiveWireDataDto(
- val fingerprint: JsonObject,
- val serverMemo: JsonObject,
-)
diff --git a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansUrlActivity.kt b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansUrlActivity.kt
deleted file mode 100644
index 6442e6411..000000000
--- a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansUrlActivity.kt
+++ /dev/null
@@ -1,34 +0,0 @@
-package eu.kanade.tachiyomi.extension.en.reaperscans
-
-import android.app.Activity
-import android.content.ActivityNotFoundException
-import android.content.Intent
-import android.os.Bundle
-import android.util.Log
-import kotlin.system.exitProcess
-
-class ReaperScansUrlActivity : Activity() {
- override fun onCreate(savedInstanceState: Bundle?) {
- super.onCreate(savedInstanceState)
- val pathSegments = intent?.data?.pathSegments
- if (pathSegments != null && pathSegments.size >= 2) {
- val id = pathSegments[1]
- val mainIntent = Intent().apply {
- action = "eu.kanade.tachiyomi.SEARCH"
- putExtra("query", ReaperScans.PREFIX_ID_SEARCH + id)
- putExtra("filter", packageName)
- }
-
- try {
- startActivity(mainIntent)
- } catch (e: ActivityNotFoundException) {
- Log.e("ReaperScansUrlActivity", e.toString())
- }
- } else {
- Log.e("ReaperScansUrlActivity", "could not parse uri from intent $intent")
- }
-
- finish()
- exitProcess(0)
- }
-}