Reaper Scans: Update to new website (#4359)

* Update reaper scans to new website

* Reaper Scans: Fix requested changes

* Reaper Scans: remove unused property.

* Reaper Scans: Fix no pages found error

they broke it already???

* Reaper Scans: Fix extra images in chapters.

* Reaper Scans: Replace scraping chapter pages with api

* Reaper Scans: Fix lint

hopefully

* HeanCms: Add cdnUrl option

for sources who host thumbnails/pages outside of the api domain

* Reaper Scans: it was a multisrc this whole time

pain.

* Reaper Scans: Remove id override

* Reaper Scans: Correct version number
This commit is contained in:
Wackery 2024-08-02 03:33:54 -07:00 committed by Draff
parent 7e3d185dab
commit 4554a0c717
No known key found for this signature in database
GPG Key ID: E8A89F3211677653
7 changed files with 19 additions and 441 deletions

View File

@ -72,6 +72,8 @@ abstract class HeanCms(
protected open val coverPath: String = ""
protected open val cdnUrl = apiUrl
protected open val mangaSubDirectory: String = "series"
protected open val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ", Locale.US)
@ -242,7 +244,7 @@ abstract class HeanCms(
val seriesResult = result.getOrNull()
?: throw Exception(intl.format("url_changed_error", name, name))
val seriesDetails = seriesResult.toSManga(apiUrl, coverPath, mangaSubDirectory)
val seriesDetails = seriesResult.toSManga(cdnUrl, coverPath, mangaSubDirectory)
return seriesDetails.apply {
status = status.takeUnless { it == SManga.UNKNOWN }
@ -345,8 +347,8 @@ abstract class HeanCms(
}
}
private fun String.toAbsoluteUrl(): String {
return if (startsWith("https://") || startsWith("http://")) this else "$apiUrl/$coverPath$this"
protected open fun String.toAbsoluteUrl(): String {
return if (startsWith("https://") || startsWith("http://")) this else "$cdnUrl/$coverPath$this"
}
override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl!!)

View File

@ -63,7 +63,7 @@ class HeanCmsSeriesDto(
) {
fun toSManga(
apiUrl: String,
cdnUrl: String,
coverPath: String,
mangaSubDirectory: String,
): SManga = SManga.create().apply {
@ -79,7 +79,7 @@ class HeanCmsSeriesDto(
.sortedBy(HeanCmsTagDto::name)
.joinToString { it.name }
thumbnail_url = thumbnail.ifEmpty { null }
?.toAbsoluteThumbnailUrl(apiUrl, coverPath)
?.toAbsoluteThumbnailUrl(cdnUrl, coverPath)
status = this@HeanCmsSeriesDto.status?.toStatus() ?: SManga.UNKNOWN
url = "/$mangaSubDirectory/$slug#$id"
}
@ -161,8 +161,8 @@ class HeanCmsGenreDto(
val name: String,
)
private fun String.toAbsoluteThumbnailUrl(apiUrl: String, coverPath: String): String {
return if (startsWith("https://") || startsWith("http://")) this else "$apiUrl/$coverPath$this"
private fun String.toAbsoluteThumbnailUrl(cdnUrl: String, coverPath: String): String {
return if (startsWith("https://") || startsWith("http://")) this else "$cdnUrl/$coverPath$this"
}
fun String.toStatus(): Int = when (this) {

View File

@ -1,22 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest package="eu.kanade.tachiyomi.extension" xmlns:android="http://schemas.android.com/apk/res/android">
<application>
<activity
android:name=".en.reaperscans.ReaperScansUrlActivity"
android:excludeFromRecents="true"
android:exported="true"
android:theme="@android:style/Theme.NoDisplay">
<intent-filter>
<action android:name="android.intent.action.VIEW" />
<category android:name="android.intent.category.DEFAULT" />
<category android:name="android.intent.category.BROWSABLE" />
<data
android:host="reaperscans.com"
android:pathPattern="/comics/..*"
android:scheme="https" />
</intent-filter>
</activity>
</application>
</manifest>

View File

@ -1,7 +1,9 @@
ext {
extName = 'Reaper Scans'
extClass = '.ReaperScans'
extVersionCode = 49
themePkg = 'heancms'
baseUrl = 'https://reaperscans.com'
overrideVersionCode = 25
}
apply from: "$rootDir/common.gradle"

View File

@ -1,363 +1,14 @@
package eu.kanade.tachiyomi.extension.en.reaperscans
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.network.interceptor.rateLimit
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.add
import kotlinx.serialization.json.addJsonObject
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import okhttp3.Headers
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.util.Calendar
import java.util.concurrent.TimeUnit
import kotlin.random.Random
import eu.kanade.tachiyomi.multisrc.heancms.HeanCms
class ReaperScans : ParsedHttpSource() {
class ReaperScans : HeanCms("Reaper Scans", "https://reaperscans.com", "en") {
override val name = "Reaper Scans"
override val versionId = 2
override val baseUrl = "https://reaperscans.com"
override val useNewChapterEndpoint = true
override val useNewQueryEndpoint = true
override val enableLogin = true
override val lang = "en"
override val id = 5177220001642863679
override val supportsLatest = true
private val json: Json by injectLazy()
override val client: OkHttpClient = network.cloudflareClient.newBuilder()
.rateLimit(1, 2, TimeUnit.SECONDS)
.addInterceptor { chain ->
val request = chain.request()
val headers = request.headers.newBuilder()
.removeAll("X-Requested-With")
.build()
chain.proceed(request.newBuilder().headers(headers).build())
}
.build()
override fun headersBuilder(): Headers.Builder = super.headersBuilder()
.set("X-Requested-With", randomString((1..20).random())) // For WebView, removed in interceptor
// Popular
override fun popularMangaRequest(page: Int) = GET("$baseUrl/comics?page=$page", headers)
override fun popularMangaNextPageSelector(): String = "button[wire:click*=nextPage]"
override fun popularMangaSelector(): String = "li"
override fun popularMangaFromElement(element: Element): SManga {
return SManga.create().apply {
element.select("a.text-white").let {
title = it.text()
setUrlWithoutDomain(it.attr("href"))
}
thumbnail_url = element.select("img").imgAttr()
}
}
// Latest
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/latest/comics?page=$page", headers)
override fun latestUpdatesNextPageSelector(): String = "button[wire:click*=nextPage]"
override fun latestUpdatesSelector(): String = ".grid > div"
override fun latestUpdatesFromElement(element: Element): SManga {
return SManga.create().apply {
element.select("p > a").let {
title = it.text().trim()
setUrlWithoutDomain(it.attr("href"))
}
thumbnail_url = element.select("img").imgAttr()
}
}
// Search
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val response = client.newCall(GET(baseUrl)).execute()
val soup = response.asJsoup()
val csrfToken = soup.selectFirst("meta[name=csrf-token]")?.attr("content")
val livewareData = soup.selectFirst("div[wire:initial-data*=comics]")
?.attr("wire:initial-data")
?.parseJson<LiveWireDataDto>()
if (csrfToken == null) error("Couldn't find csrf-token")
if (livewareData == null) error("Couldn't find LiveWireData")
val routeName = livewareData.fingerprint["name"]?.jsonPrimitive?.contentOrNull
?: error("Couldn't find routeName")
// Javascript: (Math.random() + 1).toString(36).substring(8)
val generateId = { -> "1.${Random.nextLong().toString(36)}".substring(10) } // Not exactly the same, but results in a 3-5 character string
val payload = buildJsonObject {
put("fingerprint", livewareData.fingerprint)
put("serverMemo", livewareData.serverMemo)
putJsonArray("updates") {
addJsonObject {
put("type", "syncInput")
putJsonObject("payload") {
put("id", generateId())
put("name", "query")
put("value", query)
}
}
}
}.toString().toRequestBody(JSON_MEDIA_TYPE)
val headers = Headers.Builder()
.add("x-csrf-token", csrfToken)
.add("x-livewire", "true")
.build()
return POST("$baseUrl/livewire/message/$routeName", headers, payload)
}
override fun searchMangaSelector(): String = "a[href*=/comics/]"
override fun searchMangaParse(response: Response): MangasPage {
val html = response.parseJson<LiveWireResponseDto>().effects.html
val mangas = Jsoup.parse(html, baseUrl).select(searchMangaSelector()).map { element ->
searchMangaFromElement(element)
}
return MangasPage(mangas, false)
}
override fun searchMangaFromElement(element: Element): SManga {
return SManga.create().apply {
setUrlWithoutDomain(element.attr("href"))
element.select("img").first()?.let {
thumbnail_url = it.imgAttr()
}
title = element.select("p").first()!!.text()
}
}
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
if (query.startsWith(PREFIX_ID_SEARCH)) {
val realUrl = "/comics/" + query.removePrefix(PREFIX_ID_SEARCH)
val manga = SManga.create().apply {
url = realUrl
}
return fetchMangaDetails(manga).map {
MangasPage(listOf(it.apply { url = realUrl }), false)
}
}
return super.fetchSearchManga(page, query, filters)
}
// Details
override fun mangaDetailsParse(document: Document): SManga {
return SManga.create().apply {
thumbnail_url = document.select("div > img").first()!!.imgAttr()
title = document.select("h1").first()!!.text()
status = when (document.select("dt:contains(Release Status)").next().first()!!.text()) {
"On hold" -> SManga.ON_HIATUS
"Complete" -> SManga.COMPLETED
"Ongoing" -> SManga.ONGOING
"Dropped" -> SManga.CANCELLED
else -> SManga.UNKNOWN
}
genre = mutableListOf<String>().apply {
when (document.select("dt:contains(Source Language)").next().first()!!.text()) {
"Korean" -> "Manhwa"
"Chinese" -> "Manhua"
"Japanese" -> "Manga"
else -> null
}?.let { add(it) }
}.takeIf { it.isNotEmpty() }?.joinToString(",")
description = document.select("section > div:nth-child(1) > div > p").first()!!.text()
}
}
// Chapters
private fun chapterListNextPageSelector(): String = "button[wire:click*=nextPage]"
override fun chapterListSelector() = "div[wire:id] > div > ul[role=list] > li"
override fun chapterListParse(response: Response): List<SChapter> {
val document = response.asJsoup()
val chapters = mutableListOf<SChapter>()
document.select(chapterListSelector()).forEach { chapters.add(chapterFromElement(it)) }
var hasNextPage = document.selectFirst(chapterListNextPageSelector()) != null
if (!hasNextPage) {
return chapters
}
val csrfToken = document.selectFirst("meta[name=csrf-token]")?.attr("content")
?: error("Couldn't find csrf-token")
val livewareData = document.selectFirst("div[wire:initial-data*=Models\\\\Comic]")
?.attr("wire:initial-data")
?.parseJson<LiveWireDataDto>()
?: error("Couldn't find LiveWireData")
val routeName = livewareData.fingerprint["name"]?.jsonPrimitive?.contentOrNull
?: error("Couldn't find routeName")
val fingerprint = livewareData.fingerprint
var serverMemo = livewareData.serverMemo
var pageToQuery = 2
// Javascript: (Math.random() + 1).toString(36).substring(8)
val generateId = { "1.${Random.nextLong().toString(36)}".substring(10) } // Not exactly the same, but results in a 3-5 character string
while (hasNextPage) {
val payload = buildJsonObject {
put("fingerprint", fingerprint)
put("serverMemo", serverMemo)
putJsonArray("updates") {
addJsonObject {
put("type", "callMethod")
putJsonObject("payload") {
put("id", generateId())
put("method", "gotoPage")
putJsonArray("params") {
add(pageToQuery)
add("page")
}
}
}
}
}.toString().toRequestBody(JSON_MEDIA_TYPE)
val headers = Headers.Builder()
.add("x-csrf-token", csrfToken)
.add("x-livewire", "true")
.build()
val request = POST("$baseUrl/livewire/message/$routeName", headers, payload)
val responseData = client.newCall(request).execute().parseJson<LiveWireResponseDto>()
// response contains state that we need to preserve
serverMemo = serverMemo.mergeLeft(responseData.serverMemo)
val chaptersHtml = Jsoup.parse(responseData.effects.html, baseUrl)
chaptersHtml.select(chapterListSelector()).forEach { chapters.add(chapterFromElement(it)) }
hasNextPage = chaptersHtml.selectFirst(chapterListNextPageSelector()) != null
pageToQuery++
}
return chapters
}
override fun chapterFromElement(element: Element): SChapter {
return SChapter.create().apply {
element.selectFirst("a")?.let { urlElement ->
setUrlWithoutDomain(urlElement.attr("href"))
urlElement.select("p").let {
name = it.getOrNull(0)?.text() ?: ""
date_upload = it.getOrNull(1)?.text()?.parseRelativeDate() ?: 0
}
}
}
}
// Page
override fun pageListParse(document: Document): List<Page> {
document.select("noscript").remove()
return document.select("img.max-w-full").mapIndexed { index, element ->
Page(index, imageUrl = element.imgAttr())
}
}
// Helpers
private inline fun <reified T> Response.parseJson(): T = use {
it.body.string().parseJson()
}
private inline fun <reified T> String.parseJson(): T = json.decodeFromString(this)
/**
* Recursively merges j2 onto j1 in place
* If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's
*
*/
private fun JsonObject.mergeLeft(j2: JsonObject): JsonObject = buildJsonObject {
val j1 = this@mergeLeft
j1.entries.forEach { (key, value) -> put(key, value) }
j2.entries.forEach { (key, value) ->
val j1Value = j1[key]
when {
j1Value !is JsonObject -> put(key, value)
value is JsonObject -> put(key, j1Value.mergeLeft(value))
}
}
}
/**
* Parses dates in this form: 21 hours ago
* Taken from multisrc/madara/Madara.kt
*/
private fun String.parseRelativeDate(): Long {
val number = Regex("""(\d+)""").find(this)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
contains("day") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
contains("hour") -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
contains("minute") -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
contains("second") -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
contains("week") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis
contains("month") -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
contains("year") -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
private fun Element.imgAttr(): String = when {
hasAttr("data-lazy-src") -> attr("abs:data-lazy-src")
hasAttr("data-src") -> attr("abs:data-src")
hasAttr("data-cfsrc") -> attr("abs:data-cfsrc")
else -> attr("abs:src")
}
private fun Elements.imgAttr(): String = this.first()!!.imgAttr()
private fun randomString(length: Int): String {
val charPool = ('a'..'z') + ('A'..'Z')
return List(length) { charPool.random() }.joinToString("")
}
// Unused
override fun searchMangaNextPageSelector() = throw UnsupportedOperationException()
override fun imageUrlParse(document: Document) = throw UnsupportedOperationException()
companion object {
private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType()
const val PREFIX_ID_SEARCH = "id:"
}
override val cdnUrl = "https://media.reaperscans.com/file/4SRBHm"
}

View File

@ -1,21 +0,0 @@
package eu.kanade.tachiyomi.extension.en.reaperscans
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonObject
@Serializable
data class LiveWireResponseDto(
val effects: LiveWireEffectsDto,
val serverMemo: JsonObject,
)
@Serializable
data class LiveWireEffectsDto(
val html: String,
)
@Serializable
data class LiveWireDataDto(
val fingerprint: JsonObject,
val serverMemo: JsonObject,
)

View File

@ -1,34 +0,0 @@
package eu.kanade.tachiyomi.extension.en.reaperscans
import android.app.Activity
import android.content.ActivityNotFoundException
import android.content.Intent
import android.os.Bundle
import android.util.Log
import kotlin.system.exitProcess
class ReaperScansUrlActivity : Activity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
val pathSegments = intent?.data?.pathSegments
if (pathSegments != null && pathSegments.size >= 2) {
val id = pathSegments[1]
val mainIntent = Intent().apply {
action = "eu.kanade.tachiyomi.SEARCH"
putExtra("query", ReaperScans.PREFIX_ID_SEARCH + id)
putExtra("filter", packageName)
}
try {
startActivity(mainIntent)
} catch (e: ActivityNotFoundException) {
Log.e("ReaperScansUrlActivity", e.toString())
}
} else {
Log.e("ReaperScansUrlActivity", "could not parse uri from intent $intent")
}
finish()
exitProcess(0)
}
}