From 0e9c86f9016bdf8b80156ceb883124f624372fbd Mon Sep 17 00:00:00 2001
From: AntsyLich <59261191+AntsyLich@users.noreply.github.com>
Date: Mon, 17 Oct 2022 17:47:07 +0600
Subject: [PATCH] ReaperScans: Cleanup codes (#13871)
---
src/en/reaperscans/build.gradle | 2 +-
.../extension/en/reaperscans/ReaperScans.kt | 311 ++++++++++--------
.../en/reaperscans/ReaperScansDto.kt | 21 ++
3 files changed, 188 insertions(+), 146 deletions(-)
create mode 100644 src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt
diff --git a/src/en/reaperscans/build.gradle b/src/en/reaperscans/build.gradle
index 657cc92d0..bc16be118 100644
--- a/src/en/reaperscans/build.gradle
+++ b/src/en/reaperscans/build.gradle
@@ -6,7 +6,7 @@ ext {
extName = 'Reaper Scans'
pkgNameSuffix = 'en.reaperscans'
extClass = '.ReaperScans'
- extVersionCode = 36
+ extVersionCode = 37
}
apply from: "$rootDir/common.gradle"
diff --git a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt
index 887af0be6..053f6f249 100644
--- a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt
+++ b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScans.kt
@@ -1,6 +1,7 @@
package eu.kanade.tachiyomi.extension.en.reaperscans
import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
@@ -8,17 +9,17 @@ import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.add
import kotlinx.serialization.json.addJsonObject
import kotlinx.serialization.json.buildJsonObject
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
-import okhttp3.MediaType.Companion.toMediaTypeOrNull
+import okhttp3.Headers
+import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
@@ -75,75 +76,123 @@ class ReaperScans : ParsedHttpSource() {
title = it.text().trim()
setUrlWithoutDomain(it.attr("href"))
}
- thumbnail_url = element.select("img").attr("src")
+ thumbnail_url = element.select("img").attr("abs:src")
+ }
+ }
+
+ // Search
+ override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+ val response = client.newCall(GET(baseUrl)).execute()
+ val soup = response.asJsoup()
+
+ val csrfToken = soup.selectFirst("meta[name=csrf-token]")?.attr("content")
+
+ val livewareData = soup.selectFirst("div[wire:initial-data*=frontend.global-search]")
+ ?.attr("wire:initial-data")
+ ?.parseJson<LiveWireDataDto>()
+
+ if (csrfToken == null) error("Couldn't find csrf-token")
+ if (livewareData == null) error("Couldn't find LiveWireData")
+
+ val payload = buildJsonObject {
+ put("fingerprint", livewareData.fingerprint)
+ put("serverMemo", livewareData.serverMemo)
+ putJsonArray("updates") {
+ addJsonObject {
+ put("type", "syncInput")
+ putJsonObject("payload") {
+ put("id", "03r6")
+ put("name", "query")
+ put("value", query)
+ }
+ }
+ }
+ }.toString().toRequestBody(JSON_MEDIA_TYPE)
+
+ val headers = Headers.Builder()
+ .add("x-csrf-token", csrfToken)
+ .add("x-livewire", "true")
+ .build()
+
+ return POST("$baseUrl/livewire/message/frontend.global-search", headers, payload)
+ }
+
+ override fun searchMangaSelector(): String = "a[href*=/comics/]"
+
+ override fun searchMangaParse(response: Response): MangasPage {
+ val html = response.parseJson<LiveWireResponseDto>().effects.html
+ val mangas = Jsoup.parse(html, baseUrl).select(searchMangaSelector()).map { element ->
+ searchMangaFromElement(element)
+ }
+ return MangasPage(mangas, false)
+ }
+
+ override fun searchMangaFromElement(element: Element): SManga {
+ return SManga.create().apply {
+ setUrlWithoutDomain(element.attr("href"))
+ element.select("img").first()?.let {
+ thumbnail_url = it.attr("abs:src")
+ }
+ title = element.select("p").first().text()
}
}
// Details
- override fun mangaDetailsParse(document: Document): SManga = SManga.create().apply {
- thumbnail_url = document.select("div > img").first().attr("abs:src")
- title = document.select("h1").first().text()
+ override fun mangaDetailsParse(document: Document): SManga {
+ return SManga.create().apply {
+ thumbnail_url = document.select("div > img").first().attr("abs:src")
+ title = document.select("h1").first().text()
- status = when (document.select("dt:contains(Source Status)").next().text()) {
- "On hold" -> SManga.ON_HIATUS
- "Complete" -> SManga.COMPLETED
- "Ongoing" -> SManga.ONGOING
- "Dropped" -> SManga.CANCELLED
- else -> SManga.UNKNOWN
+ status = when (document.select("dt:contains(Release Status)").next().text()) {
+ "On hold" -> SManga.ON_HIATUS
+ "Complete" -> SManga.COMPLETED
+ "Ongoing" -> SManga.ONGOING
+ "Dropped" -> SManga.CANCELLED
+ else -> SManga.UNKNOWN
+ }
+
+ genre = mutableListOf<String>().apply {
+ when (document.select("dt:contains(Source Language)").next().text()) {
+ "Korean" -> "Manhwa"
+ "Chinese" -> "Manhua"
+ "Japanese" -> "Manga"
+ else -> null
+ }?.let { add(it) }
+ }.takeIf { it.isNotEmpty() }?.joinToString(",")
+
+ description = document.select("section > div:nth-child(1) > div > p").first().text()
}
-
- val genreList = mutableListOf<String>()
- val seriesType = when (document.select("dt:contains(Source Language)").next().text()) {
- "Korean" -> "Manhwa"
- "Chinese" -> "Manhua"
- "Japanese" -> "Manga"
- else -> null
- }
- seriesType?.let { genreList.add(it) }
-
- genre = genreList.takeIf { genreList.isNotEmpty() }?.joinToString(",")
- description = document.select("section > div:nth-child(1) > div > p").first().text()
}
// Chapters
- override fun chapterListSelector() = "ul > li"
+ private fun chapterListNextPageSelector(): String = "button[wire:click*=nextPage]"
- /**
- * Recursively merges j2 onto j1 in place
- * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's
- *
- */
- private fun mergeLeft(j1: JsonObject, j2: JsonObject): JsonObject = buildJsonObject {
- j1.keys.forEach { put(it, j1[it]!!) }
- j2.keys.forEach { k ->
- when {
- j1[k] !is JsonObject -> put(k, j2[k]!!)
- j1[k] is JsonObject && j2[k] is JsonObject -> put(k, mergeLeft(j1[k]!!.jsonObject, j2[k]!!.jsonObject))
- }
- }
- }
+ override fun chapterListSelector() = "div[wire:id] > ul[role=list] > li"
override fun chapterListParse(response: Response): List<SChapter> {
var document = response.asJsoup()
val chapters = mutableListOf<SChapter>()
- document.select("div.pb-4 > div >" + chapterListSelector()).map { chapters.add(chapterFromElement(it)) }
val csrfToken = document.selectFirst("meta[name=csrf-token]")?.attr("content")
- val initialProps = document.selectFirst("div[wire:initial-data*=frontend.comic-chapters-list]")?.attr("wire:initial-data")?.let {
- json.parseToJsonElement(it)
- }
+ val livewareData = document.selectFirst("div[wire:initial-data*=frontend.comic-chapters-list]")
+ ?.attr("wire:initial-data")
+ ?.parseJson<LiveWireDataDto>()
- if (csrfToken != null && initialProps is JsonObject) {
- var serverMemo = initialProps["serverMemo"]!!.jsonObject
- val fingerprint = initialProps["fingerprint"]!!
+ if (csrfToken == null) error("Couldn't find csrf-token")
+ if (livewareData == null) error("Couldn't find LiveWireData")
- var nextPage = 2
- while (document.select(popularMangaNextPageSelector()).isNotEmpty()) {
+ val fingerprint = livewareData.fingerprint
+ var serverMemo = livewareData.serverMemo
+
+ var pageToQuery = 1
+ var hasNextPage = true
+
+ while (hasNextPage) {
+ if (pageToQuery != 1) {
val payload = buildJsonObject {
put("fingerprint", fingerprint)
put("serverMemo", serverMemo)
-// put("updates", json.parseToJsonElement("[{\"type\":\"callMethod\",\"payload\":{\"id\":\"9jhcg\",\"method\":\"gotoPage\",\"params\":[$nextPage,\"page\"]}}]"))
putJsonArray("updates") {
addJsonObject {
put("type", "callMethod")
@@ -151,133 +200,105 @@ class ReaperScans : ParsedHttpSource() {
put("id", "9jhcg")
put("method", "gotoPage")
putJsonArray("params") {
- add(nextPage)
+ add(pageToQuery)
add("page")
}
}
}
}
- }.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())
+ }.toString().toRequestBody(JSON_MEDIA_TYPE)
- val request = Request.Builder().url("$baseUrl/livewire/message/frontend.comic-chapters-list").method("POST", payload).addHeader("x-csrf-token", csrfToken).addHeader("x-livewire", "true").build()
+ val headers = Headers.Builder()
+ .add("x-csrf-token", csrfToken)
+ .add("x-livewire", "true")
+ .build()
- val response1 = client.newCall(request).execute()
- val responseText = response1.body!!.string()
+ val request = POST("$baseUrl/livewire/message/frontend.comic-chapters-list", headers, payload)
- val responseJson = json.parseToJsonElement(responseText).jsonObject
+ val responseData = client.newCall(request).execute().parseJson<LiveWireResponseDto>()
// response contains state that we need to preserve
- serverMemo = mergeLeft(serverMemo, responseJson["serverMemo"]!!.jsonObject)
-
- document = Jsoup.parse(responseJson["effects"]!!.jsonObject.get("html")?.jsonPrimitive?.content)
-
- document.select(chapterListSelector()).map { chapters.add(chapterFromElement(it)) }
- nextPage++
+ serverMemo = serverMemo.mergeLeft(responseData.serverMemo)
+ document = Jsoup.parse(responseData.effects.html, baseUrl)
}
+
+ document.select(chapterListSelector()).forEach { chapters.add(chapterFromElement(it)) }
+ hasNextPage = document.selectFirst(chapterListNextPageSelector()) != null
+ pageToQuery++
}
+
return chapters
}
override fun chapterFromElement(element: Element): SChapter {
- val chapter = SChapter.create()
- with(element) {
- select("a").first()?.let { urlElement ->
- chapter.setUrlWithoutDomain(urlElement.attr("abs:href"))
- chapter.name = urlElement.select("p").first().text()
- urlElement.select("p").takeIf { it.size > 1 }?.let {
- chapter.date_upload = parseRelativeDate(it[1].text())
+ return SChapter.create().apply {
+ element.selectFirst("a")?.let { urlElement ->
+ setUrlWithoutDomain(urlElement.attr("href"))
+ urlElement.select("p").let {
+ name = it.getOrNull(0)?.text() ?: ""
+ date_upload = it.getOrNull(1)?.text()?.parseRelativeDate() ?: 0
}
}
}
- return chapter
- }
-
- // Search
- override fun searchMangaSelector(): String = "a[href*=/comics/]"
-
- override fun searchMangaFromElement(element: Element) = SManga.create().apply {
- setUrlWithoutDomain(element.attr("href"))
- element.select("img").first()?.let {
- thumbnail_url = it.attr("abs:src")
- }
- title = element.select("p").first().text()
- }
-
- override fun searchMangaNextPageSelector(): String? = throw UnsupportedOperationException("Not Used")
-
- override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
- val response = client.newCall(GET(baseUrl)).execute()
- val soup = response.asJsoup()
-
- val csrfToken = soup.selectFirst("meta[name=csrf-token]")?.attr("content")
-
- val initialProps = soup.selectFirst("div[wire:initial-data*=frontend.global-search]")?.attr("wire:initial-data")?.let {
- json.parseToJsonElement(it)
- }
-
- if (csrfToken != null && initialProps is JsonObject) {
- val serverMemo = initialProps["serverMemo"]!!.jsonObject
- val fingerprint = initialProps["fingerprint"]!!
-
- val payload = buildJsonObject {
- put("fingerprint", fingerprint)
- put("serverMemo", serverMemo)
-// put("updates", json.parseToJsonElement("[{\"type\":\"syncInput\",\"payload\":{\"id\":\"03r6\",\"name\":\"query\",\"value\":\"$query\"}}]"))
- putJsonArray("updates") {
- addJsonObject {
- put("type", "syncInput")
- putJsonObject("payload") {
- put("id", "03r6")
- put("name", "query")
- put("value", query)
- }
- }
- }
- }.toString().toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())
-
- return Request.Builder().url("$baseUrl/livewire/message/frontend.global-search").method("POST", payload).addHeader("x-csrf-token", csrfToken).addHeader("x-livewire", "true").build()
- }
-
- throw Exception("search error")
- }
-
- override fun searchMangaParse(response: Response): MangasPage {
- val responseText = response.body!!.string()
- val responseJson = json.parseToJsonElement(responseText).jsonObject
- val document = Jsoup.parse(responseJson["effects"]!!.jsonObject.get("html")?.jsonPrimitive?.content)
- val mangas = document.select(searchMangaSelector()).map { element ->
- searchMangaFromElement(element)
- }
- return MangasPage(mangas, false)
}
// Page
- override fun pageListRequest(chapter: SChapter): Request = GET("$baseUrl${chapter.url}")
-
override fun pageListParse(document: Document): List<Page> {
return document.select("img.max-w-full").mapIndexed { index, element ->
- Page(index, "", element.attr("src"))
+ Page(index, imageUrl = element.attr("src"))
}
}
- override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not Used")
+ // Helpers
+ private inline fun <reified T> Response.parseJson(): T = use {
+ it.body?.string().orEmpty().parseJson()
+ }
- // Parses dates in this form:
- // 21 horas ago
- // Taken from multisrc/madara/Madara.kt
- private fun parseRelativeDate(date: String): Long {
- val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0
+ private inline fun <reified T> String.parseJson(): T = json.decodeFromString(this)
+
+ /**
+ * Recursively merges j2 onto j1 in place
+ * If j1 and j2 both contain keys whose values aren't both jsonObjects, j2's value overwrites j1's
+ *
+ */
+ private fun JsonObject.mergeLeft(j2: JsonObject): JsonObject = buildJsonObject {
+ val j1 = this@mergeLeft
+ j1.entries.forEach { (key, value) -> put(key, value) }
+ j2.entries.forEach { (key, value) ->
+ val j1Value = j1[key]
+ when {
+ j1Value !is JsonObject -> put(key, value)
+ value is JsonObject -> put(key, j1Value.mergeLeft(value))
+ }
+ }
+ }
+
+ /**
+ * Parses dates in this form: 21 hours ago
+ * Taken from multisrc/madara/Madara.kt
+ */
+ private fun String.parseRelativeDate(): Long {
+ val number = Regex("""(\d+)""").find(this)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
- date.contains("day") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
- date.contains("hour") -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
- date.contains("minute") -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
- date.contains("second") -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
- date.contains("week") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis
- date.contains("month") -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
- date.contains("year") -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
+ contains("day") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
+ contains("hour") -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
+ contains("minute") -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
+ contains("second") -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis
+ contains("week") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis
+ contains("month") -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
+ contains("year") -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
else -> 0
}
}
+
+ // Unused
+ override fun searchMangaNextPageSelector() = throw UnsupportedOperationException("Not Used")
+
+ override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("Not Used")
+
+ companion object {
+ private val JSON_MEDIA_TYPE = "application/json; charset=utf-8".toMediaType()
+ }
}
diff --git a/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt
new file mode 100644
index 000000000..34fde5173
--- /dev/null
+++ b/src/en/reaperscans/src/eu/kanade/tachiyomi/extension/en/reaperscans/ReaperScansDto.kt
@@ -0,0 +1,21 @@
+package eu.kanade.tachiyomi.extension.en.reaperscans
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonObject
+
+@Serializable
+data class LiveWireResponseDto(
+ val effects: LiveWireEffectsDto,
+ val serverMemo: JsonObject
+)
+
+@Serializable
+data class LiveWireEffectsDto(
+ val html: String
+)
+
+@Serializable
+data class LiveWireDataDto(
+ val fingerprint: JsonObject,
+ val serverMemo: JsonObject
+)