MangaReader.to: refactor, fixes and improvements (#13300)

* MangaReader.to: refactor, fixes and improvements

* rewrite image descrambling interceptor

* add changelog

* use contains to check dupe number instead of endsWith

* memoize calculation result instead of using lookup table

* make SeedRandom private

* add some line breaks to better align diffs

* fix media type

* use the same buffer to reduce memory allocations

* Review changes and inline stuff

* get rid of the hassle of calculating piece count
This commit is contained in:
stevenyomi 2022-09-02 04:15:21 +08:00 committed by GitHub
parent ab9984d3f4
commit 75605af7a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 284 additions and 214 deletions

View File

@ -0,0 +1,31 @@
## 1.3.3
- Appended `.to` to extension name
- Replaced dependencies
- `android.net.Uri``okhttp3.HttpUrl`
- `org.json``kotlinx.serialization`
- Refactored some code to separate files
- Image quality preference: added prompt to summary and made it take effect without restart, fixes [#12504](https://github.com/tachiyomiorg/tachiyomi-extensions/issues/12504)
- Added preference to show additional entries in volumes in list results and added code to support volumes, fixes [#12573](https://github.com/tachiyomiorg/tachiyomi-extensions/issues/12573)
- Improved parsing
- Added code to parse authors and artists
- Improved chapter list parsing
- Other improvements
- Performance boosts in selectors
- Added French, Korean and Chinese languages
- Corrected filter note type (Text → Header)
- Rewrote image descrambler
- Used fragment in URL instead of appending error-prone query parameter, hopefully fixes [#12722](https://github.com/tachiyomiorg/tachiyomi-extensions/issues/12722)
- Made interceptor singleton to be shared across languages
- Simplified code logic to make it a lot more readable, thanks to Vetle in [#9325 (comment)](https://github.com/tachiyomiorg/tachiyomi-extensions/pull/9325#issuecomment-1100950110) for code reference
- Used `javax.crypto.Cipher` for ARC4
- Memoize permutation result to reduce calculation
- Save as compressed JPG instead of PNG to avoid size bloat (original image is already compressed)
## 1.2.2
- Fixes filters causing manga list to fail to load.
## 1.2.1
- Builds on original PR and unscrambles the images.

View File

@ -1,11 +1,12 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply plugin: 'kotlinx-serialization'
ext {
extName = 'MangaReader'
extName = 'MangaReader.to'
pkgNameSuffix = 'all.mangareaderto'
extClass = '.MangaReaderFactory'
extVersionCode = 2
extVersionCode = 3
isNsfw = true
}

View File

@ -1,22 +1,26 @@
package eu.kanade.tachiyomi.extension.all.mangareaderto
import android.app.Application
import android.net.Uri
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.ConfigurableSource
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
import org.json.JSONObject
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Request
import okhttp3.Response
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.nodes.TextNode
import org.jsoup.select.Evaluator
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
@ -30,9 +34,29 @@ open class MangaReader(
override val supportsLatest = true
override val client = network.client.newBuilder()
.addInterceptor(MangaReaderImageInterceptor())
.addInterceptor(MangaReaderImageInterceptor)
.build()
private fun MangasPage.insertVolumeEntries(): MangasPage {
if (preferences.showVolume.not()) return this
val list = mangas.ifEmpty { return this }
val newList = ArrayList<SManga>(list.size * 2)
for (manga in list) {
val volume = SManga.create().apply {
url = manga.url + VOLUME_URL_SUFFIX
title = VOLUME_TITLE_PREFIX + manga.title
thumbnail_url = manga.thumbnail_url
}
newList.add(manga)
newList.add(volume)
}
return MangasPage(newList, hasNextPage)
}
override fun latestUpdatesParse(response: Response) = super.latestUpdatesParse(response).insertVolumeEntries()
override fun popularMangaParse(response: Response) = super.popularMangaParse(response).insertVolumeEntries()
override fun searchMangaParse(response: Response) = super.searchMangaParse(response).insertVolumeEntries()
override fun latestUpdatesRequest(page: Int) =
GET("$baseUrl/filter?sort=latest-updated&language=$lang&page=$page", headers)
@ -53,37 +77,37 @@ open class MangaReader(
override fun popularMangaFromElement(element: Element) =
searchMangaFromElement(element)
override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val urlBuilder = baseUrl.toHttpUrl().newBuilder()
if (query.isNotBlank()) {
Uri.parse("$baseUrl/search").buildUpon().run {
appendQueryParameter("keyword", query)
appendQueryParameter("page", page.toString())
GET(toString(), headers)
urlBuilder.addPathSegment("search").apply {
addQueryParameter("keyword", query)
addQueryParameter("page", page.toString())
}
} else {
Uri.parse("$baseUrl/filter").buildUpon().run {
appendQueryParameter("language", lang)
appendQueryParameter("page", page.toString())
urlBuilder.addPathSegment("filter").apply {
addQueryParameter("language", lang)
addQueryParameter("page", page.toString())
filters.ifEmpty(::getFilterList).forEach { filter ->
when (filter) {
is Select -> {
appendQueryParameter(filter.param, filter.selection)
addQueryParameter(filter.param, filter.selection)
}
is DateFilter -> {
filter.state.forEach {
appendQueryParameter(it.param, it.selection)
addQueryParameter(it.param, it.selection)
}
}
is GenresFilter -> {
appendQueryParameter(filter.param, filter.selection)
addQueryParameter(filter.param, filter.selection)
}
else -> Unit
}
}
GET(toString(), headers)
}
}
return Request.Builder().url(urlBuilder.build()).headers(headers).build()
}
override fun searchMangaSelector() = ".manga_list-sbs .manga-poster"
@ -92,68 +116,117 @@ open class MangaReader(
override fun searchMangaFromElement(element: Element) =
SManga.create().apply {
url = element.attr("href")
element.selectFirst(".manga-poster-img").let {
element.selectFirst(Evaluator.Tag("img")).let {
title = it.attr("alt")
thumbnail_url = it.attr("src")
}
}
private val authorSelector = ".item-head:containsOwn(Authors) ~ a"
private fun Element.parseAuthorsTo(manga: SManga) {
val authors = select(Evaluator.Tag("a"))
val text = authors.map { it.ownText().replace(",", "") }
val count = authors.size
when (count) {
0 -> return
1 -> {
manga.author = text[0]
return
}
}
val nodes = childNodes()
val authorList = ArrayList<String>(count)
val artistList = ArrayList<String>(count)
for ((index, author) in authors.withIndex()) {
val nodeIndex = nodes.indexOf(author)
val textNode = nodes.getOrNull(nodeIndex + 1) as? TextNode
val list = if (textNode != null && "(Art)" in textNode.wholeText) artistList else authorList
list.add(text[index])
}
if (authorList.isEmpty().not()) manga.author = authorList.joinToString()
if (artistList.isEmpty().not()) manga.artist = artistList.joinToString()
}
private val statusSelector = ".item-head:containsOwn(Status) + .name"
override fun mangaDetailsParse(document: Document) =
SManga.create().apply {
setUrlWithoutDomain(document.location())
document.getElementById("ani_detail").let { el ->
title = el.selectFirst(".manga-name").text().trim()
description = el.selectFirst(".description")?.text()?.trim()
thumbnail_url = el.selectFirst(".manga-poster-img").attr("src")
genre = el.select(".genres > a")?.joinToString { it.text() }
author = el.select(authorSelector)?.joinToString {
it.text().replace(",", "")
}
artist = author // TODO: separate authors and artists
status = when (el.selectFirst(statusSelector)?.text()) {
override fun mangaDetailsParse(document: Document) = SManga.create().apply {
url = document.location().removePrefix(baseUrl)
val root = document.selectFirst(Evaluator.Id("ani_detail"))
val mangaTitle = root.selectFirst(Evaluator.Tag("h2")).ownText()
title = if (url.endsWith(VOLUME_URL_SUFFIX)) VOLUME_TITLE_PREFIX + mangaTitle else mangaTitle
description = root.run {
val description = selectFirst(Evaluator.Class("description")).ownText()
when (val altTitle = selectFirst(Evaluator.Class("manga-name-or")).ownText()) {
"", mangaTitle -> description
else -> "$description\n\nAlternative Title: $altTitle"
}
}
thumbnail_url = root.selectFirst(Evaluator.Tag("img")).attr("src")
genre = root.selectFirst(Evaluator.Class("genres")).children().joinToString { it.ownText() }
for (item in root.selectFirst(Evaluator.Class("anisc-info")).children()) {
if (item.hasClass("item").not()) continue
when (item.selectFirst(Evaluator.Class("item-head")).ownText()) {
"Authors:" -> item.parseAuthorsTo(this)
"Status:" -> status = when (item.selectFirst(Evaluator.Class("name")).ownText()) {
"Finished" -> SManga.COMPLETED
"Publishing" -> SManga.ONGOING
else -> SManga.UNKNOWN
}
}
}
}
override fun chapterListRequest(manga: SManga): Request {
val url = manga.url
val id = url.removeSuffix(VOLUME_URL_SUFFIX).substringAfterLast('-')
val type = if (url.endsWith(VOLUME_URL_SUFFIX)) "vol" else "chap"
return GET("$baseUrl/ajax/manga/reading-list/$id?readingBy=$type", headers)
}
override fun chapterListSelector() = "#$lang-chapters .item"
override fun chapterListParse(response: Response): List<SChapter> {
val isVolume = response.request.url.queryParameter("readingBy") == "vol"
val container = response.parseHtmlProperty().run {
val type = if (isVolume) "volumes" else "chapters"
selectFirst(Evaluator.Id("$lang-$type")) ?: return emptyList()
}
val abbrPrefix = if (isVolume) "Vol" else "Chap"
val fullPrefix = if (isVolume) "Volume" else "Chapter"
return container.children().map { chapterFromElement(it, abbrPrefix, fullPrefix) }
}
override fun chapterFromElement(element: Element) =
throw UnsupportedOperationException("Not used.")
private fun chapterFromElement(element: Element, abbrPrefix: String, fullPrefix: String) =
SChapter.create().apply {
chapter_number = element.attr("data-number").toFloatOrNull() ?: -1f
element.selectFirst(".item-link").let {
val number = element.attr("data-number")
chapter_number = number.toFloatOrNull() ?: -1f
element.selectFirst(Evaluator.Tag("a")).let {
url = it.attr("href")
name = it.attr("title")
name = run {
val name = it.attr("title")
val prefix = "$abbrPrefix $number: "
if (name.startsWith(prefix).not()) return@run name
val realName = name.removePrefix(prefix)
if (realName.contains(number)) realName else "$fullPrefix $number: $realName"
}
}
}
private fun pageListRequest(id: String) =
GET("$baseUrl/ajax/image/list/chap/$id?quality=$quality", headers)
override fun pageListParse(document: Document): List<Page> {
val ajaxUrl = document.selectFirst(Evaluator.Id("wrapper")).run {
val readingBy = attr("data-reading-by")
val readingId = attr("data-reading-id")
"$baseUrl/ajax/image/list/$readingBy/$readingId?quality=${preferences.quality}"
}
override fun fetchPageList(chapter: SChapter) =
client.newCall(pageListRequest(chapter)).asObservableSuccess().map { res ->
res.asJsoup().getElementById("wrapper").attr("data-reading-id").let {
val call = client.newCall(pageListRequest(it))
val json = JSONObject(call.execute().body!!.string())
pageListParse(Jsoup.parse(json.getString("html")))
}
}!!
val pageDocument = client.newCall(GET(ajaxUrl, headers)).execute().parseHtmlProperty()
override fun pageListParse(document: Document): List<Page> =
document.getElementsByClass("iv-card").mapIndexed { idx, img ->
return pageDocument.getElementsByClass("iv-card").mapIndexed { index, img ->
val url = img.attr("data-url")
if (img.hasClass("shuffled")) {
Page(idx, "", "$url&shuffled=true")
} else {
Page(idx, "", url)
}
val imageUrl = if (img.hasClass("shuffled")) "$url#${MangaReaderImageInterceptor.SCRAMBLED}" else url
Page(index, imageUrl = imageUrl)
}
}
override fun imageUrlParse(document: Document) =
throw UnsupportedOperationException("Not used")
@ -162,23 +235,8 @@ open class MangaReader(
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)!!
}
private val quality by lazy {
preferences.getString("quality", "medium")!!
}
override fun setupPreferenceScreen(screen: PreferenceScreen) {
ListPreference(screen.context).apply {
key = "quality"
title = "Quality"
summary = "%s"
entries = arrayOf("Low", "Medium", "High")
entryValues = arrayOf("low", "medium", "high")
setDefaultValue("medium")
setOnPreferenceChangeListener { _, newValue ->
preferences.edit().putString("quality", newValue as String).commit()
}
}.let(screen::addPreference)
getPreferences(screen.context).forEach(screen::addPreference)
}
override fun getFilterList() =
@ -193,4 +251,9 @@ open class MangaReader(
SortFilter(),
GenresFilter()
)
private fun Response.parseHtmlProperty(): Document {
val html = Json.parseToJsonElement(body!!.string()).jsonObject["html"]!!.jsonPrimitive.content
return Jsoup.parseBodyFragment(html)
}
}

View File

@ -4,5 +4,5 @@ import eu.kanade.tachiyomi.source.SourceFactory
class MangaReaderFactory : SourceFactory {
override fun createSources() =
listOf(MangaReader("en"), MangaReader("ja"))
arrayOf("en", "fr", "ja", "ko", "zh").map(::MangaReader)
}

View File

@ -3,7 +3,7 @@ package eu.kanade.tachiyomi.extension.all.mangareaderto
import eu.kanade.tachiyomi.source.model.Filter
import java.util.Calendar
object Note : Filter.Text("NOTE: Ignored if using text search!")
object Note : Filter.Header("NOTE: Ignored if using text search!")
sealed class Select(
name: String,

View File

@ -5,188 +5,122 @@ import android.graphics.BitmapFactory
import android.graphics.Canvas
import android.graphics.Rect
import okhttp3.Interceptor
import okhttp3.MediaType.Companion.toMediaTypeOrNull
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.Response
import okhttp3.ResponseBody.Companion.toResponseBody
import java.io.ByteArrayOutputStream
import java.io.InputStream
import kotlin.math.ceil
import kotlin.math.floor
import javax.crypto.Cipher
import javax.crypto.spec.SecretKeySpec
import kotlin.math.min
class MangaReaderImageInterceptor : Interceptor {
object MangaReaderImageInterceptor : Interceptor {
private var s = IntArray(256)
private var arc4i = 0
private var arc4j = 0
private val memo = hashMapOf<Int, IntArray>()
override fun intercept(chain: Interceptor.Chain): Response {
val response = chain.proceed(chain.request())
val request = chain.request()
val response = chain.proceed(request)
// shuffled page requests should have shuffled=true query parameter
if (chain.request().url.queryParameter("shuffled") != "true")
return response
val url = request.url
// TODO: remove the query parameter check (legacy) in later versions
if (url.fragment != SCRAMBLED && url.queryParameter("shuffled") == null) return response
val image = unscrambleImage(response.body!!.byteStream())
val body = image.toResponseBody("image/png".toMediaTypeOrNull())
val image = descramble(response.body!!.byteStream())
val body = image.toResponseBody("image/jpeg".toMediaType())
return response.newBuilder()
.body(body)
.build()
}
private fun unscrambleImage(image: InputStream): ByteArray {
private fun descramble(image: InputStream): ByteArray {
// obfuscated code (imgReverser function): https://mangareader.to/js/read.min.js
// essentially, it shuffles arrays of the image slices using the key 'stay'
val bitmap = BitmapFactory.decodeStream(image)
val width = bitmap.width
val height = bitmap.height
val result = Bitmap.createBitmap(bitmap.width, bitmap.height, Bitmap.Config.ARGB_8888)
val result = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
val canvas = Canvas(result)
val horizontalParts = ceil(bitmap.width / SLICE_SIZE.toDouble()).toInt()
val totalParts = horizontalParts * ceil(bitmap.height / SLICE_SIZE.toDouble()).toInt()
// calculate slices
val slices: HashMap<Int, MutableList<Rect>> = hashMapOf()
for (i in 0 until totalParts) {
val row = floor(i / horizontalParts.toDouble()).toInt()
val x = (i - row * horizontalParts) * SLICE_SIZE
val y = row * SLICE_SIZE
val width = if (x + SLICE_SIZE <= bitmap.width) SLICE_SIZE else bitmap.width - x
val height = if (y + SLICE_SIZE <= bitmap.height) SLICE_SIZE else bitmap.height - y
val srcRect = Rect(x, y, width, height)
val key = width - height
if (!slices.containsKey(key)) {
slices[key] = mutableListOf()
val pieces = ArrayList<Piece>()
for (y in 0 until height step PIECE_SIZE) {
for (x in 0 until width step PIECE_SIZE) {
val w = min(PIECE_SIZE, width - x)
val h = min(PIECE_SIZE, height - y)
pieces.add(Piece(x, y, w, h))
}
slices[key]?.add(srcRect)
}
// handle groups of slices
for (sliceEntry in slices) {
// reset random number generator for every un-shuffle
resetRng()
val groups = pieces.groupBy { it.w shl 16 or it.h }
val currentSlices = sliceEntry.value
val sliceCount = currentSlices.count()
for (group in groups.values) {
val size = group.size
// un-shuffle slice indices
val orderedSlices = IntArray(sliceCount)
val keys = MutableList(sliceCount) { it }
val permutation = memo.getOrPut(size) {
// The key is actually "stay", but it's padded here in case the code is run in
// Oracle's JDK, where RC4 key is required to be at least 5 bytes
val random = SeedRandom("staystay")
for (i in currentSlices.indices) {
val r = floor(prng() * keys.count()).toInt()
val g = keys[r]
keys.removeAt(r)
orderedSlices[g] = i
// https://github.com/webcaetano/shuffle-seed
val indices = (0 until size).toMutableList()
IntArray(size) { indices.removeAt((random.nextDouble() * indices.size).toInt()) }
}
// draw slices
val cols = getColumnCount(currentSlices)
for ((i, original) in permutation.withIndex()) {
val src = group[i]
val dst = group[original]
val groupX = currentSlices[0].left
val groupY = currentSlices[0].top
for ((i, orderedIndex) in orderedSlices.withIndex()) {
val slice = currentSlices[i]
val row = floor((orderedIndex / cols).toDouble()).toInt()
val col = orderedIndex - row * cols
val width = slice.right
val height = slice.bottom
val x = groupX + col * width
val y = groupY + row * height
val srcRect = Rect(x, y, x + width, y + height)
val dstRect = Rect(
slice.left,
slice.top,
slice.left + width,
slice.top + height
)
val srcRect = Rect(src.x, src.y, src.x + src.w, src.y + src.h)
val dstRect = Rect(dst.x, dst.y, dst.x + dst.w, dst.y + dst.h)
canvas.drawBitmap(bitmap, srcRect, dstRect, null)
}
}
val output = ByteArrayOutputStream()
result.compress(Bitmap.CompressFormat.PNG, 100, output)
result.compress(Bitmap.CompressFormat.JPEG, 90, output)
return output.toByteArray()
}
private fun getColumnCount(slices: List<Rect>): Int {
if (slices.count() == 1) return 1
var t: Int? = null
for (i in slices.indices) {
if (t == null) t = slices[i].top
if (t != slices[i].top) {
return i
private class Piece(val x: Int, val y: Int, val w: Int, val h: Int)
// https://github.com/davidbau/seedrandom
private class SeedRandom(key: String) {
private val input = ByteArray(RC4_WIDTH)
private val buffer = ByteArray(RC4_WIDTH)
private var pos = RC4_WIDTH
private val rc4 = Cipher.getInstance("RC4").apply {
init(Cipher.ENCRYPT_MODE, SecretKeySpec(key.toByteArray(), "RC4"))
update(input, 0, RC4_WIDTH, buffer) // RC4-drop[256]
}
fun nextDouble(): Double {
var num = nextByte()
var exp = 8
while (num < 1L shl 52) {
num = num shl 8 or nextByte()
exp += 8
}
while (num >= 1L shl 53) {
num = num ushr 1
exp--
}
return Math.scalb(num.toDouble(), -exp)
}
return slices.count()
}
private fun resetRng() {
arc4i = 0
arc4j = 0
initializeS()
arc4(256) // RC4-drop[256]
}
private fun initializeS() {
val t = IntArray(256)
for (i in 0..255) {
s[i] = i
t[i] = KEY[i % KEY.size]
}
var j = 0
var tmp: Int
for (i in 0..255) {
j = (j + s[i] + t[i]) and 0xFF
tmp = s[j]
s[j] = s[i]
s[i] = tmp
private fun nextByte(): Long {
if (pos == RC4_WIDTH) {
rc4.update(input, 0, RC4_WIDTH, buffer)
pos = 0
}
return buffer[pos++].toLong() and 0xFF
}
}
private fun prng(): Double {
var n = arc4(6)
var d = 281474976710656.0 // 256^6 (start with 6 chunks in n)
var x = 0L
while (n < 4503599627370496) { // 2^52 (52 significant digits in a double)
n = (n + x) * 256
d *= 256
x = arc4(1)
if (n < 0) break // overflow
}
return (n + x) / d
}
private fun arc4(count: Int): Long {
var t: Int
var tmp: Int
var r: Long = 0
repeat(count) {
arc4i = (arc4i + 1) and 0xFF
arc4j = (arc4j + s[arc4i]) and 0xFF
tmp = s[arc4j]
s[arc4j] = s[arc4i]
s[arc4i] = tmp
t = (s[arc4i] + s[arc4j]) and 0xFF
r = r * 256 + s[t]
}
return r
}
companion object {
private val KEY = "stay".map { it.toByte().toInt() }
private const val SLICE_SIZE = 200
}
private const val RC4_WIDTH = 256
private const val PIECE_SIZE = 200
const val SCRAMBLED = "scrambled"
}

View File

@ -0,0 +1,41 @@
package eu.kanade.tachiyomi.extension.all.mangareaderto
import android.content.Context
import android.content.SharedPreferences
import androidx.preference.ListPreference
import androidx.preference.SwitchPreferenceCompat
fun getPreferences(context: Context) = arrayOf(
ListPreference(context).apply {
key = QUALITY_PREF
title = "Image quality"
summary = "Selected: %s\n" +
"Changes will not be applied to chapters that are already loaded or read " +
"until you clear the chapter cache."
entries = arrayOf("Low", "Medium", "High")
entryValues = arrayOf("low", QUALITY_MEDIUM, "high")
setDefaultValue(QUALITY_MEDIUM)
},
SwitchPreferenceCompat(context).apply {
key = SHOW_VOLUME_PREF
title = "Show manga in volumes in search result"
setDefaultValue(false)
},
)
val SharedPreferences.quality
get() =
getString(QUALITY_PREF, QUALITY_MEDIUM)!!
val SharedPreferences.showVolume
get() =
getBoolean(SHOW_VOLUME_PREF, false)
private const val QUALITY_PREF = "quality"
private const val QUALITY_MEDIUM = "medium"
private const val SHOW_VOLUME_PREF = "show_volume"
const val VOLUME_URL_SUFFIX = "#vol"
const val VOLUME_TITLE_PREFIX = "[VOL] "