Add kotlinx.serialization to MangaPlus (#1654)

* Replace ProtobufJS with Kotlinx Serialization library.

* Change the decode method to native.

* Remove unused dependencies.

* Add image cache with Images.weserv.nl.

* Add missing @Optional annotation.
This commit is contained in:
Alessandro Jean 2019-10-18 10:51:08 -03:00 committed by Carlos
parent 9b83449ea6
commit 1d5df79964
5 changed files with 246 additions and 326 deletions

View File

@ -7,6 +7,7 @@ buildscript {
dependencies {
classpath 'com.android.tools.build:gradle:3.3.2'
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
classpath "org.jetbrains.kotlin:kotlin-serialization:$kotlin_version"
}
}

View File

@ -1,18 +1,17 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply plugin: 'kotlinx-serialization'
ext {
appName = 'Tachiyomi: MANGA Plus by SHUEISHA'
pkgNameSuffix = 'all.mangaplus'
extClass = '.MangaPlusFactory'
extVersionCode = 1
extVersionCode = 2
libVersion = '1.2'
}
dependencies {
compileOnly 'com.google.code.gson:gson:2.8.2'
compileOnly 'com.github.salomonbrys.kotson:kotson:2.5.0'
compileOnly project(':duktape-stub')
implementation 'org.jetbrains.kotlinx:kotlinx-serialization-runtime:0.13.0'
}
apply from: "$rootDir/common.gradle"

View File

@ -1,13 +1,10 @@
package eu.kanade.tachiyomi.extension.all.mangaplus
import com.github.salomonbrys.kotson.*
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import com.squareup.duktape.Duktape
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.HttpSource
import kotlinx.serialization.protobuf.ProtoBuf
import okhttp3.*
import rx.Observable
import java.lang.Exception
@ -15,7 +12,7 @@ import java.util.UUID
abstract class MangaPlus(override val lang: String,
private val internalLang: String,
private val langCode: Int) : HttpSource() {
private val langCode: Language) : HttpSource() {
override val name = "Manga Plus by Shueisha"
@ -30,26 +27,7 @@ abstract class MangaPlus(override val lang: String,
.add("SESSION-TOKEN", UUID.randomUUID().toString())
override val client: OkHttpClient = network.client.newBuilder()
.addInterceptor {
var request = it.request()
if (!request.url().queryParameterNames().contains("encryptionKey")) {
return@addInterceptor it.proceed(request)
}
val encryptionKey = request.url().queryParameter("encryptionKey")!!
// Change the url and remove the encryptionKey to avoid detection.
val newUrl = request.url().newBuilder().removeAllQueryParameters("encryptionKey").build()
request = request.newBuilder().url(newUrl).build()
val response = it.proceed(request)
val image = decodeImage(encryptionKey, response.body()!!.bytes())
val body = ResponseBody.create(MediaType.parse("image/jpeg"), image)
response.newBuilder().body(body).build()
}
.addInterceptor { imageIntercept(it) }
.build()
override fun popularMangaRequest(page: Int): Request {
@ -59,16 +37,16 @@ abstract class MangaPlus(override val lang: String,
override fun popularMangaParse(response: Response): MangasPage {
val result = response.asProto()
if (result["success"] == null)
if (result.success == null)
return MangasPage(emptyList(), false)
val mangas = result["success"]["titleRankingView"]["titles"].array
.filter { it["language"].int == langCode }
val mangas = result.success.titleRankingView!!.titles
.filter { it.language == langCode }
.map {
SManga.create().apply {
title = it["name"].string
thumbnail_url = removeDuration(it["portraitImageUrl"].string)
url = "#/titles/${it["titleId"].int}"
title = it.name
thumbnail_url = getImageUrl(it.portraitImageUrl)
url = "#/titles/${it.titleId}"
}
}
@ -82,18 +60,18 @@ abstract class MangaPlus(override val lang: String,
override fun latestUpdatesParse(response: Response): MangasPage {
val result = response.asProto()
if (result["success"] == null)
if (result.success == null)
return MangasPage(emptyList(), false)
val mangas = result["success"]["webHomeView"]["groups"].array
.flatMap { it["titles"].array }
.mapNotNull { it["title"].obj }
.filter { it["language"].int == langCode }
val mangas = result.success.webHomeView!!.groups
.flatMap { it.titles }
.mapNotNull { it.title }
.filter { it.language == langCode }
.map {
SManga.create().apply {
title = it["name"].string
thumbnail_url = removeDuration(it["portraitImageUrl"].string)
url = "#/titles/${it["titleId"].int}"
title = it.name
thumbnail_url = getImageUrl(it.portraitImageUrl)
url = "#/titles/${it.titleId}"
}
}
.distinctBy { it.title }
@ -113,16 +91,16 @@ abstract class MangaPlus(override val lang: String,
override fun searchMangaParse(response: Response): MangasPage {
val result = response.asProto()
if (result["success"] == null)
if (result.success == null)
return MangasPage(emptyList(), false)
val mangas = result["success"]["allTitlesView"]["titles"].array
.filter { it["language"].int == langCode }
val mangas = result.success.allTitlesView!!.titles
.filter { it.language == langCode }
.map {
SManga.create().apply {
title = it["name"].string
thumbnail_url = removeDuration(it["portraitImageUrl"].string)
url = "#/titles/${it["titleId"].int}"
title = it.name
thumbnail_url = getImageUrl(it.portraitImageUrl)
url = "#/titles/${it.titleId}"
}
}
@ -149,18 +127,18 @@ abstract class MangaPlus(override val lang: String,
override fun mangaDetailsParse(response: Response): SManga {
val result = response.asProto()
if (result["success"] == null)
throw Exception(result["error"][mapLangToErrorProperty()]["body"].string)
if (result.success == null)
throw Exception(mapLangToErrorProperty(result.error!!).body)
val details = result["success"]["titleDetailView"].obj
val title = details["title"].obj
val details = result.success.titleDetailView!!
val title = details.title
return SManga.create().apply {
author = title["author"].string
artist = title["author"].string
description = details["overview"].string + "\n\n" + details["viewingPeriodDescription"].string
author = title.author
artist = title.author
description = details.overview + "\n\n" + details.viewingPeriodDescription
status = SManga.ONGOING
thumbnail_url = removeDuration(title["portraitImageUrl"].string)
thumbnail_url = getImageUrl(title.portraitImageUrl)
}
}
@ -169,24 +147,23 @@ abstract class MangaPlus(override val lang: String,
override fun chapterListParse(response: Response): List<SChapter> {
val result = response.asProto()
if (result["success"] == null)
throw Exception(result["error"][mapLangToErrorProperty()]["body"].string)
if (result.success == null)
throw Exception(mapLangToErrorProperty(result.error!!).body)
val titleDetailView = result["success"]["titleDetailView"].obj
val titleDetailView = result.success.titleDetailView!!
val chapters = titleDetailView["firstChapterList"].array +
(titleDetailView["lastChapterList"].nullArray ?: emptyList())
val chapters = titleDetailView.firstChapterList + titleDetailView.lastChapterList
return chapters.reversed()
// If the subTitle is null, then the chapter time expired.
.filter { it.obj["subTitle"] != null }
.filter { it.subTitle != null }
.map {
SChapter.create().apply {
name = "${it["name"].string} - ${it["subTitle"].string}"
name = "${it.name} - ${it.subTitle}"
scanlator = "Shueisha"
date_upload = 1000L * it["startTimeStamp"].long
url = "#/viewer/${it["chapterId"].int}"
chapter_number = it["name"].string.substringAfter("#").toFloatOrNull() ?: 0f
date_upload = 1000L * it.startTimeStamp
url = "#/viewer/${it.chapterId}"
chapter_number = it.name.substringAfter("#").toFloatOrNull() ?: 0f
}
}
}
@ -199,14 +176,14 @@ abstract class MangaPlus(override val lang: String,
override fun pageListParse(response: Response): List<Page> {
val result = response.asProto()
if (result["success"] == null)
throw Exception(result["error"][mapLangToErrorProperty()]["body"].string)
if (result.success == null)
throw Exception(mapLangToErrorProperty(result.error!!).body)
return result["success"]["mangaViewer"]["pages"].array
.mapNotNull { it.obj["page"].nullObj }
return result.success.mangaViewer!!.pages
.mapNotNull { it.page }
.mapIndexed { i, page ->
val encryptionKey = if (page["encryptionKey"] == null) "" else "&encryptionKey=${page["encryptionKey"].string}"
Page(i, "", "${page["imageUrl"].string}$encryptionKey")
val encryptionKey = if (page.encryptionKey == null) "" else "&encryptionKey=${page.encryptionKey}"
Page(i, "", "${page.imageUrl}$encryptionKey")
}
}
@ -225,241 +202,67 @@ abstract class MangaPlus(override val lang: String,
return GET(page.imageUrl!!, newHeaders)
}
private fun mapLangToErrorProperty(): String = when (lang) {
"es" -> "spanishPopup"
else -> "englishPopup"
private fun mapLangToErrorProperty(error: ErrorResult): Popup = when (lang) {
"es" -> error.spanishPopup
else -> error.englishPopup
}
// Maybe removing the duration parameter make the image accessible forever.
private fun removeDuration(url: String): String = url.substringBefore("&duration")
private fun getImageUrl(url: String): String {
val imageUrl = url.substringBefore("&duration")
return HttpUrl.parse(IMAGES_WESERV_URL)!!.newBuilder()
.addEncodedQueryParameter("url", imageUrl)
.toString()
}
private fun imageIntercept(chain: Interceptor.Chain): Response {
var request = chain.request()
if (!request.url().queryParameterNames().contains("encryptionKey")) {
return chain.proceed(request)
}
val encryptionKey = request.url().queryParameter("encryptionKey")!!
// Change the url and remove the encryptionKey to avoid detection.
val newUrl = request.url().newBuilder().removeAllQueryParameters("encryptionKey").build()
request = request.newBuilder().url(newUrl).build()
val response = chain.proceed(request)
val image = decodeImage(encryptionKey, response.body()!!.bytes())
val body = ResponseBody.create(MediaType.parse("image/jpeg"), image)
return response.newBuilder().body(body).build()
}
private fun decodeImage(encryptionKey: String, image: ByteArray): ByteArray {
val variablesSrc = """
var ENCRYPTION_KEY = "$encryptionKey";
var RESPONSE_BYTES = new Uint8Array([${image.joinToString()}]);
"""
val keyStream = HEX_GROUP
.findAll(encryptionKey)
.map { it.groupValues[1].toInt(16) }
.toList()
val res = Duktape.create().use {
it.evaluate(variablesSrc + IMAGE_DECRYPT_SRC) as String
val content = image
.map { it.toInt() }
.toMutableList()
val blockSizeInBytes = keyStream.size
for ((i, value) in content.iterator().withIndex()) {
content[i] = value xor keyStream[i % blockSizeInBytes]
}
return res.substringAfter("[").substringBefore("]")
.split(",")
.map { it.toInt().toByte() }
.toByteArray()
return ByteArray(content.size) { pos -> content[pos].toByte() }
}
private fun Response.asProto(): JsonObject {
val bytes = body()!!.bytes()
val messageBytes = "var BYTE_ARR = new Uint8Array([${bytes.joinToString()}]);"
val res = Duktape.create().use {
it.set("helper", DuktapeHelper::class.java, object : DuktapeHelper {
override fun getProtobuf(): String = getProtobufJSLib()
})
it.evaluate(messageBytes + PROTOBUFJS_DECODE_SRC) as String
}
return JSON_PARSER.parse(res).obj
}
private fun getProtobufJSLib(): String {
if (PROTOBUFJS == null)
PROTOBUFJS = client.newCall(GET(PROTOBUFJS_CDN, headers))
.execute().body()!!.string()
return checkNotNull(PROTOBUFJS)
}
private interface DuktapeHelper {
fun getProtobuf(): String
}
private fun Response.asProto(): MangaPlusResponse = ProtoBuf.load(MangaPlusSerializer, body()!!.bytes())
companion object {
private const val WEB_URL = "https://mangaplus.shueisha.co.jp"
private const val IMAGES_WESERV_URL = "https://images.weserv.nl"
private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36"
private val JSON_PARSER by lazy { JsonParser() }
private const val IMAGE_DECRYPT_SRC = """
function hex2bin(hex) {
return new Uint8Array(hex.match(/.{1,2}/g)
.map(function (x) { return parseInt(x, 16) }));
}
function decode(encryptionKey, bytes) {
var keystream = hex2bin(encryptionKey);
var content = bytes;
var blockSizeInBytes = keystream.length;
for (var i = 0; i < content.length; i++) {
content[i] ^= keystream[i % blockSizeInBytes];
}
return content;
}
(function() {
var decoded = decode(ENCRYPTION_KEY, RESPONSE_BYTES);
return JSON.stringify([].slice.call(decoded));
})();
"""
private var PROTOBUFJS: String? = null
private const val PROTOBUFJS_CDN = "https://cdn.rawgit.com/dcodeIO/protobuf.js/6.8.8/dist/light/protobuf.min.js"
private const val PROTOBUFJS_DECODE_SRC = """
Duktape.modSearch = function(id) {
if (id == "protobufjs")
return helper.getProtobuf();
throw new Error("Cannot find module: " + id);
}
var protobuf = require("protobufjs");
var Root = protobuf.Root;
var Type = protobuf.Type;
var Field = protobuf.Field;
var Enum = protobuf.Enum;
var OneOf = protobuf.OneOf;
var Response = new Type("Response")
.add(
new OneOf("data")
.add(new Field("success", 1, "SuccessResult"))
.add(new Field("error", 2, "ErrorResult"))
);
var ErrorResult = new Type("ErrorResult")
.add(new Field("action", 1, "Action"))
.add(new Field("englishPopup", 2, "Popup"))
.add(new Field("spanishPopup", 3, "Popup"));
var Action = new Enum("Action")
.add("DEFAULT", 0)
.add("UNAUTHORIZED", 1)
.add("MAINTAINENCE", 2)
.add("GEOIP_BLOCKING", 3);
var Popup = new Type("Popup")
.add(new Field("subject", 1, "string"))
.add(new Field("body", 2, "string"));
var SuccessResult = new Type("SuccessResult")
.add(new Field("isFeaturedUpdated", 1, "bool"))
.add(
new OneOf("data")
.add(new Field("allTitlesView", 5, "AllTitlesView"))
.add(new Field("titleRankingView", 6, "TitleRankingView"))
.add(new Field("titleDetailView", 8, "TitleDetailView"))
.add(new Field("mangaViewer", 10, "MangaViewer"))
.add(new Field("webHomeView", 11, "WebHomeView"))
);
var TitleRankingView = new Type("TitleRankingView")
.add(new Field("titles", 1, "Title", "repeated"));
var AllTitlesView = new Type("AllTitlesView")
.add(new Field("titles", 1, "Title", "repeated"));
var WebHomeView = new Type("WebHomeView")
.add(new Field("groups", 2, "UpdatedTitleGroup", "repeated"));
var TitleDetailView = new Type("TitleDetailView")
.add(new Field("title", 1, "Title"))
.add(new Field("titleImageUrl", 2, "string"))
.add(new Field("overview", 3, "string"))
.add(new Field("backgroundImageUrl", 4, "string"))
.add(new Field("nextTimeStamp", 5, "uint32"))
.add(new Field("updateTiming", 6, "UpdateTiming"))
.add(new Field("viewingPeriodDescription", 7, "string"))
.add(new Field("firstChapterList", 9, "Chapter", "repeated"))
.add(new Field("lastChapterList", 10, "Chapter", "repeated"))
.add(new Field("isSimulReleased", 14, "bool"))
.add(new Field("chaptersDescending", 17, "bool"));
var UpdateTiming = new Enum("UpdateTiming")
.add("NOT_REGULARLY", 0)
.add("MONDAY", 1)
.add("TUESDAY", 2)
.add("WEDNESDAY", 3)
.add("THURSDAY", 4)
.add("FRIDAY", 5)
.add("SATURDAY", 6)
.add("SUNDAY", 7)
.add("DAY", 8);
var MangaViewer = new Type("MangaViewer")
.add(new Field("pages", 1, "Page", "repeated"));
var Title = new Type("Title")
.add(new Field("titleId", 1, "uint32"))
.add(new Field("name", 2, "string"))
.add(new Field("author", 3, "string"))
.add(new Field("portraitImageUrl", 4, "string"))
.add(new Field("landscapeImageUrl", 5, "string"))
.add(new Field("viewCount", 6, "uint32"))
.add(new Field("language", 7, "Language", {"default": 0}));
var Language = new Enum("Language")
.add("ENGLISH", 0)
.add("SPANISH", 1);
var UpdatedTitleGroup = new Type("UpdatedTitleGroup")
.add(new Field("groupName", 1, "string"))
.add(new Field("titles", 2, "UpdatedTitle", "repeated"));
var UpdatedTitle = new Type("UpdatedTitle")
.add(new Field("title", 1, "Title"))
.add(new Field("chapterId", 2, "uint32"))
.add(new Field("chapterName", 3, "string"))
.add(new Field("chapterSubtitle", 4, "string"));
var Chapter = new Type("Chapter")
.add(new Field("titleId", 1, "uint32"))
.add(new Field("chapterId", 2, "uint32"))
.add(new Field("name", 3, "string"))
.add(new Field("subTitle", 4, "string", "optional"))
.add(new Field("startTimeStamp", 6, "uint32"))
.add(new Field("endTimeStamp", 7, "uint32"));
var Page = new Type("Page")
.add(new Field("page", 1, "MangaPage"));
var MangaPage = new Type("MangaPage")
.add(new Field("imageUrl", 1, "string"))
.add(new Field("width", 2, "uint32"))
.add(new Field("height", 3, "uint32"))
.add(new Field("encryptionKey", 5, "string", "optional"));
var root = new Root()
.define("mangaplus")
.add(Response)
.add(ErrorResult)
.add(Action)
.add(Popup)
.add(SuccessResult)
.add(TitleRankingView)
.add(AllTitlesView)
.add(WebHomeView)
.add(TitleDetailView)
.add(UpdateTiming)
.add(MangaViewer)
.add(Title)
.add(Language)
.add(UpdatedTitleGroup)
.add(UpdatedTitle)
.add(Chapter)
.add(Page)
.add(MangaPage);
function decode(arr) {
var Response = root.lookupType("Response");
var message = Response.decode(arr);
return Response.toObject(message, {defaults: true});
}
(function () {
return JSON.stringify(decode(BYTE_ARR)).replace(/\,\{\}/g, "");
})();
"""
private val HEX_GROUP = "(.{1,2})".toRegex()
}
}

View File

@ -0,0 +1,117 @@
package eu.kanade.tachiyomi.extension.all.mangaplus
import kotlinx.serialization.Optional
import kotlinx.serialization.SerialId
import kotlinx.serialization.Serializable
import kotlinx.serialization.Serializer
@Serializer(forClass=MangaPlusResponse::class)
object MangaPlusSerializer
@Serializable
data class MangaPlusResponse(
@Optional @SerialId(1) val success: SuccessResult? = null,
@Optional @SerialId(2) val error: ErrorResult? = null
)
@Serializable
data class ErrorResult(
@SerialId(1) val action: Action,
@SerialId(2) val englishPopup: Popup,
@SerialId(3) val spanishPopup: Popup
)
enum class Action { DEFAULT, UNAUTHORIZED, MAINTAINENCE, GEOIP_BLOCKING }
@Serializable
data class Popup(
@SerialId(1) val subject: String,
@SerialId(2) val body: String
)
@Serializable
data class SuccessResult(
@Optional @SerialId(1) val isFeaturedUpdated: Boolean? = false,
@Optional @SerialId(5) val allTitlesView: AllTitlesView? = null,
@Optional @SerialId(6) val titleRankingView: TitleRankingView? = null,
@Optional @SerialId(8) val titleDetailView: TitleDetailView? = null,
@Optional @SerialId(10) val mangaViewer: MangaViewer? = null,
@Optional @SerialId(11) val webHomeView: WebHomeView? = null
)
@Serializable
data class TitleRankingView(@SerialId(1) val titles: List<Title> = emptyList())
@Serializable
data class AllTitlesView(@SerialId(1) val titles: List<Title> = emptyList())
@Serializable
data class WebHomeView(@SerialId(2) val groups: List<UpdatedTitleGroup> = emptyList())
@Serializable
data class TitleDetailView(
@SerialId(1) val title: Title,
@SerialId(2) val titleImageUrl: String,
@SerialId(3) val overview: String,
@SerialId(4) val backgroundImageUrl: String,
@Optional @SerialId(5) val nextTimeStamp: Int = 0,
@Optional @SerialId(6) val updateTiming: UpdateTiming? = UpdateTiming.DAY,
@Optional @SerialId(7) val viewingPeriodDescription: String = "",
@SerialId(9) val firstChapterList: List<Chapter> = emptyList(),
@Optional @SerialId(10) val lastChapterList: List<Chapter> = emptyList(),
@Optional @SerialId(14) val isSimulReleased: Boolean = true,
@Optional @SerialId(17) val chaptersDescending: Boolean = true
)
enum class UpdateTiming { NOT_REGULARLY, MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY, DAY }
@Serializable
data class MangaViewer(@SerialId(1) val pages: List<MangaPlusPage> = emptyList())
@Serializable
data class Title(
@SerialId(1) val titleId: Int,
@SerialId(2) val name: String,
@SerialId(3) val author: String,
@SerialId(4) val portraitImageUrl: String,
@SerialId(5) val landscapeImageUrl: String,
@SerialId(6) val viewCount: Int,
@Optional @SerialId(7) val language: Language? = Language.ENGLISH
)
enum class Language(val id: Int) {
@SerialId(0) ENGLISH(0),
@SerialId(1) SPANISH(1)
}
@Serializable
data class UpdatedTitleGroup(
@SerialId(1) val groupName: String,
@SerialId(2) val titles: List<UpdatedTitle> = emptyList()
)
@Serializable
data class UpdatedTitle(
@Optional @SerialId(1) val title: Title? = null
)
@Serializable
data class Chapter(
@SerialId(1) val titleId: Int,
@SerialId(2) val chapterId: Int,
@SerialId(3) val name: String,
@Optional @SerialId(4) val subTitle: String? = null,
@SerialId(6) val startTimeStamp: Int,
@SerialId(7) val endTimeStamp: Int
)
@Serializable
data class MangaPlusPage(@Optional @SerialId(1) val page: MangaPage? = null)
@Serializable
data class MangaPage(
@SerialId(1) val imageUrl: String,
@SerialId(2) val width: Int,
@SerialId(3) val height: Int,
@Optional @SerialId(5) val encryptionKey: String? = null
)

View File

@ -7,8 +7,8 @@ class MangaPlusFactory : SourceFactory {
override fun createSources(): List<Source> = getAllMangaPlus()
}
class MangaPlusEnglish : MangaPlus("en", "eng", 0)
class MangaPlusSpanish : MangaPlus("es", "esp", 1)
class MangaPlusEnglish : MangaPlus("en", "eng", Language.ENGLISH)
class MangaPlusSpanish : MangaPlus("es", "esp", Language.SPANISH)
fun getAllMangaPlus(): List<Source> = listOf(
MangaPlusEnglish(),