From 445af0f144354023f0511033cde81bb8a1e5988d Mon Sep 17 00:00:00 2001 From: stevenyomi <95685115+stevenyomi@users.noreply.github.com> Date: Sun, 25 Sep 2022 00:01:52 +0800 Subject: [PATCH] Move unpacker lib to extension repo (#13565) It was hosted at https://github.com/stevenyomi/unpacker --- lib/unpacker/build.gradle.kts | 12 +++ .../lib/unpacker/SubstringExtractor.kt | 34 +++++++++ .../kanade/tachiyomi/lib/unpacker/Unpacker.kt | 76 +++++++++++++++++++ .../mangabz/mangabz/additional.gradle | 2 +- .../overrides/mangabz/mangabz/src/Mangabz.kt | 6 +- .../sinmh/wuqimanga/additional.gradle | 2 +- .../sinmh/wuqimanga/src/WuqiManga.kt | 2 +- .../multisrc/mangabz/MangabzGenerator.kt | 2 +- .../multisrc/sinmh/SinMHGenerator.kt | 2 +- settings.gradle.kts | 3 + src/zh/sixmh/build.gradle | 4 +- .../tachiyomi/extension/zh/sixmh/SixMH.kt | 2 +- 12 files changed, 136 insertions(+), 11 deletions(-) create mode 100644 lib/unpacker/build.gradle.kts create mode 100644 lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/SubstringExtractor.kt create mode 100644 lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/Unpacker.kt diff --git a/lib/unpacker/build.gradle.kts b/lib/unpacker/build.gradle.kts new file mode 100644 index 000000000..3aa58a627 --- /dev/null +++ b/lib/unpacker/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + `java-library` + kotlin("jvm") +} + +repositories { + mavenCentral() +} + +dependencies { + compileOnly(libs.kotlin.stdlib) +} diff --git a/lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/SubstringExtractor.kt b/lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/SubstringExtractor.kt new file mode 100644 index 000000000..a5f43e017 --- /dev/null +++ b/lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/SubstringExtractor.kt @@ -0,0 +1,34 @@ +package eu.kanade.tachiyomi.lib.unpacker + +/** + * A helper class to extract substrings efficiently. + * + * Note that all methods move [startIndex] over the ending delimiter. + */ +class SubstringExtractor(private val text: String) { + private var startIndex = 0 + + fun skipOver(str: String) { + val index = text.indexOf(str, startIndex) + if (index == -1) return + startIndex = index + str.length + } + + fun substringBefore(str: String): String { + val index = text.indexOf(str, startIndex) + if (index == -1) return "" + val result = text.substring(startIndex, index) + startIndex = index + str.length + return result + } + + fun substringBetween(left: String, right: String): String { + val index = text.indexOf(left, startIndex) + if (index == -1) return "" + val leftIndex = index + left.length + val rightIndex = text.indexOf(right, leftIndex) + if (rightIndex == -1) return "" + startIndex = rightIndex + right.length + return text.substring(leftIndex, rightIndex) + } +} diff --git a/lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/Unpacker.kt b/lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/Unpacker.kt new file mode 100644 index 000000000..1c4b5bd6c --- /dev/null +++ b/lib/unpacker/src/main/java/eu/kanade/tachiyomi/lib/unpacker/Unpacker.kt @@ -0,0 +1,76 @@ +package eu.kanade.tachiyomi.lib.unpacker + +/** + * Helper class to unpack JavaScript code compressed by [packer](http://dean.edwards.name/packer/). + * + * Source code of packer can be found [here](https://github.com/evanw/packer/blob/master/packer.js). + */ +object Unpacker { + + /** + * Unpacks JavaScript code compressed by packer. + * + * Specify [left] and [right] to unpack only the data between them. + * + * Note: single quotes `\'` in the data will be replaced with double quotes `"`. + */ + fun unpack(script: String, left: String? = null, right: String? = null): String = + unpack(SubstringExtractor(script), left, right) + + /** + * Unpacks JavaScript code compressed by packer. + * + * Specify [left] and [right] to unpack only the data between them. + * + * Note: single quotes `\'` in the data will be replaced with double quotes `"`. + */ + fun unpack(script: SubstringExtractor, left: String? = null, right: String? = null): String { + val packed = script + .substringBetween("}('", ".split('|'),0,{}))") + .replace("\\'", "\"") + + val parser = SubstringExtractor(packed) + val data: String + if (left != null && right != null) { + data = parser.substringBetween(left, right) + parser.skipOver("',") + } else { + data = parser.substringBefore("',") + } + if (data.isEmpty()) return "" + + val dictionary = parser.substringBetween("'", "'").split("|") + val size = dictionary.size + + return wordRegex.replace(data) { + val key = it.value + val index = parseRadix62(key) + if (index >= size) return@replace key + dictionary[index].ifEmpty { key } + } + } + + private val wordRegex by lazy { Regex("""\w+""") } + + private fun parseRadix62(str: String): Int { + var result = 0 + for (ch in str.toCharArray()) { + result = result * 62 + when { + ch.code <= '9'.code -> { // 0-9 + ch.code - '0'.code + } + + ch.code >= 'a'.code -> { // a-z + // ch - 'a' + 10 + ch.code - ('a'.code - 10) + } + + else -> { // A-Z + // ch - 'A' + 36 + ch.code - ('A'.code - 36) + } + } + } + return result + } +} diff --git a/multisrc/overrides/mangabz/mangabz/additional.gradle b/multisrc/overrides/mangabz/mangabz/additional.gradle index a73e30c8e..027379881 100644 --- a/multisrc/overrides/mangabz/mangabz/additional.gradle +++ b/multisrc/overrides/mangabz/mangabz/additional.gradle @@ -1,3 +1,3 @@ dependencies { - implementation 'com.github.stevenyomi:unpacker:2948449d0c' // 1.2 + implementation project(':lib-unpacker') } diff --git a/multisrc/overrides/mangabz/mangabz/src/Mangabz.kt b/multisrc/overrides/mangabz/mangabz/src/Mangabz.kt index 777662601..7e321eca3 100644 --- a/multisrc/overrides/mangabz/mangabz/src/Mangabz.kt +++ b/multisrc/overrides/mangabz/mangabz/src/Mangabz.kt @@ -2,8 +2,8 @@ package eu.kanade.tachiyomi.extension.zh.mangabz import android.app.Application import androidx.preference.PreferenceScreen -import com.github.stevenyomi.unpacker.ProgressiveParser -import com.github.stevenyomi.unpacker.Unpacker +import eu.kanade.tachiyomi.lib.unpacker.SubstringExtractor +import eu.kanade.tachiyomi.lib.unpacker.Unpacker import eu.kanade.tachiyomi.multisrc.mangabz.MangabzTheme import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess @@ -128,7 +128,7 @@ class Mangabz : MangabzTheme("Mangabz", ""), ConfigurableSource { return client.newCall(GET(page.url, headers)).asObservableSuccess().map { val script = Unpacker.unpack(it.body!!.string()) - val parser = ProgressiveParser(script) + val parser = SubstringExtractor(script) val prefix = parser.substringBetween("pix=\"", "\"") // 2 pages, or 15 if server cache is ready val paths = parser.substringBetween("[\"", "\"]").split("\",\"") diff --git a/multisrc/overrides/sinmh/wuqimanga/additional.gradle b/multisrc/overrides/sinmh/wuqimanga/additional.gradle index 7b7c4bce0..027379881 100644 --- a/multisrc/overrides/sinmh/wuqimanga/additional.gradle +++ b/multisrc/overrides/sinmh/wuqimanga/additional.gradle @@ -1,3 +1,3 @@ dependencies { - implementation 'com.github.stevenyomi:unpacker:12a09e3c1a' // 1.1 + implementation project(':lib-unpacker') } diff --git a/multisrc/overrides/sinmh/wuqimanga/src/WuqiManga.kt b/multisrc/overrides/sinmh/wuqimanga/src/WuqiManga.kt index 91f53bbaa..7c89bd17f 100644 --- a/multisrc/overrides/sinmh/wuqimanga/src/WuqiManga.kt +++ b/multisrc/overrides/sinmh/wuqimanga/src/WuqiManga.kt @@ -1,6 +1,6 @@ package eu.kanade.tachiyomi.extension.zh.wuqimanga -import com.github.stevenyomi.unpacker.Unpacker +import eu.kanade.tachiyomi.lib.unpacker.Unpacker import eu.kanade.tachiyomi.multisrc.sinmh.SinMH import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.source.model.Filter diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangabz/MangabzGenerator.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangabz/MangabzGenerator.kt index 6070ddfdd..f78c07fc2 100644 --- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangabz/MangabzGenerator.kt +++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/mangabz/MangabzGenerator.kt @@ -8,7 +8,7 @@ class MangabzGenerator : ThemeSourceGenerator { override val themePkg = "mangabz" override val baseVersionCode = 1 override val sources = listOf( - SingleLang("Mangabz", "https://mangabz.com", "zh", overrideVersionCode = 5), + SingleLang("Mangabz", "https://mangabz.com", "zh", overrideVersionCode = 6), SingleLang("vomic", "http://www.vomicmh.com", "zh", className = "Vomic"), ) diff --git a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/sinmh/SinMHGenerator.kt b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/sinmh/SinMHGenerator.kt index 7441c4b0f..d143920ba 100644 --- a/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/sinmh/SinMHGenerator.kt +++ b/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/sinmh/SinMHGenerator.kt @@ -26,7 +26,7 @@ class SinMHGenerator : ThemeSourceGenerator { ), SingleLang( name = "57Manhua", baseUrl = "http://www.wuqimh.net", lang = "zh", - className = "WuqiManga", sourceName = "57漫画", overrideVersionCode = 3 + className = "WuqiManga", sourceName = "57漫画", overrideVersionCode = 4 ), ) diff --git a/settings.gradle.kts b/settings.gradle.kts index 8cef3db0d..af6836a8e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -3,6 +3,9 @@ include(":core") include(":lib-dataimage") project(":lib-dataimage").projectDir = File("lib/dataimage") +include(":lib-unpacker") +project(":lib-unpacker").projectDir = File("lib/unpacker") + if (System.getenv("CI") == null || System.getenv("CI_MODULE_GEN") == "true") { // Local development (full project build) diff --git a/src/zh/sixmh/build.gradle b/src/zh/sixmh/build.gradle index b2784603d..28446b8bd 100644 --- a/src/zh/sixmh/build.gradle +++ b/src/zh/sixmh/build.gradle @@ -6,11 +6,11 @@ ext { extName = '6Manhua' pkgNameSuffix = 'zh.sixmh' extClass = '.SixMH' - extVersionCode = 3 + extVersionCode = 4 } apply from: "$rootDir/common.gradle" dependencies { - implementation 'com.github.stevenyomi:unpacker:12a09e3c1a' // 1.1 + implementation project(':lib-unpacker') } diff --git a/src/zh/sixmh/src/eu/kanade/tachiyomi/extension/zh/sixmh/SixMH.kt b/src/zh/sixmh/src/eu/kanade/tachiyomi/extension/zh/sixmh/SixMH.kt index c774c467c..9b0c27e3c 100644 --- a/src/zh/sixmh/src/eu/kanade/tachiyomi/extension/zh/sixmh/SixMH.kt +++ b/src/zh/sixmh/src/eu/kanade/tachiyomi/extension/zh/sixmh/SixMH.kt @@ -1,7 +1,7 @@ package eu.kanade.tachiyomi.extension.zh.sixmh -import com.github.stevenyomi.unpacker.Unpacker import eu.kanade.tachiyomi.AppInfo +import eu.kanade.tachiyomi.lib.unpacker.Unpacker import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.POST import eu.kanade.tachiyomi.network.interceptor.rateLimit