From c4326203569fee54fbc2ac2b164987a03c4d445b Mon Sep 17 00:00:00 2001 From: Chopper <156493704+choppeh@users.noreply.github.com> Date: Tue, 17 Dec 2024 02:42:06 -0300 Subject: [PATCH] Snowmtl: Adds support for multiple languages (#6563) * Move to src/all * Optimize translation * Fix image loading timeout and expired translator token * Fix extension initialization * Fix translator response --- src/{en => all}/snowmtl/AndroidManifest.xml | 0 .../snowmtl/assets/fonts/LICENSE.txt | 0 .../assets/fonts/coming_soon_regular.ttf | Bin src/{en => all}/snowmtl/build.gradle | 4 +- .../snowmtl/res/mipmap-hdpi/ic_launcher.png | Bin .../snowmtl/res/mipmap-mdpi/ic_launcher.png | Bin .../snowmtl/res/mipmap-xhdpi/ic_launcher.png | Bin .../snowmtl/res/mipmap-xxhdpi/ic_launcher.png | Bin .../res/mipmap-xxxhdpi/ic_launcher.png | Bin .../extension/all/snowmtl/SnowmltFactory.kt | 21 +++ .../extension/all}/snowmtl/SnowmltFilters.kt | 2 +- .../extension/all}/snowmtl/Snowmtl.kt | 28 +++- .../extension/all}/snowmtl/SnowmtlDto.kt | 18 ++- .../all}/snowmtl/SnowmtlUrlActivity.kt | 2 +- .../interceptors}/ComposedImageInterceptor.kt | 74 +++++---- .../interceptors/TranslationInterceptor.kt | 141 ++++++++++++++++++ .../all/snowmtl/translator/BingTranslator.kt | 139 +++++++++++++++++ .../snowmtl/translator/TranslatorEngine.kt | 6 + 18 files changed, 378 insertions(+), 57 deletions(-) rename src/{en => all}/snowmtl/AndroidManifest.xml (100%) rename src/{en => all}/snowmtl/assets/fonts/LICENSE.txt (100%) rename src/{en => all}/snowmtl/assets/fonts/coming_soon_regular.ttf (100%) rename src/{en => all}/snowmtl/build.gradle (65%) rename src/{en => all}/snowmtl/res/mipmap-hdpi/ic_launcher.png (100%) rename src/{en => all}/snowmtl/res/mipmap-mdpi/ic_launcher.png (100%) rename src/{en => all}/snowmtl/res/mipmap-xhdpi/ic_launcher.png (100%) rename src/{en => all}/snowmtl/res/mipmap-xxhdpi/ic_launcher.png (100%) rename src/{en => all}/snowmtl/res/mipmap-xxxhdpi/ic_launcher.png (100%) create mode 100644 src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmltFactory.kt rename src/{en/snowmtl/src/eu/kanade/tachiyomi/extension/en => all/snowmtl/src/eu/kanade/tachiyomi/extension/all}/snowmtl/SnowmltFilters.kt (96%) rename src/{en/snowmtl/src/eu/kanade/tachiyomi/extension/en => all/snowmtl/src/eu/kanade/tachiyomi/extension/all}/snowmtl/Snowmtl.kt (87%) rename src/{en/snowmtl/src/eu/kanade/tachiyomi/extension/en => all/snowmtl/src/eu/kanade/tachiyomi/extension/all}/snowmtl/SnowmtlDto.kt (84%) rename src/{en/snowmtl/src/eu/kanade/tachiyomi/extension/en => all/snowmtl/src/eu/kanade/tachiyomi/extension/all}/snowmtl/SnowmtlUrlActivity.kt (95%) rename src/{en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl => all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors}/ComposedImageInterceptor.kt (76%) create mode 100644 src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors/TranslationInterceptor.kt create mode 100644 src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/BingTranslator.kt create mode 100644 src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/TranslatorEngine.kt diff --git a/src/en/snowmtl/AndroidManifest.xml b/src/all/snowmtl/AndroidManifest.xml similarity index 100% rename from src/en/snowmtl/AndroidManifest.xml rename to src/all/snowmtl/AndroidManifest.xml diff --git a/src/en/snowmtl/assets/fonts/LICENSE.txt b/src/all/snowmtl/assets/fonts/LICENSE.txt similarity index 100% rename from src/en/snowmtl/assets/fonts/LICENSE.txt rename to src/all/snowmtl/assets/fonts/LICENSE.txt diff --git a/src/en/snowmtl/assets/fonts/coming_soon_regular.ttf b/src/all/snowmtl/assets/fonts/coming_soon_regular.ttf similarity index 100% rename from src/en/snowmtl/assets/fonts/coming_soon_regular.ttf rename to src/all/snowmtl/assets/fonts/coming_soon_regular.ttf diff --git a/src/en/snowmtl/build.gradle b/src/all/snowmtl/build.gradle similarity index 65% rename from src/en/snowmtl/build.gradle rename to src/all/snowmtl/build.gradle index c8ec77737..4f76663b7 100644 --- a/src/en/snowmtl/build.gradle +++ b/src/all/snowmtl/build.gradle @@ -1,7 +1,7 @@ ext { extName = 'Snow Machine Translations' - extClass = '.Snowmtl' - extVersionCode = 3 + extClass = '.SnowmltFactory' + extVersionCode = 4 isNsfw = true } diff --git a/src/en/snowmtl/res/mipmap-hdpi/ic_launcher.png b/src/all/snowmtl/res/mipmap-hdpi/ic_launcher.png similarity index 100% rename from src/en/snowmtl/res/mipmap-hdpi/ic_launcher.png rename to src/all/snowmtl/res/mipmap-hdpi/ic_launcher.png diff --git a/src/en/snowmtl/res/mipmap-mdpi/ic_launcher.png b/src/all/snowmtl/res/mipmap-mdpi/ic_launcher.png similarity index 100% rename from src/en/snowmtl/res/mipmap-mdpi/ic_launcher.png rename to src/all/snowmtl/res/mipmap-mdpi/ic_launcher.png diff --git a/src/en/snowmtl/res/mipmap-xhdpi/ic_launcher.png b/src/all/snowmtl/res/mipmap-xhdpi/ic_launcher.png similarity index 100% rename from src/en/snowmtl/res/mipmap-xhdpi/ic_launcher.png rename to src/all/snowmtl/res/mipmap-xhdpi/ic_launcher.png diff --git a/src/en/snowmtl/res/mipmap-xxhdpi/ic_launcher.png b/src/all/snowmtl/res/mipmap-xxhdpi/ic_launcher.png similarity index 100% rename from src/en/snowmtl/res/mipmap-xxhdpi/ic_launcher.png rename to src/all/snowmtl/res/mipmap-xxhdpi/ic_launcher.png diff --git a/src/en/snowmtl/res/mipmap-xxxhdpi/ic_launcher.png b/src/all/snowmtl/res/mipmap-xxxhdpi/ic_launcher.png similarity index 100% rename from src/en/snowmtl/res/mipmap-xxxhdpi/ic_launcher.png rename to src/all/snowmtl/res/mipmap-xxxhdpi/ic_launcher.png diff --git a/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmltFactory.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmltFactory.kt new file mode 100644 index 000000000..8310b61d1 --- /dev/null +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmltFactory.kt @@ -0,0 +1,21 @@ +package eu.kanade.tachiyomi.extension.all.snowmtl + +import android.os.Build +import androidx.annotation.RequiresApi +import eu.kanade.tachiyomi.source.Source +import eu.kanade.tachiyomi.source.SourceFactory + +@RequiresApi(Build.VERSION_CODES.O) +class SnowmltFactory : SourceFactory { + override fun createSources(): List = languageList.map(::Snowmtl) +} + +data class Source(val lang: String, val target: String = lang, val origin: String = "en") + +private val languageList = listOf( + Source("en"), + Source("es"), + Source("id"), + Source("it"), + Source("pt-BR", "pt"), +) diff --git a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmltFilters.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmltFilters.kt similarity index 96% rename from src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmltFilters.kt rename to src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmltFilters.kt index 22a9f51f6..104b0db3c 100644 --- a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmltFilters.kt +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmltFilters.kt @@ -1,4 +1,4 @@ -package eu.kanade.tachiyomi.extension.en.snowmtl +package eu.kanade.tachiyomi.extension.all.snowmtl import eu.kanade.tachiyomi.source.model.Filter diff --git a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/Snowmtl.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/Snowmtl.kt similarity index 87% rename from src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/Snowmtl.kt rename to src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/Snowmtl.kt index 26fd436c7..e6c460d64 100644 --- a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/Snowmtl.kt +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/Snowmtl.kt @@ -1,7 +1,11 @@ -package eu.kanade.tachiyomi.extension.en.snowmtl +package eu.kanade.tachiyomi.extension.all.snowmtl import android.os.Build import androidx.annotation.RequiresApi +import eu.kanade.tachiyomi.extension.all.snowmtl.interceptors.ComposedImageInterceptor +import eu.kanade.tachiyomi.extension.all.snowmtl.interceptors.TranslationInterceptor +import eu.kanade.tachiyomi.extension.all.snowmtl.translator.BingTranslator +import eu.kanade.tachiyomi.extension.all.snowmtl.translator.TranslatorEngine import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.interceptor.rateLimit import eu.kanade.tachiyomi.source.model.Filter @@ -23,22 +27,33 @@ import uy.kohesive.injekt.injectLazy import java.text.SimpleDateFormat import java.util.Calendar import java.util.Locale +import java.util.concurrent.TimeUnit -@RequiresApi(Build.VERSION_CODES.M) -class Snowmtl : ParsedHttpSource() { +@RequiresApi(Build.VERSION_CODES.O) +class Snowmtl( + source: Source, +) : ParsedHttpSource() { override val name = "Snow Machine Translations" override val baseUrl = "https://snowmtl.ru" - override val lang = "en" + override val lang = source.lang override val supportsLatest = true private val json: Json by injectLazy() + private val translatorClient = network.cloudflareClient.newBuilder() + .rateLimit(1, 3, TimeUnit.SECONDS) + .build() + + private val translator: TranslatorEngine = BingTranslator(translatorClient, headers) + override val client = network.cloudflareClient.newBuilder() .rateLimit(2) + .readTimeout(2, TimeUnit.MINUTES) + .addInterceptor(TranslationInterceptor(source, translator)) .addInterceptor(ComposedImageInterceptor(baseUrl, super.client)) .build() @@ -158,7 +173,9 @@ class Snowmtl : ParsedHttpSource() { dto.imageUrl.startsWith("http") -> dto.imageUrl else -> "https://${dto.imageUrl}" } - val fragment = json.encodeToString>(dto.translations) + val fragment = json.encodeToString>( + dto.dialogues.filter { it.text.isNotBlank() }, + ) Page(index, imageUrl = "$imageUrl#$fragment") } } @@ -203,6 +220,7 @@ class Snowmtl : ParsedHttpSource() { } companion object { + val PAGE_REGEX = Regex(".*?\\.(webp|png|jpg|jpeg)#\\[.*?]", RegexOption.IGNORE_CASE) const val PREFIX_SEARCH = "id:" private val dateFormat: SimpleDateFormat = SimpleDateFormat("dd MMMM yyyy", Locale.US) } diff --git a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmtlDto.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmtlDto.kt similarity index 84% rename from src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmtlDto.kt rename to src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmtlDto.kt index 288e47cf5..8ec7f0778 100644 --- a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmtlDto.kt +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmtlDto.kt @@ -1,4 +1,4 @@ -package eu.kanade.tachiyomi.extension.en.snowmtl +package eu.kanade.tachiyomi.extension.all.snowmtl import android.graphics.Color import android.os.Build @@ -18,13 +18,15 @@ import kotlinx.serialization.json.put class PageDto( @SerialName("img_url") val imageUrl: String, - @Serializable(with = TranslationsListSerializer::class) - val translations: List = emptyList(), + + @SerialName("translations") + @Serializable(with = DialogListSerializer::class) + val dialogues: List = emptyList(), ) @Serializable @RequiresApi(Build.VERSION_CODES.O) -class Translation( +data class Dialog( val x1: Float, val y1: Float, val x2: Float, @@ -55,12 +57,12 @@ class Translation( } } -private object TranslationsListSerializer : - JsonTransformingSerializer>(ListSerializer(Translation.serializer())) { +private object DialogListSerializer : + JsonTransformingSerializer>(ListSerializer(Dialog.serializer())) { override fun transformDeserialize(element: JsonElement): JsonElement { return JsonArray( element.jsonArray.map { jsonElement -> - val (coordinates, text) = getCoordinatesAndCaption(jsonElement) + val (coordinates, text) = getCoordinatesAndDialog(jsonElement) buildJsonObject { put("x1", coordinates[0]) @@ -83,7 +85,7 @@ private object TranslationsListSerializer : ) } - private fun getCoordinatesAndCaption(element: JsonElement): Pair { + private fun getCoordinatesAndDialog(element: JsonElement): Pair { return try { val arr = element.jsonArray arr[0].jsonArray to arr[1] diff --git a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmtlUrlActivity.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmtlUrlActivity.kt similarity index 95% rename from src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmtlUrlActivity.kt rename to src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmtlUrlActivity.kt index b43d33043..d793f15db 100644 --- a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/SnowmtlUrlActivity.kt +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/SnowmtlUrlActivity.kt @@ -1,4 +1,4 @@ -package eu.kanade.tachiyomi.extension.en.snowmtl +package eu.kanade.tachiyomi.extension.all.snowmtl import android.app.Activity import android.content.ActivityNotFoundException diff --git a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/ComposedImageInterceptor.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors/ComposedImageInterceptor.kt similarity index 76% rename from src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/ComposedImageInterceptor.kt rename to src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors/ComposedImageInterceptor.kt index 31d7b5556..c7e6dcd89 100644 --- a/src/en/snowmtl/src/eu/kanade/tachiyomi/extension/en/snowmtl/ComposedImageInterceptor.kt +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors/ComposedImageInterceptor.kt @@ -1,4 +1,4 @@ -package eu.kanade.tachiyomi.extension.en.snowmtl +package eu.kanade.tachiyomi.extension.all.snowmtl.interceptors import android.graphics.Bitmap import android.graphics.BitmapFactory @@ -12,6 +12,8 @@ import android.text.Layout import android.text.StaticLayout import android.text.TextPaint import androidx.annotation.RequiresApi +import eu.kanade.tachiyomi.extension.all.snowmtl.Dialog +import eu.kanade.tachiyomi.extension.all.snowmtl.Snowmtl.Companion.PAGE_REGEX import eu.kanade.tachiyomi.network.GET import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json @@ -29,7 +31,7 @@ import java.io.InputStream import kotlin.math.pow import kotlin.math.sqrt -// The Interceptor joins the captions and pages of the manga. +// The Interceptor joins the dialogues and pages of the manga. @RequiresApi(Build.VERSION_CODES.O) class ComposedImageInterceptor( baseUrl: String, @@ -44,22 +46,16 @@ class ComposedImageInterceptor( "normal" to Pair("$baseUrl/images/normal.ttf", null), ) - private val imageRegex = Regex( - "$baseUrl.*?\\.(webp|png|jpg|jpeg)#\\[.*?]", - RegexOption.IGNORE_CASE, - ) - override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() val url = request.url.toString() - val isPageImageUrl = imageRegex.containsMatchIn(url) - if (isPageImageUrl.not()) { + if (PAGE_REGEX.containsMatchIn(url).not()) { return chain.proceed(request) } - val translation = request.url.fragment?.parseAs>() - ?: throw IOException("Translation not found") + val dialogues = request.url.fragment?.parseAs>() + ?: throw IOException("Dialogues not found") val imageRequest = request.newBuilder() .url(url) @@ -78,14 +74,12 @@ class ComposedImageInterceptor( val canvas = Canvas(bitmap) - translation - .filter { it.text.isNotBlank() } - .forEach { caption -> - val textPaint = createTextPaint(selectFontFamily(caption.type)) - val dialogBox = createDialogBox(caption, textPaint, bitmap) - val y = getYAxis(textPaint, caption, dialogBox) - canvas.draw(dialogBox, caption, caption.x1, y) - } + dialogues.forEach { dialog -> + val textPaint = createTextPaint(selectFontFamily(dialog.type)) + val dialogBox = createDialogBox(dialog, textPaint, bitmap) + val y = getYAxis(textPaint, dialog, dialogBox) + canvas.draw(dialogBox, dialog, dialog.x1, y) + } val output = ByteArrayOutputStream() @@ -189,49 +183,49 @@ class ComposedImageInterceptor( /** * Adjust the text to the center of the dialog box when feasible. */ - private fun getYAxis(textPaint: TextPaint, caption: Translation, dialogBox: StaticLayout): Float { + private fun getYAxis(textPaint: TextPaint, dialog: Dialog, dialogBox: StaticLayout): Float { val fontHeight = textPaint.fontMetrics.let { it.bottom - it.top } - val dialogBoxLineCount = caption.height / fontHeight + val dialogBoxLineCount = dialog.height / fontHeight /** - * Centers text in y for captions smaller than the dialog box + * Centers text in y for dialogues smaller than the dialog box */ return when { - dialogBox.lineCount < dialogBoxLineCount -> caption.centerY - dialogBox.lineCount / 2f * fontHeight - else -> caption.y1 + dialogBox.lineCount < dialogBoxLineCount -> dialog.centerY - dialogBox.lineCount / 2f * fontHeight + else -> dialog.y1 } } - private fun createDialogBox(caption: Translation, textPaint: TextPaint, bitmap: Bitmap): StaticLayout { - var dialogBox = createBoxLayout(caption, textPaint) + private fun createDialogBox(dialog: Dialog, textPaint: TextPaint, bitmap: Bitmap): StaticLayout { + var dialogBox = createBoxLayout(dialog, textPaint) /** * The best way I've found to adjust the text in the dialog box (Especially in long dialogues) */ - while (dialogBox.height > caption.height) { + while (dialogBox.height > dialog.height) { textPaint.textSize -= 0.5f - dialogBox = createBoxLayout(caption, textPaint) + dialogBox = createBoxLayout(dialog, textPaint) } // Use source setup - if (caption.isNewApi) { - textPaint.color = caption.foregroundColor - textPaint.bgColor = caption.backgroundColor - textPaint.style = if (caption.isBold) Paint.Style.FILL_AND_STROKE else Paint.Style.FILL + if (dialog.isNewApi) { + textPaint.color = dialog.foregroundColor + textPaint.bgColor = dialog.backgroundColor + textPaint.style = if (dialog.isBold) Paint.Style.FILL_AND_STROKE else Paint.Style.FILL } /** * Forces font color correction if the background color of the dialog box and the font color are too similar. * It's a source configuration problem. */ - textPaint.adjustTextColor(caption, bitmap) + textPaint.adjustTextColor(dialog, bitmap) return dialogBox } - private fun createBoxLayout(caption: Translation, textPaint: TextPaint) = - StaticLayout.Builder.obtain(caption.text, 0, caption.text.length, textPaint, caption.width.toInt()).apply { + private fun createBoxLayout(dialog: Dialog, textPaint: TextPaint) = + StaticLayout.Builder.obtain(dialog.text, 0, dialog.text.length, textPaint, dialog.width.toInt()).apply { setAlignment(Layout.Alignment.ALIGN_CENTER) setIncludePad(false) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { @@ -240,12 +234,12 @@ class ComposedImageInterceptor( }.build() // Invert color in black dialog box. - private fun TextPaint.adjustTextColor(caption: Translation, bitmap: Bitmap) { - val pixelColor = bitmap.getPixel(caption.centerX.toInt(), caption.centerY.toInt()) + private fun TextPaint.adjustTextColor(dialog: Dialog, bitmap: Bitmap) { + val pixelColor = bitmap.getPixel(dialog.centerX.toInt(), dialog.centerY.toInt()) val inverseColor = (Color.WHITE - pixelColor) or Color.BLACK val minDistance = 80f // arbitrary - if (colorDistance(pixelColor, caption.foregroundColor) > minDistance) { + if (colorDistance(pixelColor, dialog.foregroundColor) > minDistance) { return } color = inverseColor @@ -255,10 +249,10 @@ class ComposedImageInterceptor( return json.decodeFromString(this) } - private fun Canvas.draw(layout: StaticLayout, caption: Translation, x: Float, y: Float) { + private fun Canvas.draw(layout: StaticLayout, dialog: Dialog, x: Float, y: Float) { save() translate(x, y) - rotate(caption.angle) + rotate(dialog.angle) layout.draw(this) restore() } diff --git a/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors/TranslationInterceptor.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors/TranslationInterceptor.kt new file mode 100644 index 000000000..09150a328 --- /dev/null +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/interceptors/TranslationInterceptor.kt @@ -0,0 +1,141 @@ +package eu.kanade.tachiyomi.extension.all.snowmtl.interceptors + +import android.os.Build +import androidx.annotation.RequiresApi +import eu.kanade.tachiyomi.extension.all.snowmtl.Dialog +import eu.kanade.tachiyomi.extension.all.snowmtl.Snowmtl.Companion.PAGE_REGEX +import eu.kanade.tachiyomi.extension.all.snowmtl.Source +import eu.kanade.tachiyomi.extension.all.snowmtl.translator.TranslatorEngine +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import okhttp3.Interceptor +import okhttp3.Response +import uy.kohesive.injekt.injectLazy + +@RequiresApi(Build.VERSION_CODES.O) +class TranslationInterceptor( + private val source: Source, + private val translator: TranslatorEngine, +) : Interceptor { + + private val json: Json by injectLazy() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url.toString() + + if (PAGE_REGEX.containsMatchIn(url).not() || source.target == source.origin) { + return chain.proceed(request) + } + + val dialogues = request.url.fragment?.parseAs>() + ?: return chain.proceed(request) + + val translated = translationOptimized(dialogues) + + val newRequest = request.newBuilder() + .url("${url.substringBeforeLast("#")}#${json.encodeToString(translated)}") + .build() + + return chain.proceed(newRequest) + } + + /** + * Optimizes the translation of a list of dialogues. + * This reduces the number of requests to the translator per page. + * + * @param dialogues List of Dialog objects to be translated. + * @return List of translated Dialog objects. + */ + private fun translationOptimized(dialogues: List): List { + val mapping = buildMap(dialogues) + + val tokens = tokenizeAssociatedDialog(mapping).flatMap { token -> + translator.translate(source.origin, source.target, token).split(delimiter) + } + + return replaceDialoguesWithTranslations(tokens, mapping) + } + + private fun replaceDialoguesWithTranslations( + tokens: List, + mapping: Map>, + ) = tokens.mapNotNull { token -> + val list = token.decode().parseAs>() + val key = list.first() + val text = list.last() + + mapping[key]?.second?.dialog?.copy(text = text) + } + + /** + * Tokenizes the associated dialogues. + * + * @param mapping Map of associated dialogues. + * @return List of tokens. + */ + private fun tokenizeAssociatedDialog(mapping: Map>) = + tokenizeText(mapping.map { it.value.second.content }) + + /** + * Builds a map of dialogues associated with their identifiers. + * I couldn't associate the translated dialog box with the zip method, + * because some dialog boxes aren't associated correctly + * + * @param dialogues List of Dialog objects to be mapped. + * @return Map where the key is the dialog identifier and the value is a pair containing the identifier and the associated dialog. + */ + private fun buildMap(dialogues: List): Map> { + return dialogues.map { + val payload = json.encodeToString>(listOf(it.hashCode().toString(), it.text)) + .encode() + it.hashCode().toString() to AssociatedDialog(it, payload) + }.associateBy { it.first } + } + + // Prevents the translator's response from removing quotation marks from some texts + private fun String.encode() = "\"${this}\"" + private fun String.decode() = this.substringAfter("\"").substringBeforeLast("\"") + + private val delimiter: String = "|" + + /** + * Tokenizes a list of texts based on the translator's character capacity per request + * + * @param texts List of texts to be tokenized. + * @return List of tokens. + */ + private fun tokenizeText(texts: List): List { + val tokenized = mutableListOf() + + val remainingText = buildString(translator.capacity) { + texts.forEach { text -> + if (length + text.length + delimiter.length > capacity()) { + tokenized += toString() + clear() + } + + if (isNotEmpty()) { + append(delimiter) + } + + append(text) + } + } + + if (remainingText.isNotEmpty()) { + tokenized += remainingText + } + return tokenized + } + + private inline fun String.parseAs(): T { + return json.decodeFromString(this) + } +} + +private class AssociatedDialog( + val dialog: Dialog, + val content: String, +) diff --git a/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/BingTranslator.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/BingTranslator.kt new file mode 100644 index 000000000..13a91cc0e --- /dev/null +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/BingTranslator.kt @@ -0,0 +1,139 @@ +package eu.kanade.tachiyomi.extension.all.snowmtl.translator + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.POST +import eu.kanade.tachiyomi.util.asJsoup +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import okhttp3.FormBody +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import org.jsoup.nodes.Element +import uy.kohesive.injekt.injectLazy + +class BingTranslator(private val client: OkHttpClient, private val headers: Headers) : TranslatorEngine { + + private val baseUrl = "https://www.bing.com" + + private val translatorUrl = "$baseUrl/translator" + + private val json: Json by injectLazy() + + private var tokens: TokenGroup = TokenGroup() + + override val capacity: Int = MAX_CHARS_ALLOW + + private val attempts = 3 + + override fun translate(from: String, to: String, text: String): String { + if (tokens.isNotValid() && refreshTokens().not()) { + return text + } + + repeat(attempts) { + try { + val dto = client + .newCall(translatorRequest(from, to, text)) + .execute() + .parseAs>() + + return dto.firstOrNull()?.text ?: text + } catch (e: Exception) { + refreshTokens() + } + } + return text + } + + private fun refreshTokens(): Boolean { + tokens = loadTokens() + return tokens.isValid() + } + + private fun translatorRequest(from: String, to: String, text: String): Request { + val url = "$baseUrl/ttranslatev3".toHttpUrl().newBuilder() + .addQueryParameter("isVertical", "1") + .addQueryParameter("", "") // Present in Bing URL + .addQueryParameter("IG", tokens.ig) + .addQueryParameter("IID", tokens.iid) + .build() + + val headersApi = headers.newBuilder() + .set("Accept", "*/*") + .set("Origin", baseUrl) + .set("Referer", translatorUrl) + .set("Alt-Used", baseUrl) + .build() + + val payload = FormBody.Builder() + .add("fromLang", from) + .add("to", to) + .add("text", text) + .add("tryFetchingGenderDebiasedTranslations", "true") + .add("token", tokens.token) + .add("key", tokens.key) + .build() + + return POST(url.toString(), headersApi, payload) + } + + private fun loadTokens(): TokenGroup { + val document = client.newCall(GET(translatorUrl, headers)).execute().asJsoup() + + val scripts = document.select("script") + .map(Element::data) + + val scriptOne: String = scripts.firstOrNull(TOKENS_REGEX::containsMatchIn) + ?: return TokenGroup() + + val scriptTwo: String = scripts.firstOrNull(IG_PARAM_REGEX::containsMatchIn) + ?: return TokenGroup() + + val matchOne = TOKENS_REGEX.find(scriptOne)?.groups + val matchTwo = IG_PARAM_REGEX.find(scriptTwo)?.groups + + return TokenGroup( + token = matchOne?.get("token")?.value ?: "", + key = matchOne?.get("key")?.value ?: "", + ig = matchTwo?.get("ig")?.value ?: "", + iid = document.selectFirst("div[data-iid]:not([class])")?.attr("data-iid") ?: "", + ) + } + + private inline fun Response.parseAs(): T { + return json.decodeFromStream(body.byteStream()) + } + + companion object { + val TOKENS_REGEX = """params_AbusePreventionHelper(\s+)?=(\s+)?[^\[]\[(?\d+),"(?[^"]+)""".toRegex() + val IG_PARAM_REGEX = """IG:"(?[^"]+)""".toRegex() + const val MAX_CHARS_ALLOW = 1000 + } +} + +private class TokenGroup( + val token: String = "", + val key: String = "", + val iid: String = "", + val ig: String = "", +) { + fun isNotValid() = listOf(token, key, iid, ig).any(String::isBlank) + + fun isValid() = isNotValid().not() +} + +@Serializable +private class TranslateDto( + val translations: List, +) { + val text = translations.firstOrNull()?.text ?: "" +} + +@Serializable +private class TextTranslated( + val text: String, +) diff --git a/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/TranslatorEngine.kt b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/TranslatorEngine.kt new file mode 100644 index 000000000..c1bc273a6 --- /dev/null +++ b/src/all/snowmtl/src/eu/kanade/tachiyomi/extension/all/snowmtl/translator/TranslatorEngine.kt @@ -0,0 +1,6 @@ +package eu.kanade.tachiyomi.extension.all.snowmtl.translator + +interface TranslatorEngine { + val capacity: Int + fun translate(from: String, to: String, text: String): String +}