Snowmtl: Add support to Google Translator (#7558)

* Add support to Google Translator

* Remove Request.Builder

* Remove unused code

* Refatoring

* Replace the hashmap with the when statement

* Improves readability

* Remove site color and add text outline
This commit is contained in:
Chopper 2025-02-10 11:47:06 -03:00 committed by Draff
parent 58941d9440
commit 11d6ca37c3
No known key found for this signature in database
GPG Key ID: E8A89F3211677653
9 changed files with 229 additions and 238 deletions

View File

@ -27,8 +27,6 @@ import java.io.ByteArrayOutputStream
import java.io.File
import java.io.FileOutputStream
import java.io.InputStream
import kotlin.math.pow
import kotlin.math.sqrt
// The Interceptor joins the dialogues and pages of the manga.
@RequiresApi(Build.VERSION_CODES.O)
@ -81,7 +79,7 @@ class ComposedImageInterceptor(
val textPaint = createTextPaint(selectFontFamily(dialog.type))
val dialogBox = createDialogBox(dialog, textPaint, bitmap)
val y = getYAxis(textPaint, dialog, dialogBox)
canvas.draw(dialogBox, dialog, dialog.x1, y)
canvas.draw(textPaint, dialogBox, dialog, dialog.x1, y)
}
val output = ByteArrayOutputStream()
@ -222,18 +220,9 @@ class ComposedImageInterceptor(
dialogBox = createBoxLayout(dialog, textPaint)
}
// Use source setup
if (dialog.isNewApi && language.disableSourceSettings.not()) {
textPaint.color = dialog.foregroundColor
textPaint.bgColor = dialog.backgroundColor
textPaint.style = if (dialog.isBold) Paint.Style.FILL_AND_STROKE else Paint.Style.FILL
}
/**
* Forces font color correction if the background color of the dialog box and the font color are too similar.
* It's a source configuration problem.
*/
textPaint.adjustTextColor(dialog, bitmap)
textPaint.color = Color.BLACK
textPaint.bgColor = Color.WHITE
textPaint.strokeWidth = 2F
return dialogBox
}
@ -246,59 +235,45 @@ class ComposedImageInterceptor(
setIncludePad(false)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
setBreakStrategy(LineBreaker.BREAK_STRATEGY_BALANCED)
setHyphenationFrequency(Layout.HYPHENATION_FREQUENCY_FULL)
}
}.build()
}
// Invert color in black dialog box.
private fun TextPaint.adjustTextColor(dialog: Dialog, bitmap: Bitmap) {
val pixelColor = bitmap.getPixel(dialog.centerX.toInt(), dialog.centerY.toInt())
val inverseColor = (Color.WHITE - pixelColor) or Color.BLACK
val minDistance = 80f // arbitrary
if (colorDistance(pixelColor, dialog.foregroundColor) > minDistance) {
return
}
color = inverseColor
}
private inline fun <reified T> String.parseAs(): T {
return json.decodeFromString(this)
}
private fun Canvas.draw(layout: StaticLayout, dialog: Dialog, x: Float, y: Float) {
private fun Canvas.draw(textPaint: TextPaint, layout: StaticLayout, dialog: Dialog, x: Float, y: Float) {
save()
translate(x, y)
rotate(dialog.angle)
layout.draw(this)
drawTextOutline(textPaint, layout)
drawText(textPaint, layout)
restore()
}
private fun Canvas.drawText(textPaint: TextPaint, layout: StaticLayout) {
textPaint.style = Paint.Style.FILL
layout.draw(this)
}
private fun Canvas.drawTextOutline(textPaint: TextPaint, layout: StaticLayout) {
val foregroundColor = textPaint.color
val style = textPaint.style
textPaint.color = textPaint.bgColor
textPaint.style = Paint.Style.FILL_AND_STROKE
layout.draw(this)
textPaint.color = foregroundColor
textPaint.style = style
}
// https://pixelsconverter.com/pt-to-px
private val Int.pt: Float get() = this / SCALED_DENSITY
// ============================= Utils ======================================
/**
* Calculates the Euclidean distance between two colors in RGB space.
*
* This function takes two integer values representing hexadecimal colors,
* converts them to their RGB components, and calculates the Euclidean distance
* between the two colors. The distance provides a measure of how similar or
* different the two colors are.
*
*/
private fun colorDistance(colorA: Int, colorB: Int): Double {
val a = Color.valueOf(colorA)
val b = Color.valueOf(colorB)
return sqrt(
(b.red() - a.red()).toDouble().pow(2) +
(b.green() - a.green()).toDouble().pow(2) +
(b.blue() - a.blue()).toDouble().pow(2),
)
}
companion object {
// w3: Absolute Lengths [...](https://www.w3.org/TR/css3-values/#absolute-lengths)
const val SCALED_DENSITY = 0.75f // 1px = 0.75pt

View File

@ -3,7 +3,7 @@ ext {
extClass = '.SnowmtlFactory'
themePkg = 'machinetranslations'
baseUrl = 'https://snowmtl.ru'
overrideVersionCode = 8
overrideVersionCode = 9
isNsfw = true
}

View File

@ -1,15 +1,18 @@
package eu.kanade.tachiyomi.extension.all.snowmtl
import android.os.Build
import android.widget.Toast
import androidx.annotation.RequiresApi
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import androidx.preference.SwitchPreferenceCompat
import eu.kanade.tachiyomi.extension.all.snowmtl.interceptors.TranslationInterceptor
import eu.kanade.tachiyomi.extension.all.snowmtl.translator.BingTranslator
import eu.kanade.tachiyomi.extension.all.snowmtl.translator.TranslatorEngine
import eu.kanade.tachiyomi.extension.all.snowmtl.translator.bing.BingTranslator
import eu.kanade.tachiyomi.extension.all.snowmtl.translator.google.GoogleTranslator
import eu.kanade.tachiyomi.multisrc.machinetranslations.MachineTranslations
import eu.kanade.tachiyomi.multisrc.machinetranslations.interceptors.ComposedImageInterceptor
import eu.kanade.tachiyomi.network.interceptor.rateLimit
import okhttp3.OkHttpClient
import java.util.concurrent.TimeUnit
@RequiresApi(Build.VERSION_CODES.O)
@ -22,32 +25,36 @@ class Snowmtl(
) {
override val lang = language.lang
private var disableTranslationOptimization: Boolean
get() = preferences.getBoolean(DISABLE_TRANSLATION_OPTIM_PREF, language.disableTranslationOptimization)
set(value) = preferences.edit().putBoolean(DISABLE_TRANSLATION_OPTIM_PREF, value).apply()
private val translators = arrayOf(
"Bing",
"Google",
)
private val settings: LanguageSetting get() = language.copy(
fontSize = this@Snowmtl.fontSize,
disableTranslationOptimization = this@Snowmtl.disableTranslationOptimization,
disableSourceSettings = this@Snowmtl.disableSourceSettings,
)
private val provider: String get() =
preferences.getString(TRANSLATOR_PROVIDER_PREF, translators.first())!!
private val clientUtils = network.cloudflareClient.newBuilder()
.rateLimit(3, 2, TimeUnit.SECONDS)
.build()
private val translator: TranslatorEngine = BingTranslator(clientUtils, headers)
// Keeps object state
private val composeInterceptor = ComposedImageInterceptor(baseUrl, settings)
private val translatorInterceptor = TranslationInterceptor(settings, translator)
override val useDefaultComposedImageInterceptor = false
override fun clientBuilder() = super.clientBuilder()
override fun clientBuilder(): OkHttpClient.Builder {
val translator: TranslatorEngine = when (provider) {
"Google" -> GoogleTranslator(clientUtils, headers)
else -> BingTranslator(clientUtils, headers)
}
return super.clientBuilder()
.rateLimit(3)
.addInterceptor(translatorInterceptor.apply { language = this@Snowmtl.settings })
.addInterceptor(composeInterceptor.apply { language = this@Snowmtl.settings })
.addInterceptor(TranslationInterceptor(settings, translator))
.addInterceptor(ComposedImageInterceptor(baseUrl, settings))
}
override fun setupPreferenceScreen(screen: PreferenceScreen) {
super.setupPreferenceScreen(screen)
@ -56,25 +63,35 @@ class Snowmtl(
return
}
if (language.disableTranslationOptimization.not()) {
SwitchPreferenceCompat(screen.context).apply {
key = DISABLE_TRANSLATION_OPTIM_PREF
title = "⚠ Disable translation optimization"
ListPreference(screen.context).apply {
key = TRANSLATOR_PROVIDER_PREF
title = "Translator"
entries = translators
entryValues = translators
summary = buildString {
append("Allows dialog boxes to be translated sequentially. ")
append("Avoids problems when loading some translated pages caused by the translator's text formatting. ")
append("Pages will load more slowly.")
appendLine("Engine used to translate dialog boxes")
append("\t* %s")
}
setDefaultValue(false)
setDefaultValue(translators.first())
setOnPreferenceChange { _, newValue ->
disableTranslationOptimization = newValue as Boolean
val selected = newValue as String
val index = this.findIndexOfValue(selected)
val entry = entries[index] as String
Toast.makeText(
screen.context,
"The translator has been changed to '$entry'",
Toast.LENGTH_LONG,
).show()
true
}
}.also(screen::addPreference)
}
}
companion object {
private const val DISABLE_TRANSLATION_OPTIM_PREF = "disableTranslationOptimizationPref"
private const val TRANSLATOR_PROVIDER_PREF = "translatorProviderPref"
}
}

View File

@ -11,7 +11,7 @@ class SnowmtlFactory : SourceFactory {
}
private val languageList = listOf(
LanguageSetting("ar", disableSourceSettings = true, disableTranslationOptimization = true),
LanguageSetting("ar", disableSourceSettings = true),
LanguageSetting("en"),
LanguageSetting("es"),
LanguageSetting("id"),
@ -25,5 +25,4 @@ data class LanguageSetting(
override val origin: String = "en",
override var fontSize: Int = 24,
override var disableSourceSettings: Boolean = false,
val disableTranslationOptimization: Boolean = false,
) : Language

View File

@ -6,6 +6,10 @@ import eu.kanade.tachiyomi.extension.all.snowmtl.LanguageSetting
import eu.kanade.tachiyomi.extension.all.snowmtl.translator.TranslatorEngine
import eu.kanade.tachiyomi.multisrc.machinetranslations.Dialog
import eu.kanade.tachiyomi.multisrc.machinetranslations.MachineTranslations.Companion.PAGE_REGEX
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
@ -32,11 +36,14 @@ class TranslationInterceptor(
val dialogues = request.url.fragment?.parseAs<List<Dialog>>()
?: return chain.proceed(request)
val translated = when {
language.disableTranslationOptimization -> dialogues.map {
it.replaceText(translator.translate(language.origin, language.target, it.text))
val translated = runBlocking(Dispatchers.IO) {
dialogues.map { dialog ->
async {
dialog.replaceText(
translator.translate(language.origin, language.target, dialog.text),
)
}
else -> translationOptimized(dialogues)
}.awaitAll()
}
val newRequest = request.newBuilder()
@ -46,124 +53,13 @@ class TranslationInterceptor(
return chain.proceed(newRequest)
}
/**
* Optimizes the translation of a list of dialogues.
* This reduces the number of requests to the translator per page.
*
* @param dialogues List of Dialog objects to be translated.
* @return List of translated Dialog objects.
*/
private fun translationOptimized(dialogues: List<Dialog>): List<Dialog> {
val mapping = buildMap(dialogues)
val tokens = tokenizeAssociatedDialog(mapping).flatMap { token ->
translator.translate(language.origin, language.target, token).split(delimiter)
}
return replaceDialoguesWithTranslations(tokens, mapping)
}
private fun replaceDialoguesWithTranslations(
tokens: List<String>,
mapping: Map<String, Pair<String, AssociatedDialog>>,
) = tokens.mapNotNull { token ->
val list = try {
token.decode().parseAs<List<String>>()
} catch (_: Exception) {
// The translator may return an invalid JSON, but it keeps the pattern sent.
TRANSLATOR_EXTRACT_REGEX.findAll(token).map {
listOf(
it.groups[1]?.value!!.encode(),
it.groups[3]?.value!!.let { dialog ->
dialog.takeIf { it.startsWith("\"") } ?: dialog.encode()
},
)
}.toList().flatten()
}
val key = list.first()
val text = list.last()
mapping[key]?.second?.dialog?.replaceText(text)
}
private fun Dialog.replaceText(value: String) = this.copy(
textByLanguage = mutableMapOf(
"text" to value,
),
)
/**
* Tokenizes the associated dialogues.
*
* @param mapping Map of associated dialogues.
* @return List of tokens.
*/
private fun tokenizeAssociatedDialog(mapping: Map<String, Pair<String, AssociatedDialog>>) =
tokenizeText(mapping.map { it.value.second.content })
/**
* Builds a map of dialogues associated with their identifiers.
* I couldn't associate the translated dialog box with the zip method,
* because some dialog boxes aren't associated correctly
*
* @param dialogues List of Dialog objects to be mapped.
* @return Map where the key is the dialog identifier and the value is a pair containing the identifier and the associated dialog.
*/
private fun buildMap(dialogues: List<Dialog>): Map<String, Pair<String, AssociatedDialog>> {
return dialogues.map {
val payload = json.encodeToString<List<String>>(listOf(it.hashCode().toString(), it.text))
.encode()
it.hashCode().toString() to AssociatedDialog(it, payload)
}.associateBy { it.first }
}
// Prevents the translator's response from removing quotation marks from some texts
private fun String.encode() = "\"${this}\""
private fun String.decode() = this.substringAfter("\"").substringBeforeLast("\"")
private val delimiter: String = "¦"
/**
* Tokenizes a list of texts based on the translator's character capacity per request
*
* @param texts List of texts to be tokenized.
* @return List of tokens.
*/
private fun tokenizeText(texts: List<String>): List<String> {
val tokenized = mutableListOf<String>()
val remainingText = buildString(translator.capacity) {
texts.forEach { text ->
if (length + text.length + delimiter.length > capacity()) {
tokenized += toString()
clear()
}
if (isNotEmpty()) {
append(delimiter)
}
append(text)
}
}
if (remainingText.isNotEmpty()) {
tokenized += remainingText
}
return tokenized
}
private inline fun <reified T> String.parseAs(): T {
return json.decodeFromString(this)
}
companion object {
val TRANSLATOR_EXTRACT_REGEX = """"?(-?\d+)(\\?")?,((\\?")?([^(\])]+))""".toRegex()
}
}
private class AssociatedDialog(
val dialog: Dialog,
val content: String,
)

View File

@ -1,9 +1,9 @@
package eu.kanade.tachiyomi.extension.all.snowmtl.translator
package eu.kanade.tachiyomi.extension.all.snowmtl.translator.bing
import eu.kanade.tachiyomi.extension.all.snowmtl.translator.TranslatorEngine
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import okhttp3.FormBody
@ -15,7 +15,8 @@ import okhttp3.Response
import org.jsoup.nodes.Element
import uy.kohesive.injekt.injectLazy
class BingTranslator(private val client: OkHttpClient, private val headers: Headers) : TranslatorEngine {
class BingTranslator(private val client: OkHttpClient, private val headers: Headers) :
TranslatorEngine {
private val baseUrl = "https://www.bing.com"
@ -25,7 +26,7 @@ class BingTranslator(private val client: OkHttpClient, private val headers: Head
private var tokens: TokenGroup = TokenGroup()
override val capacity: Int = MAX_CHARS_ALLOW
override val capacity: Int = 1000
private val attempts = 3
@ -33,15 +34,10 @@ class BingTranslator(private val client: OkHttpClient, private val headers: Head
if (tokens.isNotValid() && refreshTokens().not()) {
return text
}
val request = translatorRequest(from, to, text)
repeat(attempts) {
try {
val dto = client
.newCall(translatorRequest(from, to, text))
.execute()
.parseAs<List<TranslateDto>>()
return dto.firstOrNull()?.text ?: text
return fetchTranslatedText(request)
} catch (e: Exception) {
refreshTokens()
}
@ -49,6 +45,11 @@ class BingTranslator(private val client: OkHttpClient, private val headers: Head
return text
}
private fun fetchTranslatedText(request: Request): String {
return client.newCall(request).execute().parseAs<List<TranslateDto>>()
.firstOrNull()!!.text
}
private fun refreshTokens(): Boolean {
tokens = loadTokens()
return tokens.isValid()
@ -111,29 +112,5 @@ class BingTranslator(private val client: OkHttpClient, private val headers: Head
companion object {
val TOKENS_REGEX = """params_AbusePreventionHelper(\s+)?=(\s+)?[^\[]\[(\d+),"([^"]+)""".toRegex()
val IG_PARAM_REGEX = """IG:"([^"]+)""".toRegex()
const val MAX_CHARS_ALLOW = 1000
}
}
private class TokenGroup(
val token: String = "",
val key: String = "",
val iid: String = "",
val ig: String = "",
) {
fun isNotValid() = listOf(token, key, iid, ig).any(String::isBlank)
fun isValid() = isNotValid().not()
}
@Serializable
private class TranslateDto(
val translations: List<TextTranslated>,
) {
val text = translations.firstOrNull()?.text ?: ""
}
@Serializable
private class TextTranslated(
val text: String,
)

View File

@ -0,0 +1,28 @@
package eu.kanade.tachiyomi.extension.all.snowmtl.translator.bing
import kotlinx.serialization.Serializable
class BingTranslatorDto
class TokenGroup(
val token: String = "",
val key: String = "",
val iid: String = "",
val ig: String = "",
) {
fun isNotValid() = listOf(token, key, iid, ig).any(String::isBlank)
fun isValid() = isNotValid().not()
}
@Serializable
class TranslateDto(
val translations: List<TextTranslated>,
) {
val text = translations.firstOrNull()?.text ?: ""
}
@Serializable
class TextTranslated(
val text: String,
)

View File

@ -0,0 +1,84 @@
package eu.kanade.tachiyomi.extension.all.snowmtl.translator.google
import eu.kanade.tachiyomi.extension.all.snowmtl.translator.TranslatorEngine
import eu.kanade.tachiyomi.network.GET
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import uy.kohesive.injekt.injectLazy
import java.io.IOException
/**
* This client is an adaptation of the following python repository: https://github.com/ssut/py-googletrans.
*/
class GoogleTranslator(private val client: OkHttpClient, private val headers: Headers) : TranslatorEngine {
private val baseUrl: String = "https://translate.googleapis.com"
private val webpage: String = "https://translate.google.com"
private val translatorUrl = "$baseUrl/translate_a/single"
override val capacity: Int = 5000
private val json: Json by injectLazy()
override fun translate(from: String, to: String, text: String): String {
val request = translateRequest(text, from, to)
return try { fetchTranslatedText(request) } catch (_: Exception) { text }
}
private fun translateRequest(text: String, from: String, to: String): Request {
return GET(clientUrlBuilder(text, from, to).build(), headersBuilder().build())
}
private fun headersBuilder(): Headers.Builder = headers.newBuilder()
.set("Origin", webpage)
.set("Alt-Used", webpage.substringAfterLast("/"))
.set("Referer", "$webpage/")
private fun clientUrlBuilder(text: String, src: String, dest: String, token: String = "xxxx"): HttpUrl.Builder {
return translatorUrl.toHttpUrl().newBuilder()
.setQueryParameter("client", "gtx")
.setQueryParameter("sl", src)
.setQueryParameter("tl", dest)
.setQueryParameter("hl", dest)
.setQueryParameter("ie", Charsets.UTF_8.toString())
.setQueryParameter("oe", Charsets.UTF_8.toString())
.setQueryParameter("otf", "1")
.setQueryParameter("ssel", "0")
.setQueryParameter("tsel", "0")
.setQueryParameter("tk", token)
.setQueryParameter("q", text)
.apply {
arrayOf("at", "bd", "ex", "ld", "md", "qca", "rw", "rm", "ss", "t").forEach {
addQueryParameter("dt", it)
}
}
}
private fun fetchTranslatedText(request: Request): String {
val response = client.newCall(request).execute()
if (response.isSuccessful.not()) {
throw IOException("Request failed: ${response.code}")
}
return response.parseJson().let(::extractTranslatedText)
}
private fun Response.parseJson(): JsonElement = json.parseToJsonElement(this.body.string())
private fun extractTranslatedText(data: JsonElement): String {
return data.jsonArray[0].jsonArray.joinToString("") {
it.jsonArray[0].jsonPrimitive.content
}
}
}

View File

@ -0,0 +1,15 @@
package eu.kanade.tachiyomi.extension.all.snowmtl.translator.google
import okhttp3.Response
class GoogleTranslatorDto
data class Translated(
val from: String,
val to: String,
val origin: String,
val text: String,
val pronunciation: String,
val extraData: Map<String, Any?>,
val response: Response,
)