Add UnionMangas (#2091)

* Add latestUpdates, popularManga and mangaDetails to UnionMangas

* Add initial chapter request configuration

* Fix client headers request

* Add fetch chapter

* Fix chapter list parse

* Add search impl

* Fix chapter url

* Rename hash function

* Add utils functions

* Add pageList

* typo

* Cleanup.

* Add intent query

* Refactoring

* Remove password hardcode

* Replace Exception by RuntimeException

* throws message exception to user

* Add rateLimit

* Add Pageable class

* Cleanup

* Remove unicodes

* Remove nullable dto properties

* Rename variables

* Replace 'data class' with regular 'class'

* Remove try/catch. Let exceptions be thrown

* Fix search request

* Cleanup

---------

Co-authored-by: bapeey <90949336+bapeey@users.noreply.github.com>

src/all/unionmangas/AndroidManifest.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+<application>
+    <activity
+        android:name=".all.unionmangas.UnionMangasUrlActivity"
+        android:excludeFromRecents="true"
+        android:exported="true"
+        android:theme="@android:style/Theme.NoDisplay">
+        <intent-filter>
+            <action android:name="android.intent.action.VIEW" />
+
+            <category android:name="android.intent.category.DEFAULT" />
+            <category android:name="android.intent.category.BROWSABLE" />
+
+            <data android:host="https://unionmangas.xyz" />
+
+            <data android:scheme="https"/>
+            <data android:pathPattern="/manga-br/..*"/>
+
+            <data android:scheme="https"/>
+            <data android:pathPattern="/italy/..*"/>
+        </intent-filter>
+    </activity>
+</application>
+</manifest>

src/all/unionmangas/build.gradle

@@ -0,0 +1,12 @@
+ext {
+    extName = 'Union Mangas'
+    extClass = '.UnionMangasFactory'
+    extVersionCode = 1
+    isNsfw = true
+}
+
+apply from: "$rootDir/common.gradle"
+
+dependencies {
+    implementation(project(':lib:cryptoaes'))
+}

src/all/unionmangas/src/eu/kanade/tachiyomi/extension/all/unionmangas/UnionMangas.kt

@@ -0,0 +1,238 @@
+package eu.kanade.tachiyomi.extension.all.unionmangas
+
+import eu.kanade.tachiyomi.lib.cryptoaes.CryptoAES
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.interceptor.rateLimit
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.HttpSource
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import rx.Observable
+import uy.kohesive.injekt.injectLazy
+import java.security.MessageDigest
+import java.text.SimpleDateFormat
+import java.util.Date
+import java.util.Locale
+import java.util.TimeZone
+import java.util.concurrent.TimeUnit
+
+class UnionMangas(private val langOption: LanguageOption) : HttpSource() {
+    override val lang = langOption.lang
+
+    override val name: String = "Union Mangas"
+
+    override val baseUrl: String = "https://unionmangas.xyz"
+
+    override val supportsLatest = true
+
+    private val json: Json by injectLazy()
+
+    val langApiInfix = when (lang) {
+        "it" -> langOption.infix
+        else -> "v3/po"
+    }
+
+    override val client = network.client.newBuilder()
+        .rateLimit(5, 2, TimeUnit.SECONDS)
+        .build()
+
+    private fun apiHeaders(url: String): Headers {
+        val date = apiDateFormat.format(Date())
+        val path = url.toUrlWithoutDomain()
+
+        return headersBuilder()
+            .add("_hash", authorization(apiSeed, domain, date))
+            .add("_tranId", authorization(apiSeed, domain, date, path))
+            .add("_date", date)
+            .add("_domain", domain)
+            .add("_path", path)
+            .add("Origin", baseUrl)
+            .add("Host", apiUrl.removeProtocol())
+            .add("Referer", "$baseUrl/")
+            .build()
+    }
+
+    private fun authorization(vararg payloads: String): String {
+        val md = MessageDigest.getInstance("MD5")
+        val bytes = payloads.joinToString("").toByteArray()
+        val digest = md.digest(bytes)
+        return digest
+            .fold("") { str, byte -> str + "%02x".format(byte) }
+            .padStart(32, '0')
+    }
+
+    override fun chapterListParse(response: Response) = throw UnsupportedOperationException()
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        val chapters = mutableListOf<SChapter>()
+        var currentPage = 0
+        do {
+            val chaptersDto = fetchChapterListPageable(manga, currentPage)
+            chapters += chaptersDto.toSChapter(langOption)
+            currentPage++
+        } while (chaptersDto.hasNextPage())
+        return Observable.just(chapters.reversed())
+    }
+
+    private fun fetchChapterListPageable(manga: SManga, page: Int): ChapterPageDto {
+        val maxResult = 16
+        val url = "$apiUrl/api/$langApiInfix/GetChapterListFilter/${manga.slug()}/$maxResult/$page/all/ASC"
+        return client.newCall(GET(url, apiHeaders(url))).execute()
+            .parseAs<ChapterPageDto>()
+    }
+
+    override fun latestUpdatesParse(response: Response): MangasPage {
+        val nextData = response.parseNextData<LatestUpdateProps>()
+        val dto = nextData.data.latestUpdateDto
+        val mangas = dto.mangas.map { mangaParse(it, nextData.query) }
+
+        return MangasPage(
+            mangas = mangas,
+            hasNextPage = dto.hasNextPage(),
+        )
+    }
+
+    override fun latestUpdatesRequest(page: Int): Request {
+        val url = "$baseUrl/${langOption.infix}/latest-releases".toHttpUrl().newBuilder()
+            .addQueryParameter("page", "$page")
+            .build()
+        return GET(url, headers)
+    }
+
+    override fun mangaDetailsParse(response: Response): SManga {
+        val nextData = response.parseNextData<MangaDetailsProps>()
+        val dto = nextData.data.mangaDetailsDto
+        return SManga.create().apply {
+            title = dto.title
+            genre = dto.genres
+            thumbnail_url = dto.thumbnailUrl
+            url = mangaUrlParse(dto.slug, nextData.query.type)
+            status = dto.status
+        }
+    }
+
+    override fun pageListParse(response: Response): List<Page> {
+        val chaptersDto = decryptChapters(response)
+        return chaptersDto.images.mapIndexed { index, imageUrl ->
+            Page(index, imageUrl = imageUrl)
+        }
+    }
+
+    private fun decryptChapters(response: Response): ChaptersDto {
+        val document = response.asJsoup()
+        val password = findChapterPassword(document)
+        val pageListData = document.parseNextData<ChaptersProps>().data.pageListData
+        val decodedData = CryptoAES.decrypt(pageListData, password)
+        return ChaptersDto(
+            data = json.decodeFromString<ChaptersDto>(decodedData).data,
+            delimiter = langOption.pageDelimiter,
+        )
+    }
+
+    private fun findChapterPassword(document: Document): String {
+        val regxPasswordUrl = """\/pages\/%5Btype%5D\/%5Bidmanga%5D\/%5Biddetail%5D-.+\.js""".toRegex()
+        val regxFindPassword = """AES\.decrypt\(\w+,"(?<password>[^"]+)"\)""".toRegex(RegexOption.MULTILINE)
+        val jsDecryptUrl = document.select("script")
+            .map { it.absUrl("src") }
+            .first { regxPasswordUrl.find(it) != null }
+        val jsDecrypt = client.newCall(GET(jsDecryptUrl, headers)).execute().asJsoup().html()
+        return regxFindPassword.find(jsDecrypt)?.groups?.get("password")!!.value.trim()
+    }
+
+    override fun popularMangaParse(response: Response): MangasPage {
+        val dto = response.parseNextData<PopularMangaProps>()
+        val mangas = dto.data.mangas.map { it.details }.map { mangaParse(it, dto.query) }
+        return MangasPage(
+            mangas = mangas,
+            hasNextPage = false,
+        )
+    }
+
+    override fun popularMangaRequest(page: Int) = GET("$baseUrl/${langOption.infix}")
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        val maxResult = 6
+        val url = "$apiUrl/api/$langApiInfix/searchforms/$maxResult/".toHttpUrl().newBuilder()
+            .addPathSegment(query)
+            .addPathSegment("${page - 1}")
+            .build()
+        return GET(url, apiHeaders(url.toString()))
+    }
+
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        if (query.startsWith(slugPrefix)) {
+            val mangaUrl = query.substringAfter(slugPrefix)
+            return client.newCall(GET("$baseUrl/${langOption.infix}/$mangaUrl", headers))
+                .asObservableSuccess().map { response ->
+                    val manga = mangaDetailsParse(response).apply {
+                        url = mangaUrl
+                    }
+                    MangasPage(listOf(manga), false)
+                }
+        }
+        return super.fetchSearchManga(page, query, filters)
+    }
+
+    override fun imageUrlParse(response: Response): String = ""
+
+    override fun searchMangaParse(response: Response): MangasPage {
+        val mangasDto = response.parseAs<MangaListDto>().apply {
+            currentPage = response.request.url.pathSegments.last()
+        }
+
+        return MangasPage(
+            mangas = mangasDto.toSManga(langOption.infix),
+            hasNextPage = mangasDto.hasNextPage(),
+        )
+    }
+
+    private inline fun <reified T> Response.parseNextData() = asJsoup().parseNextData<T>()
+
+    private inline fun <reified T> Document.parseNextData(): NextData<T> {
+        val jsonContent = selectFirst("script#__NEXT_DATA__")!!.html()
+        return json.decodeFromString<NextData<T>>(jsonContent)
+    }
+
+    private inline fun <reified T> Response.parseAs(): T {
+        return json.decodeFromString(body.string())
+    }
+
+    private fun String.removeProtocol() = trim().replace("https://", "")
+
+    private fun SManga.slug() = this.url.split("/").last()
+
+    private fun String.toUrlWithoutDomain() = trim().replace(apiUrl, "")
+
+    private fun mangaParse(dto: MangaDto, query: QueryDto): SManga {
+        return SManga.create().apply {
+            title = dto.title
+            thumbnail_url = dto.thumbnailUrl
+            status = dto.status
+            url = mangaUrlParse(dto.slug, query.type)
+            genre = dto.genres
+        }
+    }
+
+    private fun mangaUrlParse(slug: String, pathSegment: String) = "/$pathSegment/$slug"
+
+    companion object {
+        val apiUrl = "https://api.unionmanga.xyz"
+        val apiSeed = "8e0550790c94d6abc71d738959a88d209690dc86"
+        val domain = "yaoi-chan.xyz"
+        val slugPrefix = "slug:"
+        val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH)
+        val apiDateFormat = SimpleDateFormat("EE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ENGLISH)
+            .apply { timeZone = TimeZone.getTimeZone("GMT") }
+    }
+}

src/all/unionmangas/src/eu/kanade/tachiyomi/extension/all/unionmangas/UnionMangasDto.kt

@@ -0,0 +1,149 @@
+package eu.kanade.tachiyomi.extension.all.unionmangas
+
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+class NextData<T>(val props: Props<T>, val query: QueryDto) {
+    val data get() = props.pageProps
+}
+
+@Serializable
+class Props<T>(val pageProps: T)
+
+@Serializable
+class PopularMangaProps(@SerialName("data_popular") val mangas: List<PopularMangaDto>)
+
+@Serializable
+class LatestUpdateProps(@SerialName("data_lastuppdate") val latestUpdateDto: MangaListDto)
+
+@Serializable
+class MangaDetailsProps(@SerialName("dataManga") val mangaDetailsDto: MangaDetailsDto)
+
+@Serializable
+class ChaptersProps(@SerialName("data") val pageListData: String)
+
+@Serializable
+abstract class Pageable {
+    abstract var currentPage: String?
+    abstract var totalPage: Int
+
+    fun hasNextPage() =
+        try { (currentPage!!.toInt() + 1) < totalPage } catch (_: Exception) { false }
+}
+
+@Serializable
+class ChapterPageDto(
+    val totalRecode: Int = 0,
+    override var currentPage: String?,
+    override var totalPage: Int,
+    @SerialName("data") val chapters: List<ChapterDto> = emptyList(),
+) : Pageable() {
+    fun toSChapter(langOption: LanguageOption): List<SChapter> =
+        chapters.map { chapter ->
+            SChapter.create().apply {
+                name = chapter.name
+                date_upload = chapter.date.toDate()
+                url = "/${langOption.infix}${chapter.toChapterUrl(langOption.chpPrefix)}"
+            }
+        }
+
+    private fun String.toDate(): Long =
+        try { UnionMangas.dateFormat.parse(trim())!!.time } catch (_: Exception) { 0L }
+
+    private fun ChapterDto.toChapterUrl(prefix: String) = "/${this.slugManga}/$prefix-${this.id}"
+}
+
+@Serializable
+class ChapterDto(
+    val date: String,
+    val slug: String,
+    @SerialName("idDoc") val slugManga: String,
+    @SerialName("idDetail") val id: String,
+    @SerialName("nameChapter") val name: String,
+)
+
+@Serializable
+class QueryDto(
+    val type: String,
+)
+
+@Serializable
+class MangaListDto(
+    override var currentPage: String?,
+    override var totalPage: Int,
+    @SerialName("data") val mangas: List<MangaDto>,
+) : Pageable() {
+    fun toSManga(siteLang: String) = mangas.map { dto ->
+        SManga.create().apply {
+            title = dto.title
+            thumbnail_url = dto.thumbnailUrl
+            status = dto.status
+            url = mangaUrlParse(dto.slug, siteLang)
+            genre = dto.genres
+        }
+    }
+}
+
+@Serializable
+class PopularMangaDto(
+    @SerialName("document") val details: MangaDto,
+)
+
+@Serializable
+class MangaDto(
+    @SerialName("name") val title: String,
+    @SerialName("image") private val _thumbnailUrl: String,
+    @SerialName("idDoc") val slug: String,
+    @SerialName("genres") private val _genres: String,
+    @SerialName("status") val _status: String,
+) {
+    val thumbnailUrl get() = "${UnionMangas.apiUrl}$_thumbnailUrl"
+    val genres get() = _genres.split(",").joinToString { it.trim() }
+    val status get() = toSMangaStatus(_status)
+}
+
+@Serializable
+class MangaDetailsDto(
+    @SerialName("name") val title: String,
+    @SerialName("image") private val _thumbnailUrl: String,
+    @SerialName("idDoc") val slug: String,
+    @SerialName("lsgenres") private val _genres: List<Prop>,
+    @SerialName("lsstatus") private val _status: List<Prop>,
+) {
+
+    val thumbnailUrl get() = "${UnionMangas.apiUrl}$_thumbnailUrl"
+    val genres get() = _genres.joinToString { it.name }
+    val status get() = toSMangaStatus(_status.first().name)
+
+    @Serializable
+    class Prop(
+        val name: String,
+    )
+}
+
+@Serializable
+class ChaptersDto(
+    @SerialName("dataManga") val data: PageDto,
+    private var delimiter: String = "",
+) {
+    val images get() = data.getImages(delimiter)
+}
+
+@Serializable
+class PageDto(
+    @SerialName("source") private val imgData: String,
+) {
+    fun getImages(delimiter: String): List<String> = imgData.split(delimiter)
+}
+
+private fun mangaUrlParse(slug: String, pathSegment: String) = "/$pathSegment/$slug"
+
+private fun toSMangaStatus(status: String) =
+    when (status.lowercase()) {
+        "ongoing" -> SManga.ONGOING
+        "completed" -> SManga.COMPLETED
+        else -> SManga.UNKNOWN
+    }

src/all/unionmangas/src/eu/kanade/tachiyomi/extension/all/unionmangas/UnionMangasFactory.kt

@@ -0,0 +1,15 @@
+package eu.kanade.tachiyomi.extension.all.unionmangas
+
+import eu.kanade.tachiyomi.source.Source
+import eu.kanade.tachiyomi.source.SourceFactory
+
+class UnionMangasFactory : SourceFactory {
+    override fun createSources(): List<Source> = languages.map { UnionMangas(it) }
+}
+
+class LanguageOption(val lang: String, val infix: String = lang, val chpPrefix: String, val pageDelimiter: String)
+
+val languages = listOf(
+    LanguageOption("it", "italy", "leer", ","),
+    LanguageOption("pt-BR", "manga-br", "cap", "#"),
+)

src/all/unionmangas/src/eu/kanade/tachiyomi/extension/all/unionmangas/UnionMangasUrlActivity.kt

@@ -0,0 +1,36 @@
+package eu.kanade.tachiyomi.extension.all.unionmangas
+
+import android.app.Activity
+import android.content.ActivityNotFoundException
+import android.content.Intent
+import android.os.Bundle
+import android.util.Log
+import kotlin.system.exitProcess
+
+class UnionMangasUrlActivity : Activity() {
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        val host = intent?.data?.host
+        val pathSegments = intent?.data?.pathSegments
+
+        if (host != null && pathSegments != null) {
+            val intent = Intent().apply {
+                action = "eu.kanade.tachiyomi.SEARCH"
+                putExtra("query", slug(pathSegments))
+                putExtra("filter", packageName)
+            }
+
+            try {
+                startActivity(intent)
+            } catch (e: ActivityNotFoundException) {
+                Log.e("UnionMangasUrlActivity", e.toString())
+            }
+        }
+
+        finish()
+        exitProcess(0)
+    }
+
+    private fun slug(pathSegments: List<String>) = "${UnionMangas.slugPrefix}${pathSegments.last()}"
+}