Refactor SS extension due to website redesign (#7655)

* Refactor SS extension due to website redesign.

* Remove compatibility with old URLs since id changed.

src/pt/saikaiscan/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'Saikai Scan'
     pkgNameSuffix = 'pt.saikaiscan'
     extClass = '.SaikaiScan'
-    extVersionCode = 5
+    extVersionCode = 6
     libVersion = '1.2'
 }
 

src/pt/saikaiscan/src/eu/kanade/tachiyomi/extension/pt/saikaiscan/SaikaiScan.kt

@@ -2,25 +2,30 @@ package eu.kanade.tachiyomi.extension.pt.saikaiscan
 
 import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
-import eu.kanade.tachiyomi.source.online.ParsedHttpSource
+import eu.kanade.tachiyomi.source.online.HttpSource
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
 import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
-import org.jsoup.nodes.Document
+import org.jsoup.Jsoup
-import org.jsoup.nodes.Element
+import rx.Observable
+import uy.kohesive.injekt.injectLazy
+import java.text.ParseException
+import java.text.SimpleDateFormat
+import java.util.Locale
 import java.util.concurrent.TimeUnit
 
-class SaikaiScan : ParsedHttpSource() {
+class SaikaiScan : HttpSource() {
-
-    // Hardcode the id because the language wasn't specific.
-    override val id: Long = 2686610366990303664
 
     override val name = "Saikai Scan"
 
@@ -30,121 +35,377 @@ class SaikaiScan : ParsedHttpSource() {
 
     override val supportsLatest = true
 
-    override val client: OkHttpClient = network.client.newBuilder()
+    override val client: OkHttpClient = network.cloudflareClient.newBuilder()
         .addInterceptor(RateLimitInterceptor(1, 1, TimeUnit.SECONDS))
         .build()
 
+    private val json: Json by injectLazy()
+
     override fun headersBuilder(): Headers.Builder = Headers.Builder()
-        .add("User-Agent", USER_AGENT)
         .add("Origin", baseUrl)
-        .add("Referer", baseUrl)
+        .add("Referer", "$baseUrl/")
 
-    override fun popularMangaRequest(page: Int): Request = GET(baseUrl, headers)
+    override fun popularMangaRequest(page: Int): Request {
+        val apiHeaders = headersBuilder()
+            .add("Accept", ACCEPT_JSON)
+            .build()
 
-    override fun popularMangaSelector(): String = "div#menu ul li.has_submenu:eq(3) li a"
+        val apiEndpointUrl = "$API_URL/api/stories".toHttpUrl().newBuilder()
+            .addQueryParameter("format", COMIC_FORMAT_ID)
+            .addQueryParameter("sortProperty", "pageviews")
+            .addQueryParameter("sortDirection", "desc")
+            .addQueryParameter("page", page.toString())
+            .addQueryParameter("per_page", PER_PAGE)
+            .addQueryParameter("relationships", "language,type,format")
+            .toString()
 
-    override fun popularMangaFromElement(element: Element): SManga = SManga.create().apply {
+        return GET(apiEndpointUrl, apiHeaders)
-        title = element.text().substringBeforeLast("(")
-        url = element.attr("href")
     }
 
-    override fun popularMangaNextPageSelector(): String? = null
+    override fun popularMangaParse(response: Response): MangasPage {
+        val result = json.decodeFromString<SaikaiScanPaginatedStoriesDto>(response.body!!.string())
 
-    override fun latestUpdatesRequest(page: Int): Request = GET(baseUrl, headers)
+        val mangaList = result.data!!.map(::popularMangaFromObject)
+        val hasNextPage = result.meta!!.currentPage < result.meta.lastPage
 
-    override fun latestUpdatesSelector(): String = "ul.manhuas li.manhua-item"
+        return MangasPage(mangaList, hasNextPage)
-
-    override fun latestUpdatesFromElement(element: Element): SManga = SManga.create().apply {
-        val image = element.select("div.image.lazyload")
-        val name = element.select("h3")
-
-        title = name.text().substringBeforeLast("(")
-        thumbnail_url = baseUrl + image.attr("data-src")
-        url = image.select("a").attr("href")
     }
 
-    override fun latestUpdatesNextPageSelector(): String? = null
+    private fun popularMangaFromObject(obj: SaikaiScanStoryDto): SManga = SManga.create().apply {
+        title = obj.title
+        thumbnail_url = "$IMAGE_SERVER_URL/${obj.image}"
+        url = "/comics/${obj.slug}"
+    }
+
+    override fun latestUpdatesRequest(page: Int): Request {
+        val apiHeaders = headersBuilder()
+            .add("Accept", ACCEPT_JSON)
+            .build()
+
+        val apiEndpointUrl = "$API_URL/api/lancamentos".toHttpUrl().newBuilder()
+            .addQueryParameter("format", COMIC_FORMAT_ID)
+            .addQueryParameter("page", page.toString())
+            .addQueryParameter("per_page", PER_PAGE)
+            .addQueryParameter("relationships", "language,type,format,latestReleases.separator")
+            .toString()
+
+        return GET(apiEndpointUrl, apiHeaders)
+    }
+
+    override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response)
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        val url = "$baseUrl/busca".toHttpUrlOrNull()!!.newBuilder()
+        val apiHeaders = headersBuilder()
+            .add("Accept", ACCEPT_JSON)
+            .build()
+
+        val apiEndpointUrl = "$API_URL/api/stories".toHttpUrl().newBuilder()
+            .addQueryParameter("format", COMIC_FORMAT_ID)
             .addQueryParameter("q", query)
+            .addQueryParameter("sortProperty", "pageViews")
+            .addQueryParameter("sortDirection", "desc")
+            .addQueryParameter("page", page.toString())
+            .addQueryParameter("per_page", PER_PAGE)
+            .addQueryParameter("relationships", "language,type,format")
 
-        return GET(url.toString(), headers)
+        filters.forEach { filter ->
-    }
+            when (filter) {
+                is GenreFilter -> {
+                    val genresParameter = filter.state
+                        .filter { it.state }
+                        .joinToString(",") { it.id.toString() }
+                    apiEndpointUrl.addQueryParameter("genres", genresParameter)
+                }
 
-    override fun searchMangaParse(response: Response): MangasPage {
+                is CountryFilter -> {
-        val results = super.searchMangaParse(response)
+                    if (filter.state > 0) {
-        val manhuas = results.mangas.filter { it.url.contains("/manhuas/") }
+                        apiEndpointUrl.addQueryParameter("country", filter.selected.id.toString())
+                    }
+                }
 
-        return MangasPage(manhuas, results.hasNextPage)
+                is StatusFilter -> {
-    }
+                    if (filter.state > 0) {
+                        apiEndpointUrl.addQueryParameter("status", filter.selected.id.toString())
+                    }
+                }
 
-    override fun searchMangaSelector(): String = "div#news-content ul li"
+                is SortByFilter -> {
-
+                    val sortProperty = filter.sortProperties[filter.state!!.index]
-    override fun searchMangaFromElement(element: Element): SManga = SManga.create().apply {
+                    val sortDirection = if (filter.state!!.ascending) "asc" else "desc"
-        val image = element.select("div.image.lazyload")
+                    apiEndpointUrl.setQueryParameter("sortProperty", sortProperty.slug)
-        val name = element.select("h3")
+                    apiEndpointUrl.setQueryParameter("sortDirection", sortDirection)
-
+                }
-        title = name.text().substringBeforeLast("(")
+            }
-        thumbnail_url = baseUrl + image.attr("data-src")
-        url = image.select("a").attr("href")
-    }
-
-    override fun searchMangaNextPageSelector(): String? = null
-
-    override fun mangaDetailsParse(document: Document): SManga {
-        val projectContent = document.select("div#project-content")
-        val name = projectContent.select("h2").first()
-        val cover = projectContent.select("div.cover img.lazyload")
-        val genres = projectContent.select("div.info:contains(Gênero:)")
-        val author = projectContent.select("div.info:contains(Autor:)")
-        val status = projectContent.select("div.info:contains(Status:)")
-        val summary = projectContent.select("div.summary-text")
-
-        return SManga.create().apply {
-            title = name.text()
-            thumbnail_url = baseUrl + cover.attr("data-src")
-            genre = removeLabel(genres.text())
-            this.author = removeLabel(author.text())
-            artist = removeLabel(author.text())
-            this.status = parseStatus(removeLabel(status.text()))
-            description = summary.text()
         }
+
+        return GET(apiEndpointUrl.toString(), apiHeaders)
     }
 
-    private fun parseStatus(status: String) = when {
+    override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response)
-        status.contains("Completo") -> SManga.COMPLETED
+
-        status.contains("Em Tradução", true) -> SManga.ONGOING
+    // Workaround to allow "Open in browser" use the real URL.
-        else -> SManga.UNKNOWN
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        return client.newCall(storyDetailsRequest(manga))
+            .asObservableSuccess()
+            .map { response ->
+                mangaDetailsParse(response).apply { initialized = true }
+            }
+    }
+
+    private fun storyDetailsRequest(manga: SManga): Request {
+        val storySlug = manga.url.substringAfterLast("/")
+
+        val apiHeaders = headersBuilder()
+            .add("Accept", ACCEPT_JSON)
+            .build()
+
+        val apiEndpointUrl = "$API_URL/api/stories".toHttpUrl().newBuilder()
+            .addQueryParameter("format", COMIC_FORMAT_ID)
+            .addQueryParameter("slug", storySlug)
+            .addQueryParameter("per_page", "1")
+            .addQueryParameter("relationships", "language,type,format,artists,status")
+            .toString()
+
+        return GET(apiEndpointUrl, apiHeaders)
+    }
+
+    override fun mangaDetailsParse(response: Response): SManga = SManga.create().apply {
+        val result = json.decodeFromString<SaikaiScanPaginatedStoriesDto>(response.body!!.string())
+        val story = result.data!![0]
+
+        title = story.title
+        author = story.authors.joinToString { it.name }
+        artist = story.artists.joinToString { it.name }
+        thumbnail_url = "$IMAGE_SERVER_URL/${story.image}"
+        genre = story.genres.joinToString { it.name }
+        status = story.status!!.name.toStatus()
+        description = Jsoup.parse(story.synopsis)
+            .select("p")
+            .joinToString("\n\n") { it.text() }
+    }
+
+    override fun chapterListRequest(manga: SManga): Request {
+        val storySlug = manga.url.substringAfterLast("/")
+
+        val apiHeaders = headersBuilder()
+            .add("Accept", ACCEPT_JSON)
+            .build()
+
+        val apiEndpointUrl = "$API_URL/api/stories".toHttpUrl().newBuilder()
+            .addQueryParameter("format", COMIC_FORMAT_ID)
+            .addQueryParameter("slug", storySlug)
+            .addQueryParameter("per_page", "1")
+            .addQueryParameter("relationships", "releases")
+            .toString()
+
+        return GET(apiEndpointUrl, apiHeaders)
     }
 
     override fun chapterListParse(response: Response): List<SChapter> {
-        return super.chapterListParse(response).reversed()
+        val result = json.decodeFromString<SaikaiScanPaginatedStoriesDto>(response.body!!.string())
+        val story = result.data!![0]
+
+        return story.releases
+            .filter { it.isActive == 1 }
+            .map { chapterFromObject(it, story.slug) }
+            .sortedByDescending { it.chapter_number }
     }
 
-    override fun chapterListSelector(): String = "div#project-content div.project-chapters div.chapters ul li a"
+    private fun chapterFromObject(obj: SaikaiScanReleaseDto, storySlug: String): SChapter =
+        SChapter.create().apply {
+            name = "Capítulo ${obj.chapter}" +
+                (if (obj.title.isNullOrEmpty().not()) " - ${obj.title}" else "")
+            chapter_number = obj.chapter.toFloatOrNull() ?: -1f
+            date_upload = obj.publishedAt.substringBefore(" ").toDate()
+            scanlator = this@SaikaiScan.name
+            url = "/ler/comics/$storySlug/${obj.id}/${obj.slug}"
+        }
 
-    override fun chapterFromElement(element: Element): SChapter = SChapter.create().apply {
+    override fun pageListRequest(chapter: SChapter): Request {
-        scanlator = "Saikai Scan"
+        val releaseId = chapter.url
-        chapter_number = CHAPTER_REGEX.find(element.text())?.groupValues?.get(1)?.toFloatOrNull() ?: -1f
+            .substringBeforeLast("/")
-        name = element.text()
+            .substringAfterLast("/")
-        url = element.attr("href")
+
+        val apiHeaders = headersBuilder()
+            .add("Accept", ACCEPT_JSON)
+            .build()
+
+        val apiEndpointUrl = "$API_URL/api/releases/$releaseId".toHttpUrl().newBuilder()
+            .addQueryParameter("relationships", "releaseImages")
+            .toString()
+
+        return GET(apiEndpointUrl, apiHeaders)
     }
 
-    override fun pageListParse(document: Document): List<Page> {
+    override fun pageListParse(response: Response): List<Page> {
-        val imagesBlock = document.select("div.manhua-slide div.images-block img.lazyload")
+        val result = json.decodeFromString<SaikaiScanReleaseResultDto>(response.body!!.string())
 
-        return imagesBlock
+        return result.data!!.releaseImages.mapIndexed { i, obj ->
-            .mapIndexed { i, el -> Page(i, "", el.absUrl("src")) }
+            Page(i, "", "$IMAGE_SERVER_URL/${obj.image}")
+        }
     }
 
-    override fun imageUrlParse(document: Document): String = ""
+    override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl!!)
 
-    private fun removeLabel(info: String) = info.substringAfter(":")
+    override fun imageUrlParse(response: Response): String = ""
+
+    override fun imageRequest(page: Page): Request {
+        val imageHeaders = headersBuilder()
+            .add("Accept", ACCEPT_IMAGE)
+            .build()
+
+        return GET(page.imageUrl!!, imageHeaders)
+    }
+
+    private class Genre(title: String, val id: Int) : Filter.CheckBox(title)
+
+    private class GenreFilter(genres: List<Genre>) : Filter.Group<Genre>("Gêneros", genres)
+
+    private data class Country(val name: String, val id: Int) {
+        override fun toString(): String = name
+    }
+
+    private open class EnhancedSelect<T>(name: String, values: Array<T>) : Filter.Select<T>(name, values) {
+        val selected: T
+            get() = values[state]
+    }
+
+    private class CountryFilter(countries: List<Country>) : EnhancedSelect<Country>(
+        "Nacionalidade",
+        countries.toTypedArray()
+    )
+
+    private data class Status(val name: String, val id: Int) {
+        override fun toString(): String = name
+    }
+
+    private class StatusFilter(statuses: List<Status>) : EnhancedSelect<Status>(
+        "Status",
+        statuses.toTypedArray()
+    )
+
+    private data class SortProperty(val name: String, val slug: String) {
+        override fun toString(): String = name
+    }
+
+    private class SortByFilter(val sortProperties: List<SortProperty>) : Filter.Sort(
+        "Ordenar por",
+        sortProperties.map { it.name }.toTypedArray(),
+        Selection(2, ascending = false)
+    )
+
+    // fetch('https://api.saikai.com.br/api/genres')
+    //     .then(res => res.json())
+    //     .then(res => console.log(res.data.map(g => `Genre("${g.name}", ${g.id})`).join(',\n')))
+    private fun getGenreList(): List<Genre> = listOf(
+        Genre("Ação", 1),
+        Genre("Adulto", 23),
+        Genre("Artes Marciais", 84),
+        Genre("Aventura", 2),
+        Genre("Comédia", 15),
+        Genre("Drama", 14),
+        Genre("Ecchi", 19),
+        Genre("Esportes", 42),
+        Genre("eSports", 25),
+        Genre("Fantasia", 3),
+        Genre("Ficção Cientifica", 16),
+        Genre("Histórico", 37),
+        Genre("Horror", 27),
+        Genre("Isekai", 52),
+        Genre("Josei", 40),
+        Genre("Luta", 68),
+        Genre("Magia", 11),
+        Genre("Militar", 76),
+        Genre("Mistério", 57),
+        Genre("MMORPG", 80),
+        Genre("Música", 82),
+        Genre("One-shot", 51),
+        Genre("Psicológico", 34),
+        Genre("Realidade Vitual", 18),
+        Genre("Reencarnação", 43),
+        Genre("Romance", 9),
+        Genre("RPG", 61),
+        Genre("Sci-fi", 58),
+        Genre("Seinen", 21),
+        Genre("Shoujo", 35),
+        Genre("Shounen", 26),
+        Genre("Slice of Life", 38),
+        Genre("Sobrenatural", 74),
+        Genre("Suspense", 63),
+        Genre("Tragédia", 22),
+        Genre("VRMMO", 17),
+        Genre("Wuxia", 6),
+        Genre("Xianxia", 7),
+        Genre("Xuanhuan", 48),
+        Genre("Yaoi", 41),
+        Genre("Yuri", 83)
+    )
+
+    // fetch('https://api.saikai.com.br/api/countries?hasStories=1')
+    //     .then(res => res.json())
+    //     .then(res => console.log(res.data.map(g => `Country("${g.name}", ${g.id})`).join(',\n')))
+    private fun getCountryList(): List<Country> = listOf(
+        Country("Todas", 0),
+        Country("Brasil", 32),
+        Country("China", 45),
+        Country("Coréia do Sul", 115),
+        Country("Espanha", 199),
+        Country("Estados Unidos da América", 1),
+        Country("Japão", 109),
+        Country("Portugal", 173)
+    )
+
+    // fetch('https://api.saikai.com.br/api/countries?hasStories=1')
+    //     .then(res => res.json())
+    //     .then(res => console.log(res.data.map(g => `Country("${g.name}", ${g.id})`).join(',\n')))
+    private fun getStatusList(): List<Status> = listOf(
+        Status("Todos", 0),
+        Status("Cancelado", 5),
+        Status("Concluído", 1),
+        Status("Dropado", 6),
+        Status("Em Andamento", 2),
+        Status("Hiato", 4),
+        Status("Pausado", 3)
+    )
+
+    private fun getSortProperties(): List<SortProperty> = listOf(
+        SortProperty("Título", "title"),
+        SortProperty("Quantidade de capítulos", "releases_count"),
+        SortProperty("Visualizações", "pageviews"),
+        SortProperty("Data de criação", "created_at")
+    )
+
+    override fun getFilterList(): FilterList = FilterList(
+        CountryFilter(getCountryList()),
+        StatusFilter(getStatusList()),
+        SortByFilter(getSortProperties()),
+        GenreFilter(getGenreList())
+    )
+
+    private fun String.toDate(): Long {
+        return try {
+            DATE_FORMATTER.parse(this)?.time ?: 0L
+        } catch (e: ParseException) {
+            0L
+        }
+    }
+
+    private fun String.toStatus(): Int = when (this) {
+        "Concluído" -> SManga.COMPLETED
+        "Em Andamento" -> SManga.ONGOING
+        else -> SManga.UNKNOWN
+    }
 
     companion object {
-        private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+        private const val ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
-            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36"
+        private const val ACCEPT_JSON = "application/json, text/plain, */*"
-        private val CHAPTER_REGEX = "Capítulo (\\d+)".toRegex()
+
+        private const val COMIC_FORMAT_ID = "2"
+        private const val PER_PAGE = "12"
+
+        private val DATE_FORMATTER = SimpleDateFormat("yyyy-MM-dd", Locale("pt", "BR"))
+
+        private const val API_URL = "https://api.saikai.com.br"
+        private const val IMAGE_SERVER_URL = "https://s3-alpha.saikai.com.br"
     }
 }

src/pt/saikaiscan/src/eu/kanade/tachiyomi/extension/pt/saikaiscan/SaikaiScanDto.kt

@@ -0,0 +1,63 @@
+package eu.kanade.tachiyomi.extension.pt.saikaiscan
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class SaikaiScanResultDto<T>(
+    val data: T? = null,
+    val meta: SaikaiScanMetaDto? = null
+)
+
+typealias SaikaiScanPaginatedStoriesDto = SaikaiScanResultDto<List<SaikaiScanStoryDto>>
+typealias SaikaiScanReleaseResultDto = SaikaiScanResultDto<SaikaiScanReleaseDto>
+
+@Serializable
+data class SaikaiScanMetaDto(
+    @SerialName("current_page") val currentPage: Int,
+    @SerialName("last_page") val lastPage: Int
+)
+
+@Serializable
+data class SaikaiScanStoryDto(
+    val artists: List<SaikaiScanPersonDto> = emptyList(),
+    val authors: List<SaikaiScanPersonDto> = emptyList(),
+    val genres: List<SaikaiScanGenreDto> = emptyList(),
+    val image: String,
+    val releases: List<SaikaiScanReleaseDto> = emptyList(),
+    val slug: String,
+    val status: SaikaiScanStatusDto? = null,
+    val synopsis: String,
+    val title: String
+)
+
+@Serializable
+data class SaikaiScanPersonDto(
+    val name: String
+)
+
+@Serializable
+data class SaikaiScanGenreDto(
+    val name: String
+)
+
+@Serializable
+data class SaikaiScanStatusDto(
+    val name: String
+)
+
+@Serializable
+data class SaikaiScanReleaseDto(
+    val chapter: String,
+    val id: Int,
+    @SerialName("is_active") val isActive: Int = 1,
+    @SerialName("published_at") val publishedAt: String,
+    @SerialName("release_images") val releaseImages: List<SaikaiScanReleaseImageDto> = emptyList(),
+    val slug: String,
+    val title: String? = ""
+)
+
+@Serializable
+data class SaikaiScanReleaseImageDto(
+    val image: String
+)