Pururin: use the API (#9843)

src/en/pururin/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'Pururin'
     pkgNameSuffix = 'en.pururin'
     extClass = '.Pururin'
-    extVersionCode = 4
+    extVersionCode = 5
     isNsfw = true
 }
 

src/en/pururin/src/eu/kanade/tachiyomi/extension/en/pururin/Pururin.kt

@@ -1,22 +1,26 @@
 package eu.kanade.tachiyomi.extension.en.pururin
 
-import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.network.asObservable
+import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
-import eu.kanade.tachiyomi.source.online.ParsedHttpSource
+import eu.kanade.tachiyomi.source.online.HttpSource
-import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.ExperimentalSerializationApi
-import okhttp3.OkHttpClient
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.decodeFromJsonElement
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
 import okhttp3.Request
 import okhttp3.Response
-import org.jsoup.nodes.Document
+import uy.kohesive.injekt.injectLazy
-import org.jsoup.nodes.Element
-
-class Pururin : ParsedHttpSource() {
 
+@ExperimentalSerializationApi
+class Pururin : HttpSource() {
     override val name = "Pururin"
 
     override val baseUrl = "https://pururin.to"
@@ -25,195 +29,137 @@ class Pururin : ParsedHttpSource() {
 
     override val supportsLatest = true
 
-    override val client: OkHttpClient = network.cloudflareClient
+    override val client = network.cloudflareClient
 
-    override fun latestUpdatesSelector() = "div.container div.row-gallery a"
+    private val searchUrl = "$baseUrl/api/search/advance"
 
-    override fun latestUpdatesRequest(page: Int): Request {
+    private val galleryUrl = "$baseUrl/api/contribute/gallery/info"
-        return if (page == 1) {
+
-            GET(baseUrl, headers)
+    private val json by injectLazy<Json>()
-        } else {
+
-            GET("$baseUrl/browse/newest?page=$page", headers)
+    override fun headersBuilder() = super.headersBuilder()
+        .set("Origin", baseUrl).set("X-Requested-With", "XMLHttpRequest")
+
+    override fun latestUpdatesRequest(page: Int) =
+        POST(searchUrl, headers, Search(Search.Sort.NEWEST, page))
+
+    override fun latestUpdatesParse(response: Response) =
+        searchMangaParse(response)
+
+    override fun popularMangaRequest(page: Int) =
+        POST(searchUrl, headers, Search(Search.Sort.POPULAR, page))
+
+    override fun popularMangaParse(response: Response) =
+        searchMangaParse(response)
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) =
+        filters.ifEmpty(::getFilterList).run {
+            val whitelist = mutableListOf<Int>()
+            val blacklist = mutableListOf<Int>()
+            filterIsInstance<TagGroup<*>>().forEach { group ->
+                group.state.forEach {
+                    when {
+                        it.isIncluded() -> whitelist += it.id
+                        it.isExcluded() -> blacklist += it.id
                     }
                 }
-
-    override fun latestUpdatesFromElement(element: Element): SManga {
-        val manga = SManga.create()
-
-        manga.setUrlWithoutDomain(element.attr("href"))
-        manga.title = element.select("div.title").text()
-        manga.thumbnail_url = element.select("img.card-img-top").attr("abs:data-src")
-
-        return manga
             }
-
+            val body = Search(
-    override fun latestUpdatesNextPageSelector() = "ul.pagination a.page-link[rel=next]"
+                find<SortFilter>().sort,
-
+                page,
-    override fun mangaDetailsParse(document: Document): SManga {
+                query,
-        val infoElement = document.select("div.box.box-gallery")
+                whitelist,
-        val manga = SManga.create()
+                blacklist,
-        val genres = mutableListOf<String>()
+                find<TagModeFilter>().mode,
-
+                find<PagesGroup>().range
-        document.select("tr:has(td:containsOwn(Contents)) li").forEach { element ->
-            val genre = element.text()
-            genres.add(genre)
-        }
-
-        manga.title = infoElement.select("h1").text()
-        manga.author = infoElement.select("tr:has(td:containsOwn(Artist)) a").text()
-        manga.artist = infoElement.select("tr:has(td:containsOwn(Circle)) a").text()
-        manga.status = SManga.COMPLETED
-        manga.genre = genres.joinToString(", ")
-        manga.thumbnail_url = document.select("div.cover-wrapper v-lazy-image").attr("abs:src")
-
-        manga.description = getDesc(document)
-        manga.initialized = true
-
-        return manga
-    }
-
-    private fun getDesc(document: Document): String {
-        val infoElement = document.select("div.box.box-gallery")
-        val uploader = infoElement.select("tr:has(td:containsOwn(Uploader)) .user-link")?.text()
-        val pages = infoElement.select("tr:has(td:containsOwn(Pages)) td:eq(1)").text()
-        val ratingCount = infoElement.select("tr:has(td:containsOwn(Ratings)) span[itemprop=\"ratingCount\"]")?.attr("content")
-
-        val rating = infoElement.select("tr:has(td:containsOwn(Ratings)) gallery-rating").attr(":rating")?.toFloatOrNull()?.let {
-            if (it > 5.0f) minOf(it, 5.0f) // cap rating to 5.0 for rare cases where value exceeds 5.0f
-            else it
-        }
-
-        val multiDescriptions = listOf(
-            "Convention",
-            "Parody",
-            "Circle",
-            "Category",
-            "Character",
-            "Language"
-        ).map { it to infoElement.select("tr:has(td:containsOwn($it)) a").map { v -> v.text() } }
-            .filter { !it.second.isNullOrEmpty() }
-            .map { "${it.first}: ${it.second.joinToString()}" }
-
-        val descriptions = listOf(
-            multiDescriptions.joinToString("\n\n"),
-            uploader?.let { "Uploader: $it" },
-            pages?.let { "Pages: $it" },
-            rating?.let { "Ratings: $it" + (ratingCount?.let { c -> " ($c ratings)" } ?: "") }
             )
-
+            POST(searchUrl, headers, body)
-        return descriptions.joinToString("\n\n")
-    }
-
-    override fun chapterListParse(response: Response) = with(response.asJsoup()) {
-        val mangaInfoElements = this.select(".table-gallery-info tr td:first-child").map {
-            it.text() to it.nextElementSibling()
-        }.toMap()
-
-        val chapters = this.select(".table-collection tbody tr")
-        if (!chapters.isNullOrEmpty())
-            chapters.map {
-                val details = it.select("td")
-                SChapter.create().apply {
-                    chapter_number = details[0].text().removePrefix("#").toFloat()
-                    name = details[1].select("a").text()
-                    setUrlWithoutDomain(details[1].select("a").attr("href"))
-
-                    if (it.hasClass("active") && mangaInfoElements.containsKey("Scanlator"))
-                        scanlator = mangaInfoElements.getValue("Scanlator").select("li a")?.joinToString { s -> s.text() }
-                }
-            }
-        else
-            listOf(
-                SChapter.create().apply {
-                    name = "Chapter"
-                    setUrlWithoutDomain(response.request.url.toString())
-
-                    if (mangaInfoElements.containsKey("Scanlator"))
-                        scanlator = mangaInfoElements.getValue("Scanlator").select("li a")?.joinToString { s -> s.text() }
-                }
-            )
-    }
-
-    override fun chapterListSelector(): String = throw UnsupportedOperationException("Not used")
-
-    override fun chapterFromElement(element: Element): SChapter = throw UnsupportedOperationException("Not used")
-
-    override fun pageListParse(document: Document): List<Page> {
-        val pages = mutableListOf<Page>()
-
-        // Gets gallery id from meta tags
-        val galleryUrl = document.select("meta[property='og:url']").attr("content")
-        val id = galleryUrl.substringAfter("gallery/").substringBefore('/')
-        // Gets total pages from gallery desc
-        val infoElement = document.select("div.box.box-gallery")
-        val total: Int = infoElement.select("tr:has(td:containsOwn(Pages)) td:eq(1)").text().substringBefore(' ').toInt()
-
-        for (i in 1..total) {
-            pages.add(Page(i, "", "https://cdn.pururin.to/assets/images/data/$id/$i.jpg"))
-        }
-
-        return pages
-    }
-
-    override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not used")
-
-    override fun popularMangaRequest(page: Int): Request = GET("$baseUrl/browse/most-popular?page=$page", headers)
-
-    override fun popularMangaFromElement(element: Element) = latestUpdatesFromElement(element)
-
-    override fun popularMangaSelector() = latestUpdatesSelector()
-
-    override fun popularMangaNextPageSelector() = latestUpdatesNextPageSelector()
-
-    private lateinit var tagUrl: String
-
-    // TODO: Additional filter options, specifically the type[] parameter
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        var url = "$baseUrl/search?q=$query&page=$page"
-
-        if (query.isBlank()) {
-            filters.forEach { filter ->
-                when (filter) {
-                    is Tag -> {
-                        url = if (page == 1) {
-                            "$baseUrl/search/tag?q=${filter.state}&type[]=3" // "Contents" tag
-                        } else {
-                            "$tagUrl?page=$page"
-                        }
-                    }
-                }
-            }
-        }
-
-        return GET(url, headers)
         }
 
     override fun searchMangaParse(response: Response): MangasPage {
-        return if (response.request.url.toString().contains("tag?")) {
+        val results = json.decodeFromString<Results>(
-            response.asJsoup().select("table.table tbody tr a:first-of-type").attr("abs:href").let {
+            response.jsonObject["results"]!!.jsonPrimitive.content
-                if (it.isNotEmpty()) {
+        )
-                    tagUrl = it
+        val mp = results.map {
-                    super.searchMangaParse(client.newCall(GET(tagUrl, headers)).execute())
+            SManga.create().apply {
-                } else {
+                url = it.path
-                    MangasPage(emptyList(), false)
+                title = it.title
+                thumbnail_url = CDN_URL + it.cover
             }
         }
-        } else {
+        return MangasPage(mp, results.hasNext)
-            super.searchMangaParse(response)
+    }
+
+    override fun mangaDetailsParse(response: Response): SManga {
+        val gallery = json.decodeFromJsonElement<Gallery>(
+            response.jsonObject["gallery"]!!
+        )
+        return SManga.create().apply {
+            description = gallery.description
+            artist = gallery.artists.joinToString()
+            author = gallery.authors.joinToString()
+            genre = gallery.genres.joinToString()
         }
     }
 
-    override fun searchMangaSelector() = latestUpdatesSelector()
+    override fun fetchMangaDetails(manga: SManga) =
+        client.newCall(chapterListRequest(manga))
+            .asObservableSuccess().map(::mangaDetailsParse)!!
 
-    override fun searchMangaFromElement(element: Element) = latestUpdatesFromElement(element)
+    override fun chapterListRequest(manga: SManga) =
+        POST(galleryUrl, headers, Search.info(manga.id))
 
-    override fun searchMangaNextPageSelector() = latestUpdatesNextPageSelector()
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val gallery = json.decodeFromJsonElement<Gallery>(
+            response.jsonObject["gallery"]!!
+        )
+        val chapter = SChapter.create().apply {
+            name = "Chapter"
+            url = gallery.id.toString()
+            scanlator = gallery.scanlators.joinToString()
+        }
+        return listOf(chapter)
+    }
+
+    override fun pageListRequest(chapter: SChapter) =
+        POST(galleryUrl, headers, Search.info(chapter.url))
+
+    override fun pageListParse(response: Response): List<Page> {
+        val pages = json.decodeFromJsonElement<Gallery>(
+            response.jsonObject["gallery"]!!
+        ).pages
+        return pages.mapIndexed { idx, img ->
+            Page(idx + 1, CDN_URL + img)
+        }
+    }
+
+    override fun imageUrlParse(response: Response) =
+        throw UnsupportedOperationException("Not used")
+
+    override fun fetchImageUrl(page: Page) =
+        Request.Builder().url(page.url).head().build()
+            .run(client::newCall).asObservable().map {
+                when (it.code) {
+                    200 -> page.url
+                    // try to fix images that are broken even on the site
+                    404 -> page.url.replaceAfterLast('.', "png")
+                    else -> throw Error("HTTP error ${it.code}")
+                }
+            }!!
 
     override fun getFilterList() = FilterList(
-        Filter.Header("NOTE: Ignored if using text search!"),
+        SortFilter(),
-        Filter.Separator(),
+        CategoryGroup(),
-        Tag("Tag")
+        TagModeFilter(),
+        PagesGroup(),
     )
 
-    private class Tag(name: String) : Filter.Text(name)
+    private inline val Response.jsonObject
+        get() = json.parseToJsonElement(body!!.string()).jsonObject
+
+    private inline val SManga.id get() = url.split('/')[2]
+
+    companion object {
+        private const val CDN_URL = "https://cdn.pururin.to/assets/images/data"
+    }
 }

src/en/pururin/src/eu/kanade/tachiyomi/extension/en/pururin/PururinAPI.kt

@@ -0,0 +1,66 @@
+package eu.kanade.tachiyomi.extension.en.pururin
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class Results(
+    private val current_page: Int,
+    private val data: List<Data>,
+    private val last_page: Int
+) : Iterable<Data> by data {
+    val hasNext get() = current_page != last_page
+}
+
+@Serializable
+data class Data(
+    private val id: Int,
+    val title: String,
+    private val slug: String
+) {
+    val path get() = "/gallery/$id/$slug"
+
+    val cover get() = "/$id/cover.jpg"
+}
+
+@Serializable
+data class Gallery(
+    val id: Int,
+    private val j_title: String,
+    private val alt_title: String?,
+    private val total_pages: Int,
+    private val image_extension: String,
+    private val tags: TagList
+) {
+    val description get() = "$j_title\n${alt_title ?: ""}".trim()
+
+    val pages get() = (1..total_pages).map { "/$id/$it.$image_extension" }
+
+    val genres get() = tags.Parody +
+        tags.Contents +
+        tags.Category +
+        tags.Character +
+        tags.Convention
+
+    val artists get() = tags.Artist
+
+    val authors get() = tags.Circle.ifEmpty { tags.Artist }
+
+    val scanlators get() = tags.Scanlator
+}
+
+@Serializable
+data class TagList(
+    val Artist: List<Tag>,
+    val Circle: List<Tag>,
+    val Parody: List<Tag>,
+    val Contents: List<Tag>,
+    val Category: List<Tag>,
+    val Character: List<Tag>,
+    val Scanlator: List<Tag>,
+    val Convention: List<Tag>
+)
+
+@Serializable
+data class Tag(private val name: String) {
+    override fun toString() = name
+}

src/en/pururin/src/eu/kanade/tachiyomi/extension/en/pururin/PururinFilters.kt

@@ -0,0 +1,71 @@
+package eu.kanade.tachiyomi.extension.en.pururin
+
+import eu.kanade.tachiyomi.source.model.Filter
+
+class SortFilter(
+    values: Array<Search.Sort> = Search.Sort.values()
+) : Filter.Select<Search.Sort>("Sort by", values) {
+    inline val sort get() = values[state]
+}
+
+sealed class TagFilter(
+    name: String,
+    val id: Int
+) : Filter.TriState(name)
+
+sealed class TagGroup<T : TagFilter>(
+    name: String,
+    values: List<T>
+) : Filter.Group<T>(name, values)
+
+// TODO: Artist, Circle, Contents, Parody, Character, Convention
+
+class Category(name: String, id: Int) : TagFilter(name, id)
+
+class CategoryGroup(
+    values: List<Category> = categories
+) : TagGroup<Category>("Categories", values) {
+    companion object {
+        private val categories get() = listOf(
+            Category("Doujinshi", 13003),
+            Category("Manga", 13004),
+            Category("Artist CG", 13006),
+            Category("Game CG", 13008),
+            Category("Artbook", 17783),
+            Category("Webtoon", 27939),
+        )
+    }
+}
+
+class TagModeFilter(
+    values: Array<Search.TagMode> = Search.TagMode.values()
+) : Filter.Select<Search.TagMode>("Tag mode", values) {
+    inline val mode get() = values[state]
+}
+
+class PagesFilter(
+    name: String,
+    default: Int,
+    values: Array<Int> = range,
+) : Filter.Select<Int>(name, values, default) {
+    companion object {
+        private val range get() = Array(1001) { it }
+    }
+}
+
+class PagesGroup(
+    values: List<PagesFilter> = minmax
+) : Filter.Group<PagesFilter>("Pages", values) {
+    inline val range get() = IntRange(state[0].state, state[1].state).also {
+        require(it.first <= it.last) { "'Minimum' cannot exceed 'Maximum'" }
+    }
+
+    companion object {
+        private val minmax get() = listOf(
+            PagesFilter("Minimum", 0),
+            PagesFilter("Maximum", 100)
+        )
+    }
+}
+
+inline fun <reified T> List<Filter<*>>.find() = find { it is T } as T

src/en/pururin/src/eu/kanade/tachiyomi/extension/en/pururin/Search.kt

@@ -0,0 +1,75 @@
+package eu.kanade.tachiyomi.extension.en.pururin
+
+import kotlinx.serialization.json.add
+import kotlinx.serialization.json.addJsonObject
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+import kotlinx.serialization.json.putJsonArray
+import kotlinx.serialization.json.putJsonObject
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.RequestBody.Companion.toRequestBody
+
+object Search {
+    private val jsonMime = "application/json".toMediaType()
+
+    enum class Sort(private val label: String, val id: String) {
+        NEWEST("Newest", "newest"),
+        POPULAR("Most Popular", "most-popular"),
+        RATING("Highest Rated", "highest-rated"),
+        VIEWS("Most Viewed", "most-viewed"),
+        TITLE("Title", "title");
+
+        override fun toString() = label
+    }
+
+    enum class TagMode(val id: String) {
+        AND("1"), OR("2");
+
+        override fun toString() = name
+    }
+
+    operator fun invoke(
+        sort: Sort,
+        page: Int = 1,
+        query: String = "",
+        whitelist: List<Int> = emptyList(),
+        blacklist: List<Int> = emptyList(),
+        mode: TagMode = TagMode.AND,
+        range: IntRange = 0..100
+    ) = buildJsonObject {
+        putJsonObject("search") {
+            put("sort", sort.id)
+            put("PageNumber", page)
+            putJsonObject("manga") {
+                put("string", query)
+                put("sort", "1")
+            }
+            putJsonObject("tag") {
+                putJsonObject("items") {
+                    putJsonArray("whitelisted") {
+                        whitelist.forEach {
+                            addJsonObject { put("id", it) }
+                        }
+                    }
+                    putJsonArray("blacklisted") {
+                        blacklist.forEach {
+                            addJsonObject { put("id", it) }
+                        }
+                    }
+                }
+                put("sort", mode.id)
+            }
+            putJsonObject("page") {
+                putJsonArray("range") {
+                    add(range.first)
+                    add(range.last)
+                }
+            }
+        }
+    }.toString().toRequestBody(jsonMime)
+
+    fun info(id: String) = buildJsonObject {
+        put("id", id)
+        put("type", "1")
+    }.toString().toRequestBody(jsonMime)
+}