Ninehentai json migration and other changes (#8552)

* Migration to kotlinx.serialization

* Refactoring and changed search implementation

* Add url intent

* Small fixes

src/all/ninehentai/AndroidManifest.xml

@@ -1,2 +1,23 @@
 <?xml version="1.0" encoding="utf-8"?>
-<manifest package="eu.kanade.tachiyomi.extension" />
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="eu.kanade.tachiyomi.extension">
+
+    <application>
+        <activity
+            android:name=".all.ninehentai.NineHentaiUrlActivity"
+            android:excludeFromRecents="true"
+            android:theme="@android:style/Theme.NoDisplay">
+            <intent-filter>
+                <action android:name="android.intent.action.VIEW" />
+
+                <category android:name="android.intent.category.DEFAULT" />
+                <category android:name="android.intent.category.BROWSABLE" />
+
+                <data
+                    android:host="9hentai.to"
+                    android:pathPattern="/g/..*"
+                    android:scheme="https" />
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>

src/all/ninehentai/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'NineHentai'
     pkgNameSuffix = 'all.ninehentai'
     extClass = '.NineHentai'
-    extVersionCode = 12
+    extVersionCode = 13
     libVersion = '1.2'
     containsNsfw = true
 }

src/all/ninehentai/src/eu/kanade/tachiyomi/extension/all/ninehentai/Dto.kt

@@ -1,31 +0,0 @@
-package eu.kanade.tachiyomi.extension.all.ninehentai
-
-import eu.kanade.tachiyomi.source.model.Filter
-
-data class Manga(
-    val id: Int,
-    var title: String,
-    val image_server: String,
-    val tags: List<Tag>,
-    val total_page: Int
-)
-
-class Tag(
-    val id: Int,
-    name: String,
-    val description: String = "null",
-    val type: Int = 1
-) : Filter.TriState(name)
-
-data class SearchRequest(
-    val text: String,
-    val page: Int,
-    val sort: Int,
-    val pages: Map<String, IntArray> = mapOf("range" to intArrayOf(0, 2000)),
-    val tag: Map<String, Items>
-)
-
-data class Items(
-    val included: MutableList<Tag>,
-    val excluded: MutableList<Tag>
-)

src/all/ninehentai/src/eu/kanade/tachiyomi/extension/all/ninehentai/NineHentai.kt

@@ -1,12 +1,5 @@
 package eu.kanade.tachiyomi.extension.all.ninehentai
 
-import com.github.salomonbrys.kotson.array
-import com.github.salomonbrys.kotson.fromJson
-import com.github.salomonbrys.kotson.int
-import com.github.salomonbrys.kotson.string
-import com.google.gson.Gson
-import com.google.gson.JsonElement
-import com.google.gson.JsonParser
 import eu.kanade.tachiyomi.annotations.Nsfw
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
@@ -19,12 +12,22 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.decodeFromJsonElement
+import kotlinx.serialization.json.encodeToJsonElement
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.put
 import okhttp3.MediaType.Companion.toMediaTypeOrNull
 import okhttp3.OkHttpClient
 import okhttp3.Request
-import okhttp3.RequestBody
+import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
+import org.jsoup.nodes.Element
 import rx.Observable
+import rx.schedulers.Schedulers
+import uy.kohesive.injekt.injectLazy
 import java.util.Calendar
 
 @Nsfw
@@ -40,67 +43,102 @@ class NineHentai : HttpSource() {
 
     override val client: OkHttpClient = network.cloudflareClient
 
-    private val gson = Gson()
+    private val json: Json by injectLazy()
 
-    override fun popularMangaRequest(page: Int): Request {
+    // Builds request for /api/getBooks endpoint
-        return POST(baseUrl + SEARCH_URL, headers, buildRequestBody(page = page, sort = 1))
+    private fun buildSearchRequest(
+        searchText: String = "",
+        page: Int,
+        sort: Int = 0,
+        range: List<Int> = listOf(0, 2000),
+        includedTags: List<Tag> = listOf(),
+        excludedTags: List<Tag> = listOf()
+    ): Request {
+        val request = SearchRequest(
+            text = searchText,
+            page = page - 1,    // Source starts counting from 0, not 1
+            sort = sort,
+            pages = Range(range),
+            tag = Items(
+                items = TagArrays(
+                    included = includedTags,
+                    excluded = excludedTags
+                )
+            )
+        )
+        val jsonString = buildJsonObject {
+            put("search", json.encodeToJsonElement(request))
+        }.toString()
+        return POST("$baseUrl$SEARCH_URL", headers, jsonString.toRequestBody(MEDIA_TYPE))
     }
 
-    override fun latestUpdatesRequest(page: Int): Request {
+    // Builds request for /api/getBookById endpoint
-        return POST(baseUrl + SEARCH_URL, headers, buildRequestBody(page = page))
+    private fun buildDetailRequest(id: Int): Request {
+        val jsonString = buildJsonObject { put("id", id) }.toString()
+        return POST("$baseUrl$MANGA_URL", headers, jsonString.toRequestBody(MEDIA_TYPE))
     }
 
-    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
+    // Popular and Latest
-        return client.newCall(popularMangaRequest(page))
+
-            .asObservableSuccess()
+    override fun popularMangaRequest(page: Int): Request = buildSearchRequest(page = page, sort = 1)
-            .map { response ->
+
-                getMangaList(response, page)
+    override fun popularMangaParse(response: Response): MangasPage {
+        val results = json.parseToJsonElement(response.body!!.string()).jsonObject["results"]!!.jsonArray
+        if (results.isEmpty()) return MangasPage(listOf(), false)
+        return MangasPage(
+            results.map {
+                val manga = json.decodeFromJsonElement<Manga>(it)
+                SManga.create().apply {
+                    setUrlWithoutDomain("/g/${manga.id}")
+                    title = manga.title
+                    thumbnail_url = "${manga.image_server + manga.id}/cover.jpg"
+                }
+            },
+            true
+        )
+    }
+
+    override fun latestUpdatesRequest(page: Int): Request = buildSearchRequest(page = page)
+
+    override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response)
+
+    // Manga Details
+
+    override fun mangaDetailsParse(response: Response): SManga {
+        return SManga.create().apply {
+            response.asJsoup().selectFirst("div#bigcontainer").let { info ->
+                title = info.select("h1").text()
+                thumbnail_url = info.selectFirst("div#cover v-lazy-image").attr("abs:src")
+                status = SManga.COMPLETED
+                artist = info.selectTextOrNull("div.field-name:contains(Artist:) a.tag")
+                author = info.selectTextOrNull("div.field-name:contains(Group:) a.tag") ?: artist
+                genre = info.selectTextOrNull("div.field-name:contains(Tag:) a.tag")
+                // Additional details
+                description = listOf(
+                    Pair("Alternative Title", info.selectTextOrNull("h2")),
+                    Pair("Pages", info.selectTextOrNull("div#info > div:contains(pages)")),
+                    Pair("Parody", info.selectTextOrNull("div.field-name:contains(Parody:) a.tag")),
+                    Pair("Category", info.selectTextOrNull("div.field-name:contains(Category:) a.tag")),
+                    Pair("Language", info.selectTextOrNull("div.field-name:contains(Language:) a.tag")),
+                ).filterNot { it.second.isNullOrEmpty() }.joinToString("\n\n") { "${it.first}: ${it.second}" }
             }
-    }
-
-    override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
-        return client.newCall(latestUpdatesRequest(page))
-            .asObservableSuccess()
-            .map { response ->
-                getMangaList(response, page)
-            }
-    }
-
-    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
-        return client.newCall(searchMangaRequest(page, query, filters))
-            .asObservableSuccess()
-            .map { response ->
-                getMangaList(response, page)
-            }
-    }
-
-    private fun getMangaList(response: Response, page: Int): MangasPage {
-        val jsonData = response.body!!.string()
-        val jsonObject = JsonParser().parse(jsonData).asJsonObject
-        val totalPages = jsonObject["total_count"].int
-        val results = jsonObject["results"].array
-        return MangasPage(parseSearch(results.toList()), page < totalPages)
-    }
-
-    private fun parseSearch(jsonArray: List<JsonElement>): List<SManga> {
-        val mutableList = mutableListOf<SManga>()
-        jsonArray.forEach { json ->
-            val manga = SManga.create()
-            val gsonManga = gson.fromJson<Manga>(json)
-            manga.url = "/g/${gsonManga.id}"
-            manga.title = gsonManga.title
-            manga.thumbnail_url = gsonManga.image_server + gsonManga.id + "/cover.jpg"
-            manga.genre = gsonManga.tags.filter { it.type == 1 }.joinToString { it.name }
-            manga.artist = gsonManga.tags.firstOrNull { it.type == 4 }?.name
-            manga.initialized = true
-            mutableList.add(manga)
         }
-        return mutableList
     }
 
+    // Ensures no exceptions are thrown when scraping additional details
+    private fun Element.selectTextOrNull(selector: String): String? {
+        val list = this.select(selector)
+        return if (list.isEmpty()) {
+            null
+        } else {
+            list.joinToString(", ") { it.text() }
+        }
+    }
+
+    // Chapter
+
     override fun chapterListParse(response: Response): List<SChapter> {
-        val document = response.asJsoup()
+        val time = response.asJsoup().select("div#info div time").text()
-        val time = document.select("div#info div time").text()
         return listOf(
             SChapter.create().apply {
                 name = "Chapter"
@@ -142,58 +180,19 @@ class NineHentai : HttpSource() {
         }
     }
 
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+    // Page List
-        val includedTags = mutableListOf<Tag>()
-        val excludedTags = mutableListOf<Tag>()
-        var sort = 0
-        for (filter in if (filters.isEmpty()) getFilterList() else filters) {
-            when (filter) {
-                is GenreList -> {
-                    filter.state.forEach { f ->
-                        if (!f.isIgnored()) {
-                            if (f.isExcluded())
-                                excludedTags.add(f)
-                            else
-                                includedTags.add(f)
-                        }
-                    }
-                }
-                is Sorting -> {
-                    sort = filter.state!!.index
-                }
-            }
-        }
-        return POST(baseUrl + SEARCH_URL, headers, buildRequestBody(query, page, sort, includedTags, excludedTags))
-    }
-
-    override fun mangaDetailsParse(response: Response): SManga {
-        return SManga.create().apply {
-            response.asJsoup().select("div.card-body").firstOrNull()?.let { info ->
-                title = info.select("h1").text()
-                genre = info.select("div.field-name:contains(Tag:) a.tag").joinToString { it.text() }
-                artist = info.select("div.field-name:contains(Artist:) a.tag").joinToString { it.text() }
-                thumbnail_url = info.select("div#cover v-lazy-image").attr("abs:src")
-            }
-        }
-    }
 
     override fun pageListRequest(chapter: SChapter): Request {
         val mangaId = chapter.url.substringAfter("/g/").toInt()
-        return POST(baseUrl + MANGA_URL, headers, buildIdBody(mangaId))
+        return buildDetailRequest(mangaId)
     }
 
     override fun pageListParse(response: Response): List<Page> {
-        val jsonData = response.body!!.string()
+        val resultsObj = json.parseToJsonElement(response.body!!.string()).jsonObject["results"]!!
-        val jsonObject = JsonParser().parse(jsonData).asJsonObject
+        val manga = json.decodeFromJsonElement<Manga>(resultsObj)
-        val jsonArray = jsonObject.getAsJsonObject("results")
+        val imageUrl = manga.image_server + manga.id
-        var imageUrl: String
+        var totalPages = manga.total_page
-        var totalPages: Int
+
-        var mangaId: String
-        jsonArray.let { json ->
-            mangaId = json["id"].string
-            imageUrl = json["image_server"].string + mangaId
-            totalPages = json["total_page"].int
-        }
         val pages = mutableListOf<Page>()
 
         client.newCall(
@@ -212,39 +211,158 @@ class NineHentai : HttpSource() {
         return pages
     }
 
-    private fun buildRequestBody(searchText: String = "", page: Int, sort: Int = 0, includedTags: MutableList<Tag> = arrayListOf(), excludedTags: MutableList<Tag> = arrayListOf()): RequestBody {
+    // Search
-        val json = gson.toJson(mapOf("search" to SearchRequest(text = searchText, page = page - 1, sort = sort, tag = mapOf("items" to Items(includedTags, excludedTags)))))
+
-        return RequestBody.create(MEDIA_TYPE, json)
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        val filterList = if (filters.isEmpty()) getFilterList() else filters
+        var sort = 0
+        val range = mutableListOf(0, 2000)
+        val includedTags = mutableListOf<Tag>()
+        val excludedTags = mutableListOf<Tag>()
+        for (filter in filterList) {
+            when (filter) {
+                is SortFilter -> {
+                    sort = filter.state!!.index
+                }
+                is MinPagesFilter -> {
+                    try {
+                        range[0] = filter.state.toInt()
+                    } catch (_: NumberFormatException) {
+                        // Suppress and retain default value
+                    }
+                }
+                is MaxPagesFilter -> {
+                    try {
+                        range[1] = filter.state.toInt()
+                    } catch (_: NumberFormatException) {
+                        // Suppress and retain default value
+                    }
+                }
+                is IncludedFilter -> {
+                    includedTags += getTags(filter.state, 1)
+                }
+                is ExcludedFilter -> {
+                    excludedTags += getTags(filter.state, 1)
+                }
+                is GroupFilter -> {
+                    includedTags += getTags(filter.state, 2)
+                }
+                is ParodyFilter -> {
+                    includedTags += getTags(filter.state, 3)
+                }
+                is ArtistFilter -> {
+                    includedTags += getTags(filter.state, 4)
+                }
+                is CharacterFilter -> {
+                    includedTags += getTags(filter.state, 5)
+                }
+                is CategoryFilter -> {
+                    includedTags += getTags(filter.state, 6)
+                }
+            }
+        }
+        return buildSearchRequest(
+            searchText = query,
+            page = page,
+            sort = sort,
+            range = range,
+            includedTags = includedTags,
+            excludedTags = excludedTags
+        )
     }
 
-    private fun buildIdBody(id: Int): RequestBody {
+    private fun getTags(queries: String, type: Int): List<Tag> {
-        return RequestBody.create(MEDIA_TYPE, gson.toJson(mapOf("id" to id)))
+        return queries.split(",").map(String::trim)
+            .filterNot(String::isBlank).mapNotNull { query ->
+                val jsonString = buildJsonObject {
+                    put("tag_name", query)
+                    put("tag_type", type)
+                }.toString()
+                lookupTags(jsonString)
+            }
     }
 
-    private class GenreList(tags: List<Tag>) : Filter.Group<Tag>("Tags", tags)
+    // Based on HentaiHand ext
+    private fun lookupTags(request: String): Tag? {
+        return client.newCall(POST("$baseUrl$TAG_URL", headers, request.toRequestBody(MEDIA_TYPE)))
+            .asObservableSuccess()
+            .subscribeOn(Schedulers.io())
+            .map { response ->
+                // Returns the first matched tag, or null if there are no results
+                val tagList = json.parseToJsonElement(response.body!!.string()).jsonObject["results"]!!.jsonArray.map {
+                    json.decodeFromJsonElement<Tag>(it)
+                }
+                if (tagList.isEmpty()) return@map null
+                else tagList.first()
+            }.toBlocking().first()
+    }
 
-    private class Sorting : Filter.Sort(
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
-        "Sorting",
+        if (query.startsWith("id:")) {
+            val id = query.substringAfter("id:").toInt()
+            return client.newCall(buildDetailRequest(id))
+                .asObservableSuccess()
+                .map { response ->
+                    fetchSingleManga(response)
+                }
+        }
+        return super.fetchSearchManga(page, query, filters)
+    }
+
+    private fun fetchSingleManga(response: Response): MangasPage {
+        val resultsObj = json.parseToJsonElement(response.body!!.string()).jsonObject["results"]!!
+        val manga = json.decodeFromJsonElement<Manga>(resultsObj)
+        val list = listOf(
+            SManga.create().apply {
+                setUrlWithoutDomain("/g/${manga.id}")
+                title = manga.title
+                thumbnail_url = "${manga.image_server + manga.id}/cover.jpg"
+            }
+        )
+        return MangasPage(list, false)
+    }
+
+    override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response)
+
+    // Filters
+
+    private class SortFilter : Filter.Sort(
+        "Sort by",
         arrayOf("Newest", "Popular Right now", "Most Fapped", "Most Viewed", "By Title"),
         Selection(1, false)
     )
 
+    private class MinPagesFilter : Filter.Text("Minimum Pages")
+    private class MaxPagesFilter : Filter.Text("Maximum Pages")
+    private class IncludedFilter : Filter.Text("Included Tags")
+    private class ExcludedFilter : Filter.Text("Excluded Tags")
+    private class ArtistFilter : Filter.Text("Artist")
+    private class GroupFilter : Filter.Text("Group")
+    private class ParodyFilter : Filter.Text("Parody")
+    private class CharacterFilter : Filter.Text("Character")
+    private class CategoryFilter : Filter.Text("Category")
+
     override fun getFilterList() = FilterList(
-        Sorting(),
+        Filter.Header("Search by id with \"id:\" in front of query"),
-        GenreList(NHTags.getTagsList())
+        Filter.Separator(),
+        SortFilter(),
+        MinPagesFilter(),
+        MaxPagesFilter(),
+        IncludedFilter(),
+        ExcludedFilter(),
+        ArtistFilter(),
+        GroupFilter(),
+        ParodyFilter(),
+        CharacterFilter(),
+        CategoryFilter(),
     )
 
     override fun imageUrlParse(response: Response): String = throw Exception("Not Used")
 
-    override fun popularMangaParse(response: Response): MangasPage = throw Exception("Not Used")
-
-    override fun latestUpdatesParse(response: Response): MangasPage = throw Exception("Not Used")
-
-    override fun searchMangaParse(response: Response): MangasPage = throw Exception("Not Used")
-
     companion object {
         private val MEDIA_TYPE = "application/json; charset=utf-8".toMediaTypeOrNull()
         private const val SEARCH_URL = "/api/getBook"
         private const val MANGA_URL = "/api/getBookByID"
+        private const val TAG_URL = "/api/getTag"
     }
 }

src/all/ninehentai/src/eu/kanade/tachiyomi/extension/all/ninehentai/NineHentaiDto.kt

@@ -0,0 +1,72 @@
+package eu.kanade.tachiyomi.extension.all.ninehentai
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class Manga(
+    val id: Int,
+    val title: String,
+    val image_server: String,
+    val total_page: Int
+)
+
+/*
+The basic search request JSON object looks like this:
+{
+  "search": {
+    "text": "",
+    "page": 1,
+    "sort": 1,
+    "pages": {
+      "range": [0, 2000]
+    },
+    "tag": {
+      "items": {
+        "included": [],
+        "excluded": []
+      }
+    }
+  }
+}
+*/
+
+/*
+ Sort = 0, Newest
+ Sort = 1, Popular right now
+ Sort = 2, Most Fapped
+ Sort = 3, Most Viewed
+ Sort = 4, By title
+ */
+
+@Serializable
+data class SearchRequest(
+    val text: String,
+    val page: Int,
+    val sort: Int,
+    val pages: Range,
+    val tag: Items
+)
+
+@Serializable
+data class Range(
+    val range: List<Int>
+)
+
+@Serializable
+data class Items(
+    val items: TagArrays
+)
+
+@Serializable
+data class TagArrays(
+    val included: List<Tag>,
+    val excluded: List<Tag>
+)
+
+@Serializable
+data class Tag(
+    val id: Int,
+    val name: String,
+    val description: String? = null,
+    val type: Int = 1
+)

src/all/ninehentai/src/eu/kanade/tachiyomi/extension/all/ninehentai/NineHentaiUrlActivity.kt

@@ -0,0 +1,38 @@
+package eu.kanade.tachiyomi.extension.all.ninehentai
+
+import android.app.Activity
+import android.content.ActivityNotFoundException
+import android.content.Intent
+import android.os.Bundle
+import android.util.Log
+import kotlin.system.exitProcess
+
+/**
+ * Springboard that accepts https://9hentai.to/g/xxxxxx intents and redirects them to
+ * the main Tachiyomi process.
+ */
+class NineHentaiUrlActivity : Activity() {
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        val pathSegments = intent?.data?.pathSegments
+        if (pathSegments != null && pathSegments.size > 1) {
+            val id = pathSegments[1]
+            val mainIntent = Intent().apply {
+                action = "eu.kanade.tachiyomi.SEARCH"
+                putExtra("query", "id:$id")
+                putExtra("filter", packageName)
+            }
+
+            try {
+                startActivity(mainIntent)
+            } catch (e: ActivityNotFoundException) {
+                Log.e("NineHentaiUrlActivity", e.toString())
+            }
+        } else {
+            Log.e("NineHentaiUrlActivity", "could not parse uri from intent $intent")
+        }
+
+        finish()
+        exitProcess(0)
+    }
+}