[hitomi.la] Update image url path logic and implement filter (#7654)

* [hitomi.la] Update image url path logic and implement filter

[FIX] Update image url path logic

[FEAT] Add Filter for searching with popularity support

[FIX] Replace the used json library with kotlinx.serialization

[FIX] Deeplink now support all type and can actually display result

[FEAT] Add common word detection to avoid large search result

* Fix HitomiActivity query pattern

* [hitomi.la] Update image url path logic and implement filter

[FIX] Update image url path logic

[FEAT] Add Filter for searching with popularity support

[FIX] Replace the used json library with kotlinx.serialization

[FIX] Deeplink now support all type and can actually display result

[FEAT] Add common word detection to avoid large search result

* Fix HitomiActivity query pattern

* Update extVersionCode

src/all/hitomi/AndroidManifest.xml

@@ -13,10 +13,22 @@
                 <category android:name="android.intent.category.DEFAULT" />
                 <category android:name="android.intent.category.BROWSABLE" />
 
+                <data
+                    android:host="hitomi.la"
+                    android:pathPattern="/manga/..*"
+                    android:scheme="https" />
+                <data
+                    android:host="hitomi.la"
+                    android:pathPattern="/doujinshi/..*"
+                    android:scheme="https" />
                 <data
                     android:host="hitomi.la"
                     android:pathPattern="/cg/..*"
                     android:scheme="https" />
+                <data
+                    android:host="hitomi.la"
+                    android:pathPattern="/gamecg/..*"
+                    android:scheme="https" />
             </intent-filter>
         </activity>
     </application>

src/all/hitomi/build.gradle

@@ -6,7 +6,7 @@ ext {
     extName = 'Hitomi.la'
     pkgNameSuffix = 'all.hitomi'
     extClass = '.HitomiFactory'
-    extVersionCode = 6
+    extVersionCode = 7
     libVersion = '1.2'
     containsNsfw = true
 }

src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt

@@ -5,6 +5,7 @@ import android.content.SharedPreferences
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.ConfigurableSource
+import eu.kanade.tachiyomi.source.model.Filter
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
@@ -12,13 +13,10 @@ import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.jsonArray
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
 import okhttp3.Request
 import okhttp3.Response
-import org.jsoup.nodes.Document
 import rx.Observable
 import rx.Single
 import rx.schedulers.Schedulers
@@ -90,8 +88,6 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
         ) { it.map { m -> m as SManga } }
     }
 
-    private fun Document.selectFirst(selector: String) = this.select(selector).first()
-
     private fun parseGalleryBlock(response: Response): SManga {
         val doc = response.asJsoup()
         return SManga.create().apply {
@@ -160,56 +156,134 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
 
     override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
         return if (query.startsWith(PREFIX_ID_SEARCH)) {
-            val id = query.removePrefix(PREFIX_ID_SEARCH)
+            val id = NOZOMI_ID_SIMP_PATTERN.find(
-            client.newCall(GET("$baseUrl/cg/$id", headers)).asObservableSuccess()
+                NOZOMI_ID_PATTERN
-                .map { MangasPage(listOf(mangaDetailsParse(it).apply { url = "/cg/$id" }), false) }
+                    .find(query.removePrefix(PREFIX_ID_SEARCH))!!.value
-        } else {
+            )!!.value.toInt()
-            val splitQuery = query.toLowerCase(Locale.ENGLISH).split(" ")
+            nozomiIdsToMangas(listOf(id)).map { mangas ->
-
+                MangasPage(mangas, false)
-            val positive = splitQuery.filter { !it.startsWith('-') }.toMutableList()
-            if (nozomiLang != "all") positive += "language:$nozomiLang"
-            val negative = (splitQuery - positive).map { it.removePrefix("-") }
-
-            // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
-            val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
-                .map { HitomiNozomi(client, it.first, it.second) }
-
-            var base = if (positive.isEmpty()) {
-                hn.flatMap { n -> n.getGalleryIdsFromNozomi(null, "index", "all").map { n to it.toSet() } }
-            } else {
-                val q = positive.removeAt(0)
-                hn.flatMap { n -> n.getGalleryIdsForQuery(q).map { n to it.toSet() } }
-            }
-
-            base = positive.fold(base) { acc, q ->
-                acc.flatMap { (nozomi, mangas) ->
-                    nozomi.getGalleryIdsForQuery(q).map {
-                        nozomi to mangas.intersect(it)
-                    }
-                }
-            }
-
-            base = negative.fold(base) { acc, q ->
-                acc.flatMap { (nozomi, mangas) ->
-                    nozomi.getGalleryIdsForQuery(q).map {
-                        nozomi to (mangas - it)
-                    }
-                }
-            }
-
-            base.flatMap { (_, ids) ->
-                val chunks = ids.chunked(PAGE_SIZE)
-
-                nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
-                    MangasPage(mangas, page < chunks.size)
-                }
             }.toObservable()
+        } else {
+            if (query.isBlank()) {
+                val area = filters.filterIsInstance<TypeFilter>()
+                    .joinToString("") {
+                        (it as UriPartFilter).toUriPart()
+                    }
+                val keyword = filters.filterIsInstance<Text>().toString()
+                    .replace("[", "").replace("]", "")
+                val popular = filters.filterIsInstance<SortFilter>()
+                    .joinToString("") {
+                        (it as UriPartFilter).toUriPart()
+                    } == "true"
+
+                // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
+                val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
+                    .map { HitomiNozomi(client, it.first, it.second) }
+                val base = hn.flatMap { n ->
+                    n.getGalleryIdsForQuery("$area:$keyword", nozomiLang, popular).map { n to it.toSet() }
+                }
+                base.flatMap { (_, ids) ->
+                    val chunks = ids.chunked(PAGE_SIZE)
+
+                    nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
+                        MangasPage(mangas, page < chunks.size)
+                    }
+                }.toObservable()
+            } else {
+                val splitQuery = query.toLowerCase(Locale.ENGLISH).split(" ")
+
+                val positive = splitQuery.filter {
+                    COMMON_WORDS.any { word ->
+                        it !== word
+                    } && !it.startsWith('-')
+                }.toMutableList()
+                if (nozomiLang != "all") positive += "language:$nozomiLang"
+                val negative = (splitQuery - positive).map { it.removePrefix("-") }
+
+                // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
+                val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
+                    .map { HitomiNozomi(client, it.first, it.second) }
+
+                var base = if (positive.isEmpty()) {
+                    hn.flatMap { n ->
+                        n.getGalleryIdsFromNozomi(null, "index", "all", false)
+                            .map { n to it.toSet() }
+                    }
+                } else {
+                    val q = positive.removeAt(0)
+                    hn.flatMap { n -> n.getGalleryIdsForQuery(q, nozomiLang, false).map { n to it.toSet() } }
+                }
+
+                base = positive.fold(base) { acc, q ->
+                    acc.flatMap { (nozomi, mangas) ->
+                        nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map {
+                            nozomi to mangas.intersect(it)
+                        }
+                    }
+                }
+
+                base = negative.fold(base) { acc, q ->
+                    acc.flatMap { (nozomi, mangas) ->
+                        nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map {
+                            nozomi to (mangas - it)
+                        }
+                    }
+                }
+
+                base.flatMap { (_, ids) ->
+                    val chunks = ids.chunked(PAGE_SIZE)
+
+                    nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
+                        MangasPage(mangas, page < chunks.size)
+                    }
+                }.toObservable()
+            }
         }
     }
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Not used")
     override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Not used")
 
+    // Filter
+
+    override fun getFilterList() = FilterList(
+        Filter.Header(Filter_SEARCH_MESSAGE),
+        Filter.Separator(),
+        SortFilter(),
+        TypeFilter(),
+        Text("Keyword")
+    )
+
+    private class TypeFilter : UriPartFilter(
+        "category",
+        Array(FILTER_CATEGORIES.size) { i ->
+            val category = FILTER_CATEGORIES[i]
+            Pair(category, category)
+        }
+    )
+
+    private class SortFilter : UriPartFilter(
+        "Ordered by",
+        arrayOf(
+            Pair("Date Added", "false"),
+            Pair("Popularity", "true")
+        )
+    )
+
+    private open class UriPartFilter(
+        displayName: String,
+        val pair: Array<Pair<String, String>>,
+        defaultState: Int = 0
+    ) : Filter.Select<String>(displayName, pair.map { it.first }.toTypedArray(), defaultState) {
+        open fun toUriPart() = pair[state].second
+    }
+
+    private class Text(name: String) : Filter.Text(name) {
+        override fun toString(): String {
+            return state
+        }
+    }
+
     // Details
 
     override fun mangaDetailsParse(response: Response): SManga {
@@ -217,6 +291,7 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
         fun String.replaceSpaces() = this.replace(" ", "_")
 
         return SManga.create().apply {
+            title = document.select("div.gallery h1 a").joinToString { it.text() }
             thumbnail_url = document.select("div.cover img").attr("abs:src")
             author = document.select("div.gallery h2 a").joinToString { it.text() }
             val tableInfo = document.select("table tr")
@@ -275,35 +350,30 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
     }
 
     override fun pageListParse(response: Response): List<Page> {
-        val jsonRaw = response.body!!.string().removePrefix("var galleryinfo = ")
+        val str = response.body!!.string()
-        val jsonResult = json.parseToJsonElement(jsonRaw).jsonObject
+        val json = json.decodeFromString<HitomiChapterDto>(str.removePrefix("var galleryinfo = "))
-
+        return json.files.mapIndexed { i, jsonElement ->
-        return jsonResult["files"]!!.jsonArray.mapIndexed { i, jsonEl ->
+            val hash = jsonElement.hash
-            val jsonObj = jsonEl.jsonObject
+            val ext = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) jsonElement.name.split('.').last() else "webp"
-            val hash = jsonObj["hash"]!!.jsonPrimitive.content
+            val path = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) "images" else "webp"
-            val hasWebp = jsonObj["haswebp"]!!.jsonPrimitive.content == "0"
-            val hasAvif = jsonObj["hasavif"]!!.jsonPrimitive.content == "0"
-            val ext = if (hasWebp || !hitomiAlwaysWebp())
-                jsonObj["name"]!!.jsonPrimitive.content.substringAfterLast(".") else "webp"
-            val path = if (hasWebp || !hitomiAlwaysWebp())
-                "images" else "webp"
             val hashPath1 = hash.takeLast(1)
             val hashPath2 = hash.takeLast(3).take(2)
 
             // https://ltn.hitomi.la/reader.js
             // function make_image_element()
-            val secondSubdomain = if (hasWebp && hasAvif) "b" else "a"
+            val secondSubdomain = if (jsonElement.haswebp == 0 && jsonElement.hasavif == 0) "b" else "a"
-
             Page(i, "", "https://${firstSubdomainFromGalleryId(hashPath2)}$secondSubdomain.hitomi.la/$path/$hashPath1/$hashPath2/$hash.$ext")
         }
     }
 
     // https://ltn.hitomi.la/common.js
+    // function subdomain_from_url()
+    // Change g's if statment from !isNaN(g)
     private fun firstSubdomainFromGalleryId(pathSegment: String): Char {
         var numberOfFrontends = 3
         var g = pathSegment.toInt(16)
-        if (g < 0x30) numberOfFrontends = 2
+        if (g < 0x80) numberOfFrontends = 2
-        if (g < 0x09) g = 1
+        if (g < 0x59) g = 1
 
         return (97 + g.rem(numberOfFrontends)).toChar()
     }
@@ -325,11 +395,27 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
         private const val PAGE_SIZE = 25
 
         const val PREFIX_ID_SEARCH = "id:"
+        val NOZOMI_ID_PATTERN = "[0-9]*.html".toRegex()
+        val NOZOMI_ID_SIMP_PATTERN = "[0-9]*".toRegex()
+
+        // Common English words and Japanese particles
+        private val COMMON_WORDS = listOf(
+            "a", "be", "boy", "de", "girl", "ga", "i", "is", "ka", "na",
+            "ni", "ne", "no", "suru", "to", "wa", "wo", "yo",
+            "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
+        )
 
         // From HitomiSearchMetaData
         const val LTN_BASE_URL = "https://ltn.hitomi.la"
         const val BASE_URL = "https://hitomi.la"
 
+        // Filter
+        private val FILTER_CATEGORIES = listOf(
+            "tag", "male", "female", "type",
+            "artist", "series", "character", "group"
+        )
+        private const val Filter_SEARCH_MESSAGE = "NOTE: Ignored if using text search!"
+
         // Preferences
         private const val WEBP_PREF_KEY = "HITOMI_WEBP"
         private const val WEBP_PREF_TITLE = "Webp pages"

src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt

@@ -0,0 +1,16 @@
+package eu.kanade.tachiyomi.extension.all.hitomi
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class HitomiChapterDto(
+    val files: List<HitomiFileDto> = emptyList(),
+)
+
+@Serializable
+data class HitomiFileDto(
+    val name: String,
+    val hasavif: Int,
+    val hash: String,
+    val haswebp: Int,
+)

src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt

@@ -29,26 +29,21 @@ class HitomiNozomi(
     private val tagIndexVersion: Long,
     private val galleriesIndexVersion: Long
 ) {
-    fun getGalleryIdsForQuery(query: String): Single<List<Int>> {
+    fun getGalleryIdsForQuery(query: String, language: String, popular: Boolean): Single<List<Int>> {
-        val replacedQuery = query.replace('_', ' ')
+        if (':' in query) {
-
+            val sides = query.split(':')
-        if (':' in replacedQuery) {
-            val sides = replacedQuery.split(':')
             val namespace = sides[0]
             var tag = sides[1]
 
             var area: String? = namespace
-            var language = "all"
             if (namespace == "female" || namespace == "male") {
                 area = "tag"
-                tag = replacedQuery
+                tag = query
             } else if (namespace == "language") {
-                area = null
+                return getGalleryIdsFromNozomi(null, "index", tag, popular)
-                language = tag
-                tag = "index"
             }
 
-            return getGalleryIdsFromNozomi(area, tag, language)
+            return getGalleryIdsFromNozomi(area, tag, language, popular)
         }
 
         val key = hashTerm(query)
@@ -202,10 +197,15 @@ class HitomiNozomi(
             }.toSingle()
     }
 
-    fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String): Single<List<Int>> {
+    fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String, popular: Boolean): Single<List<Int>> {
-        var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$tag-$language$NOZOMI_EXTENSION"
+        val replacedTag = tag.replace('_', ' ')
+        var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$replacedTag-$language$NOZOMI_EXTENSION"
         if (area != null) {
-            nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$tag-$language$NOZOMI_EXTENSION"
+            nozomiAddress = if (popular) {
+                "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/popular/$replacedTag-$language$NOZOMI_EXTENSION"
+            } else {
+                "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$replacedTag-$language$NOZOMI_EXTENSION"
+            }
         }
 
         return client.newCall(