From 00547b54134a5e52a1f3a9b3490db0ec160258ab Mon Sep 17 00:00:00 2001 From: vulpes310 <85771934+vulpes310@users.noreply.github.com> Date: Mon, 14 Jun 2021 10:43:17 -0700 Subject: [PATCH] [hitomi.la] Update image url path logic and implement filter (#7654) * [hitomi.la] Update image url path logic and implement filter [FIX] Update image url path logic [FEAT] Add Filter for searching with popularity support [FIX] Replace the used json library with kotlinx.serialization [FIX] Deeplink now support all type and can actually display result [FEAT] Add common word detection to avoid large search result * Fix HitomiActivity query pattern * [hitomi.la] Update image url path logic and implement filter [FIX] Update image url path logic [FEAT] Add Filter for searching with popularity support [FIX] Replace the used json library with kotlinx.serialization [FIX] Deeplink now support all type and can actually display result [FEAT] Add common word detection to avoid large search result * Fix HitomiActivity query pattern * Update extVersionCode --- src/all/hitomi/AndroidManifest.xml | 12 + src/all/hitomi/build.gradle | 2 +- .../tachiyomi/extension/all/hitomi/Hitomi.kt | 216 ++++++++++++------ .../extension/all/hitomi/HitomiDto.kt | 16 ++ .../extension/all/hitomi/HitomiNozomi.kt | 28 +-- 5 files changed, 194 insertions(+), 80 deletions(-) create mode 100644 src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt diff --git a/src/all/hitomi/AndroidManifest.xml b/src/all/hitomi/AndroidManifest.xml index a7c4ada40..b722a7165 100644 --- a/src/all/hitomi/AndroidManifest.xml +++ b/src/all/hitomi/AndroidManifest.xml @@ -13,10 +13,22 @@ + + + diff --git a/src/all/hitomi/build.gradle b/src/all/hitomi/build.gradle index df804d52f..74e886d0a 100644 --- a/src/all/hitomi/build.gradle +++ b/src/all/hitomi/build.gradle @@ -6,7 +6,7 @@ ext { extName = 'Hitomi.la' pkgNameSuffix = 'all.hitomi' extClass = '.HitomiFactory' - extVersionCode = 6 + extVersionCode = 7 libVersion = '1.2' containsNsfw = true } diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt index 257705ee8..f5f534447 100644 --- a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt @@ -5,6 +5,7 @@ import android.content.SharedPreferences import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.ConfigurableSource +import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.Page @@ -12,13 +13,10 @@ import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.util.asJsoup +import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive import okhttp3.Request import okhttp3.Response -import org.jsoup.nodes.Document import rx.Observable import rx.Single import rx.schedulers.Schedulers @@ -90,8 +88,6 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H ) { it.map { m -> m as SManga } } } - private fun Document.selectFirst(selector: String) = this.select(selector).first() - private fun parseGalleryBlock(response: Response): SManga { val doc = response.asJsoup() return SManga.create().apply { @@ -160,56 +156,134 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { return if (query.startsWith(PREFIX_ID_SEARCH)) { - val id = query.removePrefix(PREFIX_ID_SEARCH) - client.newCall(GET("$baseUrl/cg/$id", headers)).asObservableSuccess() - .map { MangasPage(listOf(mangaDetailsParse(it).apply { url = "/cg/$id" }), false) } - } else { - val splitQuery = query.toLowerCase(Locale.ENGLISH).split(" ") - - val positive = splitQuery.filter { !it.startsWith('-') }.toMutableList() - if (nozomiLang != "all") positive += "language:$nozomiLang" - val negative = (splitQuery - positive).map { it.removePrefix("-") } - - // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH) - val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv } - .map { HitomiNozomi(client, it.first, it.second) } - - var base = if (positive.isEmpty()) { - hn.flatMap { n -> n.getGalleryIdsFromNozomi(null, "index", "all").map { n to it.toSet() } } - } else { - val q = positive.removeAt(0) - hn.flatMap { n -> n.getGalleryIdsForQuery(q).map { n to it.toSet() } } - } - - base = positive.fold(base) { acc, q -> - acc.flatMap { (nozomi, mangas) -> - nozomi.getGalleryIdsForQuery(q).map { - nozomi to mangas.intersect(it) - } - } - } - - base = negative.fold(base) { acc, q -> - acc.flatMap { (nozomi, mangas) -> - nozomi.getGalleryIdsForQuery(q).map { - nozomi to (mangas - it) - } - } - } - - base.flatMap { (_, ids) -> - val chunks = ids.chunked(PAGE_SIZE) - - nozomiIdsToMangas(chunks[page - 1]).map { mangas -> - MangasPage(mangas, page < chunks.size) - } + val id = NOZOMI_ID_SIMP_PATTERN.find( + NOZOMI_ID_PATTERN + .find(query.removePrefix(PREFIX_ID_SEARCH))!!.value + )!!.value.toInt() + nozomiIdsToMangas(listOf(id)).map { mangas -> + MangasPage(mangas, false) }.toObservable() + } else { + if (query.isBlank()) { + val area = filters.filterIsInstance() + .joinToString("") { + (it as UriPartFilter).toUriPart() + } + val keyword = filters.filterIsInstance().toString() + .replace("[", "").replace("]", "") + val popular = filters.filterIsInstance() + .joinToString("") { + (it as UriPartFilter).toUriPart() + } == "true" + + // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH) + val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv } + .map { HitomiNozomi(client, it.first, it.second) } + val base = hn.flatMap { n -> + n.getGalleryIdsForQuery("$area:$keyword", nozomiLang, popular).map { n to it.toSet() } + } + base.flatMap { (_, ids) -> + val chunks = ids.chunked(PAGE_SIZE) + + nozomiIdsToMangas(chunks[page - 1]).map { mangas -> + MangasPage(mangas, page < chunks.size) + } + }.toObservable() + } else { + val splitQuery = query.toLowerCase(Locale.ENGLISH).split(" ") + + val positive = splitQuery.filter { + COMMON_WORDS.any { word -> + it !== word + } && !it.startsWith('-') + }.toMutableList() + if (nozomiLang != "all") positive += "language:$nozomiLang" + val negative = (splitQuery - positive).map { it.removePrefix("-") } + + // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH) + val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv } + .map { HitomiNozomi(client, it.first, it.second) } + + var base = if (positive.isEmpty()) { + hn.flatMap { n -> + n.getGalleryIdsFromNozomi(null, "index", "all", false) + .map { n to it.toSet() } + } + } else { + val q = positive.removeAt(0) + hn.flatMap { n -> n.getGalleryIdsForQuery(q, nozomiLang, false).map { n to it.toSet() } } + } + + base = positive.fold(base) { acc, q -> + acc.flatMap { (nozomi, mangas) -> + nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map { + nozomi to mangas.intersect(it) + } + } + } + + base = negative.fold(base) { acc, q -> + acc.flatMap { (nozomi, mangas) -> + nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map { + nozomi to (mangas - it) + } + } + } + + base.flatMap { (_, ids) -> + val chunks = ids.chunked(PAGE_SIZE) + + nozomiIdsToMangas(chunks[page - 1]).map { mangas -> + MangasPage(mangas, page < chunks.size) + } + }.toObservable() + } } } override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Not used") override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Not used") + // Filter + + override fun getFilterList() = FilterList( + Filter.Header(Filter_SEARCH_MESSAGE), + Filter.Separator(), + SortFilter(), + TypeFilter(), + Text("Keyword") + ) + + private class TypeFilter : UriPartFilter( + "category", + Array(FILTER_CATEGORIES.size) { i -> + val category = FILTER_CATEGORIES[i] + Pair(category, category) + } + ) + + private class SortFilter : UriPartFilter( + "Ordered by", + arrayOf( + Pair("Date Added", "false"), + Pair("Popularity", "true") + ) + ) + + private open class UriPartFilter( + displayName: String, + val pair: Array>, + defaultState: Int = 0 + ) : Filter.Select(displayName, pair.map { it.first }.toTypedArray(), defaultState) { + open fun toUriPart() = pair[state].second + } + + private class Text(name: String) : Filter.Text(name) { + override fun toString(): String { + return state + } + } + // Details override fun mangaDetailsParse(response: Response): SManga { @@ -217,6 +291,7 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H fun String.replaceSpaces() = this.replace(" ", "_") return SManga.create().apply { + title = document.select("div.gallery h1 a").joinToString { it.text() } thumbnail_url = document.select("div.cover img").attr("abs:src") author = document.select("div.gallery h2 a").joinToString { it.text() } val tableInfo = document.select("table tr") @@ -275,35 +350,30 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H } override fun pageListParse(response: Response): List { - val jsonRaw = response.body!!.string().removePrefix("var galleryinfo = ") - val jsonResult = json.parseToJsonElement(jsonRaw).jsonObject - - return jsonResult["files"]!!.jsonArray.mapIndexed { i, jsonEl -> - val jsonObj = jsonEl.jsonObject - val hash = jsonObj["hash"]!!.jsonPrimitive.content - val hasWebp = jsonObj["haswebp"]!!.jsonPrimitive.content == "0" - val hasAvif = jsonObj["hasavif"]!!.jsonPrimitive.content == "0" - val ext = if (hasWebp || !hitomiAlwaysWebp()) - jsonObj["name"]!!.jsonPrimitive.content.substringAfterLast(".") else "webp" - val path = if (hasWebp || !hitomiAlwaysWebp()) - "images" else "webp" + val str = response.body!!.string() + val json = json.decodeFromString(str.removePrefix("var galleryinfo = ")) + return json.files.mapIndexed { i, jsonElement -> + val hash = jsonElement.hash + val ext = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) jsonElement.name.split('.').last() else "webp" + val path = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) "images" else "webp" val hashPath1 = hash.takeLast(1) val hashPath2 = hash.takeLast(3).take(2) // https://ltn.hitomi.la/reader.js // function make_image_element() - val secondSubdomain = if (hasWebp && hasAvif) "b" else "a" - + val secondSubdomain = if (jsonElement.haswebp == 0 && jsonElement.hasavif == 0) "b" else "a" Page(i, "", "https://${firstSubdomainFromGalleryId(hashPath2)}$secondSubdomain.hitomi.la/$path/$hashPath1/$hashPath2/$hash.$ext") } } // https://ltn.hitomi.la/common.js + // function subdomain_from_url() + // Change g's if statment from !isNaN(g) private fun firstSubdomainFromGalleryId(pathSegment: String): Char { var numberOfFrontends = 3 var g = pathSegment.toInt(16) - if (g < 0x30) numberOfFrontends = 2 - if (g < 0x09) g = 1 + if (g < 0x80) numberOfFrontends = 2 + if (g < 0x59) g = 1 return (97 + g.rem(numberOfFrontends)).toChar() } @@ -325,11 +395,27 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H private const val PAGE_SIZE = 25 const val PREFIX_ID_SEARCH = "id:" + val NOZOMI_ID_PATTERN = "[0-9]*.html".toRegex() + val NOZOMI_ID_SIMP_PATTERN = "[0-9]*".toRegex() + + // Common English words and Japanese particles + private val COMMON_WORDS = listOf( + "a", "be", "boy", "de", "girl", "ga", "i", "is", "ka", "na", + "ni", "ne", "no", "suru", "to", "wa", "wo", "yo", + "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" + ) // From HitomiSearchMetaData const val LTN_BASE_URL = "https://ltn.hitomi.la" const val BASE_URL = "https://hitomi.la" + // Filter + private val FILTER_CATEGORIES = listOf( + "tag", "male", "female", "type", + "artist", "series", "character", "group" + ) + private const val Filter_SEARCH_MESSAGE = "NOTE: Ignored if using text search!" + // Preferences private const val WEBP_PREF_KEY = "HITOMI_WEBP" private const val WEBP_PREF_TITLE = "Webp pages" diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt new file mode 100644 index 000000000..a38e49982 --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt @@ -0,0 +1,16 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import kotlinx.serialization.Serializable + +@Serializable +data class HitomiChapterDto( + val files: List = emptyList(), +) + +@Serializable +data class HitomiFileDto( + val name: String, + val hasavif: Int, + val hash: String, + val haswebp: Int, +) diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt index 9443080bc..9ec7d1c62 100644 --- a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt @@ -29,26 +29,21 @@ class HitomiNozomi( private val tagIndexVersion: Long, private val galleriesIndexVersion: Long ) { - fun getGalleryIdsForQuery(query: String): Single> { - val replacedQuery = query.replace('_', ' ') - - if (':' in replacedQuery) { - val sides = replacedQuery.split(':') + fun getGalleryIdsForQuery(query: String, language: String, popular: Boolean): Single> { + if (':' in query) { + val sides = query.split(':') val namespace = sides[0] var tag = sides[1] var area: String? = namespace - var language = "all" if (namespace == "female" || namespace == "male") { area = "tag" - tag = replacedQuery + tag = query } else if (namespace == "language") { - area = null - language = tag - tag = "index" + return getGalleryIdsFromNozomi(null, "index", tag, popular) } - return getGalleryIdsFromNozomi(area, tag, language) + return getGalleryIdsFromNozomi(area, tag, language, popular) } val key = hashTerm(query) @@ -202,10 +197,15 @@ class HitomiNozomi( }.toSingle() } - fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String): Single> { - var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$tag-$language$NOZOMI_EXTENSION" + fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String, popular: Boolean): Single> { + val replacedTag = tag.replace('_', ' ') + var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$replacedTag-$language$NOZOMI_EXTENSION" if (area != null) { - nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$tag-$language$NOZOMI_EXTENSION" + nozomiAddress = if (popular) { + "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/popular/$replacedTag-$language$NOZOMI_EXTENSION" + } else { + "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$replacedTag-$language$NOZOMI_EXTENSION" + } } return client.newCall(