[hitomi.la] Update image url path logic and implement filter (#7654)
* [hitomi.la] Update image url path logic and implement filter [FIX] Update image url path logic [FEAT] Add Filter for searching with popularity support [FIX] Replace the used json library with kotlinx.serialization [FIX] Deeplink now support all type and can actually display result [FEAT] Add common word detection to avoid large search result * Fix HitomiActivity query pattern * [hitomi.la] Update image url path logic and implement filter [FIX] Update image url path logic [FEAT] Add Filter for searching with popularity support [FIX] Replace the used json library with kotlinx.serialization [FIX] Deeplink now support all type and can actually display result [FEAT] Add common word detection to avoid large search result * Fix HitomiActivity query pattern * Update extVersionCode
This commit is contained in:
vulpes310
GitHub
parent
787767942b
commit
00547b5413
|
|
@ -13,10 +13,22 @@
|
|||
<category android:name="android.intent.category.DEFAULT" />
|
||||
<category android:name="android.intent.category.BROWSABLE" />
|
||||
|
||||
<data
|
||||
android:host="hitomi.la"
|
||||
android:pathPattern="/manga/..*"
|
||||
android:scheme="https" />
|
||||
<data
|
||||
android:host="hitomi.la"
|
||||
android:pathPattern="/doujinshi/..*"
|
||||
android:scheme="https" />
|
||||
<data
|
||||
android:host="hitomi.la"
|
||||
android:pathPattern="/cg/..*"
|
||||
android:scheme="https" />
|
||||
<data
|
||||
android:host="hitomi.la"
|
||||
android:pathPattern="/gamecg/..*"
|
||||
android:scheme="https" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
</application>
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ ext {
|
|||
extName = 'Hitomi.la'
|
||||
pkgNameSuffix = 'all.hitomi'
|
||||
extClass = '.HitomiFactory'
|
||||
extVersionCode = 6
|
||||
extVersionCode = 7
|
||||
libVersion = '1.2'
|
||||
containsNsfw = true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import android.content.SharedPreferences
|
|||
import eu.kanade.tachiyomi.network.GET
|
||||
import eu.kanade.tachiyomi.network.asObservableSuccess
|
||||
import eu.kanade.tachiyomi.source.ConfigurableSource
|
||||
import eu.kanade.tachiyomi.source.model.Filter
|
||||
import eu.kanade.tachiyomi.source.model.FilterList
|
||||
import eu.kanade.tachiyomi.source.model.MangasPage
|
||||
import eu.kanade.tachiyomi.source.model.Page
|
||||
|
|
@ -12,13 +13,10 @@ import eu.kanade.tachiyomi.source.model.SChapter
|
|||
import eu.kanade.tachiyomi.source.model.SManga
|
||||
import eu.kanade.tachiyomi.source.online.HttpSource
|
||||
import eu.kanade.tachiyomi.util.asJsoup
|
||||
import kotlinx.serialization.decodeFromString
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.jsonArray
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import org.jsoup.nodes.Document
|
||||
import rx.Observable
|
||||
import rx.Single
|
||||
import rx.schedulers.Schedulers
|
||||
|
|
@ -90,8 +88,6 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
) { it.map { m -> m as SManga } }
|
||||
}
|
||||
|
||||
private fun Document.selectFirst(selector: String) = this.select(selector).first()
|
||||
|
||||
private fun parseGalleryBlock(response: Response): SManga {
|
||||
val doc = response.asJsoup()
|
||||
return SManga.create().apply {
|
||||
|
|
@ -160,13 +156,47 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
|
||||
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
|
||||
return if (query.startsWith(PREFIX_ID_SEARCH)) {
|
||||
val id = query.removePrefix(PREFIX_ID_SEARCH)
|
||||
client.newCall(GET("$baseUrl/cg/$id", headers)).asObservableSuccess()
|
||||
.map { MangasPage(listOf(mangaDetailsParse(it).apply { url = "/cg/$id" }), false) }
|
||||
val id = NOZOMI_ID_SIMP_PATTERN.find(
|
||||
NOZOMI_ID_PATTERN
|
||||
.find(query.removePrefix(PREFIX_ID_SEARCH))!!.value
|
||||
)!!.value.toInt()
|
||||
nozomiIdsToMangas(listOf(id)).map { mangas ->
|
||||
MangasPage(mangas, false)
|
||||
}.toObservable()
|
||||
} else {
|
||||
if (query.isBlank()) {
|
||||
val area = filters.filterIsInstance<TypeFilter>()
|
||||
.joinToString("") {
|
||||
(it as UriPartFilter).toUriPart()
|
||||
}
|
||||
val keyword = filters.filterIsInstance<Text>().toString()
|
||||
.replace("[", "").replace("]", "")
|
||||
val popular = filters.filterIsInstance<SortFilter>()
|
||||
.joinToString("") {
|
||||
(it as UriPartFilter).toUriPart()
|
||||
} == "true"
|
||||
|
||||
// TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
|
||||
val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
|
||||
.map { HitomiNozomi(client, it.first, it.second) }
|
||||
val base = hn.flatMap { n ->
|
||||
n.getGalleryIdsForQuery("$area:$keyword", nozomiLang, popular).map { n to it.toSet() }
|
||||
}
|
||||
base.flatMap { (_, ids) ->
|
||||
val chunks = ids.chunked(PAGE_SIZE)
|
||||
|
||||
nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
|
||||
MangasPage(mangas, page < chunks.size)
|
||||
}
|
||||
}.toObservable()
|
||||
} else {
|
||||
val splitQuery = query.toLowerCase(Locale.ENGLISH).split(" ")
|
||||
|
||||
val positive = splitQuery.filter { !it.startsWith('-') }.toMutableList()
|
||||
val positive = splitQuery.filter {
|
||||
COMMON_WORDS.any { word ->
|
||||
it !== word
|
||||
} && !it.startsWith('-')
|
||||
}.toMutableList()
|
||||
if (nozomiLang != "all") positive += "language:$nozomiLang"
|
||||
val negative = (splitQuery - positive).map { it.removePrefix("-") }
|
||||
|
||||
|
|
@ -175,15 +205,18 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
.map { HitomiNozomi(client, it.first, it.second) }
|
||||
|
||||
var base = if (positive.isEmpty()) {
|
||||
hn.flatMap { n -> n.getGalleryIdsFromNozomi(null, "index", "all").map { n to it.toSet() } }
|
||||
hn.flatMap { n ->
|
||||
n.getGalleryIdsFromNozomi(null, "index", "all", false)
|
||||
.map { n to it.toSet() }
|
||||
}
|
||||
} else {
|
||||
val q = positive.removeAt(0)
|
||||
hn.flatMap { n -> n.getGalleryIdsForQuery(q).map { n to it.toSet() } }
|
||||
hn.flatMap { n -> n.getGalleryIdsForQuery(q, nozomiLang, false).map { n to it.toSet() } }
|
||||
}
|
||||
|
||||
base = positive.fold(base) { acc, q ->
|
||||
acc.flatMap { (nozomi, mangas) ->
|
||||
nozomi.getGalleryIdsForQuery(q).map {
|
||||
nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map {
|
||||
nozomi to mangas.intersect(it)
|
||||
}
|
||||
}
|
||||
|
|
@ -191,7 +224,7 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
|
||||
base = negative.fold(base) { acc, q ->
|
||||
acc.flatMap { (nozomi, mangas) ->
|
||||
nozomi.getGalleryIdsForQuery(q).map {
|
||||
nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map {
|
||||
nozomi to (mangas - it)
|
||||
}
|
||||
}
|
||||
|
|
@ -206,10 +239,51 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
}.toObservable()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Not used")
|
||||
override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Not used")
|
||||
|
||||
// Filter
|
||||
|
||||
override fun getFilterList() = FilterList(
|
||||
Filter.Header(Filter_SEARCH_MESSAGE),
|
||||
Filter.Separator(),
|
||||
SortFilter(),
|
||||
TypeFilter(),
|
||||
Text("Keyword")
|
||||
)
|
||||
|
||||
private class TypeFilter : UriPartFilter(
|
||||
"category",
|
||||
Array(FILTER_CATEGORIES.size) { i ->
|
||||
val category = FILTER_CATEGORIES[i]
|
||||
Pair(category, category)
|
||||
}
|
||||
)
|
||||
|
||||
private class SortFilter : UriPartFilter(
|
||||
"Ordered by",
|
||||
arrayOf(
|
||||
Pair("Date Added", "false"),
|
||||
Pair("Popularity", "true")
|
||||
)
|
||||
)
|
||||
|
||||
private open class UriPartFilter(
|
||||
displayName: String,
|
||||
val pair: Array<Pair<String, String>>,
|
||||
defaultState: Int = 0
|
||||
) : Filter.Select<String>(displayName, pair.map { it.first }.toTypedArray(), defaultState) {
|
||||
open fun toUriPart() = pair[state].second
|
||||
}
|
||||
|
||||
private class Text(name: String) : Filter.Text(name) {
|
||||
override fun toString(): String {
|
||||
return state
|
||||
}
|
||||
}
|
||||
|
||||
// Details
|
||||
|
||||
override fun mangaDetailsParse(response: Response): SManga {
|
||||
|
|
@ -217,6 +291,7 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
fun String.replaceSpaces() = this.replace(" ", "_")
|
||||
|
||||
return SManga.create().apply {
|
||||
title = document.select("div.gallery h1 a").joinToString { it.text() }
|
||||
thumbnail_url = document.select("div.cover img").attr("abs:src")
|
||||
author = document.select("div.gallery h2 a").joinToString { it.text() }
|
||||
val tableInfo = document.select("table tr")
|
||||
|
|
@ -275,35 +350,30 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
}
|
||||
|
||||
override fun pageListParse(response: Response): List<Page> {
|
||||
val jsonRaw = response.body!!.string().removePrefix("var galleryinfo = ")
|
||||
val jsonResult = json.parseToJsonElement(jsonRaw).jsonObject
|
||||
|
||||
return jsonResult["files"]!!.jsonArray.mapIndexed { i, jsonEl ->
|
||||
val jsonObj = jsonEl.jsonObject
|
||||
val hash = jsonObj["hash"]!!.jsonPrimitive.content
|
||||
val hasWebp = jsonObj["haswebp"]!!.jsonPrimitive.content == "0"
|
||||
val hasAvif = jsonObj["hasavif"]!!.jsonPrimitive.content == "0"
|
||||
val ext = if (hasWebp || !hitomiAlwaysWebp())
|
||||
jsonObj["name"]!!.jsonPrimitive.content.substringAfterLast(".") else "webp"
|
||||
val path = if (hasWebp || !hitomiAlwaysWebp())
|
||||
"images" else "webp"
|
||||
val str = response.body!!.string()
|
||||
val json = json.decodeFromString<HitomiChapterDto>(str.removePrefix("var galleryinfo = "))
|
||||
return json.files.mapIndexed { i, jsonElement ->
|
||||
val hash = jsonElement.hash
|
||||
val ext = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) jsonElement.name.split('.').last() else "webp"
|
||||
val path = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) "images" else "webp"
|
||||
val hashPath1 = hash.takeLast(1)
|
||||
val hashPath2 = hash.takeLast(3).take(2)
|
||||
|
||||
// https://ltn.hitomi.la/reader.js
|
||||
// function make_image_element()
|
||||
val secondSubdomain = if (hasWebp && hasAvif) "b" else "a"
|
||||
|
||||
val secondSubdomain = if (jsonElement.haswebp == 0 && jsonElement.hasavif == 0) "b" else "a"
|
||||
Page(i, "", "https://${firstSubdomainFromGalleryId(hashPath2)}$secondSubdomain.hitomi.la/$path/$hashPath1/$hashPath2/$hash.$ext")
|
||||
}
|
||||
}
|
||||
|
||||
// https://ltn.hitomi.la/common.js
|
||||
// function subdomain_from_url()
|
||||
// Change g's if statment from !isNaN(g)
|
||||
private fun firstSubdomainFromGalleryId(pathSegment: String): Char {
|
||||
var numberOfFrontends = 3
|
||||
var g = pathSegment.toInt(16)
|
||||
if (g < 0x30) numberOfFrontends = 2
|
||||
if (g < 0x09) g = 1
|
||||
if (g < 0x80) numberOfFrontends = 2
|
||||
if (g < 0x59) g = 1
|
||||
|
||||
return (97 + g.rem(numberOfFrontends)).toChar()
|
||||
}
|
||||
|
|
@ -325,11 +395,27 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
|
|||
private const val PAGE_SIZE = 25
|
||||
|
||||
const val PREFIX_ID_SEARCH = "id:"
|
||||
val NOZOMI_ID_PATTERN = "[0-9]*.html".toRegex()
|
||||
val NOZOMI_ID_SIMP_PATTERN = "[0-9]*".toRegex()
|
||||
|
||||
// Common English words and Japanese particles
|
||||
private val COMMON_WORDS = listOf(
|
||||
"a", "be", "boy", "de", "girl", "ga", "i", "is", "ka", "na",
|
||||
"ni", "ne", "no", "suru", "to", "wa", "wo", "yo",
|
||||
"b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
|
||||
)
|
||||
|
||||
// From HitomiSearchMetaData
|
||||
const val LTN_BASE_URL = "https://ltn.hitomi.la"
|
||||
const val BASE_URL = "https://hitomi.la"
|
||||
|
||||
// Filter
|
||||
private val FILTER_CATEGORIES = listOf(
|
||||
"tag", "male", "female", "type",
|
||||
"artist", "series", "character", "group"
|
||||
)
|
||||
private const val Filter_SEARCH_MESSAGE = "NOTE: Ignored if using text search!"
|
||||
|
||||
// Preferences
|
||||
private const val WEBP_PREF_KEY = "HITOMI_WEBP"
|
||||
private const val WEBP_PREF_TITLE = "Webp pages"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
package eu.kanade.tachiyomi.extension.all.hitomi
|
||||
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
@Serializable
|
||||
data class HitomiChapterDto(
|
||||
val files: List<HitomiFileDto> = emptyList(),
|
||||
)
|
||||
|
||||
@Serializable
|
||||
data class HitomiFileDto(
|
||||
val name: String,
|
||||
val hasavif: Int,
|
||||
val hash: String,
|
||||
val haswebp: Int,
|
||||
)
|
||||
|
|
@ -29,26 +29,21 @@ class HitomiNozomi(
|
|||
private val tagIndexVersion: Long,
|
||||
private val galleriesIndexVersion: Long
|
||||
) {
|
||||
fun getGalleryIdsForQuery(query: String): Single<List<Int>> {
|
||||
val replacedQuery = query.replace('_', ' ')
|
||||
|
||||
if (':' in replacedQuery) {
|
||||
val sides = replacedQuery.split(':')
|
||||
fun getGalleryIdsForQuery(query: String, language: String, popular: Boolean): Single<List<Int>> {
|
||||
if (':' in query) {
|
||||
val sides = query.split(':')
|
||||
val namespace = sides[0]
|
||||
var tag = sides[1]
|
||||
|
||||
var area: String? = namespace
|
||||
var language = "all"
|
||||
if (namespace == "female" || namespace == "male") {
|
||||
area = "tag"
|
||||
tag = replacedQuery
|
||||
tag = query
|
||||
} else if (namespace == "language") {
|
||||
area = null
|
||||
language = tag
|
||||
tag = "index"
|
||||
return getGalleryIdsFromNozomi(null, "index", tag, popular)
|
||||
}
|
||||
|
||||
return getGalleryIdsFromNozomi(area, tag, language)
|
||||
return getGalleryIdsFromNozomi(area, tag, language, popular)
|
||||
}
|
||||
|
||||
val key = hashTerm(query)
|
||||
|
|
@ -202,10 +197,15 @@ class HitomiNozomi(
|
|||
}.toSingle()
|
||||
}
|
||||
|
||||
fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String): Single<List<Int>> {
|
||||
var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$tag-$language$NOZOMI_EXTENSION"
|
||||
fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String, popular: Boolean): Single<List<Int>> {
|
||||
val replacedTag = tag.replace('_', ' ')
|
||||
var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$replacedTag-$language$NOZOMI_EXTENSION"
|
||||
if (area != null) {
|
||||
nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$tag-$language$NOZOMI_EXTENSION"
|
||||
nozomiAddress = if (popular) {
|
||||
"$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/popular/$replacedTag-$language$NOZOMI_EXTENSION"
|
||||
} else {
|
||||
"$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$replacedTag-$language$NOZOMI_EXTENSION"
|
||||
}
|
||||
}
|
||||
|
||||
return client.newCall(
|
||||
|
|
|
|||
Loading…
Reference in New Issue