Draff
/
Tachiyomi-Extensions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[hitomi.la] Update image url path logic and implement filter (#7654) 

* [hitomi.la] Update image url path logic and implement filter

[FIX] Update image url path logic

[FEAT] Add Filter for searching with popularity support

[FIX] Replace the used json library with kotlinx.serialization

[FIX] Deeplink now support all type and can actually display result

[FEAT] Add common word detection to avoid large search result

* Fix HitomiActivity query pattern

* [hitomi.la] Update image url path logic and implement filter

[FIX] Update image url path logic

[FEAT] Add Filter for searching with popularity support

[FIX] Replace the used json library with kotlinx.serialization

[FIX] Deeplink now support all type and can actually display result

[FEAT] Add common word detection to avoid large search result

* Fix HitomiActivity query pattern

* Update extVersionCode
This commit is contained in:
vulpes310 2021-06-14 10:43:17 -07:00 committed by GitHub
parent 787767942b
commit 00547b5413
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 194 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/all/hitomi/AndroidManifest.xml
View File

@ -13,10 +13,22 @@
<category android:name="android.intent.category.DEFAULT" />
<category android:name="android.intent.category.BROWSABLE" />
<data
android:host="hitomi.la"
android:pathPattern="/manga/..*"
android:scheme="https" />
<data
android:host="hitomi.la"
android:pathPattern="/doujinshi/..*"
android:scheme="https" />
<data
android:host="hitomi.la"
android:pathPattern="/cg/..*"
android:scheme="https" />
<data
android:host="hitomi.la"
android:pathPattern="/gamecg/..*"
android:scheme="https" />
</intent-filter>
</activity>
</application>

2
src/all/hitomi/build.gradle
View File

@ -6,7 +6,7 @@ ext {
extName = 'Hitomi.la'
pkgNameSuffix = 'all.hitomi'
extClass = '.HitomiFactory'
extVersionCode = 6
extVersionCode = 7
libVersion = '1.2'
containsNsfw = true
}

216
src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt
View File

@ -5,6 +5,7 @@ import android.content.SharedPreferences
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.ConfigurableSource
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
@ -12,13 +13,10 @@ import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
import rx.Observable
import rx.Single
import rx.schedulers.Schedulers
@ -90,8 +88,6 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
) { it.map { m -> m as SManga } }
}
private fun Document.selectFirst(selector: String) = this.select(selector).first()
private fun parseGalleryBlock(response: Response): SManga {
val doc = response.asJsoup()
return SManga.create().apply {
@ -160,56 +156,134 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
return if (query.startsWith(PREFIX_ID_SEARCH)) {
val id = query.removePrefix(PREFIX_ID_SEARCH)
client.newCall(GET("$baseUrl/cg/$id", headers)).asObservableSuccess()
.map { MangasPage(listOf(mangaDetailsParse(it).apply { url = "/cg/$id" }), false) }
} else {
val splitQuery = query.toLowerCase(Locale.ENGLISH).split(" ")
val positive = splitQuery.filter { !it.startsWith('-') }.toMutableList()
if (nozomiLang != "all") positive += "language:$nozomiLang"
val negative = (splitQuery - positive).map { it.removePrefix("-") }
// TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
.map { HitomiNozomi(client, it.first, it.second) }
var base = if (positive.isEmpty()) {
hn.flatMap { n -> n.getGalleryIdsFromNozomi(null, "index", "all").map { n to it.toSet() } }
} else {
val q = positive.removeAt(0)
hn.flatMap { n -> n.getGalleryIdsForQuery(q).map { n to it.toSet() } }
}
base = positive.fold(base) { acc, q ->
acc.flatMap { (nozomi, mangas) ->
nozomi.getGalleryIdsForQuery(q).map {
nozomi to mangas.intersect(it)
}
}
}
base = negative.fold(base) { acc, q ->
acc.flatMap { (nozomi, mangas) ->
nozomi.getGalleryIdsForQuery(q).map {
nozomi to (mangas - it)
}
}
}
base.flatMap { (_, ids) ->
val chunks = ids.chunked(PAGE_SIZE)
nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
MangasPage(mangas, page < chunks.size)
}
val id = NOZOMI_ID_SIMP_PATTERN.find(
NOZOMI_ID_PATTERN
.find(query.removePrefix(PREFIX_ID_SEARCH))!!.value
)!!.value.toInt()
nozomiIdsToMangas(listOf(id)).map { mangas ->
MangasPage(mangas, false)
}.toObservable()
} else {
if (query.isBlank()) {
val area = filters.filterIsInstance<TypeFilter>()
.joinToString("") {
(it as UriPartFilter).toUriPart()
}
val keyword = filters.filterIsInstance<Text>().toString()
.replace("[", "").replace("]", "")
val popular = filters.filterIsInstance<SortFilter>()
.joinToString("") {
(it as UriPartFilter).toUriPart()
} == "true"
// TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
.map { HitomiNozomi(client, it.first, it.second) }
val base = hn.flatMap { n ->
n.getGalleryIdsForQuery("$area:$keyword", nozomiLang, popular).map { n to it.toSet() }
}
base.flatMap { (_, ids) ->
val chunks = ids.chunked(PAGE_SIZE)
nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
MangasPage(mangas, page < chunks.size)
}
}.toObservable()
} else {
val splitQuery = query.toLowerCase(Locale.ENGLISH).split(" ")
val positive = splitQuery.filter {
COMMON_WORDS.any { word ->
it !== word
} && !it.startsWith('-')
}.toMutableList()
if (nozomiLang != "all") positive += "language:$nozomiLang"
val negative = (splitQuery - positive).map { it.removePrefix("-") }
// TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
.map { HitomiNozomi(client, it.first, it.second) }
var base = if (positive.isEmpty()) {
hn.flatMap { n ->
n.getGalleryIdsFromNozomi(null, "index", "all", false)
.map { n to it.toSet() }
}
} else {
val q = positive.removeAt(0)
hn.flatMap { n -> n.getGalleryIdsForQuery(q, nozomiLang, false).map { n to it.toSet() } }
}
base = positive.fold(base) { acc, q ->
acc.flatMap { (nozomi, mangas) ->
nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map {
nozomi to mangas.intersect(it)
}
}
}
base = negative.fold(base) { acc, q ->
acc.flatMap { (nozomi, mangas) ->
nozomi.getGalleryIdsForQuery(q, nozomiLang, false).map {
nozomi to (mangas - it)
}
}
}
base.flatMap { (_, ids) ->
val chunks = ids.chunked(PAGE_SIZE)
nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
MangasPage(mangas, page < chunks.size)
}
}.toObservable()
}
}
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Not used")
override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Not used")
// Filter
override fun getFilterList() = FilterList(
Filter.Header(Filter_SEARCH_MESSAGE),
Filter.Separator(),
SortFilter(),
TypeFilter(),
Text("Keyword")
)
private class TypeFilter : UriPartFilter(
"category",
Array(FILTER_CATEGORIES.size) { i ->
val category = FILTER_CATEGORIES[i]
Pair(category, category)
}
)
private class SortFilter : UriPartFilter(
"Ordered by",
arrayOf(
Pair("Date Added", "false"),
Pair("Popularity", "true")
)
)
private open class UriPartFilter(
displayName: String,
val pair: Array<Pair<String, String>>,
defaultState: Int = 0
) : Filter.Select<String>(displayName, pair.map { it.first }.toTypedArray(), defaultState) {
open fun toUriPart() = pair[state].second
}
private class Text(name: String) : Filter.Text(name) {
override fun toString(): String {
return state
}
}
// Details
override fun mangaDetailsParse(response: Response): SManga {
@ -217,6 +291,7 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
fun String.replaceSpaces() = this.replace(" ", "_")
return SManga.create().apply {
title = document.select("div.gallery h1 a").joinToString { it.text() }
thumbnail_url = document.select("div.cover img").attr("abs:src")
author = document.select("div.gallery h2 a").joinToString { it.text() }
val tableInfo = document.select("table tr")
@ -275,35 +350,30 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
}
override fun pageListParse(response: Response): List<Page> {
val jsonRaw = response.body!!.string().removePrefix("var galleryinfo = ")
val jsonResult = json.parseToJsonElement(jsonRaw).jsonObject
return jsonResult["files"]!!.jsonArray.mapIndexed { i, jsonEl ->
val jsonObj = jsonEl.jsonObject
val hash = jsonObj["hash"]!!.jsonPrimitive.content
val hasWebp = jsonObj["haswebp"]!!.jsonPrimitive.content == "0"
val hasAvif = jsonObj["hasavif"]!!.jsonPrimitive.content == "0"
val ext = if (hasWebp || !hitomiAlwaysWebp())
jsonObj["name"]!!.jsonPrimitive.content.substringAfterLast(".") else "webp"
val path = if (hasWebp || !hitomiAlwaysWebp())
"images" else "webp"
val str = response.body!!.string()
val json = json.decodeFromString<HitomiChapterDto>(str.removePrefix("var galleryinfo = "))
return json.files.mapIndexed { i, jsonElement ->
val hash = jsonElement.hash
val ext = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) jsonElement.name.split('.').last() else "webp"
val path = if (jsonElement.haswebp == 0 || !hitomiAlwaysWebp()) "images" else "webp"
val hashPath1 = hash.takeLast(1)
val hashPath2 = hash.takeLast(3).take(2)
// https://ltn.hitomi.la/reader.js
// function make_image_element()
val secondSubdomain = if (hasWebp && hasAvif) "b" else "a"
val secondSubdomain = if (jsonElement.haswebp == 0 && jsonElement.hasavif == 0) "b" else "a"
Page(i, "", "https://${firstSubdomainFromGalleryId(hashPath2)}$secondSubdomain.hitomi.la/$path/$hashPath1/$hashPath2/$hash.$ext")
}
}
// https://ltn.hitomi.la/common.js
// function subdomain_from_url()
// Change g's if statment from !isNaN(g)
private fun firstSubdomainFromGalleryId(pathSegment: String): Char {
var numberOfFrontends = 3
var g = pathSegment.toInt(16)
if (g < 0x30) numberOfFrontends = 2
if (g < 0x09) g = 1
if (g < 0x80) numberOfFrontends = 2
if (g < 0x59) g = 1
return (97 + g.rem(numberOfFrontends)).toChar()
}
@ -325,11 +395,27 @@ open class Hitomi(override val lang: String, private val nozomiLang: String) : H
private const val PAGE_SIZE = 25
const val PREFIX_ID_SEARCH = "id:"
val NOZOMI_ID_PATTERN = "[0-9]*.html".toRegex()
val NOZOMI_ID_SIMP_PATTERN = "[0-9]*".toRegex()
// Common English words and Japanese particles
private val COMMON_WORDS = listOf(
"a", "be", "boy", "de", "girl", "ga", "i", "is", "ka", "na",
"ni", "ne", "no", "suru", "to", "wa", "wo", "yo",
"b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
)
// From HitomiSearchMetaData
const val LTN_BASE_URL = "https://ltn.hitomi.la"
const val BASE_URL = "https://hitomi.la"
// Filter
private val FILTER_CATEGORIES = listOf(
"tag", "male", "female", "type",
"artist", "series", "character", "group"
)
private const val Filter_SEARCH_MESSAGE = "NOTE: Ignored if using text search!"
// Preferences
private const val WEBP_PREF_KEY = "HITOMI_WEBP"
private const val WEBP_PREF_TITLE = "Webp pages"

16
src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt Normal file
View File

@ -0,0 +1,16 @@
package eu.kanade.tachiyomi.extension.all.hitomi
import kotlinx.serialization.Serializable
@Serializable
data class HitomiChapterDto(
val files: List<HitomiFileDto> = emptyList(),
)
@Serializable
data class HitomiFileDto(
val name: String,
val hasavif: Int,
val hash: String,
val haswebp: Int,
)

28
src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt
View File

@ -29,26 +29,21 @@ class HitomiNozomi(
private val tagIndexVersion: Long,
private val galleriesIndexVersion: Long
) {
fun getGalleryIdsForQuery(query: String): Single<List<Int>> {
val replacedQuery = query.replace('_', ' ')
if (':' in replacedQuery) {
val sides = replacedQuery.split(':')
fun getGalleryIdsForQuery(query: String, language: String, popular: Boolean): Single<List<Int>> {
if (':' in query) {
val sides = query.split(':')
val namespace = sides[0]
var tag = sides[1]
var area: String? = namespace
var language = "all"
if (namespace == "female" || namespace == "male") {
area = "tag"
tag = replacedQuery
tag = query
} else if (namespace == "language") {
area = null
language = tag
tag = "index"
return getGalleryIdsFromNozomi(null, "index", tag, popular)
}
return getGalleryIdsFromNozomi(area, tag, language)
return getGalleryIdsFromNozomi(area, tag, language, popular)
}
val key = hashTerm(query)
@ -202,10 +197,15 @@ class HitomiNozomi(
}.toSingle()
}
fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String): Single<List<Int>> {
var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$tag-$language$NOZOMI_EXTENSION"
fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String, popular: Boolean): Single<List<Int>> {
val replacedTag = tag.replace('_', ' ')
var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$replacedTag-$language$NOZOMI_EXTENSION"
if (area != null) {
nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$tag-$language$NOZOMI_EXTENSION"
nozomiAddress = if (popular) {
"$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/popular/$replacedTag-$language$NOZOMI_EXTENSION"
} else {
"$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$replacedTag-$language$NOZOMI_EXTENSION"
}
}
return client.newCall(