diff --git a/src/all/hitomi/AndroidManifest.xml b/src/all/hitomi/AndroidManifest.xml new file mode 100644 index 000000000..8072ee00d --- /dev/null +++ b/src/all/hitomi/AndroidManifest.xml @@ -0,0 +1,2 @@ + + diff --git a/src/all/hitomi/build.gradle b/src/all/hitomi/build.gradle new file mode 100644 index 000000000..c32dfc2a7 --- /dev/null +++ b/src/all/hitomi/build.gradle @@ -0,0 +1,8 @@ +ext { + extName = 'Hitomi' + extClass = '.HitomiFactory' + extVersionCode = 25 + isNsfw = true +} + +apply from: "$rootDir/common.gradle" diff --git a/src/all/hitomi/res/mipmap-hdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 000000000..4c66ae35d Binary files /dev/null and b/src/all/hitomi/res/mipmap-hdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-mdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 000000000..87efcf752 Binary files /dev/null and b/src/all/hitomi/res/mipmap-mdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-xhdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 000000000..6be7a86f6 Binary files /dev/null and b/src/all/hitomi/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-xxhdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 000000000..6a56665d6 Binary files /dev/null and b/src/all/hitomi/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-xxxhdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 000000000..55a36d806 Binary files /dev/null and b/src/all/hitomi/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt new file mode 100644 index 000000000..27c4a26d9 --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt @@ -0,0 +1,615 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.await +import eu.kanade.tachiyomi.source.model.Filter +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.MangasPage +import eu.kanade.tachiyomi.source.model.Page +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.model.UpdateStrategy +import eu.kanade.tachiyomi.source.online.HttpSource +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json +import okhttp3.Call +import okhttp3.Request +import okhttp3.Response +import rx.Observable +import uy.kohesive.injekt.injectLazy +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import java.text.SimpleDateFormat +import java.util.LinkedList +import java.util.Locale +import kotlin.math.min + +@OptIn(ExperimentalUnsignedTypes::class) +class Hitomi( + override val lang: String, + private val nozomiLang: String, +) : HttpSource() { + + override val name = "Hitomi" + + private val domain = "hitomi.la" + + override val baseUrl = "https://$domain" + + private val ltnUrl = "https://ltn.$domain" + + override val supportsLatest = true + + private val json: Json by injectLazy() + + override val client = network.cloudflareClient + + override fun headersBuilder() = super.headersBuilder() + .set("referer", "$baseUrl/") + .set("origin", baseUrl) + + override fun fetchPopularManga(page: Int): Observable = Observable.fromCallable { + runBlocking { getPopularManga(page) } + } + + private suspend fun getPopularManga(page: Int): MangasPage { + val entries = getGalleryIDsFromNozomi("popular", "today", nozomiLang, page.nextPageRange()) + .toMangaList() + + return MangasPage(entries, entries.size >= 24) + } + + override fun fetchLatestUpdates(page: Int): Observable = Observable.fromCallable { + runBlocking { getLatestUpdates(page) } + } + + private suspend fun getLatestUpdates(page: Int): MangasPage { + val entries = getGalleryIDsFromNozomi(null, "index", nozomiLang, page.nextPageRange()) + .toMangaList() + + return MangasPage(entries, entries.size >= 24) + } + + private lateinit var searchResponse: List + + override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable = Observable.fromCallable { + runBlocking { getSearchManga(page, query, filters) } + } + + private suspend fun getSearchManga(page: Int, query: String, filters: FilterList): MangasPage { + if (page == 1) { + searchResponse = hitomiSearch( + query.trim(), + filters.filterIsInstance().firstOrNull()?.state == 0, + nozomiLang, + ).toList() + } + + val end = min(page * 25, searchResponse.size) + val entries = searchResponse.subList((page - 1) * 25, end) + .toMangaList() + + return MangasPage(entries, end != searchResponse.size) + } + + private class SortFilter : Filter.Select("Sort By", arrayOf("Popularity", "Updated")) + + override fun getFilterList(): FilterList { + return FilterList(SortFilter()) + } + + private fun Int.nextPageRange(): LongRange { + val byteOffset = ((this - 1) * 25) * 4L + return byteOffset.until(byteOffset + 100) + } + + private suspend fun getRangedResponse(url: String, range: LongRange?): ByteArray { + val rangeHeaders = when (range) { + null -> headers + else -> headersBuilder() + .set("Range", "bytes=${range.first}-${range.last}") + .build() + } + + return client.newCall(GET(url, rangeHeaders)).awaitSuccess().use { it.body.bytes() } + } + + private suspend fun hitomiSearch( + query: String, + sortByPopularity: Boolean = false, + language: String = "all", + ): Set = + coroutineScope { + val terms = query + .trim() + .replace(Regex("""^\?"""), "") + .lowercase() + .split(Regex("\\s+")) + .map { + it.replace('_', ' ') + } + + val positiveTerms = LinkedList() + val negativeTerms = LinkedList() + + for (term in terms) { + if (term.startsWith("-")) { + negativeTerms.push(term.removePrefix("-")) + } else if (term.isNotBlank()) { + positiveTerms.push(term) + } + } + + val positiveResults = positiveTerms.map { + async { + runCatching { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val negativeResults = negativeTerms.map { + async { + runCatching { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val results = when { + sortByPopularity -> getGalleryIDsFromNozomi(null, "popular", language) + positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) + else -> emptySet() + }.toMutableSet() + + fun filterPositive(newResults: Set) { + when { + results.isEmpty() -> results.addAll(newResults) + else -> results.retainAll(newResults) + } + } + + fun filterNegative(newResults: Set) { + results.removeAll(newResults) + } + + // positive results + positiveResults.forEach { + filterPositive(it.await()) + } + + // negative results + negativeResults.forEach { + filterNegative(it.await()) + } + + results + } + + // search.js + private suspend fun getGalleryIDsForQuery( + query: String, + language: String = "all", + ): Set { + query.replace("_", " ").let { + if (it.indexOf(':') > -1) { + val sides = it.split(":") + val ns = sides[0] + var tag = sides[1] + + var area: String? = ns + var lang = language + when (ns) { + "female", "male" -> { + area = "tag" + tag = it + } + + "language" -> { + area = null + lang = tag + tag = "index" + } + } + + return getGalleryIDsFromNozomi(area, tag, lang) + } + + val key = hashTerm(it) + val node = getGalleryNodeAtAddress(0) + val data = bSearch(key, node) ?: return emptySet() + + return getGalleryIDsFromData(data) + } + } + + private suspend fun getGalleryIDsFromData(data: Pair): Set { + val url = "$ltnUrl/galleriesindex/galleries.$galleriesIndexVersion.data" + val (offset, length) = data + require(length in 1..100000000) { + "Length $length is too long" + } + + val inbuf = getRangedResponse(url, offset.until(offset + length)) + + val galleryIDs = mutableSetOf() + + val buffer = + ByteBuffer + .wrap(inbuf) + .order(ByteOrder.BIG_ENDIAN) + + val numberOfGalleryIDs = buffer.int + + val expectedLength = numberOfGalleryIDs * 4 + 4 + + require(numberOfGalleryIDs in 1..10000000) { + "number_of_galleryids $numberOfGalleryIDs is too long" + } + require(inbuf.size == expectedLength) { + "inbuf.byteLength ${inbuf.size} != expected_length $expectedLength" + } + + for (i in 0.until(numberOfGalleryIDs)) + galleryIDs.add(buffer.int) + + return galleryIDs + } + + private tailrec suspend fun bSearch( + key: UByteArray, + node: Node, + ): Pair? { + fun compareArrayBuffers( + dv1: UByteArray, + dv2: UByteArray, + ): Int { + val top = min(dv1.size, dv2.size) + + for (i in 0.until(top)) { + if (dv1[i] < dv2[i]) { + return -1 + } else if (dv1[i] > dv2[i]) { + return 1 + } + } + + return 0 + } + + fun locateKey( + key: UByteArray, + node: Node, + ): Pair { + for (i in node.keys.indices) { + val cmpResult = compareArrayBuffers(key, node.keys[i]) + + if (cmpResult <= 0) { + return Pair(cmpResult == 0, i) + } + } + + return Pair(false, node.keys.size) + } + + fun isLeaf(node: Node): Boolean { + for (subnode in node.subNodeAddresses) + if (subnode != 0L) { + return false + } + + return true + } + + if (node.keys.isEmpty()) { + return null + } + + val (there, where) = locateKey(key, node) + if (there) { + return node.datas[where] + } else if (isLeaf(node)) { + return null + } + + val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where]) + return bSearch(key, nextNode) + } + + private suspend fun getGalleryIDsFromNozomi( + area: String?, + tag: String, + language: String, + range: LongRange? = null, + ): Set { + val nozomiAddress = when (area) { + null -> "$ltnUrl/$tag-$language.nozomi" + else -> "$ltnUrl/$area/$tag-$language.nozomi" + } + + val bytes = getRangedResponse(nozomiAddress, range) + val nozomi = mutableSetOf() + + val arrayBuffer = ByteBuffer + .wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + + while (arrayBuffer.hasRemaining()) + nozomi.add(arrayBuffer.int) + + return nozomi + } + + private val galleriesIndexVersion by lazy { + client.newCall( + GET("$ltnUrl/galleriesindex/version?_=${System.currentTimeMillis()}", headers), + ).execute().use { it.body.string() } + } + + private data class Node( + val keys: List, + val datas: List>, + val subNodeAddresses: List, + ) + + private fun decodeNode(data: ByteArray): Node { + val buffer = ByteBuffer + .wrap(data) + .order(ByteOrder.BIG_ENDIAN) + + val uData = data.toUByteArray() + + val numberOfKeys = buffer.int + val keys = ArrayList() + + for (i in 0.until(numberOfKeys)) { + val keySize = buffer.int + + if (keySize == 0 || keySize > 32) { + throw Exception("fatal: !keySize || keySize > 32") + } + + keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize))) + buffer.position(buffer.position() + keySize) + } + + val numberOfDatas = buffer.int + val datas = ArrayList>() + + for (i in 0.until(numberOfDatas)) { + val offset = buffer.long + val length = buffer.int + + datas.add(Pair(offset, length)) + } + + val numberOfSubNodeAddresses = 16 + 1 + val subNodeAddresses = ArrayList() + + for (i in 0.until(numberOfSubNodeAddresses)) { + val subNodeAddress = buffer.long + subNodeAddresses.add(subNodeAddress) + } + + return Node(keys, datas, subNodeAddresses) + } + + private suspend fun getGalleryNodeAtAddress(address: Long): Node { + val url = "$ltnUrl/galleriesindex/galleries.$galleriesIndexVersion.index" + + val nodedata = getRangedResponse(url, address.until(address + 464)) + + return decodeNode(nodedata) + } + + private fun hashTerm(term: String): UByteArray { + return sha256(term.toByteArray()).copyOfRange(0, 4).toUByteArray() + } + + private fun sha256(data: ByteArray): ByteArray { + return MessageDigest.getInstance("SHA-256").digest(data) + } + + private suspend fun Collection.toMangaList() = coroutineScope { + map { id -> + async { + runCatching { + client.newCall(GET("$ltnUrl/galleries/$id.js", headers)) + .awaitSuccess() + .parseScriptAs() + .toSManga() + }.getOrNull() + } + }.awaitAll().filterNotNull() + } + + private suspend fun Gallery.toSManga() = SManga.create().apply { + title = this@toSManga.title + url = galleryurl + author = groups?.joinToString { it.formatted } + artist = artists?.joinToString { it.formatted } + genre = tags?.joinToString { it.formatted } + thumbnail_url = files.first().let { + val hash = it.hash + val imageId = imageIdFromHash(hash) + val subDomain = 'a' + subdomainOffset(imageId) + + "https://${subDomain}tn.$domain/webpbigtn/${thumbPathFromHash(hash)}/$hash.webp" + } + description = buildString { + characters?.joinToString { it.formatted }?.let { + append("Characters: ", it, "\n") + } + parodys?.joinToString { it.formatted }?.let { + append("Parodies: ", it, "\n") + } + } + status = SManga.COMPLETED + update_strategy = UpdateStrategy.ONLY_FETCH_ONCE + initialized = true + } + + override fun mangaDetailsRequest(manga: SManga): Request { + val id = manga.url + .substringAfterLast("-") + .substringBefore(".") + + return GET("$ltnUrl/galleries/$id.js", headers) + } + + override fun mangaDetailsParse(response: Response): SManga { + return response.parseScriptAs().let { + runBlocking { it.toSManga() } + } + } + + override fun getMangaUrl(manga: SManga) = baseUrl + manga.url + + override fun chapterListRequest(manga: SManga): Request { + val id = manga.url + .substringAfterLast("-") + .substringBefore(".") + + return GET("$ltnUrl/galleries/$id.js#${manga.url}", headers) + } + + override fun chapterListParse(response: Response): List { + val gallery = response.parseScriptAs() + val mangaUrl = response.request.url.fragment!! + + return listOf( + SChapter.create().apply { + name = "Chapter" + url = mangaUrl + scanlator = gallery.type + date_upload = runCatching { + dateFormat.parse(gallery.date.substringBeforeLast("-"))!!.time + }.getOrDefault(0L) + }, + ) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) + + override fun getChapterUrl(chapter: SChapter) = baseUrl + chapter.url + + override fun pageListRequest(chapter: SChapter): Request { + val id = chapter.url + .substringAfterLast("-") + .substringBefore(".") + + return GET("$ltnUrl/galleries/$id.js", headers) + } + + override fun pageListParse(response: Response): List { + val gallery = response.parseScriptAs() + + return gallery.files.mapIndexed { idx, img -> + runBlocking { + val hash = img.hash + val commonId = commonImageId() + val imageId = imageIdFromHash(hash) + val subDomain = 'a' + subdomainOffset(imageId) + + Page( + idx, + "$baseUrl/reader/$id.html", + "https://${subDomain}a.$domain/webp/$commonId$imageId/$hash.webp", + ) + } + } + } + + override fun imageRequest(page: Page): Request { + val imageHeaders = headersBuilder() + .set("Accept", "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8") + .set("Referer", page.url) + .build() + + return GET(page.imageUrl!!, imageHeaders) + } + + private inline fun Response.parseScriptAs(): T = + parseAs { it.substringAfter("var galleryinfo = ") } + + private inline fun Response.parseAs(transform: (String) -> String = { body -> body }): T { + val body = use { it.body.string() } + val transformed = transform(body) + + return json.decodeFromString(transformed) + } + + private suspend fun Call.awaitSuccess() = + await().also { + require(it.isSuccessful) { + it.close() + "HTTP error ${it.code}" + } + } + + // ------------------ gg.js ------------------ + private var scriptLastRetrieval: Long? = null + private val mutex = Mutex() + private var subdomainOffsetDefault = 0 + private val subdomainOffsetMap = mutableMapOf() + private var commonImageId = "" + + private suspend fun refreshScript() = mutex.withLock { + if (scriptLastRetrieval == null || (scriptLastRetrieval!! + 60000) < System.currentTimeMillis()) { + val ggScript = client.newCall( + GET("$ltnUrl/gg.js?_=${System.currentTimeMillis()}", headers), + ).awaitSuccess().use { it.body.string() } + + subdomainOffsetDefault = Regex("var o = (\\d)").find(ggScript)!!.groupValues[1].toInt() + val o = Regex("o = (\\d); break;").find(ggScript)!!.groupValues[1].toInt() + + subdomainOffsetMap.clear() + Regex("case (\\d+):").findAll(ggScript).forEach { + val case = it.groupValues[1].toInt() + subdomainOffsetMap[case] = o + } + + commonImageId = Regex("b: '(.+)'").find(ggScript)!!.groupValues[1] + + scriptLastRetrieval = System.currentTimeMillis() + } + } + + // m <-- gg.js + private suspend fun subdomainOffset(imageId: Int): Int { + refreshScript() + return subdomainOffsetMap[imageId] ?: subdomainOffsetDefault + } + + // b <-- gg.js + private suspend fun commonImageId(): String { + refreshScript() + return commonImageId + } + + // s <-- gg.js + private fun imageIdFromHash(hash: String): Int { + val match = Regex("(..)(.)$").find(hash) + return match!!.groupValues.let { it[2] + it[1] }.toInt(16) + } + + // real_full_path_from_hash <-- common.js + private fun thumbPathFromHash(hash: String): String { + return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1") + } + + override fun popularMangaParse(response: Response) = throw UnsupportedOperationException() + override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException() + override fun latestUpdatesRequest(page: Int) = throw UnsupportedOperationException() + override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException() + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException() + override fun searchMangaParse(response: Response) = throw UnsupportedOperationException() + override fun imageUrlParse(response: Response) = throw UnsupportedOperationException() +} diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt new file mode 100644 index 000000000..0e5a6d5be --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiDto.kt @@ -0,0 +1,82 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonPrimitive + +@Serializable +data class Gallery( + val galleryurl: String, + val title: String, + val date: String, + val type: String, + val tags: List?, + val artists: List?, + val groups: List?, + val characters: List?, + val parodys: List?, + val files: List, +) + +@Serializable +data class ImageFile( + val hash: String, +) + +@Serializable +data class Tag( + val female: JsonPrimitive?, + val male: JsonPrimitive?, + val tag: String, +) { + val formatted get() = if (female?.content == "1") { + "${tag.toCamelCase()} (Female)" + } else if (male?.content == "1") { + "${tag.toCamelCase()} (Male)" + } else { + tag.toCamelCase() + } +} + +@Serializable +data class Artist( + val artist: String, +) { + val formatted get() = artist.toCamelCase() +} + +@Serializable +data class Group( + val group: String, +) { + val formatted get() = group.toCamelCase() +} + +@Serializable +data class Character( + val character: String, +) { + val formatted get() = character.toCamelCase() +} + +@Serializable +data class Parody( + val parody: String, +) { + val formatted get() = parody.toCamelCase() +} + +private fun String.toCamelCase(): String { + val result = StringBuilder(length) + var capitalize = true + for (char in this) { + result.append( + if (capitalize) { + char.uppercase() + } else { + char.lowercase() + }, + ) + capitalize = char.isWhitespace() + } + return result.toString() +} diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiFactory.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiFactory.kt new file mode 100644 index 000000000..fd3667021 --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiFactory.kt @@ -0,0 +1,34 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import eu.kanade.tachiyomi.source.SourceFactory + +class HitomiFactory : SourceFactory { + override fun createSources() = listOf( + Hitomi("all", "all"), + Hitomi("en", "english"), + Hitomi("id", "indonesian"), + Hitomi("jv", "javanese"), + Hitomi("ca", "catalan"), + Hitomi("ceb", "cebuano"), + Hitomi("cs", "czech"), + Hitomi("da", "danish"), + Hitomi("de", "german"), + Hitomi("et", "estonian"), + Hitomi("es", "spanish"), + Hitomi("eo", "esperanto"), + Hitomi("fr", "french"), + Hitomi("it", "italian"), + Hitomi("hi", "hindi"), + Hitomi("hu", "hungarian"), + Hitomi("pl", "polish"), + Hitomi("pt", "portuguese"), + Hitomi("vi", "vietnamese"), + Hitomi("tr", "turkish"), + Hitomi("ru", "russian"), + Hitomi("uk", "ukrainian"), + Hitomi("ar", "arabic"), + Hitomi("ko", "korean"), + Hitomi("zh", "chinese"), + Hitomi("ja", "japanese"), + ) +}