diff --git a/src/all/hitomi/build.gradle b/src/all/hitomi/build.gradle new file mode 100644 index 000000000..66828b9c2 --- /dev/null +++ b/src/all/hitomi/build.gradle @@ -0,0 +1,13 @@ +apply plugin: 'com.android.application' +apply plugin: 'kotlin-android' + +ext { + extName = 'Hitomi.la' + pkgNameSuffix = 'all.hitomi' + extClass = '.HitomiFactory' + extVersionCode = 1 + libVersion = '1.2' + containsNsfw = true +} + +apply from: "$rootDir/common.gradle" diff --git a/src/all/hitomi/res/mipmap-hdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 000000000..a534633b8 Binary files /dev/null and b/src/all/hitomi/res/mipmap-hdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-mdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 000000000..5c6f3b8f8 Binary files /dev/null and b/src/all/hitomi/res/mipmap-mdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-xhdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 000000000..f2477f79a Binary files /dev/null and b/src/all/hitomi/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-xxhdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 000000000..48549d3f4 Binary files /dev/null and b/src/all/hitomi/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/mipmap-xxxhdpi/ic_launcher.png b/src/all/hitomi/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 000000000..97f74cc71 Binary files /dev/null and b/src/all/hitomi/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/src/all/hitomi/res/web_hi_res_512.png b/src/all/hitomi/res/web_hi_res_512.png new file mode 100644 index 000000000..a851e8798 Binary files /dev/null and b/src/all/hitomi/res/web_hi_res_512.png differ diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/ByteCursor.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/ByteCursor.kt new file mode 100644 index 000000000..1c6b03061 --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/ByteCursor.kt @@ -0,0 +1,93 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import java.nio.ByteBuffer + +/** + * Simple cursor for use on byte arrays + * @author nulldev + */ +class ByteCursor(val content: ByteArray) { + var index = -1 + private set + private var mark = -1 + + fun mark() { + mark = index + } + + fun jumpToMark() { + index = mark + } + + fun jumpToIndex(index: Int) { + this.index = index + } + + fun next(): Byte { + return content[++index] + } + + fun next(count: Int): ByteArray { + val res = content.sliceArray(index + 1..index + count) + skip(count) + return res + } + + // Used to perform conversions + private fun byteBuffer(count: Int): ByteBuffer { + return ByteBuffer.wrap(next(count)) + } + + // Epic hack to get an unsigned short properly... + fun fakeNextShortInt(): Int = ByteBuffer + .wrap(arrayOf(0x00, 0x00, *next(2).toTypedArray()).toByteArray()) + .getInt(0) + + // fun nextShort(): Short = byteBuffer(2).getShort(0) + fun nextInt(): Int = byteBuffer(4).getInt(0) + fun nextLong(): Long = byteBuffer(8).getLong(0) + fun nextFloat(): Float = byteBuffer(4).getFloat(0) + fun nextDouble(): Double = byteBuffer(8).getDouble(0) + + fun skip(count: Int) { + index += count + } + + fun expect(vararg bytes: Byte) { + if (bytes.size > remaining()) { + throw IllegalStateException("Unexpected end of content!") + } + + for (i in 0..bytes.lastIndex) { + val expected = bytes[i] + val actual = content[index + i + 1] + + if (expected != actual) { + throw IllegalStateException("Unexpected content (expected: $expected, actual: $actual)!") + } + } + + index += bytes.size + } + + fun checkEqual(vararg bytes: Byte): Boolean { + if (bytes.size > remaining()) { + return false + } + + for (i in 0..bytes.lastIndex) { + val expected = bytes[i] + val actual = content[index + i + 1] + + if (expected != actual) { + return false + } + } + + return true + } + + fun atEnd() = index >= content.size - 1 + + fun remaining() = content.size - index - 1 +} diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt new file mode 100644 index 000000000..64fe75f32 --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt @@ -0,0 +1,392 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import android.app.Application +import android.content.SharedPreferences +import android.support.v7.preference.CheckBoxPreference +import android.support.v7.preference.PreferenceScreen +import androidx.preference.CheckBoxPreference as AndroidXCheckBoxPreference +import androidx.preference.PreferenceScreen as AndroidXPreferenceScreen +import com.github.salomonbrys.kotson.array +import com.github.salomonbrys.kotson.get +import com.github.salomonbrys.kotson.string +import com.google.gson.JsonParser +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservableSuccess +import eu.kanade.tachiyomi.source.ConfigurableSource +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.MangasPage +import eu.kanade.tachiyomi.source.model.Page +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.online.HttpSource +import eu.kanade.tachiyomi.util.asJsoup +import okhttp3.Request +import okhttp3.Response +import org.jsoup.nodes.Document +import rx.Observable +import rx.Single +import rx.schedulers.Schedulers +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get + +/** + * Ported from TachiyomiSy + * Original work by NerdNumber9 for TachiyomiEH + */ + +open class Hitomi(override val lang: String, private val nozomiLang: String) : HttpSource(), ConfigurableSource { + + override val supportsLatest = true + + override val name = if (nozomiLang == "all") "Hitomi.la unfiltered" else "Hitomi.la" + + override val baseUrl = BASE_URL + + // Popular + + override fun fetchPopularManga(page: Int): Observable { + return client.newCall(popularMangaRequest(page)) + .asObservableSuccess() + .flatMap { responseToMangas(it) } + } + + override fun popularMangaRequest(page: Int) = HitomiNozomi.rangedGet( + "$LTN_BASE_URL/popular-$nozomiLang.nozomi", + 100L * (page - 1), + 99L + 100 * (page - 1) + ) + + private fun responseToMangas(response: Response): Observable { + val range = response.header("Content-Range")!! + val total = range.substringAfter('/').toLong() + val end = range.substringBefore('/').substringAfter('-').toLong() + val body = response.body()!! + return parseNozomiPage(body.bytes()) + .map { + MangasPage(it, end < total - 1) + } + } + + private fun parseNozomiPage(array: ByteArray): Observable> { + val cursor = ByteCursor(array) + val ids = (1..array.size / 4).map { + cursor.nextInt() + } + + return nozomiIdsToMangas(ids).toObservable() + } + + private fun nozomiIdsToMangas(ids: List): Single> { + return Single.zip( + ids.map { int -> + client.newCall(GET("$LTN_BASE_URL/galleryblock/$int.html")) + .asObservableSuccess() + .subscribeOn(Schedulers.io()) // Perform all these requests in parallel + .map { parseGalleryBlock(it) } + .toSingle() + } + ) { it.map { m -> m as SManga } } + } + + private fun Document.selectFirst(selector: String) = this.select(selector).first() + + private fun parseGalleryBlock(response: Response): SManga { + val doc = response.asJsoup() + return SManga.create().apply { + val titleElement = doc.selectFirst("h1") + title = titleElement.text() + thumbnail_url = "https:" + if (useHqThumbPref()) { + doc.selectFirst("img").attr("srcset").substringBefore(' ') + } else { + doc.selectFirst("img").attr("src") + } + url = titleElement.child(0).attr("href") + } + } + + override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Not used") + + // Latest + + override fun fetchLatestUpdates(page: Int): Observable { + return client.newCall(latestUpdatesRequest(page)) + .asObservableSuccess() + .flatMap { responseToMangas(it) } + } + + override fun latestUpdatesRequest(page: Int) = HitomiNozomi.rangedGet( + "$LTN_BASE_URL/index-$nozomiLang.nozomi", + 100L * (page - 1), + 99L + 100 * (page - 1) + ) + + override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException("Not used") + + // Search + + private var cachedTagIndexVersion: Long? = null + private var tagIndexVersionCacheTime: Long = 0 + private fun tagIndexVersion(): Single { + val sCachedTagIndexVersion = cachedTagIndexVersion + return if (sCachedTagIndexVersion == null || + tagIndexVersionCacheTime + INDEX_VERSION_CACHE_TIME_MS < System.currentTimeMillis() + ) { + HitomiNozomi.getIndexVersion(client, "tagindex").subscribeOn(Schedulers.io()).doOnNext { + cachedTagIndexVersion = it + tagIndexVersionCacheTime = System.currentTimeMillis() + }.toSingle() + } else { + Single.just(sCachedTagIndexVersion) + } + } + + private var cachedGalleryIndexVersion: Long? = null + private var galleryIndexVersionCacheTime: Long = 0 + private fun galleryIndexVersion(): Single { + val sCachedGalleryIndexVersion = cachedGalleryIndexVersion + return if (sCachedGalleryIndexVersion == null || + galleryIndexVersionCacheTime + INDEX_VERSION_CACHE_TIME_MS < System.currentTimeMillis() + ) { + HitomiNozomi.getIndexVersion(client, "galleriesindex").subscribeOn(Schedulers.io()).doOnNext { + cachedGalleryIndexVersion = it + galleryIndexVersionCacheTime = System.currentTimeMillis() + }.toSingle() + } else { + Single.just(sCachedGalleryIndexVersion) + } + } + + override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { + val splitQuery = query.split(" ") + + val positive = splitQuery.filter { !it.startsWith('-') }.toMutableList() + if (nozomiLang != "all") positive += "language:$nozomiLang" + val negative = (splitQuery - positive).map { it.removePrefix("-") } + + // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH) + val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv } + .map { HitomiNozomi(client, it.first, it.second) } + + var base = if (positive.isEmpty()) { + hn.flatMap { n -> n.getGalleryIdsFromNozomi(null, "index", "all").map { n to it.toSet() } } + } else { + val q = positive.removeAt(0) + hn.flatMap { n -> n.getGalleryIdsForQuery(q).map { n to it.toSet() } } + } + + base = positive.fold(base) { acc, q -> + acc.flatMap { (nozomi, mangas) -> + nozomi.getGalleryIdsForQuery(q).map { + nozomi to mangas.intersect(it) + } + } + } + + base = negative.fold(base) { acc, q -> + acc.flatMap { (nozomi, mangas) -> + nozomi.getGalleryIdsForQuery(q).map { + nozomi to (mangas - it) + } + } + } + + return base.flatMap { (_, ids) -> + val chunks = ids.chunked(PAGE_SIZE) + + nozomiIdsToMangas(chunks[page - 1]).map { mangas -> + MangasPage(mangas, page < chunks.size) + } + }.toObservable() + } + + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Not used") + override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Not used") + + // Details + + override fun mangaDetailsParse(response: Response): SManga { + val document = response.asJsoup() + fun String.replaceSpaces() = this.replace(" ", "_") + + return SManga.create().apply { + thumbnail_url = document.select("div.cover img").attr("abs:src") + author = document.select("div.gallery h2 a").joinToString { it.text() } + val tableInfo = document.select("table tr") + .map { tr -> + val key = tr.select("td:first-child").text() + val value = with(tr.select("td:last-child a")) { + when (key) { + "Series", "Characters" -> { + if (text().isNotEmpty()) + joinToString { "${attr("href").removePrefix("/").substringBefore("/")}:${it.text().replaceSpaces()}" } else null + } + "Tags" -> joinToString { element -> + element.text().let { + when { + it.contains("♀") -> "female:${it.substringBeforeLast(" ").replaceSpaces()}" + it.contains("♂") -> "male:${it.substringBeforeLast(" ").replaceSpaces()}" + else -> it + } + } + } + else -> joinToString { it.text() } + } + } + Pair(key, value) + } + .plus(Pair("Date uploaded", document.select("div.gallery span.date").text())) + .toMap() + description = tableInfo.filterNot { it.value.isNullOrEmpty() || it.key in listOf("Series", "Characters", "Tags") }.entries.joinToString("\n") { "${it.key}: ${it.value}" } + genre = listOfNotNull(tableInfo["Series"], tableInfo["Characters"], tableInfo["Tags"]).joinToString() + } + } + + // Chapters + + override fun fetchChapterList(manga: SManga): Observable> { + return Observable.just( + listOf( + SChapter.create().apply { + url = manga.url + name = "Chapter" + chapter_number = 0.0f + } + ) + ) + } + + override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Not used") + + // Pages + + private fun hlIdFromUrl(url: String) = + url.split('/').last().split('-').last().substringBeforeLast('.') + + override fun pageListRequest(chapter: SChapter): Request { + return GET("$LTN_BASE_URL/galleries/${hlIdFromUrl(chapter.url)}.js") + } + + private val jsonParser = JsonParser() + + override fun pageListParse(response: Response): List { + val str = response.body()!!.string() + val json = jsonParser.parse(str.removePrefix("var galleryinfo = ")) + return json["files"].array.mapIndexed { i, jsonElement -> + val hash = jsonElement["hash"].string + val ext = if (jsonElement["haswebp"].string == "0" || !hitomiAlwaysWebp()) jsonElement["name"].string.split('.').last() else "webp" + val path = if (jsonElement["haswebp"].string == "0" || !hitomiAlwaysWebp()) "images" else "webp" + val hashPath1 = hash.takeLast(1) + val hashPath2 = hash.takeLast(3).take(2) + Page(i, "", "https://${subdomainFromGalleryId(hashPath2)}a.hitomi.la/$path/$hashPath1/$hashPath2/$hash.$ext") + } + } + + // https://ltn.hitomi.la/common.js + private fun subdomainFromGalleryId(pathSegment: String): Char { + var numberOfFrontends = 3 + var g = pathSegment.toInt(16) + if (g < 0x30) numberOfFrontends = 2 + if (g < 0x09) g = 1 + + return (97 + g.rem(numberOfFrontends)).toChar() + } + + override fun imageRequest(page: Page): Request { + val request = super.imageRequest(page) + val hlId = request.url().pathSegments().let { + it[it.lastIndex - 1] + } + return request.newBuilder() + .header("Referer", "$BASE_URL/reader/$hlId.html") + .build() + } + + override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Not used") + + companion object { + private const val INDEX_VERSION_CACHE_TIME_MS = 1000 * 60 * 10 + private const val PAGE_SIZE = 25 + + // From HitomiSearchMetaData + const val LTN_BASE_URL = "https://ltn.hitomi.la" + const val BASE_URL = "https://hitomi.la" + + // Preferences + private const val WEBP_PREF_KEY = "HITOMI_WEBP" + private const val WEBP_PREF_TITLE = "Webp pages" + private const val WEBP_PREF_SUMMARY = "Download webp pages instead of jpeg (when available)" + private const val WEBP_PREF_DEFAULT_VALUE = true + + private const val COVER_PREF_KEY = "HITOMI_COVERS" + private const val COVER_PREF_TITLE = "Use HQ covers" + private const val COVER_PREF_SUMMARY = "See HQ covers while browsing" + private const val COVER_PREF_DEFAULT_VALUE = true + } + + // Preferences + + private val preferences: SharedPreferences by lazy { + Injekt.get().getSharedPreferences("source_$id", 0x0000) + } + + override fun setupPreferenceScreen(screen: PreferenceScreen) { + val webpPref = CheckBoxPreference(screen.context).apply { + key = "${WEBP_PREF_KEY}_$lang" + title = WEBP_PREF_TITLE + summary = WEBP_PREF_SUMMARY + setDefaultValue(WEBP_PREF_DEFAULT_VALUE) + + setOnPreferenceChangeListener { _, newValue -> + val checkValue = newValue as Boolean + preferences.edit().putBoolean("${WEBP_PREF_KEY}_$lang", checkValue).commit() + } + } + + val coverPref = CheckBoxPreference(screen.context).apply { + key = "${COVER_PREF_KEY}_$lang" + title = COVER_PREF_TITLE + summary = COVER_PREF_SUMMARY + setDefaultValue(COVER_PREF_DEFAULT_VALUE) + + setOnPreferenceChangeListener { _, newValue -> + val checkValue = newValue as Boolean + preferences.edit().putBoolean("${COVER_PREF_KEY}_$lang", checkValue).commit() + } + } + + screen.addPreference(webpPref) + screen.addPreference(coverPref) + } + + override fun setupPreferenceScreen(screen: AndroidXPreferenceScreen) { + val webpPref = AndroidXCheckBoxPreference(screen.context).apply { + key = "${WEBP_PREF_KEY}_$lang" + title = WEBP_PREF_TITLE + summary = WEBP_PREF_SUMMARY + setDefaultValue(WEBP_PREF_DEFAULT_VALUE) + + setOnPreferenceChangeListener { _, newValue -> + val checkValue = newValue as Boolean + preferences.edit().putBoolean("${WEBP_PREF_KEY}_$lang", checkValue).commit() + } + } + + val coverPref = AndroidXCheckBoxPreference(screen.context).apply { + key = "${COVER_PREF_KEY}_$lang" + title = COVER_PREF_TITLE + summary = COVER_PREF_SUMMARY + setDefaultValue(COVER_PREF_DEFAULT_VALUE) + + setOnPreferenceChangeListener { _, newValue -> + val checkValue = newValue as Boolean + preferences.edit().putBoolean("${COVER_PREF_KEY}_$lang", checkValue).commit() + } + } + + screen.addPreference(webpPref) + screen.addPreference(coverPref) + } + + private fun hitomiAlwaysWebp(): Boolean = preferences.getBoolean("${WEBP_PREF_KEY}_$lang", WEBP_PREF_DEFAULT_VALUE) + private fun useHqThumbPref(): Boolean = preferences.getBoolean("${COVER_PREF_KEY}_$lang", COVER_PREF_DEFAULT_VALUE) +} diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiFactory.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiFactory.kt new file mode 100644 index 000000000..e2703a990 --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiFactory.kt @@ -0,0 +1,57 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import eu.kanade.tachiyomi.annotations.Nsfw +import eu.kanade.tachiyomi.source.Source +import eu.kanade.tachiyomi.source.SourceFactory + +@Nsfw +class HitomiFactory : SourceFactory { + override fun createSources(): List = languageList + .filterNot { it.first.isEmpty() } + .map { Hitomi(it.first, it.second) } +} + +/** + * These should all be valid languages but I was too lazy to look up all the language codes + * Replace an empty string with a valid language code to enable that language + */ +private val languageList = listOf( + Pair("other", "all"), // all languages + Pair("id", "indonesian"), + Pair("", "catalan"), + Pair("", "cebuano"), + Pair("", "czech"), + Pair("", "danish"), + Pair("de", "german"), + Pair("", "estonian"), + Pair("en", "english"), + Pair("es", "spanish"), + Pair("", "esperanto"), + Pair("fr", "french"), + Pair("it", "italian"), + Pair("", "latin"), + Pair("", "hungarian"), + Pair("", "dutch"), + Pair("", "norwegian"), + Pair("pl", "polish"), + Pair("pt-BR", "portuguese"), + Pair("", "romanian"), + Pair("", "albanian"), + Pair("", "slovak"), + Pair("", "finnish"), + Pair("", "swedish"), + Pair("", "tagalog"), + Pair("vi", "vietnamese"), + Pair("tr", "turkish"), + Pair("", "greek"), + Pair("", "mongolian"), + Pair("ru", "russian"), + Pair("", "ukrainian"), + Pair("", "hebrew"), + Pair("ar", "arabic"), + Pair("", "persian"), + Pair("th", "thai"), + Pair("ko", "korean"), + Pair("zh", "chinese"), + Pair("ja", "japanese") +) diff --git a/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt new file mode 100644 index 000000000..b66923a03 --- /dev/null +++ b/src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt @@ -0,0 +1,257 @@ +package eu.kanade.tachiyomi.extension.all.hitomi + +import eu.kanade.tachiyomi.extension.all.hitomi.Hitomi.Companion.LTN_BASE_URL +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservable +import eu.kanade.tachiyomi.network.asObservableSuccess +import java.security.MessageDigest +import okhttp3.Headers +import okhttp3.OkHttpClient +import okhttp3.Request +import rx.Observable +import rx.Single + +private typealias HashedTerm = ByteArray + +private data class DataPair(val offset: Long, val length: Int) +private data class Node( + val keys: List, + val datas: List, + val subnodeAddresses: List +) + +/** + * Kotlin port of the hitomi.la search algorithm + * @author NerdNumber9 + */ +class HitomiNozomi( + private val client: OkHttpClient, + private val tagIndexVersion: Long, + private val galleriesIndexVersion: Long +) { + fun getGalleryIdsForQuery(query: String): Single> { + val replacedQuery = query.replace('_', ' ') + + if (':' in replacedQuery) { + val sides = replacedQuery.split(':') + val namespace = sides[0] + var tag = sides[1] + + var area: String? = namespace + var language = "all" + if (namespace == "female" || namespace == "male") { + area = "tag" + tag = replacedQuery + } else if (namespace == "language") { + area = null + language = tag + tag = "index" + } + + return getGalleryIdsFromNozomi(area, tag, language) + } + + val key = hashTerm(query) + val field = "galleries" + + return getNodeAtAddress(field, 0).flatMap { node -> + if (node == null) { + Single.just(null) + } else { + BSearch(field, key, node).flatMap { data -> + if (data == null) { + Single.just(null) + } else { + getGalleryIdsFromData(data) + } + } + } + } + } + + private fun getGalleryIdsFromData(data: DataPair?): Single> { + if (data == null) { + return Single.just(emptyList()) + } + + val url = "$LTN_BASE_URL/$GALLERIES_INDEX_DIR/galleries.$galleriesIndexVersion.data" + val (offset, length) = data + if (length > 100000000 || length <= 0) { + return Single.just(emptyList()) + } + + return client.newCall(rangedGet(url, offset, offset + length - 1)) + .asObservable() + .map { + it.body()?.bytes() ?: ByteArray(0) + } + .onErrorReturn { ByteArray(0) } + .map { inbuf -> + if (inbuf.isEmpty()) { + return@map emptyList() + } + + val view = ByteCursor(inbuf) + val numberOfGalleryIds = view.nextInt() + + val expectedLength = numberOfGalleryIds * 4 + 4 + + if (numberOfGalleryIds > 10000000 || + numberOfGalleryIds <= 0 || + inbuf.size != expectedLength + ) { + return@map emptyList() + } + + (1..numberOfGalleryIds).map { + view.nextInt() + } + }.toSingle() + } + + @Suppress("FunctionName") + private fun BSearch(field: String, key: ByteArray, node: Node?): Single { + fun compareByteArrays(dv1: ByteArray, dv2: ByteArray): Int { + val top = dv1.size.coerceAtMost(dv2.size) + for (i in 0 until top) { + val dv1i = dv1[i].toInt() and 0xFF + val dv2i = dv2[i].toInt() and 0xFF + if (dv1i < dv2i) { + return -1 + } else if (dv1i > dv2i) { + return 1 + } + } + return 0 + } + + fun locateKey(key: ByteArray, node: Node): Pair { + var cmpResult = -1 + var lastI = 0 + for (nodeKey in node.keys) { + cmpResult = compareByteArrays(key, nodeKey) + if (cmpResult <= 0) break + lastI++ + } + return (cmpResult == 0) to lastI + } + + fun isLeaf(node: Node): Boolean { + return !node.subnodeAddresses.any { + it != 0L + } + } + + if (node == null || node.keys.isEmpty()) { + return Single.just(null) + } + + val (there, where) = locateKey(key, node) + if (there) { + return Single.just(node.datas[where]) + } else if (isLeaf(node)) { + return Single.just(null) + } + + return getNodeAtAddress(field, node.subnodeAddresses[where]).flatMap { newNode -> + BSearch(field, key, newNode) + } + } + + private fun decodeNode(data: ByteArray): Node { + val view = ByteCursor(data) + + val numberOfKeys = view.nextInt() + + val keys = (1..numberOfKeys).map { + val keySize = view.nextInt() + view.next(keySize) + } + + val numberOfDatas = view.nextInt() + val datas = (1..numberOfDatas).map { + val offset = view.nextLong() + val length = view.nextInt() + DataPair(offset, length) + } + + val numberOfSubnodeAddresses = B + 1 + val subnodeAddresses = (1..numberOfSubnodeAddresses).map { + view.nextLong() + } + + return Node(keys, datas, subnodeAddresses) + } + + private fun getNodeAtAddress(field: String, address: Long): Single { + var url = "$LTN_BASE_URL/$INDEX_DIR/$field.$tagIndexVersion.index" + if (field == "galleries") { + url = "$LTN_BASE_URL/$GALLERIES_INDEX_DIR/galleries.$galleriesIndexVersion.index" + } + + return client.newCall(rangedGet(url, address, address + MAX_NODE_SIZE - 1)) + .asObservableSuccess() + .map { + it.body()?.bytes() ?: ByteArray(0) + } + .onErrorReturn { ByteArray(0) } + .map { nodedata -> + if (nodedata.isNotEmpty()) { + decodeNode(nodedata) + } else null + }.toSingle() + } + + fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String): Single> { + var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$tag-$language$NOZOMI_EXTENSION" + if (area != null) { + nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$tag-$language$NOZOMI_EXTENSION" + } + + return client.newCall( + Request.Builder() + .url(nozomiAddress) + .build() + ) + .asObservableSuccess() + .map { resp -> + val body = resp.body()!!.bytes() + val cursor = ByteCursor(body) + (1..body.size / 4).map { + cursor.nextInt() + } + }.toSingle() + } + + private fun hashTerm(query: String): HashedTerm { + val md = MessageDigest.getInstance("SHA-256") + md.update(query.toByteArray(HASH_CHARSET)) + return md.digest().copyOf(4) + } + + companion object { + private const val INDEX_DIR = "tagindex" + private const val GALLERIES_INDEX_DIR = "galleriesindex" + private const val COMPRESSED_NOZOMI_PREFIX = "n" + private const val NOZOMI_EXTENSION = ".nozomi" + private const val MAX_NODE_SIZE = 464 + private const val B = 16 + + private val HASH_CHARSET = Charsets.UTF_8 + + fun rangedGet(url: String, rangeBegin: Long, rangeEnd: Long?): Request { + return GET( + url, + Headers.Builder() + .add("Range", "bytes=$rangeBegin-${rangeEnd ?: ""}") + .build() + ) + } + + fun getIndexVersion(httpClient: OkHttpClient, name: String): Observable { + return httpClient.newCall(GET("$LTN_BASE_URL/$name/version?_=${System.currentTimeMillis()}")) + .asObservableSuccess() + .map { it.body()!!.string().toLong() } + } + } +}