Hitomi - ported from TachiyomiSy/EH (#4079)

* Hitomi - ported from TachiyomiSy/EH

* enable a couple more languages

* nsfw annotation

* fix missing import

src/all/hitomi/build.gradle

@@ -0,0 +1,13 @@
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+
+ext {
+    extName = 'Hitomi.la'
+    pkgNameSuffix = 'all.hitomi'
+    extClass = '.HitomiFactory'
+    extVersionCode = 1
+    libVersion = '1.2'
+    containsNsfw = true
+}
+
+apply from: "$rootDir/common.gradle"

src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/ByteCursor.kt

@@ -0,0 +1,93 @@
+package eu.kanade.tachiyomi.extension.all.hitomi
+
+import java.nio.ByteBuffer
+
+/**
+ * Simple cursor for use on byte arrays
+ * @author nulldev
+ */
+class ByteCursor(val content: ByteArray) {
+    var index = -1
+        private set
+    private var mark = -1
+
+    fun mark() {
+        mark = index
+    }
+
+    fun jumpToMark() {
+        index = mark
+    }
+
+    fun jumpToIndex(index: Int) {
+        this.index = index
+    }
+
+    fun next(): Byte {
+        return content[++index]
+    }
+
+    fun next(count: Int): ByteArray {
+        val res = content.sliceArray(index + 1..index + count)
+        skip(count)
+        return res
+    }
+
+    // Used to perform conversions
+    private fun byteBuffer(count: Int): ByteBuffer {
+        return ByteBuffer.wrap(next(count))
+    }
+
+    // Epic hack to get an unsigned short properly...
+    fun fakeNextShortInt(): Int = ByteBuffer
+        .wrap(arrayOf(0x00, 0x00, *next(2).toTypedArray()).toByteArray())
+        .getInt(0)
+
+    //    fun nextShort(): Short = byteBuffer(2).getShort(0)
+    fun nextInt(): Int = byteBuffer(4).getInt(0)
+    fun nextLong(): Long = byteBuffer(8).getLong(0)
+    fun nextFloat(): Float = byteBuffer(4).getFloat(0)
+    fun nextDouble(): Double = byteBuffer(8).getDouble(0)
+
+    fun skip(count: Int) {
+        index += count
+    }
+
+    fun expect(vararg bytes: Byte) {
+        if (bytes.size > remaining()) {
+            throw IllegalStateException("Unexpected end of content!")
+        }
+
+        for (i in 0..bytes.lastIndex) {
+            val expected = bytes[i]
+            val actual = content[index + i + 1]
+
+            if (expected != actual) {
+                throw IllegalStateException("Unexpected content (expected: $expected, actual: $actual)!")
+            }
+        }
+
+        index += bytes.size
+    }
+
+    fun checkEqual(vararg bytes: Byte): Boolean {
+        if (bytes.size > remaining()) {
+            return false
+        }
+
+        for (i in 0..bytes.lastIndex) {
+            val expected = bytes[i]
+            val actual = content[index + i + 1]
+
+            if (expected != actual) {
+                return false
+            }
+        }
+
+        return true
+    }
+
+    fun atEnd() = index >= content.size - 1
+
+    fun remaining() = content.size - index - 1
+}

src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/Hitomi.kt

@@ -0,0 +1,392 @@
+package eu.kanade.tachiyomi.extension.all.hitomi
+
+import android.app.Application
+import android.content.SharedPreferences
+import android.support.v7.preference.CheckBoxPreference
+import android.support.v7.preference.PreferenceScreen
+import androidx.preference.CheckBoxPreference as AndroidXCheckBoxPreference
+import androidx.preference.PreferenceScreen as AndroidXPreferenceScreen
+import com.github.salomonbrys.kotson.array
+import com.github.salomonbrys.kotson.get
+import com.github.salomonbrys.kotson.string
+import com.google.gson.JsonParser
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.source.ConfigurableSource
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.HttpSource
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import rx.Observable
+import rx.Single
+import rx.schedulers.Schedulers
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+
+/**
+ * Ported from TachiyomiSy
+ * Original work by NerdNumber9 for TachiyomiEH
+ */
+
+open class Hitomi(override val lang: String, private val nozomiLang: String) : HttpSource(), ConfigurableSource {
+
+    override val supportsLatest = true
+
+    override val name = if (nozomiLang == "all") "Hitomi.la unfiltered" else "Hitomi.la"
+
+    override val baseUrl = BASE_URL
+
+    // Popular
+
+    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
+        return client.newCall(popularMangaRequest(page))
+            .asObservableSuccess()
+            .flatMap { responseToMangas(it) }
+    }
+
+    override fun popularMangaRequest(page: Int) = HitomiNozomi.rangedGet(
+        "$LTN_BASE_URL/popular-$nozomiLang.nozomi",
+        100L * (page - 1),
+        99L + 100 * (page - 1)
+    )
+
+    private fun responseToMangas(response: Response): Observable<MangasPage> {
+        val range = response.header("Content-Range")!!
+        val total = range.substringAfter('/').toLong()
+        val end = range.substringBefore('/').substringAfter('-').toLong()
+        val body = response.body()!!
+        return parseNozomiPage(body.bytes())
+            .map {
+                MangasPage(it, end < total - 1)
+            }
+    }
+
+    private fun parseNozomiPage(array: ByteArray): Observable<List<SManga>> {
+        val cursor = ByteCursor(array)
+        val ids = (1..array.size / 4).map {
+            cursor.nextInt()
+        }
+
+        return nozomiIdsToMangas(ids).toObservable()
+    }
+
+    private fun nozomiIdsToMangas(ids: List<Int>): Single<List<SManga>> {
+        return Single.zip(
+            ids.map { int ->
+                client.newCall(GET("$LTN_BASE_URL/galleryblock/$int.html"))
+                    .asObservableSuccess()
+                    .subscribeOn(Schedulers.io()) // Perform all these requests in parallel
+                    .map { parseGalleryBlock(it) }
+                    .toSingle()
+            }
+        ) { it.map { m -> m as SManga } }
+    }
+
+    private fun Document.selectFirst(selector: String) = this.select(selector).first()
+
+    private fun parseGalleryBlock(response: Response): SManga {
+        val doc = response.asJsoup()
+        return SManga.create().apply {
+            val titleElement = doc.selectFirst("h1")
+            title = titleElement.text()
+            thumbnail_url = "https:" + if (useHqThumbPref()) {
+                doc.selectFirst("img").attr("srcset").substringBefore(' ')
+            } else {
+                doc.selectFirst("img").attr("src")
+            }
+            url = titleElement.child(0).attr("href")
+        }
+    }
+
+    override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Not used")
+
+    // Latest
+
+    override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
+        return client.newCall(latestUpdatesRequest(page))
+            .asObservableSuccess()
+            .flatMap { responseToMangas(it) }
+    }
+
+    override fun latestUpdatesRequest(page: Int) = HitomiNozomi.rangedGet(
+        "$LTN_BASE_URL/index-$nozomiLang.nozomi",
+        100L * (page - 1),
+        99L + 100 * (page - 1)
+    )
+
+    override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException("Not used")
+
+    // Search
+
+    private var cachedTagIndexVersion: Long? = null
+    private var tagIndexVersionCacheTime: Long = 0
+    private fun tagIndexVersion(): Single<Long> {
+        val sCachedTagIndexVersion = cachedTagIndexVersion
+        return if (sCachedTagIndexVersion == null ||
+            tagIndexVersionCacheTime + INDEX_VERSION_CACHE_TIME_MS < System.currentTimeMillis()
+        ) {
+            HitomiNozomi.getIndexVersion(client, "tagindex").subscribeOn(Schedulers.io()).doOnNext {
+                cachedTagIndexVersion = it
+                tagIndexVersionCacheTime = System.currentTimeMillis()
+            }.toSingle()
+        } else {
+            Single.just(sCachedTagIndexVersion)
+        }
+    }
+
+    private var cachedGalleryIndexVersion: Long? = null
+    private var galleryIndexVersionCacheTime: Long = 0
+    private fun galleryIndexVersion(): Single<Long> {
+        val sCachedGalleryIndexVersion = cachedGalleryIndexVersion
+        return if (sCachedGalleryIndexVersion == null ||
+            galleryIndexVersionCacheTime + INDEX_VERSION_CACHE_TIME_MS < System.currentTimeMillis()
+        ) {
+            HitomiNozomi.getIndexVersion(client, "galleriesindex").subscribeOn(Schedulers.io()).doOnNext {
+                cachedGalleryIndexVersion = it
+                galleryIndexVersionCacheTime = System.currentTimeMillis()
+            }.toSingle()
+        } else {
+            Single.just(sCachedGalleryIndexVersion)
+        }
+    }
+
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        val splitQuery = query.split(" ")
+
+        val positive = splitQuery.filter { !it.startsWith('-') }.toMutableList()
+        if (nozomiLang != "all") positive += "language:$nozomiLang"
+        val negative = (splitQuery - positive).map { it.removePrefix("-") }
+
+        // TODO Cache the results coming out of HitomiNozomi (this TODO dates back to TachiyomiEH)
+        val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv }
+            .map { HitomiNozomi(client, it.first, it.second) }
+
+        var base = if (positive.isEmpty()) {
+            hn.flatMap { n -> n.getGalleryIdsFromNozomi(null, "index", "all").map { n to it.toSet() } }
+        } else {
+            val q = positive.removeAt(0)
+            hn.flatMap { n -> n.getGalleryIdsForQuery(q).map { n to it.toSet() } }
+        }
+
+        base = positive.fold(base) { acc, q ->
+            acc.flatMap { (nozomi, mangas) ->
+                nozomi.getGalleryIdsForQuery(q).map {
+                    nozomi to mangas.intersect(it)
+                }
+            }
+        }
+
+        base = negative.fold(base) { acc, q ->
+            acc.flatMap { (nozomi, mangas) ->
+                nozomi.getGalleryIdsForQuery(q).map {
+                    nozomi to (mangas - it)
+                }
+            }
+        }
+
+            return base.flatMap { (_, ids) ->
+                val chunks = ids.chunked(PAGE_SIZE)
+
+                nozomiIdsToMangas(chunks[page - 1]).map { mangas ->
+                    MangasPage(mangas, page < chunks.size)
+                }
+            }.toObservable()
+    }
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Not used")
+    override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Not used")
+
+    // Details
+
+    override fun mangaDetailsParse(response: Response): SManga {
+        val document = response.asJsoup()
+        fun String.replaceSpaces() = this.replace(" ", "_")
+
+        return SManga.create().apply {
+            thumbnail_url = document.select("div.cover img").attr("abs:src")
+            author = document.select("div.gallery h2 a").joinToString { it.text() }
+            val tableInfo = document.select("table tr")
+                .map { tr ->
+                    val key = tr.select("td:first-child").text()
+                    val value = with(tr.select("td:last-child a")) {
+                        when (key) {
+                            "Series", "Characters" -> {
+                                if (text().isNotEmpty())
+                                    joinToString { "${attr("href").removePrefix("/").substringBefore("/")}:${it.text().replaceSpaces()}" } else null
+                            }
+                            "Tags" -> joinToString { element ->
+                                element.text().let {
+                                    when {
+                                        it.contains("♀") -> "female:${it.substringBeforeLast(" ").replaceSpaces()}"
+                                        it.contains("♂") -> "male:${it.substringBeforeLast(" ").replaceSpaces()}"
+                                        else -> it
+                                    }
+                                }
+                            }
+                            else -> joinToString { it.text() }
+                        }
+                    }
+                    Pair(key, value)
+                }
+                .plus(Pair("Date uploaded", document.select("div.gallery span.date").text()))
+                .toMap()
+            description = tableInfo.filterNot { it.value.isNullOrEmpty() || it.key in listOf("Series", "Characters", "Tags") }.entries.joinToString("\n") { "${it.key}: ${it.value}" }
+            genre = listOfNotNull(tableInfo["Series"], tableInfo["Characters"], tableInfo["Tags"]).joinToString()
+        }
+    }
+
+    // Chapters
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        return Observable.just(
+            listOf(
+                SChapter.create().apply {
+                    url = manga.url
+                    name = "Chapter"
+                    chapter_number = 0.0f
+                }
+            )
+        )
+    }
+
+    override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Not used")
+
+    // Pages
+
+    private fun hlIdFromUrl(url: String) =
+        url.split('/').last().split('-').last().substringBeforeLast('.')
+
+    override fun pageListRequest(chapter: SChapter): Request {
+        return GET("$LTN_BASE_URL/galleries/${hlIdFromUrl(chapter.url)}.js")
+    }
+
+    private val jsonParser = JsonParser()
+
+    override fun pageListParse(response: Response): List<Page> {
+        val str = response.body()!!.string()
+        val json = jsonParser.parse(str.removePrefix("var galleryinfo = "))
+        return json["files"].array.mapIndexed { i, jsonElement ->
+            val hash = jsonElement["hash"].string
+            val ext = if (jsonElement["haswebp"].string == "0" || !hitomiAlwaysWebp()) jsonElement["name"].string.split('.').last() else "webp"
+            val path = if (jsonElement["haswebp"].string == "0" || !hitomiAlwaysWebp()) "images" else "webp"
+            val hashPath1 = hash.takeLast(1)
+            val hashPath2 = hash.takeLast(3).take(2)
+            Page(i, "", "https://${subdomainFromGalleryId(hashPath2)}a.hitomi.la/$path/$hashPath1/$hashPath2/$hash.$ext")
+        }
+    }
+
+    // https://ltn.hitomi.la/common.js
+    private fun subdomainFromGalleryId(pathSegment: String): Char {
+        var numberOfFrontends = 3
+        var g = pathSegment.toInt(16)
+        if (g < 0x30) numberOfFrontends = 2
+        if (g < 0x09) g = 1
+
+        return (97 + g.rem(numberOfFrontends)).toChar()
+    }
+
+    override fun imageRequest(page: Page): Request {
+        val request = super.imageRequest(page)
+        val hlId = request.url().pathSegments().let {
+            it[it.lastIndex - 1]
+        }
+        return request.newBuilder()
+            .header("Referer", "$BASE_URL/reader/$hlId.html")
+            .build()
+    }
+
+    override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Not used")
+
+    companion object {
+        private const val INDEX_VERSION_CACHE_TIME_MS = 1000 * 60 * 10
+        private const val PAGE_SIZE = 25
+
+        // From HitomiSearchMetaData
+        const val LTN_BASE_URL = "https://ltn.hitomi.la"
+        const val BASE_URL = "https://hitomi.la"
+
+        // Preferences
+        private const val WEBP_PREF_KEY = "HITOMI_WEBP"
+        private const val WEBP_PREF_TITLE = "Webp pages"
+        private const val WEBP_PREF_SUMMARY = "Download webp pages instead of jpeg (when available)"
+        private const val WEBP_PREF_DEFAULT_VALUE = true
+
+        private const val COVER_PREF_KEY = "HITOMI_COVERS"
+        private const val COVER_PREF_TITLE = "Use HQ covers"
+        private const val COVER_PREF_SUMMARY = "See HQ covers while browsing"
+        private const val COVER_PREF_DEFAULT_VALUE = true
+    }
+
+    // Preferences
+
+    private val preferences: SharedPreferences by lazy {
+        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
+    }
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        val webpPref = CheckBoxPreference(screen.context).apply {
+            key = "${WEBP_PREF_KEY}_$lang"
+            title = WEBP_PREF_TITLE
+            summary = WEBP_PREF_SUMMARY
+            setDefaultValue(WEBP_PREF_DEFAULT_VALUE)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit().putBoolean("${WEBP_PREF_KEY}_$lang", checkValue).commit()
+            }
+        }
+
+        val coverPref = CheckBoxPreference(screen.context).apply {
+            key = "${COVER_PREF_KEY}_$lang"
+            title = COVER_PREF_TITLE
+            summary = COVER_PREF_SUMMARY
+            setDefaultValue(COVER_PREF_DEFAULT_VALUE)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit().putBoolean("${COVER_PREF_KEY}_$lang", checkValue).commit()
+            }
+        }
+
+        screen.addPreference(webpPref)
+        screen.addPreference(coverPref)
+    }
+
+    override fun setupPreferenceScreen(screen: AndroidXPreferenceScreen) {
+        val webpPref = AndroidXCheckBoxPreference(screen.context).apply {
+            key = "${WEBP_PREF_KEY}_$lang"
+            title = WEBP_PREF_TITLE
+            summary = WEBP_PREF_SUMMARY
+            setDefaultValue(WEBP_PREF_DEFAULT_VALUE)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit().putBoolean("${WEBP_PREF_KEY}_$lang", checkValue).commit()
+            }
+        }
+
+        val coverPref = AndroidXCheckBoxPreference(screen.context).apply {
+            key = "${COVER_PREF_KEY}_$lang"
+            title = COVER_PREF_TITLE
+            summary = COVER_PREF_SUMMARY
+            setDefaultValue(COVER_PREF_DEFAULT_VALUE)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val checkValue = newValue as Boolean
+                preferences.edit().putBoolean("${COVER_PREF_KEY}_$lang", checkValue).commit()
+            }
+        }
+
+        screen.addPreference(webpPref)
+        screen.addPreference(coverPref)
+    }
+
+    private fun hitomiAlwaysWebp(): Boolean = preferences.getBoolean("${WEBP_PREF_KEY}_$lang", WEBP_PREF_DEFAULT_VALUE)
+    private fun useHqThumbPref(): Boolean = preferences.getBoolean("${COVER_PREF_KEY}_$lang", COVER_PREF_DEFAULT_VALUE)
+}

src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiFactory.kt

@@ -0,0 +1,57 @@
+package eu.kanade.tachiyomi.extension.all.hitomi
+
+import eu.kanade.tachiyomi.annotations.Nsfw
+import eu.kanade.tachiyomi.source.Source
+import eu.kanade.tachiyomi.source.SourceFactory
+
+@Nsfw
+class HitomiFactory : SourceFactory {
+    override fun createSources(): List<Source> = languageList
+        .filterNot { it.first.isEmpty() }
+        .map { Hitomi(it.first, it.second) }
+}
+
+/**
+ * These should all be valid languages but I was too lazy to look up all the language codes
+ * Replace an empty string with a valid language code to enable that language
+ */
+private val languageList = listOf(
+    Pair("other", "all"), // all languages
+    Pair("id", "indonesian"),
+    Pair("", "catalan"),
+    Pair("", "cebuano"),
+    Pair("", "czech"),
+    Pair("", "danish"),
+    Pair("de", "german"),
+    Pair("", "estonian"),
+    Pair("en", "english"),
+    Pair("es", "spanish"),
+    Pair("", "esperanto"),
+    Pair("fr", "french"),
+    Pair("it", "italian"),
+    Pair("", "latin"),
+    Pair("", "hungarian"),
+    Pair("", "dutch"),
+    Pair("", "norwegian"),
+    Pair("pl", "polish"),
+    Pair("pt-BR", "portuguese"),
+    Pair("", "romanian"),
+    Pair("", "albanian"),
+    Pair("", "slovak"),
+    Pair("", "finnish"),
+    Pair("", "swedish"),
+    Pair("", "tagalog"),
+    Pair("vi", "vietnamese"),
+    Pair("tr", "turkish"),
+    Pair("", "greek"),
+    Pair("", "mongolian"),
+    Pair("ru", "russian"),
+    Pair("", "ukrainian"),
+    Pair("", "hebrew"),
+    Pair("ar", "arabic"),
+    Pair("", "persian"),
+    Pair("th", "thai"),
+    Pair("ko", "korean"),
+    Pair("zh", "chinese"),
+    Pair("ja", "japanese")
+)

src/all/hitomi/src/eu/kanade/tachiyomi/extension/all/hitomi/HitomiNozomi.kt

@@ -0,0 +1,257 @@
+package eu.kanade.tachiyomi.extension.all.hitomi
+
+import eu.kanade.tachiyomi.extension.all.hitomi.Hitomi.Companion.LTN_BASE_URL
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservable
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import java.security.MessageDigest
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import rx.Observable
+import rx.Single
+
+private typealias HashedTerm = ByteArray
+
+private data class DataPair(val offset: Long, val length: Int)
+private data class Node(
+    val keys: List<ByteArray>,
+    val datas: List<DataPair>,
+    val subnodeAddresses: List<Long>
+)
+
+/**
+ * Kotlin port of the hitomi.la search algorithm
+ * @author NerdNumber9
+ */
+class HitomiNozomi(
+    private val client: OkHttpClient,
+    private val tagIndexVersion: Long,
+    private val galleriesIndexVersion: Long
+) {
+    fun getGalleryIdsForQuery(query: String): Single<List<Int>> {
+        val replacedQuery = query.replace('_', ' ')
+
+        if (':' in replacedQuery) {
+            val sides = replacedQuery.split(':')
+            val namespace = sides[0]
+            var tag = sides[1]
+
+            var area: String? = namespace
+            var language = "all"
+            if (namespace == "female" || namespace == "male") {
+                area = "tag"
+                tag = replacedQuery
+            } else if (namespace == "language") {
+                area = null
+                language = tag
+                tag = "index"
+            }
+
+            return getGalleryIdsFromNozomi(area, tag, language)
+        }
+
+        val key = hashTerm(query)
+        val field = "galleries"
+
+        return getNodeAtAddress(field, 0).flatMap { node ->
+            if (node == null) {
+                Single.just(null)
+            } else {
+                BSearch(field, key, node).flatMap { data ->
+                    if (data == null) {
+                        Single.just(null)
+                    } else {
+                        getGalleryIdsFromData(data)
+                    }
+                }
+            }
+        }
+    }
+
+    private fun getGalleryIdsFromData(data: DataPair?): Single<List<Int>> {
+        if (data == null) {
+            return Single.just(emptyList())
+        }
+
+        val url = "$LTN_BASE_URL/$GALLERIES_INDEX_DIR/galleries.$galleriesIndexVersion.data"
+        val (offset, length) = data
+        if (length > 100000000 || length <= 0) {
+            return Single.just(emptyList())
+        }
+
+        return client.newCall(rangedGet(url, offset, offset + length - 1))
+            .asObservable()
+            .map {
+                it.body()?.bytes() ?: ByteArray(0)
+            }
+            .onErrorReturn { ByteArray(0) }
+            .map { inbuf ->
+                if (inbuf.isEmpty()) {
+                    return@map emptyList<Int>()
+                }
+
+                val view = ByteCursor(inbuf)
+                val numberOfGalleryIds = view.nextInt()
+
+                val expectedLength = numberOfGalleryIds * 4 + 4
+
+                if (numberOfGalleryIds > 10000000 ||
+                    numberOfGalleryIds <= 0 ||
+                    inbuf.size != expectedLength
+                ) {
+                    return@map emptyList<Int>()
+                }
+
+                (1..numberOfGalleryIds).map {
+                    view.nextInt()
+                }
+            }.toSingle()
+    }
+
+    @Suppress("FunctionName")
+    private fun BSearch(field: String, key: ByteArray, node: Node?): Single<DataPair?> {
+        fun compareByteArrays(dv1: ByteArray, dv2: ByteArray): Int {
+            val top = dv1.size.coerceAtMost(dv2.size)
+            for (i in 0 until top) {
+                val dv1i = dv1[i].toInt() and 0xFF
+                val dv2i = dv2[i].toInt() and 0xFF
+                if (dv1i < dv2i) {
+                    return -1
+                } else if (dv1i > dv2i) {
+                    return 1
+                }
+            }
+            return 0
+        }
+
+        fun locateKey(key: ByteArray, node: Node): Pair<Boolean, Int> {
+            var cmpResult = -1
+            var lastI = 0
+            for (nodeKey in node.keys) {
+                cmpResult = compareByteArrays(key, nodeKey)
+                if (cmpResult <= 0) break
+                lastI++
+            }
+            return (cmpResult == 0) to lastI
+        }
+
+        fun isLeaf(node: Node): Boolean {
+            return !node.subnodeAddresses.any {
+                it != 0L
+            }
+        }
+
+        if (node == null || node.keys.isEmpty()) {
+            return Single.just(null)
+        }
+
+        val (there, where) = locateKey(key, node)
+        if (there) {
+            return Single.just(node.datas[where])
+        } else if (isLeaf(node)) {
+            return Single.just(null)
+        }
+
+        return getNodeAtAddress(field, node.subnodeAddresses[where]).flatMap { newNode ->
+            BSearch(field, key, newNode)
+        }
+    }
+
+    private fun decodeNode(data: ByteArray): Node {
+        val view = ByteCursor(data)
+
+        val numberOfKeys = view.nextInt()
+
+        val keys = (1..numberOfKeys).map {
+            val keySize = view.nextInt()
+            view.next(keySize)
+        }
+
+        val numberOfDatas = view.nextInt()
+        val datas = (1..numberOfDatas).map {
+            val offset = view.nextLong()
+            val length = view.nextInt()
+            DataPair(offset, length)
+        }
+
+        val numberOfSubnodeAddresses = B + 1
+        val subnodeAddresses = (1..numberOfSubnodeAddresses).map {
+            view.nextLong()
+        }
+
+        return Node(keys, datas, subnodeAddresses)
+    }
+
+    private fun getNodeAtAddress(field: String, address: Long): Single<Node?> {
+        var url = "$LTN_BASE_URL/$INDEX_DIR/$field.$tagIndexVersion.index"
+        if (field == "galleries") {
+            url = "$LTN_BASE_URL/$GALLERIES_INDEX_DIR/galleries.$galleriesIndexVersion.index"
+        }
+
+        return client.newCall(rangedGet(url, address, address + MAX_NODE_SIZE - 1))
+            .asObservableSuccess()
+            .map {
+                it.body()?.bytes() ?: ByteArray(0)
+            }
+            .onErrorReturn { ByteArray(0) }
+            .map { nodedata ->
+                if (nodedata.isNotEmpty()) {
+                    decodeNode(nodedata)
+                } else null
+            }.toSingle()
+    }
+
+    fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String): Single<List<Int>> {
+        var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$tag-$language$NOZOMI_EXTENSION"
+        if (area != null) {
+            nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$tag-$language$NOZOMI_EXTENSION"
+        }
+
+        return client.newCall(
+            Request.Builder()
+                .url(nozomiAddress)
+                .build()
+        )
+            .asObservableSuccess()
+            .map { resp ->
+                val body = resp.body()!!.bytes()
+                val cursor = ByteCursor(body)
+                (1..body.size / 4).map {
+                    cursor.nextInt()
+                }
+            }.toSingle()
+    }
+
+    private fun hashTerm(query: String): HashedTerm {
+        val md = MessageDigest.getInstance("SHA-256")
+        md.update(query.toByteArray(HASH_CHARSET))
+        return md.digest().copyOf(4)
+    }
+
+    companion object {
+        private const val INDEX_DIR = "tagindex"
+        private const val GALLERIES_INDEX_DIR = "galleriesindex"
+        private const val COMPRESSED_NOZOMI_PREFIX = "n"
+        private const val NOZOMI_EXTENSION = ".nozomi"
+        private const val MAX_NODE_SIZE = 464
+        private const val B = 16
+
+        private val HASH_CHARSET = Charsets.UTF_8
+
+        fun rangedGet(url: String, rangeBegin: Long, rangeEnd: Long?): Request {
+            return GET(
+                url,
+                Headers.Builder()
+                    .add("Range", "bytes=$rangeBegin-${rangeEnd ?: ""}")
+                    .build()
+            )
+        }
+
+        fun getIndexVersion(httpClient: OkHttpClient, name: String): Observable<Long> {
+            return httpClient.newCall(GET("$LTN_BASE_URL/$name/version?_=${System.currentTimeMillis()}"))
+                .asObservableSuccess()
+                .map { it.body()!!.string().toLong() }
+        }
+    }
+}