More work on Hitomi.la

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt

@@ -155,9 +155,9 @@ object PreferenceKeys {
 
     const val eh_showSettingsUploadWarning = "eh_showSettingsUploadWarning1"
 
-    const val eh_hl_refreshFrequency = "eh_nh_refresh_frequency"
+    const val eh_hl_refreshFrequency = "eh_lh_refresh_frequency"
 
-    const val eh_hl_lastRefresh = "eh_nh_last_refresh"
+    const val eh_hl_lastRefresh = "eh_lh_last_refresh"
 
     const val eh_expandFilters = "eh_expand_filters"
 }

app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt

@@ -5,6 +5,7 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.source.online.all.EHentai
+import eu.kanade.tachiyomi.source.online.all.Hitomi
 import eu.kanade.tachiyomi.source.online.all.NHentai
 import eu.kanade.tachiyomi.source.online.all.PervEden
 import eu.kanade.tachiyomi.source.online.english.*
@@ -88,8 +89,7 @@ open class SourceManager(private val context: Context) {
         exSrcs += NHentai(context)
         exSrcs += HentaiCafe()
         exSrcs += Tsumino(context)
-        // Mysteriously broken
+        exSrcs += Hitomi(context)
-//        exSrcs += Hitomi(context)
         return exSrcs
     }
 }

app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt

@@ -1,9 +1,11 @@
 package eu.kanade.tachiyomi.source.online.all
 
 import android.content.Context
+import android.os.HandlerThread
 import com.github.salomonbrys.kotson.*
 import com.google.gson.JsonObject
 import com.google.gson.JsonParser
+import com.google.gson.stream.JsonReader
 import com.squareup.duktape.Duktape
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
@@ -14,38 +16,71 @@ import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.source.online.LewdSource
 import eu.kanade.tachiyomi.util.asJsoup
 import exh.HITOMI_SOURCE_ID
+import exh.metadata.EMULATED_TAG_NAMESPACE
 import exh.metadata.models.HitomiGalleryMetadata
 import exh.metadata.models.HitomiGalleryMetadata.Companion.BASE_URL
-import exh.metadata.models.HitomiGalleryMetadata.Companion.urlFromHlId
+import exh.metadata.models.HitomiGalleryMetadata.Companion.hlIdFromUrl
+import exh.metadata.models.HitomiPage
+import exh.metadata.models.HitomiSkeletonGalleryMetadata
 import exh.metadata.models.Tag
+import exh.metadata.nullIfBlank
+import exh.search.SearchEngine
+import exh.util.*
+import io.realm.Realm
+import io.realm.RealmConfiguration
+import io.realm.RealmResults
 import okhttp3.Request
 import okhttp3.Response
 import org.jsoup.nodes.Document
+import org.jsoup.nodes.Element
 import rx.Observable
+import rx.Scheduler
+import rx.android.schedulers.AndroidSchedulers
+import rx.schedulers.Schedulers
+import rx.subjects.AsyncSubject
+import timber.log.Timber
 import uy.kohesive.injekt.injectLazy
-import java.io.File
+import java.text.SimpleDateFormat
+import java.util.*
+import java.util.concurrent.ConcurrentLinkedQueue
+import java.util.concurrent.LinkedBlockingQueue
 import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.thread
 
+/**
+ * WTF is going on in this class?
+ */
 class Hitomi(private val context: Context)
-    :HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiGallery> {
+    :HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiSkeletonGalleryMetadata> {
+    private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) { JsonParser() }
+    private val searchEngine by lazy { SearchEngine() }
+
+    private val queryCache = mutableMapOf<String, RealmResults<HitomiSkeletonGalleryMetadata>>()
+    private val queryWorkQueue = LinkedBlockingQueue<Triple<String, Int, AsyncSubject<List<HitomiSkeletonGalleryMetadata>>>>()
+    private var searchWorker: Thread? = null
+
+    private var parseToMangaScheduler: Scheduler? = null
+
     override fun queryAll() = HitomiGalleryMetadata.EmptyQuery()
     override fun queryFromUrl(url: String) = HitomiGalleryMetadata.UrlQuery(url)
 
-    override val metaParser: HitomiGalleryMetadata.(HitomiGallery) -> Unit = {
+    override val metaParser: HitomiGalleryMetadata.(HitomiSkeletonGalleryMetadata) -> Unit = {
-        hlId = it.id.toString()
+        hlId = it.hlId
-        title = it.name
+        thumbnailUrl = it.thumbnailUrl
-        thumbnailUrl = resolveImage("//g.hitomi.la/galleries/$hlId/001.jpg")
+        artist = it.artist
-        artist = it.artists.firstOrNull()
+        group = it.group
-        group = it.groups.firstOrNull()
         type = it.type
-        languageSimple = it.language
+        language = it.language
+        languageSimple = it.languageSimple
         series.clear()
-        series.addAll(it.parodies)
+        series.addAll(it.series)
         characters.clear()
         characters.addAll(it.characters)
-
+        buyLink = it.buyLink
+        uploadDate = it.uploadDate
         tags.clear()
-        it.tags.mapTo(tags) { Tag(it.key, it.value) }
+        tags.addAll(it.tags)
+        title = it.title
     }
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Unused method called!")
@@ -54,38 +89,160 @@ class Hitomi(private val context: Context)
 
     override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
 
-    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+    /** >>> PARSE TO MANGA SCHEDULER <<< **/
-        return loadGalleryMetadata(manga.url).map {
+    /*
-            parseToManga(queryFromUrl(manga.url), it)
+    Realm becomes very, very slow after you start opening and closing Realms rapidly.
+    By keeping a global Realm open at all times, we can migitate this.
+    Realms are per-thread so we create our own RxJava scheduler to schedule realm-heavy
+    operations on.
+     */
+    @Synchronized
+    private fun startParseToMangaScheduler() {
+        if(parseToMangaScheduler != null) return
+
+        val thread = object : HandlerThread("parse-to-manga-thread") {
+            override fun onLooperPrepared() {
+                // Open permanent Realm instance on this thread!
+                Realm.getDefaultInstance()
             }
         }
 
+        thread.start()
+        parseToMangaScheduler = AndroidSchedulers.from(thread.looper)
+    }
+
+    private fun parseToMangaScheduler(): Scheduler {
+        startParseToMangaScheduler()
+        return parseToMangaScheduler!!
+    }
+
+    /** >>> SEARCH WORKER <<< **/
+    /*
+    Running RealmResults.size on a new RealmResults object is very, very slow.
+    By caching our RealmResults in memory, we avoid creating many new RealmResults objects,
+    thus speeding up RealmResults.size.
+
+    Realms are per-thread and RealmReults are bound to Realms. Therefore we create a
+    permanent thread that will open a permanent realm and wait for requests to load RealmResults.
+     */
+
+    @Synchronized
+    private fun startSearchWorker() {
+        if(searchWorker != null) return
+
+        searchWorker = thread {
+            ensureCacheLoaded().toBlocking().first()
+
+            getCacheRealm().use { realm ->
+                Timber.d("[SW] New search worker thread started!")
+                while (true) {
+                    Timber.d("[SW] Waiting for next query!")
+                    val next = queryWorkQueue.take()
+                    Timber.d("[SW] Found new query (page ${next.second}): ${next.first}")
+
+                    if(queryCache[next.first] == null) {
+                        val first = realm.where(HitomiSkeletonGalleryMetadata::class.java).findFirst()
+
+                        if (first == null) {
+                            next.third.onNext(emptyList())
+                            next.third.onCompleted()
+                            continue
+                        }
+
+                        val parsed = searchEngine.parseQuery(next.first)
+                        val filtered = searchEngine.filterResults(realm.where(HitomiSkeletonGalleryMetadata::class.java),
+                                parsed,
+                                first.titleFields).findAll()
+
+                        queryCache[next.first] = filtered
+                    }
+
+                    val filtered = queryCache[next.first]!!
+
+                    val beginIndex = (next.second - 1) * PAGE_SIZE
+                    if (beginIndex > filtered.lastIndex) {
+                        next.third.onNext(emptyList())
+                        next.third.onCompleted()
+                        continue
+                    }
+
+                    // Chunk into pages of 100
+                    val res = realm.copyFromRealm(filtered.subList(beginIndex,
+                            Math.min(next.second * PAGE_SIZE, filtered.size)))
+
+                    next.third.onNext(res)
+                    next.third.onCompleted()
+                }
+            }
+        }
+    }
+
+    private fun trySearch(page: Int, query: String): Observable<List<HitomiSkeletonGalleryMetadata>> {
+        startSearchWorker()
+
+        val subject = AsyncSubject.create<List<HitomiSkeletonGalleryMetadata>>()
+        queryWorkQueue.clear()
+        queryWorkQueue.add(Triple(query, page, subject))
+        return subject
+    }
+
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        return trySearch(page, query).map {
+            val res = it.map {
+                SManga.create().apply {
+                    setUrlWithoutDomain(it.url!!)
+
+                    title = it.title!!
+
+                    it.thumbnailUrl?.let {
+                        thumbnail_url = it
+                    }
+                }
+            }
+
+            MangasPage(res, it.isNotEmpty())
+        }
+    }
+
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        return lazyLoadMetaPages(HitomiGalleryMetadata.hlIdFromUrl(manga.url), true)
+                .map {
+                    manga.copyFrom(parseToManga(queryFromUrl(manga.url), it.first))
+                    manga
+                }
+                .subscribeOn(parseToMangaScheduler())
+    }
+
     override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
         return lazyLoadMeta(queryFromUrl(manga.url),
-            loadAllGalleryMetadata().map {
+                lazyLoadMetaPages(hlIdFromUrl(manga.url), false).map { it.first }
-                val mid = HitomiGalleryMetadata.hlIdFromUrl(manga.url)
-                it.find { it.id.toString() == mid }
-            }
         ).map {
             listOf(SChapter.create().apply {
-                url = "$BASE_URL/reader/${it.hlId}.html"
+                url = readerUrl(it.hlId!!)
 
                 name = "Chapter"
 
                 chapter_number = 1f
+
+                it.uploadDate?.let {
+                    date_upload = it
+                }
             })
         }
     }
 
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
+        val hlId = chapter.url.substringAfterLast('/').removeSuffix(".html")
+        return lazyLoadMetaPages(hlId, false).map { (_, it) ->
+            it.mapIndexed { index, s ->
+                Page(index, s, s)
+            }
+        }
+    }
+
     override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
 
-    override fun pageListParse(response: Response): List<Page> {
+    override fun pageListParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
-        val doc = response.asJsoup()
-        return doc.select(".img-url").mapIndexed { index, element ->
-            val resolved = resolveImage(element.text())
-            Page(index, resolved, resolved)
-        }
-    }
 
     override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
 
@@ -101,52 +258,236 @@ class Hitomi(private val context: Context)
 
     private val prefs: PreferencesHelper by injectLazy()
 
-    private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) {
-        JsonParser()
-    }
-
     private val cacheLock = ReentrantLock()
 
-    private var metaCache: List<HitomiGallery>? = null
-
     override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html")
 
     override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
     override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
 
-    override fun latestUpdatesRequest(page: Int) = GET("$BASE_URL/index-all-2.html")
+    override fun latestUpdatesRequest(page: Int) = GET("$BASE_URL/index-all-$page.html")
 
-    private fun resolveMangaIds(doc: Document, data: List<HitomiGallery>): List<HitomiGallery> {
+    private fun parsePage(doc: Document): List<HitomiSkeletonGalleryMetadata> {
-        return doc.select(".gallery-content > div > a").mapNotNull {
+        return doc.select(".gallery-content > div").map {
-            val id = HitomiGalleryMetadata.hlIdFromUrl(it.attr("href"))
+            HitomiSkeletonGalleryMetadata().apply {
-            data.find { it.id.toString() == id }
+                it.select("h1 > a").let {
+                    url = it.attr("href")
+
+                    title = it.text()
+                }
+
+                thumbnailUrl = "https:" + it.select(".dj-img1 > img").attr("src")
+            }
+        }
+    }
+
+    fun readerUrl(hlId: String) = "$BASE_URL/reader/$hlId.html"
+
+    private fun lazyLoadMetaPages(hlId: String, forceReload: Boolean):
+            Observable<Pair<HitomiSkeletonGalleryMetadata, List<String>>> {
+        val pages = defRealm { realm ->
+            val rres = realm.where(HitomiPage::class.java)
+                    .equalTo(HitomiPage::gallery.name, hlId)
+                    .findAllSorted(HitomiPage::index.name)
+
+            if (rres.isNotEmpty())
+                rres.map(HitomiPage::url)
+            else null
+        }
+
+        val meta = getCacheRealm().use {
+            val res = it.where(HitomiSkeletonGalleryMetadata::class.java)
+                    .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
+                    .findFirst()
+
+            // Force reload if no thumbnail
+            if(res?.thumbnailUrl == null) null else res
+        }
+
+        if(pages != null && meta != null && !forceReload) {
+            return Observable.just(meta to pages)
+        }
+
+        val loc = "$BASE_URL/galleries/$hlId.html"
+        val req = GET(loc)
+
+        return client.newCall(req).asObservableSuccess().map {
+            val doc = it.asJsoup()
+
+            Duktape.create().use { duck ->
+                val thumbs = doc.getElementsByTag("script").find {
+                    it.html().startsWith("var thumbnails")
+                }
+
+                val parsedThumbs = jsonParser.parse(thumbs!!.html()
+                        .removePrefix("var thumbnails = ")
+                        .removeSuffix(";")).array
+
+                // Get pages (drop last element as its always null)
+                val newPages = parsedThumbs.take(parsedThumbs.size() - 1).mapIndexed { index, item ->
+                    val itemName = item.string
+                            .substringAfterLast('/')
+                            .removeSuffix(".jpg")
+
+                    val url = "//a.hitomi.la/galleries/$hlId/$itemName"
+
+                    val resolved = resolveImage(duck, url)
+                    HitomiPage().apply {
+                        gallery = hlId
+                        this.index = index
+                        this.url = resolved
+                    }
+                }
+
+                // Parse meta
+                val galleryParent = doc.select(".gallery")
+
+                val newMeta = HitomiSkeletonGalleryMetadata().apply {
+                    url = loc
+
+                    title = galleryParent.select("h1 > a").text()
+
+                    artist = galleryParent.select("h2 > .comma-list > li").joinToString { it.text() }.nullIfBlank()
+
+                    thumbnailUrl = "https:" + doc.select(".cover img").attr("src")
+
+                    uploadDate = DATE_FORMAT.parse(doc.select(".date").text()).time
+
+                    galleryParent.select(".gallery-info tr").forEach {
+                        val content = it.child(1)
+
+                        when(it.child(0).text().toLowerCase()) {
+                            "group" -> group = content.text().trim()
+                            "type" -> type = content.text().trim()
+                            "language" -> {
+                                language = content.text().trim()
+                                languageSimple = content.select("a")
+                                        .attr("href")
+                                        .split("-").getOrNull(1) ?: "speechless"
+                            }
+                            "series" -> {
+                                series.clear()
+                                series.addAll(content.select("li").map(Element::text))
+                            }
+                            "characters" -> {
+                                characters.clear()
+                                characters.addAll(content.select("li").map(Element::text))
+                            }
+                            "tags" -> {
+                                tags.clear()
+                                tags.addAll(content.select("li").map {
+                                    val txt = it.text()
+
+                                    val ns: String
+                                    val name: String
+
+                                    when {
+                                        txt.endsWith(CHAR_MALE) -> {
+                                            ns = "male"
+                                            name = txt.removeSuffix(CHAR_MALE).trim()
+                                        }
+                                        txt.endsWith(CHAR_FEMALE) -> {
+                                            ns = "female"
+                                            name = txt.removeSuffix(CHAR_FEMALE).trim()
+                                        }
+                                        else -> {
+                                            ns = EMULATED_TAG_NAMESPACE
+                                            name = txt.trim()
+                                        }
+                                    }
+
+                                    Tag(ns, name)
+                                })
+                            }
+                        }
+                    }
+
+                    // Inject pseudo tags
+                    fun String?.nullNaTag(name: String) {
+                        if(this == null || this == NOT_AVAILABLE) return
+
+                        tags.add(Tag(name, this))
+                    }
+
+                    group.nullNaTag("group")
+                    artist.nullNaTag("artist")
+                    languageSimple.nullNaTag("language")
+                    series.forEach {
+                        it.nullNaTag("parody")
+                    }
+                    characters.forEach {
+                        it.nullNaTag("character")
+                    }
+                    type.nullNaTag("category")
+                }
+
+                realmTrans {
+                    // Delete old pages
+                    it.where(HitomiPage::class.java)
+                            .equalTo(HitomiPage::gallery.name, hlId)
+                            .findAll().deleteAllFromRealm()
+
+                    // Add new pages
+                    it.insert(newPages)
+                }
+
+                getCacheRealm().useTrans {
+                    // Delete old meta
+                    it.where(HitomiSkeletonGalleryMetadata::class.java)
+                            .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
+                            .findAll().deleteAllFromRealm()
+
+                    // Add new meta
+                    it.insert(newMeta)
+                }
+
+                newMeta to newPages.map(HitomiPage::url)
+            }
         }
     }
 
     private fun fetchAndResolveRequest(request: Request): Observable<MangasPage> {
-        return loadAllGalleryMetadata().flatMap {
+        //Begin pre-loading cache
-            client.newCall(request)
+        ensureCacheLoaded(false).subscribeOn(Schedulers.computation()).subscribe()
+
+        return client.newCall(request)
                 .asObservableSuccess()
                 .map { response ->
                     val doc = response.asJsoup()
-                        val res = resolveMangaIds(doc, it)
+
+                    val res = getCacheRealm().use { realm ->
+                        parsePage(doc).map {
+                            it
+                        }
+                    }
                     val sManga = res.map {
-                            parseToManga(queryFromUrl(urlFromHlId(it.id.toString())), it)
+                        SManga.create().apply {
-                        }
+                            setUrlWithoutDomain(it.url!!)
-                        val hasNextPage = doc.select(".page-container > ul > li:last-child > a").isNotEmpty()
+
-                        MangasPage(sManga, hasNextPage)
+                            title = it.title!!
+
+                            it.thumbnailUrl?.let {
+                                thumbnail_url = it
                             }
                         }
                     }
+                    val pagingScript = doc.getElementsByTag("script").map { it.html().trim() }.find {
+                        it.startsWith("insert_paging")
+                    } ?: ""
+
+                    val curPage = pagingScript.substringAfterLast("', ").substringBefore(',').toInt()
+                    val endPage = pagingScript.substringAfterLast(", ").removeSuffix(");").toInt()
+
+                    MangasPage(sManga, curPage < endPage)
+                }
+
+    }
 
     override fun fetchPopularManga(page: Int)
             = fetchAndResolveRequest(popularMangaRequest(page))
     override fun fetchLatestUpdates(page: Int)
             = fetchAndResolveRequest(latestUpdatesRequest(page))
 
-    private fun galleryFile(index: Int)
-            = File(context.cacheDir.absoluteFile, "hitomi/galleries$index.json")
-
     private fun shouldRefreshGalleryFiles(): Boolean {
         val timeDiff = System.currentTimeMillis() - prefs.eh_hl_lastRefresh().getOrDefault()
         return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L
@@ -161,82 +502,121 @@ class Hitomi(private val context: Context)
         }
     }
 
-    private fun loadGalleryMetadata(url: String): Observable<HitomiGallery> {
+    private fun loadGalleryMetadata(url: String): Observable<HitomiSkeletonGalleryMetadata> {
-        return loadAllGalleryMetadata().map {
         val mid = HitomiGalleryMetadata.hlIdFromUrl(url)
-            it.find { it.id.toString() == mid }
+
+        return ensureCacheLoaded().map {
+            getCacheRealm().use { realm ->
+                findCacheMetadataById(realm, mid)
+            }
         }
     }
 
-    private fun loadAllGalleryMetadata(): Observable<List<HitomiGallery>> {
+    private fun findCacheMetadataById(realm: Realm, hlId: String): HitomiSkeletonGalleryMetadata? {
+        return realm.where(HitomiSkeletonGalleryMetadata::class.java)
+                .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
+                .findFirst()?.let { realm.copyFromRealm(it) }
+    }
+
+    private fun ensureCacheLoaded(blocking: Boolean = true): Observable<Any> {
+        return Observable.fromCallable {
+            if(!blocking && cacheLock.isLocked) return@fromCallable Any()
+
+            lockCache {
                 val shouldRefresh = shouldRefreshGalleryFiles()
+                getCacheRealm().useTrans { realm ->
+                    if (!realm.isEmpty && !shouldRefresh)
+                        return@fromCallable Any()
 
-        metaCache?.let {
+                    realm.deleteAll()
-            if(!shouldRefresh) {
-                return Observable.just(metaCache)
-            }
                 }
 
-        var obs: Observable<List<String>> = Observable.just(emptyList())
+                val cores = Runtime.getRuntime().availableProcessors()
+                Timber.d("Starting $cores threads to parse hitomi.la gallery data...")
 
-        var refresh = false
+                val workQueue = ConcurrentLinkedQueue<Int>((0 until GALLERY_CHUNK_COUNT).toList())
+                val threads = mutableListOf<Thread>()
+
+                for(threadIndex in 1 .. cores) {
+                    threads += thread {
+                        getCacheRealm().use { realm ->
+                            while (true) {
+                                val i = workQueue.poll() ?: break
+
+                                Timber.d("[$threadIndex] Downloading + parsing hitomi.la gallery data ${i + 1}/$GALLERY_CHUNK_COUNT...")
 
-        for (i in 0 until GALLERY_CHUNK_COUNT) {
-            val cacheFile = galleryFile(i)
-            val newObs = if(shouldRefresh || !cacheFile.exists()) {
                                 val url = "https://ltn.hitomi.la/galleries$i.json"
 
-                refresh = true
+                                val resp = client.newCall(GET(url)).execute().body()!!
 
-                client.newCall(GET(url)).asObservableSuccess().map {
+                                val out = mutableListOf<HitomiSkeletonGalleryMetadata>()
-                    it.body()!!.string().apply {
+
-                        lockCache {
+                                JsonReader(resp.charStream()).use { reader ->
-                            cacheFile.parentFile.mkdirs()
+                                    reader.beginArray()
-                            cacheFile.writeText(this)
+
+                                    while (reader.hasNext()) {
+                                        val gallery = HitomiGallery.fromJson(reader.nextJsonObject())
+                                        val meta = HitomiSkeletonGalleryMetadata()
+                                        gallery.addToGalleryMeta(meta)
+
+                                        out.add(meta)
                                     }
                                 }
+
+                                Timber.d("[$threadIndex] Saving hitomi.la gallery data ${i + 1}/$GALLERY_CHUNK_COUNT...")
+
+                                realm.trans {
+                                    realm.insert(out)
+                                }
                             }
-            } else {
-                // Load galleries from cache
-                Observable.fromCallable {
-                    lockCache {
-                        cacheFile.readText()
                         }
                     }
                 }
 
-            obs = obs.flatMap { l ->
+                threads.forEach(Thread::join)
-                newObs.map {
-                    l + it
-                }
-            }
-        }
 
-        // Update refresh time if we refreshed
+                // Update refresh time
-        if(refresh)
                 prefs.eh_hl_lastRefresh().set(System.currentTimeMillis())
+            }
 
-        return obs.map {
+            return@fromCallable Any()
-            val res = it.flatMap {
-                jsonParser.parse(it).array.map {
-                    HitomiGallery.fromJson(it.obj)
         }
     }
 
-            metaCache = res
+    private fun resolveImage(duktape: Duktape, url: String): String {
-            res
+        return "https:" + duktape.evaluate(IMAGE_RESOLVER.replace(IMAGE_RESOLVER_URL_VAR, url)) as String
+    }
+
+    private fun HitomiGallery.addToGalleryMeta(meta: HitomiSkeletonGalleryMetadata) {
+        with(meta) {
+            hlId = id.toString()
+            title = name
+            // Intentionally avoid setting thumbnails
+            // We need another request to get them anyways
+            artist = artists.firstOrNull()
+            group = groups.firstOrNull()
+            type = this@addToGalleryMeta.type
+            languageSimple = language
+            series.clear()
+            series.addAll(parodies)
+            characters.clear()
+            characters.addAll(this@addToGalleryMeta.characters)
+
+            tags.clear()
+            this@addToGalleryMeta.tags.mapTo(tags) { Tag(it.key, it.value) }
         }
     }
 
-    private fun resolveImage(url: String): String {
+    private fun getCacheRealm() = Realm.getInstance(REALM_CONFIG)
-        return Duktape.create().use {
-            it.evaluate(IMAGE_RESOLVER.replace(IMAGE_RESOLVER_URL_VAR, url)) as String
-        }
-    }
 
     companion object {
+        private val PAGE_SIZE = 25
+        private val CHAR_MALE = "♂"
+        private val CHAR_FEMALE = "♀"
         private val GALLERY_CHUNK_COUNT = 20
         private val IMAGE_RESOLVER_URL_VAR = "%IMAGE_URL%"
+        private val NOT_AVAILABLE = "N/A"
+        private val DATE_FORMAT = SimpleDateFormat("yyyy-MM-dd HH:mm:ssX", Locale.US)
         private val IMAGE_RESOLVER = """
             (function() {
 var adapose = false; // Currently not sure what this does, it switches out frontend URL when we right click???
@@ -272,6 +652,11 @@ function url_from_url(url, base) {
 return url_from_url('$IMAGE_RESOLVER_URL_VAR');
 })();
             """.trimIndent()
+
+        private val REALM_CONFIG = RealmConfiguration.Builder()
+                .name("hitomi-cache")
+                .deleteRealmIfMigrationNeeded()
+                .build()
     }
 }
 
@@ -297,7 +682,7 @@ data class HitomiGallery(val artists: List<String>,
                     if(str.contains(":"))
                         str.substringBefore(':') to str.substringAfter(':')
                     else
-                        "tag" to str
+                        EMULATED_TAG_NAMESPACE to str
                 } ?: emptyMap(),
                 obj.mapNullStringList("c"),
                 obj["type"].string,

app/src/main/java/exh/metadata/models/HitomiGalleryMetadata.kt

@@ -44,9 +44,7 @@ open class HitomiGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
     override var tags: RealmList<Tag> = RealmList()
 
     // Sites does not show uploader
-    override var uploader: String?
+    override var uploader: String? = "admin"
-        get() = "admin"
-        set(value) {}
 
     var url get() = hlId?.let { urlFromHlId(it) }
         set(a) {
@@ -148,6 +146,6 @@ open class HitomiGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
                 = url.split('/').last().substringBeforeLast('.')
 
         fun urlFromHlId(id: String)
-                = "$BASE_URL/galleries/$id"
+                = "$BASE_URL/galleries/$id.html"
     }
 }

app/src/main/java/exh/metadata/models/HitomiPage.kt

@@ -0,0 +1,14 @@
+package exh.metadata.models
+
+import io.realm.RealmObject
+import io.realm.annotations.Index
+import io.realm.annotations.RealmClass
+
+@RealmClass
+open class HitomiPage: RealmObject() {
+    @Index lateinit var gallery: String
+
+    @Index var index: Int = -1
+
+    lateinit var url: String
+}

app/src/main/java/exh/metadata/models/HitomiSkeletonGalleryMetadata.kt

@@ -0,0 +1,67 @@
+package exh.metadata.models
+
+import eu.kanade.tachiyomi.source.model.SManga
+import exh.metadata.models.HitomiGalleryMetadata.Companion.hlIdFromUrl
+import exh.metadata.models.HitomiGalleryMetadata.Companion.urlFromHlId
+import io.realm.RealmList
+import io.realm.RealmObject
+import io.realm.annotations.Ignore
+import io.realm.annotations.Index
+import io.realm.annotations.RealmClass
+
+@RealmClass
+open class HitomiSkeletonGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
+    override var uuid: String
+        set(value) {}
+        get() = throw UnsupportedOperationException()
+
+    var hlId: String? = null
+
+    var thumbnailUrl: String? = null
+
+    var artist: String? = null
+
+    var group: String? = null
+
+    var type: String? = null
+
+    var language: String? = null
+
+    var languageSimple: String? = null
+
+    var series: RealmList<String> = RealmList()
+
+    var characters: RealmList<String> = RealmList()
+
+    var buyLink: String? = null
+
+    var uploadDate: Long? = null
+
+    override var tags: RealmList<Tag> = RealmList()
+
+    // Sites does not show uploader
+    override var uploader: String? = "admin"
+
+    var url get() = hlId?.let { urlFromHlId(it) }
+        set(a) {
+            a?.let {
+                hlId = hlIdFromUrl(a)
+            }
+        }
+
+    override var mangaId: Long? = null
+
+    @Index
+    var title: String? = null
+
+    override fun getTitles() = listOfNotNull(title)
+
+    @Ignore
+    override val titleFields = listOf(
+            ::title.name
+    )
+    override fun copyTo(manga: SManga) {
+        throw UnsupportedOperationException("This operation cannot be performed on skeleton galleries!")
+    }
+}
+

app/src/main/java/exh/util/JsonReaderObjectReader.kt

@@ -0,0 +1,66 @@
+package exh.util
+
+/**
+ * Reads entire `JsonObject`s and `JsonArray`s from `JsonReader`s
+ *
+ * @author nulldev
+ */
+
+import com.google.gson.JsonArray
+import com.google.gson.JsonNull
+import com.google.gson.JsonObject
+import com.google.gson.stream.JsonReader
+import com.google.gson.stream.JsonToken
+import java.math.BigDecimal
+
+fun JsonReader.nextJsonObject(): JsonObject {
+    beginObject()
+
+    val obj = JsonObject()
+
+    while(hasNext()) {
+        val name = nextName()
+
+        when(peek()) {
+            JsonToken.BEGIN_ARRAY -> obj.add(name, nextJsonArray())
+            JsonToken.BEGIN_OBJECT -> obj.add(name, nextJsonObject())
+            JsonToken.NULL -> {
+                nextNull()
+                obj.add(name, JsonNull.INSTANCE)
+            }
+            JsonToken.BOOLEAN -> obj.addProperty(name, nextBoolean())
+            JsonToken.NUMBER -> obj.addProperty(name, BigDecimal(nextString()))
+            JsonToken.STRING -> obj.addProperty(name, nextString())
+            else -> skipValue()
+        }
+    }
+
+    endObject()
+
+    return obj
+}
+
+fun JsonReader.nextJsonArray(): JsonArray {
+    beginArray()
+
+    val arr = JsonArray()
+
+    while(hasNext()) {
+        when(peek()) {
+            JsonToken.BEGIN_ARRAY -> arr.add(nextJsonArray())
+            JsonToken.BEGIN_OBJECT -> arr.add(nextJsonObject())
+            JsonToken.NULL -> {
+                nextNull()
+                arr.add(JsonNull.INSTANCE)
+            }
+            JsonToken.BOOLEAN -> arr.add(nextBoolean())
+            JsonToken.NUMBER -> arr.add(BigDecimal(nextString()))
+            JsonToken.STRING -> arr.add(nextString())
+            else -> skipValue()
+        }
+    }
+
+    endArray()
+
+    return arr
+}

app/src/main/java/exh/util/RealmUtil.kt

@@ -41,5 +41,17 @@ inline fun <T> Realm.trans(block: () -> T): T {
     }
 }
 
+inline fun <T> Realm.useTrans(block: (Realm) -> T): T {
+    return use {
+        trans {
+            block(this)
+        }
+    }
+}
+
 fun <T : RealmModel> Realm.createUUIDObj(clazz: Class<T>)
     = createObject(clazz, UUID.randomUUID().toString())!!
+
+inline fun <reified T : RealmModel> Realm.createUUIDObj()
+        = createUUIDObj(T::class.java)
+