Add A-B swapping for hitomi.la search database

This commit is contained in:
NerdNumber9 2018-04-14 23:23:55 -04:00
parent 4bd965a795
commit 7aa8abdd98
4 changed files with 102 additions and 59 deletions

View File

@ -159,5 +159,7 @@ object PreferenceKeys {
const val eh_hl_lastRefresh = "eh_lh_last_refresh" const val eh_hl_lastRefresh = "eh_lh_last_refresh"
const val eh_hl_lastRealmIndex = "eh_hl_lastRealmIndex"
const val eh_expandFilters = "eh_expand_filters" const val eh_expandFilters = "eh_expand_filters"
} }

View File

@ -229,6 +229,8 @@ class PreferencesHelper(val context: Context) {
fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24") fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24")
fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L) fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L)
fun eh_hl_lastRealmIndex() = rxPrefs.getInteger(Keys.eh_hl_lastRealmIndex, -1)
// <-- EH // <-- EH
fun eh_expandFilters() = rxPrefs.getBoolean(Keys.eh_expandFilters, false) fun eh_expandFilters() = rxPrefs.getBoolean(Keys.eh_expandFilters, false)

View File

@ -54,6 +54,7 @@ class Hitomi(private val context: Context)
:HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiSkeletonGalleryMetadata> { :HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiSkeletonGalleryMetadata> {
private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) { JsonParser() } private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) { JsonParser() }
private val searchEngine by lazy { SearchEngine() } private val searchEngine by lazy { SearchEngine() }
private val prefs: PreferencesHelper by injectLazy()
private val queryCache = mutableMapOf<String, RealmResults<HitomiSkeletonGalleryMetadata>>() private val queryCache = mutableMapOf<String, RealmResults<HitomiSkeletonGalleryMetadata>>()
private val queryWorkQueue = LinkedBlockingQueue<Triple<String, Int, AsyncSubject<List<HitomiSkeletonGalleryMetadata>>>>() private val queryWorkQueue = LinkedBlockingQueue<Triple<String, Int, AsyncSubject<List<HitomiSkeletonGalleryMetadata>>>>()
@ -122,7 +123,7 @@ class Hitomi(private val context: Context)
By caching our RealmResults in memory, we avoid creating many new RealmResults objects, By caching our RealmResults in memory, we avoid creating many new RealmResults objects,
thus speeding up RealmResults.size. thus speeding up RealmResults.size.
Realms are per-thread and RealmReults are bound to Realms. Therefore we create a Realms are per-thread and RealmResults are bound to Realms. Therefore we create a
permanent thread that will open a permanent realm and wait for requests to load RealmResults. permanent thread that will open a permanent realm and wait for requests to load RealmResults.
*/ */
@ -133,46 +134,48 @@ class Hitomi(private val context: Context)
searchWorker = thread { searchWorker = thread {
ensureCacheLoaded().toBlocking().first() ensureCacheLoaded().toBlocking().first()
getCacheRealm().use { realm -> val realms = arrayOf(getCacheRealm(0), getCacheRealm(1))
Timber.d("[SW] New search worker thread started!")
while (true) {
Timber.d("[SW] Waiting for next query!")
val next = queryWorkQueue.take()
Timber.d("[SW] Found new query (page ${next.second}): ${next.first}")
if(queryCache[next.first] == null) { Timber.d("[SW] New search worker thread started!")
val first = realm.where(HitomiSkeletonGalleryMetadata::class.java).findFirst() while (true) {
val realm = realms[prefs.eh_hl_lastRealmIndex().getOrDefault()]
if (first == null) { Timber.d("[SW] Waiting for next query!")
next.third.onNext(emptyList()) val next = queryWorkQueue.take()
next.third.onCompleted() Timber.d("[SW] Found new query (page ${next.second}): ${next.first}")
continue
}
val parsed = searchEngine.parseQuery(next.first) if(queryCache[next.first] == null) {
val filtered = searchEngine.filterResults(realm.where(HitomiSkeletonGalleryMetadata::class.java), val first = realm.where(HitomiSkeletonGalleryMetadata::class.java).findFirst()
parsed,
first.titleFields).findAll()
queryCache[next.first] = filtered if (first == null) {
}
val filtered = queryCache[next.first]!!
val beginIndex = (next.second - 1) * PAGE_SIZE
if (beginIndex > filtered.lastIndex) {
next.third.onNext(emptyList()) next.third.onNext(emptyList())
next.third.onCompleted() next.third.onCompleted()
continue continue
} }
// Chunk into pages of 100 val parsed = searchEngine.parseQuery(next.first)
val res = realm.copyFromRealm(filtered.subList(beginIndex, val filtered = searchEngine.filterResults(realm.where(HitomiSkeletonGalleryMetadata::class.java),
Math.min(next.second * PAGE_SIZE, filtered.size))) parsed,
first.titleFields).findAll()
next.third.onNext(res) queryCache[next.first] = filtered
next.third.onCompleted()
} }
val filtered = queryCache[next.first]!!
val beginIndex = (next.second - 1) * PAGE_SIZE
if (beginIndex > filtered.lastIndex) {
next.third.onNext(emptyList())
next.third.onCompleted()
continue
}
// Chunk into pages of 100
val res = realm.copyFromRealm(filtered.subList(beginIndex,
Math.min(next.second * PAGE_SIZE, filtered.size)))
next.third.onNext(res)
next.third.onCompleted()
} }
} }
} }
@ -256,9 +259,7 @@ class Hitomi(private val context: Context)
override val supportsLatest = true override val supportsLatest = true
private val prefs: PreferencesHelper by injectLazy() private val cacheLocks = arrayOf(ReentrantLock(), ReentrantLock())
private val cacheLock = ReentrantLock()
override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html") override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html")
@ -295,7 +296,7 @@ class Hitomi(private val context: Context)
else null else null
} }
val meta = getCacheRealm().use { val meta = getAvailableCacheRealm()?.use {
val res = it.where(HitomiSkeletonGalleryMetadata::class.java) val res = it.where(HitomiSkeletonGalleryMetadata::class.java)
.equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId) .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
.findFirst() .findFirst()
@ -431,14 +432,16 @@ class Hitomi(private val context: Context)
it.insert(newPages) it.insert(newPages)
} }
getCacheRealm().useTrans { (0 .. 1).map { getCacheRealm(it) }.forEach {
// Delete old meta it.useTrans {
it.where(HitomiSkeletonGalleryMetadata::class.java) // Delete old meta
.equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId) it.where(HitomiSkeletonGalleryMetadata::class.java)
.findAll().deleteAllFromRealm() .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
.findAll().deleteAllFromRealm()
// Add new meta // Add new meta
it.insert(newMeta) it.insert(newMeta)
}
} }
newMeta to newPages.map(HitomiPage::url) newMeta to newPages.map(HitomiPage::url)
@ -455,10 +458,8 @@ class Hitomi(private val context: Context)
.map { response -> .map { response ->
val doc = response.asJsoup() val doc = response.asJsoup()
val res = getCacheRealm().use { realm -> val res = parsePage(doc).map {
parsePage(doc).map { it
it
}
} }
val sManga = res.map { val sManga = res.map {
SManga.create().apply { SManga.create().apply {
@ -493,12 +494,12 @@ class Hitomi(private val context: Context)
return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L
} }
private inline fun <T> lockCache(block: () -> T): T { private inline fun <T> lockCache(index: Int, block: () -> T): T {
cacheLock.lock() cacheLocks[index].lock()
try { try {
return block() return block()
} finally { } finally {
cacheLock.unlock() cacheLocks[index].unlock()
} }
} }
@ -506,7 +507,7 @@ class Hitomi(private val context: Context)
val mid = HitomiGalleryMetadata.hlIdFromUrl(url) val mid = HitomiGalleryMetadata.hlIdFromUrl(url)
return ensureCacheLoaded().map { return ensureCacheLoaded().map {
getCacheRealm().use { realm -> getAvailableCacheRealm()?.use { realm ->
findCacheMetadataById(realm, mid) findCacheMetadataById(realm, mid)
} }
} }
@ -520,11 +521,19 @@ class Hitomi(private val context: Context)
private fun ensureCacheLoaded(blocking: Boolean = true): Observable<Any> { private fun ensureCacheLoaded(blocking: Boolean = true): Observable<Any> {
return Observable.fromCallable { return Observable.fromCallable {
if(!blocking && cacheLock.isLocked) return@fromCallable Any() if(prefs.eh_hl_lastRealmIndex().getOrDefault() >= 0) { return@fromCallable Any() }
lockCache { val nextRealmIndex = when(prefs.eh_hl_lastRealmIndex().getOrDefault()) {
0 -> 1
1 -> 0
else -> 0
}
if(!blocking && cacheLocks[nextRealmIndex].isLocked) return@fromCallable Any()
lockCache(nextRealmIndex) {
val shouldRefresh = shouldRefreshGalleryFiles() val shouldRefresh = shouldRefreshGalleryFiles()
getCacheRealm().useTrans { realm -> getCacheRealm(nextRealmIndex).useTrans { realm ->
if (!realm.isEmpty && !shouldRefresh) if (!realm.isEmpty && !shouldRefresh)
return@fromCallable Any() return@fromCallable Any()
@ -539,7 +548,7 @@ class Hitomi(private val context: Context)
for(threadIndex in 1 .. cores) { for(threadIndex in 1 .. cores) {
threads += thread { threads += thread {
getCacheRealm().use { realm -> getCacheRealm(nextRealmIndex).use { realm ->
while (true) { while (true) {
val i = workQueue.poll() ?: break val i = workQueue.poll() ?: break
@ -577,6 +586,11 @@ class Hitomi(private val context: Context)
// Update refresh time // Update refresh time
prefs.eh_hl_lastRefresh().set(System.currentTimeMillis()) prefs.eh_hl_lastRefresh().set(System.currentTimeMillis())
// Update last refreshed realm
prefs.eh_hl_lastRealmIndex().set(nextRealmIndex)
Timber.d("Successfully refreshed realm #$nextRealmIndex!")
} }
return@fromCallable Any() return@fromCallable Any()
@ -607,7 +621,35 @@ class Hitomi(private val context: Context)
} }
} }
private fun getCacheRealm() = Realm.getInstance(REALM_CONFIG) private fun <T> getAndLockAvailableCacheRealm(block: (Realm) -> T): T? {
val index = prefs.eh_hl_lastRealmIndex().getOrDefault()
return if(index >= 0) {
val cache = getCacheRealm(index)
lockCache(index) {
block(cache)
}
} else {
null
}
}
private fun getAvailableCacheRealm(): Realm? {
val index = prefs.eh_hl_lastRealmIndex().getOrDefault()
return if(index >= 0) {
getCacheRealm(index)
} else {
null
}
}
private fun getCacheRealm(index: Int) = Realm.getInstance(getRealmConfig(index))
private fun getRealmConfig(index: Int) = RealmConfiguration.Builder()
.name("hitomi-cache-$index")
.deleteRealmIfMigrationNeeded()
.build()
companion object { companion object {
private val PAGE_SIZE = 25 private val PAGE_SIZE = 25
@ -652,11 +694,6 @@ function url_from_url(url, base) {
return url_from_url('$IMAGE_RESOLVER_URL_VAR'); return url_from_url('$IMAGE_RESOLVER_URL_VAR');
})(); })();
""".trimIndent() """.trimIndent()
private val REALM_CONFIG = RealmConfiguration.Builder()
.name("hitomi-cache")
.deleteRealmIfMigrationNeeded()
.build()
} }
} }

View File

@ -20,12 +20,14 @@ import android.widget.Toast
import com.afollestad.materialdialogs.MaterialDialog import com.afollestad.materialdialogs.MaterialDialog
import com.bumptech.glide.load.engine.DiskCacheStrategy import com.bumptech.glide.load.engine.DiskCacheStrategy
import com.bumptech.glide.load.resource.bitmap.RoundedCorners import com.bumptech.glide.load.resource.bitmap.RoundedCorners
import com.bumptech.glide.load.resource.bitmap.TransformationUtils.centerCrop
import com.bumptech.glide.request.target.SimpleTarget import com.bumptech.glide.request.target.SimpleTarget
import com.bumptech.glide.request.transition.Transition import com.bumptech.glide.request.transition.Transition
import com.jakewharton.rxbinding.support.v4.widget.refreshes import com.jakewharton.rxbinding.support.v4.widget.refreshes
import com.jakewharton.rxbinding.view.clicks import com.jakewharton.rxbinding.view.clicks
import com.jakewharton.rxbinding.view.longClicks import com.jakewharton.rxbinding.view.longClicks
import eu.kanade.tachiyomi.R import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.R.id.*
import eu.kanade.tachiyomi.data.database.models.Category import eu.kanade.tachiyomi.data.database.models.Category
import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.glide.GlideApp import eu.kanade.tachiyomi.data.glide.GlideApp