Add A-B swapping for hitomi.la search database
This commit is contained in:
parent
4bd965a795
commit
7aa8abdd98
@ -159,5 +159,7 @@ object PreferenceKeys {
|
|||||||
|
|
||||||
const val eh_hl_lastRefresh = "eh_lh_last_refresh"
|
const val eh_hl_lastRefresh = "eh_lh_last_refresh"
|
||||||
|
|
||||||
|
const val eh_hl_lastRealmIndex = "eh_hl_lastRealmIndex"
|
||||||
|
|
||||||
const val eh_expandFilters = "eh_expand_filters"
|
const val eh_expandFilters = "eh_expand_filters"
|
||||||
}
|
}
|
||||||
|
@ -229,6 +229,8 @@ class PreferencesHelper(val context: Context) {
|
|||||||
fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24")
|
fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24")
|
||||||
|
|
||||||
fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L)
|
fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L)
|
||||||
|
|
||||||
|
fun eh_hl_lastRealmIndex() = rxPrefs.getInteger(Keys.eh_hl_lastRealmIndex, -1)
|
||||||
// <-- EH
|
// <-- EH
|
||||||
|
|
||||||
fun eh_expandFilters() = rxPrefs.getBoolean(Keys.eh_expandFilters, false)
|
fun eh_expandFilters() = rxPrefs.getBoolean(Keys.eh_expandFilters, false)
|
||||||
|
@ -54,6 +54,7 @@ class Hitomi(private val context: Context)
|
|||||||
:HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiSkeletonGalleryMetadata> {
|
:HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiSkeletonGalleryMetadata> {
|
||||||
private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) { JsonParser() }
|
private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) { JsonParser() }
|
||||||
private val searchEngine by lazy { SearchEngine() }
|
private val searchEngine by lazy { SearchEngine() }
|
||||||
|
private val prefs: PreferencesHelper by injectLazy()
|
||||||
|
|
||||||
private val queryCache = mutableMapOf<String, RealmResults<HitomiSkeletonGalleryMetadata>>()
|
private val queryCache = mutableMapOf<String, RealmResults<HitomiSkeletonGalleryMetadata>>()
|
||||||
private val queryWorkQueue = LinkedBlockingQueue<Triple<String, Int, AsyncSubject<List<HitomiSkeletonGalleryMetadata>>>>()
|
private val queryWorkQueue = LinkedBlockingQueue<Triple<String, Int, AsyncSubject<List<HitomiSkeletonGalleryMetadata>>>>()
|
||||||
@ -122,7 +123,7 @@ class Hitomi(private val context: Context)
|
|||||||
By caching our RealmResults in memory, we avoid creating many new RealmResults objects,
|
By caching our RealmResults in memory, we avoid creating many new RealmResults objects,
|
||||||
thus speeding up RealmResults.size.
|
thus speeding up RealmResults.size.
|
||||||
|
|
||||||
Realms are per-thread and RealmReults are bound to Realms. Therefore we create a
|
Realms are per-thread and RealmResults are bound to Realms. Therefore we create a
|
||||||
permanent thread that will open a permanent realm and wait for requests to load RealmResults.
|
permanent thread that will open a permanent realm and wait for requests to load RealmResults.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@ -133,46 +134,48 @@ class Hitomi(private val context: Context)
|
|||||||
searchWorker = thread {
|
searchWorker = thread {
|
||||||
ensureCacheLoaded().toBlocking().first()
|
ensureCacheLoaded().toBlocking().first()
|
||||||
|
|
||||||
getCacheRealm().use { realm ->
|
val realms = arrayOf(getCacheRealm(0), getCacheRealm(1))
|
||||||
Timber.d("[SW] New search worker thread started!")
|
|
||||||
while (true) {
|
|
||||||
Timber.d("[SW] Waiting for next query!")
|
|
||||||
val next = queryWorkQueue.take()
|
|
||||||
Timber.d("[SW] Found new query (page ${next.second}): ${next.first}")
|
|
||||||
|
|
||||||
if(queryCache[next.first] == null) {
|
Timber.d("[SW] New search worker thread started!")
|
||||||
val first = realm.where(HitomiSkeletonGalleryMetadata::class.java).findFirst()
|
while (true) {
|
||||||
|
val realm = realms[prefs.eh_hl_lastRealmIndex().getOrDefault()]
|
||||||
|
|
||||||
if (first == null) {
|
Timber.d("[SW] Waiting for next query!")
|
||||||
next.third.onNext(emptyList())
|
val next = queryWorkQueue.take()
|
||||||
next.third.onCompleted()
|
Timber.d("[SW] Found new query (page ${next.second}): ${next.first}")
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
val parsed = searchEngine.parseQuery(next.first)
|
if(queryCache[next.first] == null) {
|
||||||
val filtered = searchEngine.filterResults(realm.where(HitomiSkeletonGalleryMetadata::class.java),
|
val first = realm.where(HitomiSkeletonGalleryMetadata::class.java).findFirst()
|
||||||
parsed,
|
|
||||||
first.titleFields).findAll()
|
|
||||||
|
|
||||||
queryCache[next.first] = filtered
|
if (first == null) {
|
||||||
}
|
|
||||||
|
|
||||||
val filtered = queryCache[next.first]!!
|
|
||||||
|
|
||||||
val beginIndex = (next.second - 1) * PAGE_SIZE
|
|
||||||
if (beginIndex > filtered.lastIndex) {
|
|
||||||
next.third.onNext(emptyList())
|
next.third.onNext(emptyList())
|
||||||
next.third.onCompleted()
|
next.third.onCompleted()
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Chunk into pages of 100
|
val parsed = searchEngine.parseQuery(next.first)
|
||||||
val res = realm.copyFromRealm(filtered.subList(beginIndex,
|
val filtered = searchEngine.filterResults(realm.where(HitomiSkeletonGalleryMetadata::class.java),
|
||||||
Math.min(next.second * PAGE_SIZE, filtered.size)))
|
parsed,
|
||||||
|
first.titleFields).findAll()
|
||||||
|
|
||||||
next.third.onNext(res)
|
queryCache[next.first] = filtered
|
||||||
next.third.onCompleted()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
val filtered = queryCache[next.first]!!
|
||||||
|
|
||||||
|
val beginIndex = (next.second - 1) * PAGE_SIZE
|
||||||
|
if (beginIndex > filtered.lastIndex) {
|
||||||
|
next.third.onNext(emptyList())
|
||||||
|
next.third.onCompleted()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunk into pages of 100
|
||||||
|
val res = realm.copyFromRealm(filtered.subList(beginIndex,
|
||||||
|
Math.min(next.second * PAGE_SIZE, filtered.size)))
|
||||||
|
|
||||||
|
next.third.onNext(res)
|
||||||
|
next.third.onCompleted()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -256,9 +259,7 @@ class Hitomi(private val context: Context)
|
|||||||
|
|
||||||
override val supportsLatest = true
|
override val supportsLatest = true
|
||||||
|
|
||||||
private val prefs: PreferencesHelper by injectLazy()
|
private val cacheLocks = arrayOf(ReentrantLock(), ReentrantLock())
|
||||||
|
|
||||||
private val cacheLock = ReentrantLock()
|
|
||||||
|
|
||||||
override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html")
|
override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html")
|
||||||
|
|
||||||
@ -295,7 +296,7 @@ class Hitomi(private val context: Context)
|
|||||||
else null
|
else null
|
||||||
}
|
}
|
||||||
|
|
||||||
val meta = getCacheRealm().use {
|
val meta = getAvailableCacheRealm()?.use {
|
||||||
val res = it.where(HitomiSkeletonGalleryMetadata::class.java)
|
val res = it.where(HitomiSkeletonGalleryMetadata::class.java)
|
||||||
.equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
|
.equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
|
||||||
.findFirst()
|
.findFirst()
|
||||||
@ -431,14 +432,16 @@ class Hitomi(private val context: Context)
|
|||||||
it.insert(newPages)
|
it.insert(newPages)
|
||||||
}
|
}
|
||||||
|
|
||||||
getCacheRealm().useTrans {
|
(0 .. 1).map { getCacheRealm(it) }.forEach {
|
||||||
// Delete old meta
|
it.useTrans {
|
||||||
it.where(HitomiSkeletonGalleryMetadata::class.java)
|
// Delete old meta
|
||||||
.equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
|
it.where(HitomiSkeletonGalleryMetadata::class.java)
|
||||||
.findAll().deleteAllFromRealm()
|
.equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId)
|
||||||
|
.findAll().deleteAllFromRealm()
|
||||||
|
|
||||||
// Add new meta
|
// Add new meta
|
||||||
it.insert(newMeta)
|
it.insert(newMeta)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
newMeta to newPages.map(HitomiPage::url)
|
newMeta to newPages.map(HitomiPage::url)
|
||||||
@ -455,10 +458,8 @@ class Hitomi(private val context: Context)
|
|||||||
.map { response ->
|
.map { response ->
|
||||||
val doc = response.asJsoup()
|
val doc = response.asJsoup()
|
||||||
|
|
||||||
val res = getCacheRealm().use { realm ->
|
val res = parsePage(doc).map {
|
||||||
parsePage(doc).map {
|
it
|
||||||
it
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
val sManga = res.map {
|
val sManga = res.map {
|
||||||
SManga.create().apply {
|
SManga.create().apply {
|
||||||
@ -493,12 +494,12 @@ class Hitomi(private val context: Context)
|
|||||||
return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L
|
return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L
|
||||||
}
|
}
|
||||||
|
|
||||||
private inline fun <T> lockCache(block: () -> T): T {
|
private inline fun <T> lockCache(index: Int, block: () -> T): T {
|
||||||
cacheLock.lock()
|
cacheLocks[index].lock()
|
||||||
try {
|
try {
|
||||||
return block()
|
return block()
|
||||||
} finally {
|
} finally {
|
||||||
cacheLock.unlock()
|
cacheLocks[index].unlock()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -506,7 +507,7 @@ class Hitomi(private val context: Context)
|
|||||||
val mid = HitomiGalleryMetadata.hlIdFromUrl(url)
|
val mid = HitomiGalleryMetadata.hlIdFromUrl(url)
|
||||||
|
|
||||||
return ensureCacheLoaded().map {
|
return ensureCacheLoaded().map {
|
||||||
getCacheRealm().use { realm ->
|
getAvailableCacheRealm()?.use { realm ->
|
||||||
findCacheMetadataById(realm, mid)
|
findCacheMetadataById(realm, mid)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -520,11 +521,19 @@ class Hitomi(private val context: Context)
|
|||||||
|
|
||||||
private fun ensureCacheLoaded(blocking: Boolean = true): Observable<Any> {
|
private fun ensureCacheLoaded(blocking: Boolean = true): Observable<Any> {
|
||||||
return Observable.fromCallable {
|
return Observable.fromCallable {
|
||||||
if(!blocking && cacheLock.isLocked) return@fromCallable Any()
|
if(prefs.eh_hl_lastRealmIndex().getOrDefault() >= 0) { return@fromCallable Any() }
|
||||||
|
|
||||||
lockCache {
|
val nextRealmIndex = when(prefs.eh_hl_lastRealmIndex().getOrDefault()) {
|
||||||
|
0 -> 1
|
||||||
|
1 -> 0
|
||||||
|
else -> 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!blocking && cacheLocks[nextRealmIndex].isLocked) return@fromCallable Any()
|
||||||
|
|
||||||
|
lockCache(nextRealmIndex) {
|
||||||
val shouldRefresh = shouldRefreshGalleryFiles()
|
val shouldRefresh = shouldRefreshGalleryFiles()
|
||||||
getCacheRealm().useTrans { realm ->
|
getCacheRealm(nextRealmIndex).useTrans { realm ->
|
||||||
if (!realm.isEmpty && !shouldRefresh)
|
if (!realm.isEmpty && !shouldRefresh)
|
||||||
return@fromCallable Any()
|
return@fromCallable Any()
|
||||||
|
|
||||||
@ -539,7 +548,7 @@ class Hitomi(private val context: Context)
|
|||||||
|
|
||||||
for(threadIndex in 1 .. cores) {
|
for(threadIndex in 1 .. cores) {
|
||||||
threads += thread {
|
threads += thread {
|
||||||
getCacheRealm().use { realm ->
|
getCacheRealm(nextRealmIndex).use { realm ->
|
||||||
while (true) {
|
while (true) {
|
||||||
val i = workQueue.poll() ?: break
|
val i = workQueue.poll() ?: break
|
||||||
|
|
||||||
@ -577,6 +586,11 @@ class Hitomi(private val context: Context)
|
|||||||
|
|
||||||
// Update refresh time
|
// Update refresh time
|
||||||
prefs.eh_hl_lastRefresh().set(System.currentTimeMillis())
|
prefs.eh_hl_lastRefresh().set(System.currentTimeMillis())
|
||||||
|
|
||||||
|
// Update last refreshed realm
|
||||||
|
prefs.eh_hl_lastRealmIndex().set(nextRealmIndex)
|
||||||
|
|
||||||
|
Timber.d("Successfully refreshed realm #$nextRealmIndex!")
|
||||||
}
|
}
|
||||||
|
|
||||||
return@fromCallable Any()
|
return@fromCallable Any()
|
||||||
@ -607,7 +621,35 @@ class Hitomi(private val context: Context)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun getCacheRealm() = Realm.getInstance(REALM_CONFIG)
|
private fun <T> getAndLockAvailableCacheRealm(block: (Realm) -> T): T? {
|
||||||
|
val index = prefs.eh_hl_lastRealmIndex().getOrDefault()
|
||||||
|
|
||||||
|
return if(index >= 0) {
|
||||||
|
val cache = getCacheRealm(index)
|
||||||
|
lockCache(index) {
|
||||||
|
block(cache)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getAvailableCacheRealm(): Realm? {
|
||||||
|
val index = prefs.eh_hl_lastRealmIndex().getOrDefault()
|
||||||
|
|
||||||
|
return if(index >= 0) {
|
||||||
|
getCacheRealm(index)
|
||||||
|
} else {
|
||||||
|
null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun getCacheRealm(index: Int) = Realm.getInstance(getRealmConfig(index))
|
||||||
|
|
||||||
|
private fun getRealmConfig(index: Int) = RealmConfiguration.Builder()
|
||||||
|
.name("hitomi-cache-$index")
|
||||||
|
.deleteRealmIfMigrationNeeded()
|
||||||
|
.build()
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
private val PAGE_SIZE = 25
|
private val PAGE_SIZE = 25
|
||||||
@ -652,11 +694,6 @@ function url_from_url(url, base) {
|
|||||||
return url_from_url('$IMAGE_RESOLVER_URL_VAR');
|
return url_from_url('$IMAGE_RESOLVER_URL_VAR');
|
||||||
})();
|
})();
|
||||||
""".trimIndent()
|
""".trimIndent()
|
||||||
|
|
||||||
private val REALM_CONFIG = RealmConfiguration.Builder()
|
|
||||||
.name("hitomi-cache")
|
|
||||||
.deleteRealmIfMigrationNeeded()
|
|
||||||
.build()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -20,12 +20,14 @@ import android.widget.Toast
|
|||||||
import com.afollestad.materialdialogs.MaterialDialog
|
import com.afollestad.materialdialogs.MaterialDialog
|
||||||
import com.bumptech.glide.load.engine.DiskCacheStrategy
|
import com.bumptech.glide.load.engine.DiskCacheStrategy
|
||||||
import com.bumptech.glide.load.resource.bitmap.RoundedCorners
|
import com.bumptech.glide.load.resource.bitmap.RoundedCorners
|
||||||
|
import com.bumptech.glide.load.resource.bitmap.TransformationUtils.centerCrop
|
||||||
import com.bumptech.glide.request.target.SimpleTarget
|
import com.bumptech.glide.request.target.SimpleTarget
|
||||||
import com.bumptech.glide.request.transition.Transition
|
import com.bumptech.glide.request.transition.Transition
|
||||||
import com.jakewharton.rxbinding.support.v4.widget.refreshes
|
import com.jakewharton.rxbinding.support.v4.widget.refreshes
|
||||||
import com.jakewharton.rxbinding.view.clicks
|
import com.jakewharton.rxbinding.view.clicks
|
||||||
import com.jakewharton.rxbinding.view.longClicks
|
import com.jakewharton.rxbinding.view.longClicks
|
||||||
import eu.kanade.tachiyomi.R
|
import eu.kanade.tachiyomi.R
|
||||||
|
import eu.kanade.tachiyomi.R.id.*
|
||||||
import eu.kanade.tachiyomi.data.database.models.Category
|
import eu.kanade.tachiyomi.data.database.models.Category
|
||||||
import eu.kanade.tachiyomi.data.database.models.Manga
|
import eu.kanade.tachiyomi.data.database.models.Manga
|
||||||
import eu.kanade.tachiyomi.data.glide.GlideApp
|
import eu.kanade.tachiyomi.data.glide.GlideApp
|
||||||
|
Loading…
x
Reference in New Issue
Block a user