Draff/TachiyomiSY-Plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync is now essentially fully atomic on the client side

Browse Source 
Sync now aborts on most errors instead of continuing
Sync now holds screen, CPU and WIFI locks
This commit is contained in:
NerdNumber9 2018-02-02 12:11:09 -05:00
parent e9bea8ed47
commit 8f51abfc97
9 changed files with 128 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt
View File

@ -12,7 +12,6 @@ import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.LewdSource
import eu.kanade.tachiyomi.util.asJsoup
import exh.metadata.EX_DATE_FORMAT
import exh.metadata.ignore
import exh.metadata.models.ExGalleryMetadata
import exh.metadata.models.Tag
import exh.metadata.nullIfBlank
@ -20,6 +19,7 @@ import exh.metadata.parseHumanReadableByteCount
import exh.ui.login.LoginController
import exh.util.UriFilter
import exh.util.UriGroup
import exh.util.ignore
import exh.util.urlImportFetchSearchManga
import okhttp3.CacheControl
import okhttp3.Headers

34
app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
View File

@ -571,6 +571,8 @@ class LibraryController(
favSyncDialog?.dismiss()
favSyncDialog = null
oldSyncStatus = null
//Clear flags
releaseSyncLocks()
}
private fun buildDialog() = activity?.let {
@ -586,13 +588,25 @@ class LibraryController(
?.show()
}
private fun takeSyncLocks() {
activity?.window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
}
private fun releaseSyncLocks() {
activity?.window?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
}
private fun updateSyncStatus(status: FavoritesSyncStatus) {
when(status) {
is FavoritesSyncStatus.Idle -> {
releaseSyncLocks()
favSyncDialog?.dismiss()
favSyncDialog = null
}
is FavoritesSyncStatus.Error -> {
releaseSyncLocks()
favSyncDialog?.dismiss()
favSyncDialog = buildDialog()
?.title("Favorites sync error")
@ -606,6 +620,8 @@ class LibraryController(
}
is FavoritesSyncStatus.Processing,
is FavoritesSyncStatus.Initializing -> {
takeSyncLocks()
if(favSyncDialog == null || (oldSyncStatus != null
&& oldSyncStatus !is FavoritesSyncStatus.Initializing
&& oldSyncStatus !is FavoritesSyncStatus.Processing))
@ -613,24 +629,6 @@ class LibraryController(
favSyncDialog?.setContent(status.message)
}
is FavoritesSyncStatus.Complete -> {
favSyncDialog?.dismiss()
if(status.errors.isNotEmpty()) {
favSyncDialog = buildDialog()
?.title("Favorites sync complete with errors")
?.content("Some errors occurred during the sync process:\n\n"
+ status.errors.joinToString("\n"))
?.cancelable(false)
?.positiveText("Ok")
?.onPositive { _, _ ->
presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle())
}
?.show()
} else {
presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle())
}
}
}
oldSyncStatus = status
}

9
app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsEhController.kt
View File

@ -10,6 +10,7 @@ import eu.kanade.tachiyomi.util.toast
import exh.favorites.FavoritesIntroDialog
import exh.favorites.LocalFavoritesStorage
import exh.ui.login.LoginController
import exh.util.trans
import rx.android.schedulers.AndroidSchedulers
import rx.schedulers.Schedulers
@ -163,7 +164,13 @@ class SettingsEhController : SettingsController() {
.content("Resetting the sync state can cause your next sync to be extremely slow.")
.positiveText("Yes")
.onPositive { _, _ ->
LocalFavoritesStorage().clearSnapshots()
LocalFavoritesStorage().apply {
getRealm().use {
it.trans {
clearSnapshots(it)
}
}
}
it.toast("Sync state reset", Toast.LENGTH_LONG)
}
.negativeText("No")

7
app/src/main/java/eu/kanade/tachiyomi/util/ContextExtensions.kt
View File

@ -10,6 +10,7 @@ import android.content.IntentFilter
import android.content.pm.PackageManager
import android.content.res.Resources
import android.net.ConnectivityManager
import android.net.wifi.WifiManager
import android.os.PowerManager
import android.support.annotation.StringRes
import android.support.v4.app.NotificationCompat
@ -116,6 +117,12 @@ val Context.connectivityManager: ConnectivityManager
val Context.powerManager: PowerManager
get() = getSystemService(Context.POWER_SERVICE) as PowerManager
/**
* Property to get the wifi manager from the context.
*/
val Context.wifiManager: WifiManager
get() = applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager
/**
* Function used to send a local broadcast asynchronous
*

2
app/src/main/java/exh/favorites/FavoritesIntroDialog.kt
View File

@ -24,7 +24,7 @@ class FavoritesIntroDialog {
<br><br>
2. The favorite categories on ExHentai correspond to the <b>first 10 categories in the app</b> (excluding the 'Default' category). <i>Galleries in other categories will <b>NOT</b> be synced!</i>
<br><br>
3. <font color='red'><b>ENSURE YOU HAVE A STABLE INTERNET CONNECTION WHEN SYNC IS IN PROGRESS!</b></font> If the internet disconnects while the app is syncing, your favorites may be left in a <i>partially-synced state</i>. This could be disastrous and can cause you to lose favorites. Backup regularly and be aware that <i>I will not be responsible for any loss of data</i>...
3. <font color='red'><b>ENSURE YOU HAVE A STABLE INTERNET CONNECTION WHEN SYNC IS IN PROGRESS!</b></font> If the internet disconnects while the app is syncing, your favorites may be left in a <i>partially-synced state</i>.
<br><br>
4. Keep the app open while favorites are syncing. Android will close apps that are in the background sometimes and that could be bad if it happens while the app is syncing.
<br><br>

94
app/src/main/java/exh/favorites/FavoritesSyncHelper.kt
View File

@ -1,6 +1,8 @@
package exh.favorites
import android.content.Context
import android.net.wifi.WifiManager
import android.os.PowerManager
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Category
import eu.kanade.tachiyomi.data.database.models.Manga
@ -9,10 +11,14 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.all.EHentai
import eu.kanade.tachiyomi.util.powerManager
import eu.kanade.tachiyomi.util.wifiManager
import exh.EH_METADATA_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.GalleryAddEvent
import exh.GalleryAdder
import exh.util.ignore
import exh.util.trans
import okhttp3.FormBody
import okhttp3.Request
import rx.subjects.BehaviorSubject
@ -22,7 +28,7 @@ import uy.kohesive.injekt.api.get
import uy.kohesive.injekt.injectLazy
import kotlin.concurrent.thread
class FavoritesSyncHelper(context: Context) {
class FavoritesSyncHelper(val context: Context) {
private val db: DatabaseHelper by injectLazy()
private val prefs: PreferencesHelper by injectLazy()
@ -39,6 +45,9 @@ class FavoritesSyncHelper(context: Context) {
private var lastThrottleTime: Long = 0
private var throttleTime: Long = 0
private var wifiLock: WifiManager.WifiLock? = null
private var wakeLock: PowerManager.WakeLock? = null
val status = BehaviorSubject.create<FavoritesSyncStatus>(FavoritesSyncStatus.Idle())
@Synchronized
@ -69,30 +78,44 @@ class FavoritesSyncHelper(context: Context) {
return
}
val errors = mutableListOf<String>()
try {
db.inTransaction {
status.onNext(FavoritesSyncStatus.Processing("Calculating remote changes"))
val remoteChanges = storage.getChangedRemoteEntries(favorites.first)
val localChanges = if(prefs.eh_readOnlySync().getOrDefault()) {
null //Do not build local changes if they are not going to be applied
} else {
status.onNext(FavoritesSyncStatus.Processing("Calculating local changes"))
storage.getChangedDbEntries()
//Take wake + wifi locks
wakeLock?.release()
wakeLock = ignore {
context.powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK,
"ExhFavoritesSyncWakelock")
}
wifiLock?.release()
wifiLock = ignore {
context.wifiManager.createWifiLock(WifiManager.WIFI_MODE_FULL,
"ExhFavoritesSyncWifi")
}
storage.getRealm().use { realm ->
realm.trans {
db.inTransaction {
status.onNext(FavoritesSyncStatus.Processing("Calculating remote changes"))
val remoteChanges = storage.getChangedRemoteEntries(realm, favorites.first)
val localChanges = if(prefs.eh_readOnlySync().getOrDefault()) {
null //Do not build local changes if they are not going to be applied
} else {
status.onNext(FavoritesSyncStatus.Processing("Calculating local changes"))
storage.getChangedDbEntries(realm)
}
//Apply remote categories
status.onNext(FavoritesSyncStatus.Processing("Updating category names"))
applyRemoteCategories(favorites.second)
//Apply change sets
applyChangeSetToLocal(remoteChanges)
if(localChanges != null)
applyChangeSetToRemote(localChanges)
status.onNext(FavoritesSyncStatus.Processing("Cleaning up"))
storage.snapshotEntries(realm)
}
}
//Apply remote categories
status.onNext(FavoritesSyncStatus.Processing("Updating category names"))
applyRemoteCategories(favorites.second)
//Apply change sets
applyChangeSetToLocal(remoteChanges, errors)
if(localChanges != null)
applyChangeSetToRemote(localChanges, errors)
status.onNext(FavoritesSyncStatus.Processing("Cleaning up"))
storage.snapshotEntries()
}
} catch(e: IgnoredException) {
//Do not display error as this error has already been reported
@ -102,9 +125,13 @@ class FavoritesSyncHelper(context: Context) {
status.onNext(FavoritesSyncStatus.Error("Unknown error: ${e.message}"))
Timber.e(e, "Sync error!")
return
} finally {
//Release wake + wifi locks
ignore { wakeLock?.release() }
ignore { wifiLock?.release() }
}
status.onNext(FavoritesSyncStatus.Complete(errors))
status.onNext(FavoritesSyncStatus.Idle())
}
private fun applyRemoteCategories(categories: List<String>) {
@ -149,7 +176,7 @@ class FavoritesSyncHelper(context: Context) {
db.insertCategories(newLocalCategories).executeAsBlocking()
}
private fun addGalleryRemote(gallery: FavoriteEntry, errors: MutableList<String>) {
private fun addGalleryRemote(gallery: FavoriteEntry) {
val url = "${exh.baseUrl}/gallerypopups.php?gid=${gallery.gid}&t=${gallery.token}&act=addfav"
val request = Request.Builder()
@ -163,11 +190,12 @@ class FavoritesSyncHelper(context: Context) {
.build()
if(!explicitlyRetryExhRequest(10, request)) {
errors += "Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!"
status.onNext(FavoritesSyncStatus.Error("Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!"))
throw IgnoredException()
}
}
private fun explicitlyRetryExhRequest(retryCount: Int, request: Request, minDelay: Int = 0): Boolean {
private fun explicitlyRetryExhRequest(retryCount: Int, request: Request): Boolean {
var success = false
for(i in 1 .. retryCount) {
@ -186,7 +214,7 @@ class FavoritesSyncHelper(context: Context) {
return success
}
private fun applyChangeSetToRemote(changeSet: ChangeSet, errors: MutableList<String>) {
private fun applyChangeSetToRemote(changeSet: ChangeSet) {
//Apply removals
if(changeSet.removed.isNotEmpty()) {
status.onNext(FavoritesSyncStatus.Processing("Removing ${changeSet.removed.size} galleries from remote server"))
@ -221,11 +249,11 @@ class FavoritesSyncHelper(context: Context) {
throttle()
addGalleryRemote(it, errors)
addGalleryRemote(it)
}
}
private fun applyChangeSetToLocal(changeSet: ChangeSet, errors: MutableList<String>) {
private fun applyChangeSetToLocal(changeSet: ChangeSet) {
val removedManga = mutableListOf<Manga>()
//Apply removals
@ -266,10 +294,11 @@ class FavoritesSyncHelper(context: Context) {
EXH_SOURCE_ID)
if(result is GalleryAddEvent.Fail) {
errors += "Failed to add gallery to local database: " + when (result) {
status.onNext(FavoritesSyncStatus.Error("Failed to add gallery to local database: " + when (result) {
is GalleryAddEvent.Fail.Error -> "'${it.title}' ${result.logMessage}"
is GalleryAddEvent.Fail.UnknownType -> "'${it.title}' (${result.galleryUrl}) is not a valid gallery!"
}
}))
throw IgnoredException()
} else if(result is GalleryAddEvent.Success) {
insertedMangaCategories += MangaCategory.create(result.manga,
categories[it.category])
@ -318,5 +347,4 @@ sealed class FavoritesSyncStatus(val message: String) {
(message + "\n\nSync is currently throttling (to avoid being banned from ExHentai) and may take a long to complete.")
else
message)
class Complete(val errors: List<String>) : FavoritesSyncStatus("Sync complete!")
}

66
app/src/main/java/exh/favorites/LocalFavoritesStorage.kt
View File

@ -6,7 +6,6 @@ import eu.kanade.tachiyomi.source.online.all.EHentai
import exh.EH_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.metadata.models.ExGalleryMetadata
import exh.util.trans
import io.realm.Realm
import io.realm.RealmConfiguration
import uy.kohesive.injekt.injectLazy
@ -19,11 +18,10 @@ class LocalFavoritesStorage {
.deleteRealmIfMigrationNeeded()
.build()
private val realm
get() = Realm.getInstance(realmConfig)
fun getRealm() = Realm.getInstance(realmConfig)
fun getChangedDbEntries()
= getChangedEntries(
fun getChangedDbEntries(realm: Realm)
= getChangedEntries(realm,
parseToFavoriteEntries(
loadDbCategories(
db.getFavoriteMangas()
@ -33,8 +31,8 @@ class LocalFavoritesStorage {
)
)
fun getChangedRemoteEntries(entries: List<EHentai.ParsedManga>)
= getChangedEntries(
fun getChangedRemoteEntries(realm: Realm, entries: List<EHentai.ParsedManga>)
= getChangedEntries(realm,
parseToFavoriteEntries(
entries.asSequence().map {
Pair(it.fav, it.manga.apply {
@ -44,7 +42,7 @@ class LocalFavoritesStorage {
)
)
fun snapshotEntries() {
fun snapshotEntries(realm: Realm) {
val dbMangas = parseToFavoriteEntries(
loadDbCategories(
db.getFavoriteMangas()
@ -53,43 +51,33 @@ class LocalFavoritesStorage {
)
)
realm.use { realm ->
realm.trans {
//Delete old snapshot
realm.delete(FavoriteEntry::class.java)
//Delete old snapshot
realm.delete(FavoriteEntry::class.java)
//Insert new snapshots
realm.copyToRealm(dbMangas.toList())
}
}
//Insert new snapshots
realm.copyToRealm(dbMangas.toList())
}
fun clearSnapshots() {
realm.use {
it.trans {
it.delete(FavoriteEntry::class.java)
}
}
fun clearSnapshots(realm: Realm) {
realm.delete(FavoriteEntry::class.java)
}
private fun getChangedEntries(entries: Sequence<FavoriteEntry>): ChangeSet {
return realm.use { realm ->
val terminated = entries.toList()
private fun getChangedEntries(realm: Realm, entries: Sequence<FavoriteEntry>): ChangeSet {
val terminated = entries.toList()
val added = terminated.filter {
realm.queryRealmForEntry(it) == null
}
val removed = realm.where(FavoriteEntry::class.java)
.findAll()
.filter {
queryListForEntry(terminated, it) == null
}.map {
realm.copyFromRealm(it)
}
ChangeSet(added, removed)
val added = terminated.filter {
realm.queryRealmForEntry(it) == null
}
val removed = realm.where(FavoriteEntry::class.java)
.findAll()
.filter {
queryListForEntry(terminated, it) == null
}.map {
realm.copyFromRealm(it)
}
return ChangeSet(added, removed)
}
private fun Realm.queryRealmForEntry(entry: FavoriteEntry)
@ -100,7 +88,7 @@ class LocalFavoritesStorage {
.findFirst()
private fun queryListForEntry(list: List<FavoriteEntry>, entry: FavoriteEntry)
= list.find {
= list.find {
it.gid == entry.gid
&& it.token == entry.token
&& it.category == entry.category

4
app/src/main/java/exh/metadata/MetadataUtil.kt
View File

@ -43,10 +43,6 @@ fun String?.nullIfBlank(): String? = if(isNullOrBlank())
else
this
fun <T> ignore(expr: () -> T): T? {
return try { expr() } catch (t: Throwable) { null }
}
fun <K,V> Set<Map.Entry<K,V>>.forEach(action: (K, V) -> Unit) {
forEach { action(it.key, it.value) }
}

7
app/src/main/java/exh/util/ExceptionUtil.kt Normal file
View File

@ -0,0 +1,7 @@
package exh.util
inline fun <T> ignore(expr: () -> T): T? {
return try { expr() } catch (t: Throwable) { null }
}