From d892f2f7f49b36bb6fe314898db31e7a99781f35 Mon Sep 17 00:00:00 2001 From: NerdNumber9 Date: Wed, 31 Jan 2018 22:39:55 -0500 Subject: [PATCH] Initial implementation of favorites syncing General code cleanup Fix some cases of duplicate galleries (not completely fixed) --- .../tachiyomi/source/online/all/EHentai.kt | 67 ++--- .../tachiyomi/ui/library/LibraryController.kt | 105 ++++++- .../tachiyomi/ui/library/LibraryPresenter.kt | 5 + app/src/main/java/exh/FavoritesSyncHelper.kt | 137 --------- .../main/java/exh/FavoritesSyncManager.java | 192 ------------ app/src/main/java/exh/GalleryAdder.kt | 18 +- .../main/java/exh/favorites/FavoriteEntry.kt | 23 ++ .../java/exh/favorites/FavoritesSyncHelper.kt | 274 ++++++++++++++++++ .../exh/favorites/LocalFavoritesStorage.kt | 132 +++++++++ .../exh/metadata/models/ExGalleryMetadata.kt | 18 +- .../java/exh/metadata/models/GalleryQuery.kt | 6 +- 11 files changed, 588 insertions(+), 389 deletions(-) delete mode 100755 app/src/main/java/exh/FavoritesSyncHelper.kt delete mode 100755 app/src/main/java/exh/FavoritesSyncManager.java create mode 100644 app/src/main/java/exh/favorites/FavoriteEntry.kt create mode 100644 app/src/main/java/exh/favorites/FavoritesSyncHelper.kt create mode 100644 app/src/main/java/exh/favorites/LocalFavoritesStorage.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt index a3015d141..75b20c4f7 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt @@ -11,21 +11,26 @@ import eu.kanade.tachiyomi.source.model.* import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.LewdSource import eu.kanade.tachiyomi.util.asJsoup -import exh.metadata.* +import exh.metadata.EX_DATE_FORMAT +import exh.metadata.ignore import exh.metadata.models.ExGalleryMetadata import exh.metadata.models.Tag +import exh.metadata.nullIfBlank +import exh.metadata.parseHumanReadableByteCount +import exh.ui.login.LoginController +import exh.util.UriFilter +import exh.util.UriGroup +import exh.util.urlImportFetchSearchManga +import okhttp3.CacheControl +import okhttp3.Headers +import okhttp3.Request import okhttp3.Response +import org.jsoup.nodes.Document import org.jsoup.nodes.Element import rx.Observable import uy.kohesive.injekt.injectLazy import java.net.URLEncoder import java.util.* -import exh.ui.login.LoginController -import okhttp3.CacheControl -import okhttp3.Headers -import okhttp3.Request -import org.jsoup.nodes.Document -import exh.util.* class EHentai(override val id: Long, val exh: Boolean, @@ -51,14 +56,14 @@ class EHentai(override val id: Long, /** * Gallery list entry */ - data class ParsedManga(val fav: String?, val manga: Manga) + data class ParsedManga(val fav: Int, val manga: Manga) fun extendedGenericMangaParse(doc: Document) = with(doc) { //Parse mangas val parsedMangas = select(".gtr0,.gtr1").map { ParsedManga( - fav = it.select(".itd .it3 > .i[id]").first()?.attr("title"), + fav = parseFavoritesStyle(it.select(".itd .it3 > .i[id]").first()?.attr("style")), manga = Manga.create(id).apply { //Get title it.select(".itd .it5 a").first()?.apply { @@ -85,6 +90,14 @@ class EHentai(override val id: Long, Pair(parsedMangas, hasNextPage) } + fun parseFavoritesStyle(style: String?): Int { + val offset = style?.substringAfterLast("background-position:0px ") + ?.removeSuffix("px; cursor:pointer") + ?.toIntOrNull() ?: return -1 + + return (offset + 2)/-19 + } + /** * Parse a list of galleries */ @@ -287,9 +300,7 @@ class EHentai(override val id: Long, throw UnsupportedOperationException("Unused method was called somehow!") } - //Too lazy to write return type - fun fetchFavorites() = { - //Used to get "s" cookie + fun fetchFavorites(): Pair, List> { val favoriteUrl = "$baseUrl/favorites.php" val result = mutableListOf() var page = 1 @@ -308,22 +319,23 @@ class EHentai(override val id: Long, //Parse fav names if (favNames == null) - favNames = doc.getElementsByClass("nosel").first().children().filter { - it.children().size >= 3 - }.mapNotNull { it.child(2).text() } + favNames = doc.select(".fp:not(.fps)").mapNotNull { + it.child(2).text() + } //Next page page++ } while (parsed.second) - Pair(result as List, favNames!!) - }() + + return Pair(result as List, favNames!!) + } val cookiesHeader by lazy { val cookies: MutableMap = mutableMapOf() if(prefs.enableExhentai().getOrDefault()) { - cookies.put(LoginController.MEMBER_ID_COOKIE, prefs.memberIdVal().get()!!) - cookies.put(LoginController.PASS_HASH_COOKIE, prefs.passHashVal().get()!!) - cookies.put(LoginController.IGNEOUS_COOKIE, prefs.igneousVal().get()!!) + cookies[LoginController.MEMBER_ID_COOKIE] = prefs.memberIdVal().get()!! + cookies[LoginController.PASS_HASH_COOKIE] = prefs.passHashVal().get()!! + cookies[LoginController.IGNEOUS_COOKIE] = prefs.igneousVal().get()!! } //Setup settings @@ -458,20 +470,5 @@ class EHentai(override val id: Long, companion object { val QUERY_PREFIX = "?f_apply=Apply+Filter" val TR_SUFFIX = "TR" - - fun getCookies(cookies: String): Map? { - val foundCookies = HashMap() - for (cookie in cookies.split(";".toRegex()).dropLastWhile(String::isEmpty).toTypedArray()) { - val splitCookie = cookie.split("=".toRegex()).dropLastWhile(String::isEmpty).toTypedArray() - if (splitCookie.size < 2) { - return null - } - val trimmedKey = splitCookie[0].trim { it <= ' ' } - if (!foundCookies.containsKey(trimmedKey)) { - foundCookies.put(trimmedKey, splitCookie[1].trim { it <= ' ' }) - } - } - return foundCookies - } } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt index a145f8124..235f80f2b 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt @@ -12,6 +12,7 @@ import android.support.v7.app.AppCompatActivity import android.support.v7.view.ActionMode import android.support.v7.widget.SearchView import android.view.* +import com.afollestad.materialdialogs.MaterialDialog import com.bluelinelabs.conductor.ControllerChangeHandler import com.bluelinelabs.conductor.ControllerChangeType import com.f2prateek.rx.preferences.Preference @@ -36,7 +37,7 @@ import eu.kanade.tachiyomi.ui.migration.MigrationController import eu.kanade.tachiyomi.util.inflate import eu.kanade.tachiyomi.util.toast import eu.kanade.tachiyomi.widget.DrawerSwipeCloseListener -import exh.FavoritesSyncHelper +import exh.favorites.FavoritesSyncStatus import exh.metadata.loadAllMetadata import exh.metadata.models.SearchableGalleryMetadata import io.realm.Realm @@ -133,8 +134,12 @@ class LibraryController( var realm: Realm? = null //Cached metadata var meta: Map, RealmResults>? = null + //Sync dialog + private var favSyncDialog: MaterialDialog? = null + //Old sync status + private var oldSyncStatus: FavoritesSyncStatus? = null //Favorites - val favorites by lazy { FavoritesSyncHelper(activity!!) } + private var favoritesSyncSubscription: Subscription? = null // <-- EH init { @@ -406,7 +411,7 @@ class LibraryController( router.pushController(MigrationController().withFadeTransaction()) } R.id.action_download_favorites -> { - favorites.guiSyncFavorites { } + presenter.favoritesSync.runSync() } else -> return super.onOptionsItemSelected(item) } @@ -531,6 +536,100 @@ class LibraryController( } } + override fun onAttach(view: View) { + super.onAttach(view) + + // --> EXH + cleanupSyncState() + favoritesSyncSubscription = + presenter.favoritesSync.status + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { + updateSyncStatus(it) + } + // <-- EXH + } + + override fun onDetach(view: View) { + super.onDetach(view) + + //EXH + cleanupSyncState() + } + + // --> EXH + private fun cleanupSyncState() { + favoritesSyncSubscription?.unsubscribe() + favoritesSyncSubscription = null + //Close sync status + favSyncDialog?.dismiss() + favSyncDialog = null + oldSyncStatus = null + } + + private fun buildDialog() = activity?.let { + MaterialDialog.Builder(it) + } + + private fun showSyncProgressDialog() { + favSyncDialog?.dismiss() + favSyncDialog = buildDialog() + ?.title("Favorites syncing") + ?.cancelable(false) + ?.progress(true, 0) + ?.show() + } + + private fun updateSyncStatus(status: FavoritesSyncStatus) { + when(status) { + is FavoritesSyncStatus.Idle -> { + favSyncDialog?.dismiss() + favSyncDialog = null + } + is FavoritesSyncStatus.Error -> { + favSyncDialog?.dismiss() + favSyncDialog = buildDialog() + ?.title("Favorites sync error") + ?.content("An error occurred during the sync process: ${status.message}") + ?.cancelable(false) + ?.positiveText("Ok") + ?.onPositive { _, _ -> + presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle()) + } + ?.show() + } + is FavoritesSyncStatus.Processing, + is FavoritesSyncStatus.Initializing -> { + if(favSyncDialog == null || (oldSyncStatus != null + && oldSyncStatus !is FavoritesSyncStatus.Initializing + && oldSyncStatus !is FavoritesSyncStatus.Processing)) + showSyncProgressDialog() + + favSyncDialog?.setContent(status.message) + } + is FavoritesSyncStatus.Complete -> { + favSyncDialog?.dismiss() + + if(status.errors.isNotEmpty()) { + favSyncDialog = buildDialog() + ?.title("Favorites sync complete with errors") + ?.content("Some errors occurred during the sync process:\n\n" + + status.errors.joinToString("\n")) + ?.cancelable(false) + ?.positiveText("Ok") + ?.onPositive { _, _ -> + presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle()) + } + ?.show() + } else { + presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle()) + } + } + } + oldSyncStatus = status + } + // <-- EXH + override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) { if (requestCode == REQUEST_IMAGE_OPEN) { if (data == null || resultCode != Activity.RESULT_OK) return diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryPresenter.kt index dbb2d606e..a816e0289 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryPresenter.kt @@ -17,6 +17,7 @@ import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter import eu.kanade.tachiyomi.util.combineLatest import eu.kanade.tachiyomi.util.isNullOrUnsubscribed +import exh.favorites.FavoritesSyncHelper import rx.Observable import rx.Subscription import rx.android.schedulers.AndroidSchedulers @@ -76,6 +77,10 @@ class LibraryPresenter( */ private var librarySubscription: Subscription? = null + // --> EXH + val favoritesSync = FavoritesSyncHelper(context) + // <-- EXH + override fun onCreate(savedState: Bundle?) { super.onCreate(savedState) subscribeLibrary() diff --git a/app/src/main/java/exh/FavoritesSyncHelper.kt b/app/src/main/java/exh/FavoritesSyncHelper.kt deleted file mode 100755 index db1528a03..000000000 --- a/app/src/main/java/exh/FavoritesSyncHelper.kt +++ /dev/null @@ -1,137 +0,0 @@ -package exh - -import android.app.Activity -import android.support.v7.app.AlertDialog -import com.afollestad.materialdialogs.MaterialDialog -import eu.kanade.tachiyomi.data.database.DatabaseHelper -import eu.kanade.tachiyomi.data.database.models.Category -import eu.kanade.tachiyomi.data.database.models.Manga -import eu.kanade.tachiyomi.data.database.models.MangaCategory -import eu.kanade.tachiyomi.data.preference.PreferencesHelper -import eu.kanade.tachiyomi.data.preference.getOrDefault -import eu.kanade.tachiyomi.source.SourceManager -import eu.kanade.tachiyomi.source.online.all.EHentai -import eu.kanade.tachiyomi.util.syncChaptersWithSource -import timber.log.Timber -import uy.kohesive.injekt.injectLazy -import kotlin.concurrent.thread - -class FavoritesSyncHelper(val activity: Activity) { - - val db: DatabaseHelper by injectLazy() - - val sourceManager: SourceManager by injectLazy() - - val prefs: PreferencesHelper by injectLazy() - - fun guiSyncFavorites(onComplete: () -> Unit) { - //ExHentai must be enabled/user must be logged in - if (!prefs.enableExhentai().getOrDefault()) { - AlertDialog.Builder(activity).setTitle("Error") - .setMessage("You are not logged in! Please log in and try again!") - .setPositiveButton("Ok") { dialog, _ -> dialog.dismiss() }.show() - return - } - val dialog = MaterialDialog.Builder(activity) - .progress(true, 0) - .title("Downloading favorites") - .content("Please wait...") - .cancelable(false) - .show() - thread { - var error = false - try { - syncFavorites() - } catch (e: Exception) { - error = true - Timber.e(e, "Could not sync favorites!") - } - - dialog.dismiss() - - activity.runOnUiThread { - if (error) - MaterialDialog.Builder(activity) - .title("Error") - .content("There was an error downloading your favorites, please try again later!") - .positiveText("Ok") - .show() - onComplete() - } - } - } - - fun syncFavorites() { - val onlineSources = sourceManager.getOnlineSources() - var ehSource: EHentai? = null - var exSource: EHentai? = null - onlineSources.forEach { - if(it.id == EH_SOURCE_ID) - ehSource = it as EHentai - else if(it.id == EXH_SOURCE_ID) - exSource = it as EHentai - } - - (exSource ?: ehSource)?.let { source -> - val favResponse = source.fetchFavorites() - val ourCategories = db.getCategories().executeAsBlocking().toMutableList() - val ourMangas = db.getMangas().executeAsBlocking().filter { - it.source == EH_SOURCE_ID || it.source == EXH_SOURCE_ID - }.toMutableList() - //Add required categories (categories do not sync upwards) - favResponse.second.filter { theirCategory -> - ourCategories.find { - it.name.endsWith(theirCategory) - } == null - }.map { - Category.create(it) - }.let { - db.inTransaction { - //Insert new categories - db.insertCategories(it).executeAsBlocking().results().entries.filter { - it.value.wasInserted() - }.forEach { it.key.id = it.value.insertedId()!!.toInt() } - - val categoryMap = (it + ourCategories).associateBy { it.name } - - //Insert new mangas - val mangaToInsert = mutableListOf() - favResponse.first.map { - val category = categoryMap[it.fav]!! - var manga = it.manga - val alreadyHaveManga = ourMangas.find { - it.url == manga.url - }?.apply { - manga = this - } != null - if (!alreadyHaveManga) { - ourMangas.add(manga) - mangaToInsert.add(manga) - } - manga.favorite = true - Pair(manga, category) - }.apply { - //Insert mangas - db.insertMangas(mangaToInsert).executeAsBlocking().results().entries.filter { - it.value.wasInserted() - }.forEach { manga -> - manga.key.id = manga.value.insertedId() - try { - source.fetchChapterList(manga.key).map { - syncChaptersWithSource(db, it, manga.key, source) - }.toBlocking().first() - } catch (e: Exception) { - Timber.w(e, "Failed to update chapters for gallery: ${manga.key.title}!") - } - } - - //Set categories - val categories = map { MangaCategory.create(it.first, it.second) } - val mangas = map { it.first } - db.setMangaCategories(categories, mangas) - } - } - } - } - } -} \ No newline at end of file diff --git a/app/src/main/java/exh/FavoritesSyncManager.java b/app/src/main/java/exh/FavoritesSyncManager.java deleted file mode 100755 index badc70704..000000000 --- a/app/src/main/java/exh/FavoritesSyncManager.java +++ /dev/null @@ -1,192 +0,0 @@ -package exh; - -import android.app.ProgressDialog; -import android.content.Context; -import android.content.DialogInterface; -import android.os.Handler; -import android.os.Looper; -import android.support.v7.app.AlertDialog; - -import com.pushtorefresh.storio.sqlite.operations.put.PutResult; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import eu.kanade.tachiyomi.data.database.DatabaseHelper; -import eu.kanade.tachiyomi.data.database.models.Category; -import eu.kanade.tachiyomi.data.database.models.Manga; -import eu.kanade.tachiyomi.data.database.models.MangaCategory; -import eu.kanade.tachiyomi.source.online.all.EHentai; -import kotlin.Pair; -//import eu.kanade.tachiyomi.data.source.online.english.EHentai; - -public class FavoritesSyncManager { - /*Context context; - DatabaseHelper db; - - public FavoritesSyncManager(Context context, DatabaseHelper db) { - this.context = context; - this.db = db; - } - - public void guiSyncFavorites(final Runnable onComplete) { - if(!DialogLogin.isLoggedIn(context, false)) { - new AlertDialog.Builder(context).setTitle("Error") - .setMessage("You are not logged in! Please log in and try again!") - .setPositiveButton("Ok", new DialogInterface.OnClickListener() { - @Override - public void onClick(DialogInterface dialog, int which) { - dialog.dismiss(); - } - }).show(); - return; - } - final ProgressDialog dialog = ProgressDialog.show(context, "Downloading Favorites", "Please wait...", true, false); - new Thread(new Runnable() { - @Override - public void run() { - Handler mainLooper = new Handler(Looper.getMainLooper()); - try { - syncFavorites(); - } catch (Exception e) { - mainLooper.post(new Runnable() { - @Override - public void run() { - new AlertDialog.Builder(context) - .setTitle("Error") - .setMessage("There was an error downloading your favorites, please try again later!") - .setPositiveButton("Ok", new DialogInterface.OnClickListener() { - @Override - public void onClick(DialogInterface dialog, int which) { - dialog.dismiss(); - } - }).show(); - } - }); - e.printStackTrace(); - } - dialog.dismiss(); - mainLooper.post(onComplete); - } - }).start(); - }*/ -/* - public void syncFavorites() throws IOException { - Pair favResponse = EHentai.fetchFavorites(context); - Map> favorites = favResponse.favs; - List ourCategories = new ArrayList<>(db.getCategories().executeAsBlocking()); - List ourMangas = new ArrayList<>(db.getMangas().executeAsBlocking()); - //Add required categories (categories do not sync upwards) - List categoriesToInsert = new ArrayList<>(); - for (String theirCategory : favorites.keySet()) { - boolean haveCategory = false; - for (Category category : ourCategories) { - if (category.getName().endsWith(theirCategory)) { - haveCategory = true; - } - } - if (!haveCategory) { - Category category = Category.Companion.create(theirCategory); - ourCategories.add(category); - categoriesToInsert.add(category); - } - } - if (!categoriesToInsert.isEmpty()) { - for(Map.Entry result : db.insertCategories(categoriesToInsert).executeAsBlocking().results().entrySet()) { - if(result.getValue().wasInserted()) { - result.getKey().setId(result.getValue().insertedId().intValue()); - } - } - } - //Build category map - Map categoryMap = new HashMap<>(); - for (Category category : ourCategories) { - categoryMap.put(category.getName(), category); - } - //Insert new mangas - List mangaToInsert = new ArrayList<>(); - Map mangaToSetCategories = new HashMap<>(); - for (Map.Entry> entry : favorites.entrySet()) { - Category category = categoryMap.get(entry.getKey()); - for (Manga manga : entry.getValue()) { - boolean alreadyHaveManga = false; - for (Manga ourManga : ourMangas) { - if (ourManga.getUrl().equals(manga.getUrl())) { - alreadyHaveManga = true; - manga = ourManga; - break; - } - } - if (!alreadyHaveManga) { - ourMangas.add(manga); - mangaToInsert.add(manga); - } - mangaToSetCategories.put(manga, category); - manga.setFavorite(true); - } - } - for (Map.Entry results : db.insertMangas(mangaToInsert).executeAsBlocking().results().entrySet()) { - if(results.getValue().wasInserted()) { - results.getKey().setId(results.getValue().insertedId()); - } - } - for(Map.Entry entry : mangaToSetCategories.entrySet()) { - db.setMangaCategories(Collections.singletonList(MangaCategory.Companion.create(entry.getKey(), entry.getValue())), - Collections.singletonList(entry.getKey())); - }*/ - //Determines what - /*Map> toUpload = new HashMap<>(); - for (Manga manga : ourMangas) { - if(manga.getFavorite()) { - boolean remoteHasManga = false; - for (List remoteMangas : favorites.values()) { - for (Manga remoteManga : remoteMangas) { - if (remoteManga.getUrl().equals(manga.getUrl())) { - remoteHasManga = true; - break; - } - } - } - if (!remoteHasManga) { - List mangaCategories = db.getCategoriesForManga(manga).executeAsBlocking(); - for (Category category : mangaCategories) { - int categoryIndex = favResponse.favCategories.indexOf(category.getName()); - if (categoryIndex >= 0) { - List uploadMangas = toUpload.get(categoryIndex); - if (uploadMangas == null) { - uploadMangas = new ArrayList<>(); - toUpload.put(categoryIndex, uploadMangas); - } - uploadMangas.add(manga); - } - } - } - } - }*/ - /********** NON-FUNCTIONAL, modifygids[] CANNOT ADD NEW FAVORITES! (or as of my testing it can't, maybe I'll do more testing)**/ - /*PreferencesHelper helper = new PreferencesHelper(context); - for(Map.Entry> entry : toUpload.entrySet()) { - FormBody.Builder formBody = new FormBody.Builder() - .add("ddact", "fav" + entry.getKey()); - for(Manga manga : entry.getValue()) { - List splitUrl = new ArrayList<>(Arrays.asList(manga.getUrl().split("/"))); - splitUrl.removeAll(Collections.singleton("")); - if(splitUrl.size() < 2) { - continue; - } - formBody.add("modifygids[]", splitUrl.get(1).trim()); - } - formBody.add("apply", "Apply"); - Request request = RequestsKt.POST(EHentai.buildFavoritesBase(context, helper.getPrefs()).favoritesBase, - EHentai.getHeadersBuilder(helper).build(), - formBody.build(), - RequestsKt.getDEFAULT_CACHE_CONTROL()); - Response response = NetworkManager.getInstance().getClient().newCall(request).execute(); - Util.d("EHentai", response.body().string()); - }*/ -// } -} diff --git a/app/src/main/java/exh/GalleryAdder.kt b/app/src/main/java/exh/GalleryAdder.kt index bf0bbaecb..c082f35e7 100755 --- a/app/src/main/java/exh/GalleryAdder.kt +++ b/app/src/main/java/exh/GalleryAdder.kt @@ -10,8 +10,7 @@ import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.network.NetworkHelper import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.util.syncChaptersWithSource -import exh.metadata.models.* -import exh.util.defRealm +import exh.metadata.models.ExGalleryMetadata import okhttp3.MediaType import okhttp3.Request import okhttp3.RequestBody @@ -131,7 +130,7 @@ class GalleryAdder { ?: return GalleryAddEvent.Fail.Error(url, "Could not find EH source!") val cleanedUrl = when(source) { - EH_SOURCE_ID, EXH_SOURCE_ID -> getUrlWithoutDomain(realUrl) + EH_SOURCE_ID, EXH_SOURCE_ID -> ExGalleryMetadata.normalizeUrl(getUrlWithoutDomain(realUrl)) NHENTAI_SOURCE_ID -> realUrl //nhentai uses URLs directly (oops, my bad when implementing this source) PERV_EDEN_EN_SOURCE_ID, PERV_EDEN_IT_SOURCE_ID -> getUrlWithoutDomain(realUrl) @@ -152,19 +151,6 @@ class GalleryAdder { manga.copyFrom(newManga) manga.title = newManga.title //Forcibly copy title as copyFrom does not copy title - //Apply metadata - defRealm { realm -> - when (source) { - EH_SOURCE_ID, EXH_SOURCE_ID -> ExGalleryMetadata.UrlQuery(realUrl, isExSource(source)) - NHENTAI_SOURCE_ID -> NHentaiMetadata.UrlQuery(realUrl) - PERV_EDEN_EN_SOURCE_ID, - PERV_EDEN_IT_SOURCE_ID -> PervEdenGalleryMetadata.UrlQuery(realUrl, PervEdenLang.source(source)) - HENTAI_CAFE_SOURCE_ID -> HentaiCafeMetadata.UrlQuery(realUrl) - TSUMINO_SOURCE_ID -> TsuminoMetadata.UrlQuery(realUrl) - else -> return GalleryAddEvent.Fail.UnknownType(url) - }.query(realm).findFirst() - } - if (fav) manga.favorite = true db.insertManga(manga).executeAsBlocking().insertedId()?.let { diff --git a/app/src/main/java/exh/favorites/FavoriteEntry.kt b/app/src/main/java/exh/favorites/FavoriteEntry.kt new file mode 100644 index 000000000..099bae2e3 --- /dev/null +++ b/app/src/main/java/exh/favorites/FavoriteEntry.kt @@ -0,0 +1,23 @@ +package exh.favorites + +import exh.metadata.models.ExGalleryMetadata +import io.realm.RealmObject +import io.realm.annotations.Index +import io.realm.annotations.PrimaryKey +import io.realm.annotations.RealmClass +import java.util.* + +@RealmClass +open class FavoriteEntry : RealmObject() { + @PrimaryKey var id: String = UUID.randomUUID().toString() + + var title: String? = null + + @Index lateinit var gid: String + + @Index lateinit var token: String + + @Index var category: Int = -1 + + fun getUrl() = ExGalleryMetadata.normalizeUrl(gid, token) +} diff --git a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt new file mode 100644 index 000000000..0af17749b --- /dev/null +++ b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt @@ -0,0 +1,274 @@ +package exh.favorites + +import android.content.Context +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Category +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.database.models.MangaCategory +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import eu.kanade.tachiyomi.source.SourceManager +import eu.kanade.tachiyomi.source.online.all.EHentai +import exh.EH_METADATA_SOURCE_ID +import exh.EXH_SOURCE_ID +import exh.GalleryAddEvent +import exh.GalleryAdder +import okhttp3.FormBody +import okhttp3.Request +import rx.subjects.BehaviorSubject +import timber.log.Timber +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import uy.kohesive.injekt.injectLazy +import kotlin.concurrent.thread + +class FavoritesSyncHelper(context: Context) { + private val db: DatabaseHelper by injectLazy() + + private val prefs: PreferencesHelper by injectLazy() + + private val exh by lazy { + Injekt.get().get(EXH_SOURCE_ID) as? EHentai + ?: EHentai(0, true, context) + } + + private val storage = LocalFavoritesStorage() + + private val galleryAdder = GalleryAdder() + + val status = BehaviorSubject.create(FavoritesSyncStatus.Idle()) + + @Synchronized + fun runSync() { + if(status.value !is FavoritesSyncStatus.Idle) { + return + } + + status.onNext(FavoritesSyncStatus.Initializing()) + + thread { beginSync() } + } + + private fun beginSync() { + //Check if logged in + if(!prefs.enableExhentai().getOrDefault()) { + status.onNext(FavoritesSyncStatus.Error("Please log in!")) + return + } + + //Download remote favorites + val favorites = try { + status.onNext(FavoritesSyncStatus.Processing("Downloading favorites from remote server")) + exh.fetchFavorites() + } catch(e: Exception) { + status.onNext(FavoritesSyncStatus.Error("Failed to fetch favorites from remote server!")) + Timber.e(e, "Could not fetch favorites!") + return + } + + val errors = mutableListOf() + + try { + db.inTransaction { + val remoteChanges = storage.getChangedRemoteEntries(favorites.first) + val localChanges = storage.getChangedDbEntries() + + //Apply remote categories + status.onNext(FavoritesSyncStatus.Processing("Updating category names")) + applyRemoteCategories(favorites.second) + + //Apply ChangeSets + applyChangeSetToLocal(remoteChanges, errors) + applyChangeSetToRemote(localChanges, errors) + + status.onNext(FavoritesSyncStatus.Processing("Cleaning up")) + storage.snapshotEntries() + } + } catch(e: IgnoredException) { + //Do not display error as this error has already been reported + Timber.w(e, "Ignoring exception!") + return + } catch (e: Exception) { + status.onNext(FavoritesSyncStatus.Error("Unknown error: ${e.message}")) + Timber.e(e, "Sync error!") + return + } + + status.onNext(FavoritesSyncStatus.Complete(errors)) + } + + private fun applyRemoteCategories(categories: List) { + val localCategories = db.getCategories().executeAsBlocking() + + val newLocalCategories = localCategories.toMutableList() + + var changed = false + + categories.forEachIndexed { index, remote -> + val local = localCategories.getOrElse(index) { + changed = true + + Category.create(remote).apply { + order = index + + //Going through categories list from front to back + //If category does not exist, list size <= category index + //Thus, we can just add it here and not worry about indexing + newLocalCategories += this + } + } + + if(local.name != remote) { + changed = true + + local.name = remote + } + } + + //Ensure consistent ordering + newLocalCategories.forEachIndexed { index, category -> + if(category.order != index) { + changed = true + + category.order = index + } + } + + //Only insert categories if changed + if(changed) + db.insertCategories(newLocalCategories).executeAsBlocking() + } + + private fun addGalleryRemote(gallery: FavoriteEntry, errors: MutableList) { + val url = "${exh.baseUrl}/gallerypopups.php?gid=${gallery.gid}&t=${gallery.token}&act=addfav" + + val request = Request.Builder() + .url(url) + .post(FormBody.Builder() + .add("favcat", gallery.category.toString()) + .add("favnote", "") + .add("apply", "Add to Favorites") + .add("update", "1") + .build()) + .build() + + if(!explicitlyRetryExhRequest(10, request)) { + errors += "Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!" + } + } + + private fun explicitlyRetryExhRequest(retryCount: Int, request: Request): Boolean { + var success = false + + for(i in 1 .. retryCount) { + try { + val resp = exh.client.newCall(request).execute() + + if (resp.isSuccessful) { + success = true + break + } + } catch (e: Exception) { + Timber.e(e, "Sync network error!") + } + } + + return success + } + + private fun applyChangeSetToRemote(changeSet: ChangeSet, errors: MutableList) { + //Apply removals + if(changeSet.removed.isNotEmpty()) { + status.onNext(FavoritesSyncStatus.Processing("Removing ${changeSet.removed.size} galleries from remote server")) + + val formBody = FormBody.Builder() + .add("ddact", "delete") + .add("apply", "Apply") + + //Add change set to form + changeSet.removed.forEach { + formBody.add("modifygids[]", it.gid) + } + + val request = Request.Builder() + .url("https://exhentai.org/favorites.php") + .post(formBody.build()) + .build() + + if(!explicitlyRetryExhRequest(10, request)) { + status.onNext(FavoritesSyncStatus.Error("Unable to delete galleries from the remote servers!")) + + //It is still safe to stop here so crash + throw IgnoredException() + } + } + + //Apply additions + changeSet.added.forEachIndexed { index, it -> + status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to remote server")) + + addGalleryRemote(it, errors) + } + } + + private fun applyChangeSetToLocal(changeSet: ChangeSet, errors: MutableList) { + val removedManga = mutableListOf() + + //Apply removals + changeSet.removed.forEachIndexed { index, it -> + status.onNext(FavoritesSyncStatus.Processing("Removing gallery ${index + 1} of ${changeSet.removed.size} from local library")) + val url = it.getUrl() + + //Consider both EX and EH sources + listOf(db.getManga(url, EXH_SOURCE_ID), + db.getManga(url, EH_METADATA_SOURCE_ID)).forEach { + val manga = it.executeAsBlocking() + + if(manga?.favorite == true) { + manga.favorite = false + db.updateMangaFavorite(manga).executeAsBlocking() + removedManga += manga + } + } + } + + db.deleteOldMangasCategories(removedManga).executeAsBlocking() + + val insertedMangaCategories = mutableListOf() + val insertedMangaCategoriesMangas = mutableListOf() + val categories = db.getCategories().executeAsBlocking() + + //Apply additions + changeSet.added.forEachIndexed { index, it -> + status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to local library")) + + //Import using gallery adder + val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}", + true, + EXH_SOURCE_ID) + + if(result is GalleryAddEvent.Fail) { + errors += "Failed to add gallery to local database: " + when (result) { + is GalleryAddEvent.Fail.Error -> "'${it.title}' ${result.logMessage}" + is GalleryAddEvent.Fail.UnknownType -> "'${it.title}' (${result.galleryUrl}) is not a valid gallery!" + } + } else if(result is GalleryAddEvent.Success) { + insertedMangaCategories += MangaCategory.create(result.manga, + categories[it.category]) + insertedMangaCategoriesMangas += result.manga + } + } + + db.setMangaCategories(insertedMangaCategories, insertedMangaCategoriesMangas) + } + + class IgnoredException : RuntimeException() +} + +sealed class FavoritesSyncStatus(val message: String) { + class Error(message: String) : FavoritesSyncStatus(message) + class Idle : FavoritesSyncStatus("Waiting for sync to start") + class Initializing : FavoritesSyncStatus("Initializing sync") + class Processing(message: String) : FavoritesSyncStatus(message) + class Complete(val errors: List) : FavoritesSyncStatus("Sync complete!") +} diff --git a/app/src/main/java/exh/favorites/LocalFavoritesStorage.kt b/app/src/main/java/exh/favorites/LocalFavoritesStorage.kt new file mode 100644 index 000000000..64e605bb1 --- /dev/null +++ b/app/src/main/java/exh/favorites/LocalFavoritesStorage.kt @@ -0,0 +1,132 @@ +package exh.favorites + +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.source.online.all.EHentai +import exh.EH_SOURCE_ID +import exh.EXH_SOURCE_ID +import exh.metadata.models.ExGalleryMetadata +import exh.util.trans +import io.realm.Realm +import io.realm.RealmConfiguration +import uy.kohesive.injekt.injectLazy + +class LocalFavoritesStorage { + private val db: DatabaseHelper by injectLazy() + + private val realmConfig = RealmConfiguration.Builder() + .name("fav-sync") + .deleteRealmIfMigrationNeeded() + .build() + + private val realm + get() = Realm.getInstance(realmConfig) + + fun getChangedDbEntries() + = getChangedEntries( + parseToFavoriteEntries( + loadDbCategories( + db.getFavoriteMangas() + .executeAsBlocking() + .asSequence() + ) + ) + ) + + fun getChangedRemoteEntries(entries: List) + = getChangedEntries( + parseToFavoriteEntries( + entries.asSequence().map { + Pair(it.fav, it.manga.apply { + favorite = true + }) + } + ) + ) + + fun snapshotEntries() { + val dbMangas = parseToFavoriteEntries( + loadDbCategories( + db.getFavoriteMangas() + .executeAsBlocking() + .asSequence() + ) + ) + + realm.use { realm -> + realm.trans { + //Delete old snapshot + realm.delete(FavoriteEntry::class.java) + + //Insert new snapshots + realm.copyToRealm(dbMangas.toList()) + } + } + } + + private fun getChangedEntries(entries: Sequence): ChangeSet { + return realm.use { realm -> + val terminated = entries.toList() + + val added = terminated.filter { + realm.queryRealmForEntry(it) == null + } + + val removed = realm.where(FavoriteEntry::class.java) + .findAll() + .filter { + queryListForEntry(terminated, it) == null + }.map { + realm.copyFromRealm(it) + } + + ChangeSet(added, removed) + } + } + + private fun Realm.queryRealmForEntry(entry: FavoriteEntry) + = where(FavoriteEntry::class.java) + .equalTo(FavoriteEntry::gid.name, entry.gid) + .equalTo(FavoriteEntry::token.name, entry.token) + .equalTo(FavoriteEntry::category.name, entry.category) + .findFirst() + + private fun queryListForEntry(list: List, entry: FavoriteEntry) + = list.find { + it.gid == entry.gid + && it.token == entry.token + && it.category == entry.category + } + + private fun loadDbCategories(manga: Sequence): Sequence> { + val dbCategories = db.getCategories().executeAsBlocking() + + return manga.filter(this::validateDbManga).mapNotNull { + val category = db.getCategoriesForManga(it).executeAsBlocking() + + Pair(dbCategories.indexOf(category.firstOrNull() + ?: return@mapNotNull null), it) + } + } + + private fun parseToFavoriteEntries(manga: Sequence>) + = manga.filter { + validateDbManga(it.second) + }.mapNotNull { + FavoriteEntry().apply { + title = it.second.title + gid = ExGalleryMetadata.galleryId(it.second.url) + token = ExGalleryMetadata.galleryToken(it.second.url) + category = it.first + + if(this.category > 9) + return@mapNotNull null + } + } + + private fun validateDbManga(manga: Manga) + = manga.favorite && (manga.source == EH_SOURCE_ID || manga.source == EXH_SOURCE_ID) +} + +data class ChangeSet(val added: List, + val removed: List) diff --git a/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt b/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt index 9e3d63ea9..d4453f28d 100755 --- a/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt +++ b/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt @@ -26,6 +26,10 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata { override var uuid: String = UUID.randomUUID().toString() var url: String? = null + set(value) { + //Ensure that URLs are always formatted in the same way to reduce duplicate galleries + field = value?.let { normalizeUrl(it) } + } @Index var gId: String? = null @@ -60,7 +64,7 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata { override var tags: RealmList = RealmList() - override fun getTitles() = listOf(title, altTitle).filterNotNull() + override fun getTitles() = listOfNotNull(title, altTitle) @Ignore override val titleFields = TITLE_FIELDS @@ -93,7 +97,7 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata { } override fun copyTo(manga: SManga) { - url?.let { manga.url = it } + url?.let { manga.url = normalizeUrl(it) } thumbnailUrl?.let { manga.thumbnail_url = it } //No title bug? @@ -118,8 +122,8 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata { ONGOING_SUFFIX.find { t.endsWith(it, ignoreCase = true) }?.let { - manga.status = SManga.ONGOING - } + manga.status = SManga.ONGOING + } } //Build a nice looking description out of what we know @@ -165,6 +169,12 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata { fun galleryToken(url: String) = splitGalleryUrl(url).last() + fun normalizeUrl(id: String, token: String) + = "/g/$id/$token/?nw=always" + + fun normalizeUrl(url: String) + = normalizeUrl(galleryId(url), galleryToken(url)) + val TITLE_FIELDS = listOf( ExGalleryMetadata::title.name, ExGalleryMetadata::altTitle.name diff --git a/app/src/main/java/exh/metadata/models/GalleryQuery.kt b/app/src/main/java/exh/metadata/models/GalleryQuery.kt index 521a5dbd7..70293fa11 100755 --- a/app/src/main/java/exh/metadata/models/GalleryQuery.kt +++ b/app/src/main/java/exh/metadata/models/GalleryQuery.kt @@ -1,6 +1,8 @@ package exh.metadata.models -import io.realm.* +import io.realm.Case +import io.realm.Realm +import io.realm.RealmQuery import java.util.* import kotlin.reflect.KClass import kotlin.reflect.KProperty @@ -58,7 +60,7 @@ abstract class GalleryQuery(val clazz: KClass) is Long -> newMeta.equalTo(n, v) is Short -> newMeta.equalTo(n, v) is String -> newMeta.equalTo(n, v, Case.INSENSITIVE) - else -> throw IllegalArgumentException("Unknown type: ${v::class.qualifiedName}!") + else -> throw IllegalArgumentException("Unknown type: ${v::class.java.name}!") } } }