Draff/TachiyomiSY-Plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial implementation of favorites syncing

Browse Source 
General code cleanup
Fix some cases of duplicate galleries (not completely fixed)
This commit is contained in:
NerdNumber9 2018-01-31 22:39:55 -05:00
parent f18b32626a
commit d892f2f7f4
11 changed files with 588 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt
View File

@ -11,21 +11,26 @@ import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.LewdSource
import eu.kanade.tachiyomi.util.asJsoup
import exh.metadata.*
import exh.metadata.EX_DATE_FORMAT
import exh.metadata.ignore
import exh.metadata.models.ExGalleryMetadata
import exh.metadata.models.Tag
import exh.metadata.nullIfBlank
import exh.metadata.parseHumanReadableByteCount
import exh.ui.login.LoginController
import exh.util.UriFilter
import exh.util.UriGroup
import exh.util.urlImportFetchSearchManga
import okhttp3.CacheControl
import okhttp3.Headers
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.net.URLEncoder
import java.util.*
import exh.ui.login.LoginController
import okhttp3.CacheControl
import okhttp3.Headers
import okhttp3.Request
import org.jsoup.nodes.Document
import exh.util.*
class EHentai(override val id: Long,
val exh: Boolean,
@ -51,14 +56,14 @@ class EHentai(override val id: Long,
/**
* Gallery list entry
*/
data class ParsedManga(val fav: String?, val manga: Manga)
data class ParsedManga(val fav: Int, val manga: Manga)
fun extendedGenericMangaParse(doc: Document)
= with(doc) {
//Parse mangas
val parsedMangas = select(".gtr0,.gtr1").map {
ParsedManga(
fav = it.select(".itd .it3 > .i[id]").first()?.attr("title"),
fav = parseFavoritesStyle(it.select(".itd .it3 > .i[id]").first()?.attr("style")),
manga = Manga.create(id).apply {
//Get title
it.select(".itd .it5 a").first()?.apply {
@ -85,6 +90,14 @@ class EHentai(override val id: Long,
Pair(parsedMangas, hasNextPage)
}
fun parseFavoritesStyle(style: String?): Int {
val offset = style?.substringAfterLast("background-position:0px ")
?.removeSuffix("px; cursor:pointer")
?.toIntOrNull() ?: return -1
return (offset + 2)/-19
}
/**
* Parse a list of galleries
*/
@ -287,9 +300,7 @@ class EHentai(override val id: Long,
throw UnsupportedOperationException("Unused method was called somehow!")
}
//Too lazy to write return type
fun fetchFavorites() = {
//Used to get "s" cookie
fun fetchFavorites(): Pair<List<ParsedManga>, List<String>> {
val favoriteUrl = "$baseUrl/favorites.php"
val result = mutableListOf<ParsedManga>()
var page = 1
@ -308,22 +319,23 @@ class EHentai(override val id: Long,
//Parse fav names
if (favNames == null)
favNames = doc.getElementsByClass("nosel").first().children().filter {
it.children().size >= 3
}.mapNotNull { it.child(2).text() }
favNames = doc.select(".fp:not(.fps)").mapNotNull {
it.child(2).text()
}
//Next page
page++
} while (parsed.second)
Pair(result as List<ParsedManga>, favNames!!)
}()
return Pair(result as List<ParsedManga>, favNames!!)
}
val cookiesHeader by lazy {
val cookies: MutableMap<String, String> = mutableMapOf()
if(prefs.enableExhentai().getOrDefault()) {
cookies.put(LoginController.MEMBER_ID_COOKIE, prefs.memberIdVal().get()!!)
cookies.put(LoginController.PASS_HASH_COOKIE, prefs.passHashVal().get()!!)
cookies.put(LoginController.IGNEOUS_COOKIE, prefs.igneousVal().get()!!)
cookies[LoginController.MEMBER_ID_COOKIE] = prefs.memberIdVal().get()!!
cookies[LoginController.PASS_HASH_COOKIE] = prefs.passHashVal().get()!!
cookies[LoginController.IGNEOUS_COOKIE] = prefs.igneousVal().get()!!
}
//Setup settings
@ -458,20 +470,5 @@ class EHentai(override val id: Long,
companion object {
val QUERY_PREFIX = "?f_apply=Apply+Filter"
val TR_SUFFIX = "TR"
fun getCookies(cookies: String): Map<String, String>? {
val foundCookies = HashMap<String, String>()
for (cookie in cookies.split(";".toRegex()).dropLastWhile(String::isEmpty).toTypedArray()) {
val splitCookie = cookie.split("=".toRegex()).dropLastWhile(String::isEmpty).toTypedArray()
if (splitCookie.size < 2) {
return null
}
val trimmedKey = splitCookie[0].trim { it <= ' ' }
if (!foundCookies.containsKey(trimmedKey)) {
foundCookies.put(trimmedKey, splitCookie[1].trim { it <= ' ' })
}
}
return foundCookies
}
}
}

105
app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
View File

@ -12,6 +12,7 @@ import android.support.v7.app.AppCompatActivity
import android.support.v7.view.ActionMode
import android.support.v7.widget.SearchView
import android.view.*
import com.afollestad.materialdialogs.MaterialDialog
import com.bluelinelabs.conductor.ControllerChangeHandler
import com.bluelinelabs.conductor.ControllerChangeType
import com.f2prateek.rx.preferences.Preference
@ -36,7 +37,7 @@ import eu.kanade.tachiyomi.ui.migration.MigrationController
import eu.kanade.tachiyomi.util.inflate
import eu.kanade.tachiyomi.util.toast
import eu.kanade.tachiyomi.widget.DrawerSwipeCloseListener
import exh.FavoritesSyncHelper
import exh.favorites.FavoritesSyncStatus
import exh.metadata.loadAllMetadata
import exh.metadata.models.SearchableGalleryMetadata
import io.realm.Realm
@ -133,8 +134,12 @@ class LibraryController(
var realm: Realm? = null
//Cached metadata
var meta: Map<KClass<out SearchableGalleryMetadata>, RealmResults<out SearchableGalleryMetadata>>? = null
//Sync dialog
private var favSyncDialog: MaterialDialog? = null
//Old sync status
private var oldSyncStatus: FavoritesSyncStatus? = null
//Favorites
val favorites by lazy { FavoritesSyncHelper(activity!!) }
private var favoritesSyncSubscription: Subscription? = null
// <-- EH
init {
@ -406,7 +411,7 @@ class LibraryController(
router.pushController(MigrationController().withFadeTransaction())
}
R.id.action_download_favorites -> {
favorites.guiSyncFavorites { }
presenter.favoritesSync.runSync()
}
else -> return super.onOptionsItemSelected(item)
}
@ -531,6 +536,100 @@ class LibraryController(
}
}
override fun onAttach(view: View) {
super.onAttach(view)
// --> EXH
cleanupSyncState()
favoritesSyncSubscription =
presenter.favoritesSync.status
.observeOn(AndroidSchedulers.mainThread())
.subscribe {
updateSyncStatus(it)
}
// <-- EXH
}
override fun onDetach(view: View) {
super.onDetach(view)
//EXH
cleanupSyncState()
}
// --> EXH
private fun cleanupSyncState() {
favoritesSyncSubscription?.unsubscribe()
favoritesSyncSubscription = null
//Close sync status
favSyncDialog?.dismiss()
favSyncDialog = null
oldSyncStatus = null
}
private fun buildDialog() = activity?.let {
MaterialDialog.Builder(it)
}
private fun showSyncProgressDialog() {
favSyncDialog?.dismiss()
favSyncDialog = buildDialog()
?.title("Favorites syncing")
?.cancelable(false)
?.progress(true, 0)
?.show()
}
private fun updateSyncStatus(status: FavoritesSyncStatus) {
when(status) {
is FavoritesSyncStatus.Idle -> {
favSyncDialog?.dismiss()
favSyncDialog = null
}
is FavoritesSyncStatus.Error -> {
favSyncDialog?.dismiss()
favSyncDialog = buildDialog()
?.title("Favorites sync error")
?.content("An error occurred during the sync process: ${status.message}")
?.cancelable(false)
?.positiveText("Ok")
?.onPositive { _, _ ->
presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle())
}
?.show()
}
is FavoritesSyncStatus.Processing,
is FavoritesSyncStatus.Initializing -> {
if(favSyncDialog == null || (oldSyncStatus != null
&& oldSyncStatus !is FavoritesSyncStatus.Initializing
&& oldSyncStatus !is FavoritesSyncStatus.Processing))
showSyncProgressDialog()
favSyncDialog?.setContent(status.message)
}
is FavoritesSyncStatus.Complete -> {
favSyncDialog?.dismiss()
if(status.errors.isNotEmpty()) {
favSyncDialog = buildDialog()
?.title("Favorites sync complete with errors")
?.content("Some errors occurred during the sync process:\n\n"
+ status.errors.joinToString("\n"))
?.cancelable(false)
?.positiveText("Ok")
?.onPositive { _, _ ->
presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle())
}
?.show()
} else {
presenter.favoritesSync.status.onNext(FavoritesSyncStatus.Idle())
}
}
}
oldSyncStatus = status
}
// <-- EXH
override fun onActivityResult(requestCode: Int, resultCode: Int, data: Intent?) {
if (requestCode == REQUEST_IMAGE_OPEN) {
if (data == null || resultCode != Activity.RESULT_OK) return

5
app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryPresenter.kt
View File

@ -17,6 +17,7 @@ import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter
import eu.kanade.tachiyomi.util.combineLatest
import eu.kanade.tachiyomi.util.isNullOrUnsubscribed
import exh.favorites.FavoritesSyncHelper
import rx.Observable
import rx.Subscription
import rx.android.schedulers.AndroidSchedulers
@ -76,6 +77,10 @@ class LibraryPresenter(
*/
private var librarySubscription: Subscription? = null
// --> EXH
val favoritesSync = FavoritesSyncHelper(context)
// <-- EXH
override fun onCreate(savedState: Bundle?) {
super.onCreate(savedState)
subscribeLibrary()

137
app/src/main/java/exh/FavoritesSyncHelper.kt
View File

@ -1,137 +0,0 @@
package exh
import android.app.Activity
import android.support.v7.app.AlertDialog
import com.afollestad.materialdialogs.MaterialDialog
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Category
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.database.models.MangaCategory
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.all.EHentai
import eu.kanade.tachiyomi.util.syncChaptersWithSource
import timber.log.Timber
import uy.kohesive.injekt.injectLazy
import kotlin.concurrent.thread
class FavoritesSyncHelper(val activity: Activity) {
val db: DatabaseHelper by injectLazy()
val sourceManager: SourceManager by injectLazy()
val prefs: PreferencesHelper by injectLazy()
fun guiSyncFavorites(onComplete: () -> Unit) {
//ExHentai must be enabled/user must be logged in
if (!prefs.enableExhentai().getOrDefault()) {
AlertDialog.Builder(activity).setTitle("Error")
.setMessage("You are not logged in! Please log in and try again!")
.setPositiveButton("Ok") { dialog, _ -> dialog.dismiss() }.show()
return
}
val dialog = MaterialDialog.Builder(activity)
.progress(true, 0)
.title("Downloading favorites")
.content("Please wait...")
.cancelable(false)
.show()
thread {
var error = false
try {
syncFavorites()
} catch (e: Exception) {
error = true
Timber.e(e, "Could not sync favorites!")
}
dialog.dismiss()
activity.runOnUiThread {
if (error)
MaterialDialog.Builder(activity)
.title("Error")
.content("There was an error downloading your favorites, please try again later!")
.positiveText("Ok")
.show()
onComplete()
}
}
}
fun syncFavorites() {
val onlineSources = sourceManager.getOnlineSources()
var ehSource: EHentai? = null
var exSource: EHentai? = null
onlineSources.forEach {
if(it.id == EH_SOURCE_ID)
ehSource = it as EHentai
else if(it.id == EXH_SOURCE_ID)
exSource = it as EHentai
}
(exSource ?: ehSource)?.let { source ->
val favResponse = source.fetchFavorites()
val ourCategories = db.getCategories().executeAsBlocking().toMutableList()
val ourMangas = db.getMangas().executeAsBlocking().filter {
it.source == EH_SOURCE_ID || it.source == EXH_SOURCE_ID
}.toMutableList()
//Add required categories (categories do not sync upwards)
favResponse.second.filter { theirCategory ->
ourCategories.find {
it.name.endsWith(theirCategory)
} == null
}.map {
Category.create(it)
}.let {
db.inTransaction {
//Insert new categories
db.insertCategories(it).executeAsBlocking().results().entries.filter {
it.value.wasInserted()
}.forEach { it.key.id = it.value.insertedId()!!.toInt() }
val categoryMap = (it + ourCategories).associateBy { it.name }
//Insert new mangas
val mangaToInsert = mutableListOf<Manga>()
favResponse.first.map {
val category = categoryMap[it.fav]!!
var manga = it.manga
val alreadyHaveManga = ourMangas.find {
it.url == manga.url
}?.apply {
manga = this
} != null
if (!alreadyHaveManga) {
ourMangas.add(manga)
mangaToInsert.add(manga)
}
manga.favorite = true
Pair(manga, category)
}.apply {
//Insert mangas
db.insertMangas(mangaToInsert).executeAsBlocking().results().entries.filter {
it.value.wasInserted()
}.forEach { manga ->
manga.key.id = manga.value.insertedId()
try {
source.fetchChapterList(manga.key).map {
syncChaptersWithSource(db, it, manga.key, source)
}.toBlocking().first()
} catch (e: Exception) {
Timber.w(e, "Failed to update chapters for gallery: ${manga.key.title}!")
}
}
//Set categories
val categories = map { MangaCategory.create(it.first, it.second) }
val mangas = map { it.first }
db.setMangaCategories(categories, mangas)
}
}
}
}
}
}

192
app/src/main/java/exh/FavoritesSyncManager.java
View File

@ -1,192 +0,0 @@
package exh;
import android.app.ProgressDialog;
import android.content.Context;
import android.content.DialogInterface;
import android.os.Handler;
import android.os.Looper;
import android.support.v7.app.AlertDialog;
import com.pushtorefresh.storio.sqlite.operations.put.PutResult;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import eu.kanade.tachiyomi.data.database.DatabaseHelper;
import eu.kanade.tachiyomi.data.database.models.Category;
import eu.kanade.tachiyomi.data.database.models.Manga;
import eu.kanade.tachiyomi.data.database.models.MangaCategory;
import eu.kanade.tachiyomi.source.online.all.EHentai;
import kotlin.Pair;
//import eu.kanade.tachiyomi.data.source.online.english.EHentai;
public class FavoritesSyncManager {
/*Context context;
DatabaseHelper db;
public FavoritesSyncManager(Context context, DatabaseHelper db) {
this.context = context;
this.db = db;
}
public void guiSyncFavorites(final Runnable onComplete) {
if(!DialogLogin.isLoggedIn(context, false)) {
new AlertDialog.Builder(context).setTitle("Error")
.setMessage("You are not logged in! Please log in and try again!")
.setPositiveButton("Ok", new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog, int which) {
dialog.dismiss();
}
}).show();
return;
}
final ProgressDialog dialog = ProgressDialog.show(context, "Downloading Favorites", "Please wait...", true, false);
new Thread(new Runnable() {
@Override
public void run() {
Handler mainLooper = new Handler(Looper.getMainLooper());
try {
syncFavorites();
} catch (Exception e) {
mainLooper.post(new Runnable() {
@Override
public void run() {
new AlertDialog.Builder(context)
.setTitle("Error")
.setMessage("There was an error downloading your favorites, please try again later!")
.setPositiveButton("Ok", new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog, int which) {
dialog.dismiss();
}
}).show();
}
});
e.printStackTrace();
}
dialog.dismiss();
mainLooper.post(onComplete);
}
}).start();
}*/
/*
public void syncFavorites() throws IOException {
Pair favResponse = EHentai.fetchFavorites(context);
Map<String, List<Manga>> favorites = favResponse.favs;
List<Category> ourCategories = new ArrayList<>(db.getCategories().executeAsBlocking());
List<Manga> ourMangas = new ArrayList<>(db.getMangas().executeAsBlocking());
//Add required categories (categories do not sync upwards)
List<Category> categoriesToInsert = new ArrayList<>();
for (String theirCategory : favorites.keySet()) {
boolean haveCategory = false;
for (Category category : ourCategories) {
if (category.getName().endsWith(theirCategory)) {
haveCategory = true;
}
}
if (!haveCategory) {
Category category = Category.Companion.create(theirCategory);
ourCategories.add(category);
categoriesToInsert.add(category);
}
}
if (!categoriesToInsert.isEmpty()) {
for(Map.Entry<Category, PutResult> result : db.insertCategories(categoriesToInsert).executeAsBlocking().results().entrySet()) {
if(result.getValue().wasInserted()) {
result.getKey().setId(result.getValue().insertedId().intValue());
}
}
}
//Build category map
Map<String, Category> categoryMap = new HashMap<>();
for (Category category : ourCategories) {
categoryMap.put(category.getName(), category);
}
//Insert new mangas
List<Manga> mangaToInsert = new ArrayList<>();
Map<Manga, Category> mangaToSetCategories = new HashMap<>();
for (Map.Entry<String, List<Manga>> entry : favorites.entrySet()) {
Category category = categoryMap.get(entry.getKey());
for (Manga manga : entry.getValue()) {
boolean alreadyHaveManga = false;
for (Manga ourManga : ourMangas) {
if (ourManga.getUrl().equals(manga.getUrl())) {
alreadyHaveManga = true;
manga = ourManga;
break;
}
}
if (!alreadyHaveManga) {
ourMangas.add(manga);
mangaToInsert.add(manga);
}
mangaToSetCategories.put(manga, category);
manga.setFavorite(true);
}
}
for (Map.Entry<Manga, PutResult> results : db.insertMangas(mangaToInsert).executeAsBlocking().results().entrySet()) {
if(results.getValue().wasInserted()) {
results.getKey().setId(results.getValue().insertedId());
}
}
for(Map.Entry<Manga, Category> entry : mangaToSetCategories.entrySet()) {
db.setMangaCategories(Collections.singletonList(MangaCategory.Companion.create(entry.getKey(), entry.getValue())),
Collections.singletonList(entry.getKey()));
}*/
//Determines what
/*Map<Integer, List<Manga>> toUpload = new HashMap<>();
for (Manga manga : ourMangas) {
if(manga.getFavorite()) {
boolean remoteHasManga = false;
for (List<Manga> remoteMangas : favorites.values()) {
for (Manga remoteManga : remoteMangas) {
if (remoteManga.getUrl().equals(manga.getUrl())) {
remoteHasManga = true;
break;
}
}
}
if (!remoteHasManga) {
List<Category> mangaCategories = db.getCategoriesForManga(manga).executeAsBlocking();
for (Category category : mangaCategories) {
int categoryIndex = favResponse.favCategories.indexOf(category.getName());
if (categoryIndex >= 0) {
List<Manga> uploadMangas = toUpload.get(categoryIndex);
if (uploadMangas == null) {
uploadMangas = new ArrayList<>();
toUpload.put(categoryIndex, uploadMangas);
}
uploadMangas.add(manga);
}
}
}
}
}*/
/********** NON-FUNCTIONAL, modifygids[] CANNOT ADD NEW FAVORITES! (or as of my testing it can't, maybe I'll do more testing)**/
/*PreferencesHelper helper = new PreferencesHelper(context);
for(Map.Entry<Integer, List<Manga>> entry : toUpload.entrySet()) {
FormBody.Builder formBody = new FormBody.Builder()
.add("ddact", "fav" + entry.getKey());
for(Manga manga : entry.getValue()) {
List<String> splitUrl = new ArrayList<>(Arrays.asList(manga.getUrl().split("/")));
splitUrl.removeAll(Collections.singleton(""));
if(splitUrl.size() < 2) {
continue;
}
formBody.add("modifygids[]", splitUrl.get(1).trim());
}
formBody.add("apply", "Apply");
Request request = RequestsKt.POST(EHentai.buildFavoritesBase(context, helper.getPrefs()).favoritesBase,
EHentai.getHeadersBuilder(helper).build(),
formBody.build(),
RequestsKt.getDEFAULT_CACHE_CONTROL());
Response response = NetworkManager.getInstance().getClient().newCall(request).execute();
Util.d("EHentai", response.body().string());
}*/
// }
}

18
app/src/main/java/exh/GalleryAdder.kt
View File

@ -10,8 +10,7 @@ import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.util.syncChaptersWithSource
import exh.metadata.models.*
import exh.util.defRealm
import exh.metadata.models.ExGalleryMetadata
import okhttp3.MediaType
import okhttp3.Request
import okhttp3.RequestBody
@ -131,7 +130,7 @@ class GalleryAdder {
?: return GalleryAddEvent.Fail.Error(url, "Could not find EH source!")
val cleanedUrl = when(source) {
EH_SOURCE_ID, EXH_SOURCE_ID -> getUrlWithoutDomain(realUrl)
EH_SOURCE_ID, EXH_SOURCE_ID -> ExGalleryMetadata.normalizeUrl(getUrlWithoutDomain(realUrl))
NHENTAI_SOURCE_ID -> realUrl //nhentai uses URLs directly (oops, my bad when implementing this source)
PERV_EDEN_EN_SOURCE_ID,
PERV_EDEN_IT_SOURCE_ID -> getUrlWithoutDomain(realUrl)
@ -152,19 +151,6 @@ class GalleryAdder {
manga.copyFrom(newManga)
manga.title = newManga.title //Forcibly copy title as copyFrom does not copy title
//Apply metadata
defRealm { realm ->
when (source) {
EH_SOURCE_ID, EXH_SOURCE_ID -> ExGalleryMetadata.UrlQuery(realUrl, isExSource(source))
NHENTAI_SOURCE_ID -> NHentaiMetadata.UrlQuery(realUrl)
PERV_EDEN_EN_SOURCE_ID,
PERV_EDEN_IT_SOURCE_ID -> PervEdenGalleryMetadata.UrlQuery(realUrl, PervEdenLang.source(source))
HENTAI_CAFE_SOURCE_ID -> HentaiCafeMetadata.UrlQuery(realUrl)
TSUMINO_SOURCE_ID -> TsuminoMetadata.UrlQuery(realUrl)
else -> return GalleryAddEvent.Fail.UnknownType(url)
}.query(realm).findFirst()
}
if (fav) manga.favorite = true
db.insertManga(manga).executeAsBlocking().insertedId()?.let {

23
app/src/main/java/exh/favorites/FavoriteEntry.kt Normal file
View File

@ -0,0 +1,23 @@
package exh.favorites
import exh.metadata.models.ExGalleryMetadata
import io.realm.RealmObject
import io.realm.annotations.Index
import io.realm.annotations.PrimaryKey
import io.realm.annotations.RealmClass
import java.util.*
@RealmClass
open class FavoriteEntry : RealmObject() {
@PrimaryKey var id: String = UUID.randomUUID().toString()
var title: String? = null
@Index lateinit var gid: String
@Index lateinit var token: String
@Index var category: Int = -1
fun getUrl() = ExGalleryMetadata.normalizeUrl(gid, token)
}

274
app/src/main/java/exh/favorites/FavoritesSyncHelper.kt Normal file
View File

@ -0,0 +1,274 @@
package exh.favorites
import android.content.Context
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Category
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.database.models.MangaCategory
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.all.EHentai
import exh.EH_METADATA_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.GalleryAddEvent
import exh.GalleryAdder
import okhttp3.FormBody
import okhttp3.Request
import rx.subjects.BehaviorSubject
import timber.log.Timber
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import uy.kohesive.injekt.injectLazy
import kotlin.concurrent.thread
class FavoritesSyncHelper(context: Context) {
private val db: DatabaseHelper by injectLazy()
private val prefs: PreferencesHelper by injectLazy()
private val exh by lazy {
Injekt.get<SourceManager>().get(EXH_SOURCE_ID) as? EHentai
?: EHentai(0, true, context)
}
private val storage = LocalFavoritesStorage()
private val galleryAdder = GalleryAdder()
val status = BehaviorSubject.create<FavoritesSyncStatus>(FavoritesSyncStatus.Idle())
@Synchronized
fun runSync() {
if(status.value !is FavoritesSyncStatus.Idle) {
return
}
status.onNext(FavoritesSyncStatus.Initializing())
thread { beginSync() }
}
private fun beginSync() {
//Check if logged in
if(!prefs.enableExhentai().getOrDefault()) {
status.onNext(FavoritesSyncStatus.Error("Please log in!"))
return
}
//Download remote favorites
val favorites = try {
status.onNext(FavoritesSyncStatus.Processing("Downloading favorites from remote server"))
exh.fetchFavorites()
} catch(e: Exception) {
status.onNext(FavoritesSyncStatus.Error("Failed to fetch favorites from remote server!"))
Timber.e(e, "Could not fetch favorites!")
return
}
val errors = mutableListOf<String>()
try {
db.inTransaction {
val remoteChanges = storage.getChangedRemoteEntries(favorites.first)
val localChanges = storage.getChangedDbEntries()
//Apply remote categories
status.onNext(FavoritesSyncStatus.Processing("Updating category names"))
applyRemoteCategories(favorites.second)
//Apply ChangeSets
applyChangeSetToLocal(remoteChanges, errors)
applyChangeSetToRemote(localChanges, errors)
status.onNext(FavoritesSyncStatus.Processing("Cleaning up"))
storage.snapshotEntries()
}
} catch(e: IgnoredException) {
//Do not display error as this error has already been reported
Timber.w(e, "Ignoring exception!")
return
} catch (e: Exception) {
status.onNext(FavoritesSyncStatus.Error("Unknown error: ${e.message}"))
Timber.e(e, "Sync error!")
return
}
status.onNext(FavoritesSyncStatus.Complete(errors))
}
private fun applyRemoteCategories(categories: List<String>) {
val localCategories = db.getCategories().executeAsBlocking()
val newLocalCategories = localCategories.toMutableList()
var changed = false
categories.forEachIndexed { index, remote ->
val local = localCategories.getOrElse(index) {
changed = true
Category.create(remote).apply {
order = index
//Going through categories list from front to back
//If category does not exist, list size <= category index
//Thus, we can just add it here and not worry about indexing
newLocalCategories += this
}
}
if(local.name != remote) {
changed = true
local.name = remote
}
}
//Ensure consistent ordering
newLocalCategories.forEachIndexed { index, category ->
if(category.order != index) {
changed = true
category.order = index
}
}
//Only insert categories if changed
if(changed)
db.insertCategories(newLocalCategories).executeAsBlocking()
}
private fun addGalleryRemote(gallery: FavoriteEntry, errors: MutableList<String>) {
val url = "${exh.baseUrl}/gallerypopups.php?gid=${gallery.gid}&t=${gallery.token}&act=addfav"
val request = Request.Builder()
.url(url)
.post(FormBody.Builder()
.add("favcat", gallery.category.toString())
.add("favnote", "")
.add("apply", "Add to Favorites")
.add("update", "1")
.build())
.build()
if(!explicitlyRetryExhRequest(10, request)) {
errors += "Unable to add gallery to remote server: '${gallery.title}' (GID: ${gallery.gid})!"
}
}
private fun explicitlyRetryExhRequest(retryCount: Int, request: Request): Boolean {
var success = false
for(i in 1 .. retryCount) {
try {
val resp = exh.client.newCall(request).execute()
if (resp.isSuccessful) {
success = true
break
}
} catch (e: Exception) {
Timber.e(e, "Sync network error!")
}
}
return success
}
private fun applyChangeSetToRemote(changeSet: ChangeSet, errors: MutableList<String>) {
//Apply removals
if(changeSet.removed.isNotEmpty()) {
status.onNext(FavoritesSyncStatus.Processing("Removing ${changeSet.removed.size} galleries from remote server"))
val formBody = FormBody.Builder()
.add("ddact", "delete")
.add("apply", "Apply")
//Add change set to form
changeSet.removed.forEach {
formBody.add("modifygids[]", it.gid)
}
val request = Request.Builder()
.url("https://exhentai.org/favorites.php")
.post(formBody.build())
.build()
if(!explicitlyRetryExhRequest(10, request)) {
status.onNext(FavoritesSyncStatus.Error("Unable to delete galleries from the remote servers!"))
//It is still safe to stop here so crash
throw IgnoredException()
}
}
//Apply additions
changeSet.added.forEachIndexed { index, it ->
status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to remote server"))
addGalleryRemote(it, errors)
}
}
private fun applyChangeSetToLocal(changeSet: ChangeSet, errors: MutableList<String>) {
val removedManga = mutableListOf<Manga>()
//Apply removals
changeSet.removed.forEachIndexed { index, it ->
status.onNext(FavoritesSyncStatus.Processing("Removing gallery ${index + 1} of ${changeSet.removed.size} from local library"))
val url = it.getUrl()
//Consider both EX and EH sources
listOf(db.getManga(url, EXH_SOURCE_ID),
db.getManga(url, EH_METADATA_SOURCE_ID)).forEach {
val manga = it.executeAsBlocking()
if(manga?.favorite == true) {
manga.favorite = false
db.updateMangaFavorite(manga).executeAsBlocking()
removedManga += manga
}
}
}
db.deleteOldMangasCategories(removedManga).executeAsBlocking()
val insertedMangaCategories = mutableListOf<MangaCategory>()
val insertedMangaCategoriesMangas = mutableListOf<Manga>()
val categories = db.getCategories().executeAsBlocking()
//Apply additions
changeSet.added.forEachIndexed { index, it ->
status.onNext(FavoritesSyncStatus.Processing("Adding gallery ${index + 1} of ${changeSet.added.size} to local library"))
//Import using gallery adder
val result = galleryAdder.addGallery("${exh.baseUrl}${it.getUrl()}",
true,
EXH_SOURCE_ID)
if(result is GalleryAddEvent.Fail) {
errors += "Failed to add gallery to local database: " + when (result) {
is GalleryAddEvent.Fail.Error -> "'${it.title}' ${result.logMessage}"
is GalleryAddEvent.Fail.UnknownType -> "'${it.title}' (${result.galleryUrl}) is not a valid gallery!"
}
} else if(result is GalleryAddEvent.Success) {
insertedMangaCategories += MangaCategory.create(result.manga,
categories[it.category])
insertedMangaCategoriesMangas += result.manga
}
}
db.setMangaCategories(insertedMangaCategories, insertedMangaCategoriesMangas)
}
class IgnoredException : RuntimeException()
}
sealed class FavoritesSyncStatus(val message: String) {
class Error(message: String) : FavoritesSyncStatus(message)
class Idle : FavoritesSyncStatus("Waiting for sync to start")
class Initializing : FavoritesSyncStatus("Initializing sync")
class Processing(message: String) : FavoritesSyncStatus(message)
class Complete(val errors: List<String>) : FavoritesSyncStatus("Sync complete!")
}

132
app/src/main/java/exh/favorites/LocalFavoritesStorage.kt Normal file
View File

@ -0,0 +1,132 @@
package exh.favorites
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.source.online.all.EHentai
import exh.EH_SOURCE_ID
import exh.EXH_SOURCE_ID
import exh.metadata.models.ExGalleryMetadata
import exh.util.trans
import io.realm.Realm
import io.realm.RealmConfiguration
import uy.kohesive.injekt.injectLazy
class LocalFavoritesStorage {
private val db: DatabaseHelper by injectLazy()
private val realmConfig = RealmConfiguration.Builder()
.name("fav-sync")
.deleteRealmIfMigrationNeeded()
.build()
private val realm
get() = Realm.getInstance(realmConfig)
fun getChangedDbEntries()
= getChangedEntries(
parseToFavoriteEntries(
loadDbCategories(
db.getFavoriteMangas()
.executeAsBlocking()
.asSequence()
)
)
)
fun getChangedRemoteEntries(entries: List<EHentai.ParsedManga>)
= getChangedEntries(
parseToFavoriteEntries(
entries.asSequence().map {
Pair(it.fav, it.manga.apply {
favorite = true
})
}
)
)
fun snapshotEntries() {
val dbMangas = parseToFavoriteEntries(
loadDbCategories(
db.getFavoriteMangas()
.executeAsBlocking()
.asSequence()
)
)
realm.use { realm ->
realm.trans {
//Delete old snapshot
realm.delete(FavoriteEntry::class.java)
//Insert new snapshots
realm.copyToRealm(dbMangas.toList())
}
}
}
private fun getChangedEntries(entries: Sequence<FavoriteEntry>): ChangeSet {
return realm.use { realm ->
val terminated = entries.toList()
val added = terminated.filter {
realm.queryRealmForEntry(it) == null
}
val removed = realm.where(FavoriteEntry::class.java)
.findAll()
.filter {
queryListForEntry(terminated, it) == null
}.map {
realm.copyFromRealm(it)
}
ChangeSet(added, removed)
}
}
private fun Realm.queryRealmForEntry(entry: FavoriteEntry)
= where(FavoriteEntry::class.java)
.equalTo(FavoriteEntry::gid.name, entry.gid)
.equalTo(FavoriteEntry::token.name, entry.token)
.equalTo(FavoriteEntry::category.name, entry.category)
.findFirst()
private fun queryListForEntry(list: List<FavoriteEntry>, entry: FavoriteEntry)
= list.find {
it.gid == entry.gid
&& it.token == entry.token
&& it.category == entry.category
}
private fun loadDbCategories(manga: Sequence<Manga>): Sequence<Pair<Int, Manga>> {
val dbCategories = db.getCategories().executeAsBlocking()
return manga.filter(this::validateDbManga).mapNotNull {
val category = db.getCategoriesForManga(it).executeAsBlocking()
Pair(dbCategories.indexOf(category.firstOrNull()
?: return@mapNotNull null), it)
}
}
private fun parseToFavoriteEntries(manga: Sequence<Pair<Int, Manga>>)
= manga.filter {
validateDbManga(it.second)
}.mapNotNull {
FavoriteEntry().apply {
title = it.second.title
gid = ExGalleryMetadata.galleryId(it.second.url)
token = ExGalleryMetadata.galleryToken(it.second.url)
category = it.first
if(this.category > 9)
return@mapNotNull null
}
}
private fun validateDbManga(manga: Manga)
= manga.favorite && (manga.source == EH_SOURCE_ID || manga.source == EXH_SOURCE_ID)
}
data class ChangeSet(val added: List<FavoriteEntry>,
val removed: List<FavoriteEntry>)

18
app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt
View File

@ -26,6 +26,10 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
override var uuid: String = UUID.randomUUID().toString()
var url: String? = null
set(value) {
//Ensure that URLs are always formatted in the same way to reduce duplicate galleries
field = value?.let { normalizeUrl(it) }
}
@Index
var gId: String? = null
@ -60,7 +64,7 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
override var tags: RealmList<Tag> = RealmList()
override fun getTitles() = listOf(title, altTitle).filterNotNull()
override fun getTitles() = listOfNotNull(title, altTitle)
@Ignore
override val titleFields = TITLE_FIELDS
@ -93,7 +97,7 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
}
override fun copyTo(manga: SManga) {
url?.let { manga.url = it }
url?.let { manga.url = normalizeUrl(it) }
thumbnailUrl?.let { manga.thumbnail_url = it }
//No title bug?
@ -118,8 +122,8 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
ONGOING_SUFFIX.find {
t.endsWith(it, ignoreCase = true)
}?.let {
manga.status = SManga.ONGOING
}
manga.status = SManga.ONGOING
}
}
//Build a nice looking description out of what we know
@ -165,6 +169,12 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
fun galleryToken(url: String) =
splitGalleryUrl(url).last()
fun normalizeUrl(id: String, token: String)
= "/g/$id/$token/?nw=always"
fun normalizeUrl(url: String)
= normalizeUrl(galleryId(url), galleryToken(url))
val TITLE_FIELDS = listOf(
ExGalleryMetadata::title.name,
ExGalleryMetadata::altTitle.name

6
app/src/main/java/exh/metadata/models/GalleryQuery.kt
View File

@ -1,6 +1,8 @@
package exh.metadata.models
import io.realm.*
import io.realm.Case
import io.realm.Realm
import io.realm.RealmQuery
import java.util.*
import kotlin.reflect.KClass
import kotlin.reflect.KProperty
@ -58,7 +60,7 @@ abstract class GalleryQuery<T : SearchableGalleryMetadata>(val clazz: KClass<T>)
is Long -> newMeta.equalTo(n, v)
is Short -> newMeta.equalTo(n, v)
is String -> newMeta.equalTo(n, v, Case.INSENSITIVE)
else -> throw IllegalArgumentException("Unknown type: ${v::class.qualifiedName}!")
else -> throw IllegalArgumentException("Unknown type: ${v::class.java.name}!")
}
}
}