diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt index f0ceffc11..20c1ef9a8 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt @@ -437,8 +437,10 @@ class LibraryController( R.id.action_move_to_category -> showChangeMangaCategoriesDialog() R.id.action_delete -> showDeleteMangaDialog() R.id.action_auto_source_migration -> { + router.pushController(MigrationDesignController.create( + selectedMangas.mapNotNull { it.id } + ).withFadeTransaction()) destroyActionModeIfNeeded() - router.pushController(MigrationDesignController().withFadeTransaction()) } else -> return false } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt index b52b23aea..6d0c80e75 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt @@ -252,6 +252,8 @@ class MangaInfoController : NucleusController(), private fun setMangaInfo(manga: Manga, source: Source?) { val view = view ?: return + // TODO Duplicated in MigrationProcedureAdapter + //update full title TextView. manga_full_title.text = if (manga.title.isBlank()) { view.context.getString(R.string.unknown) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/migration/MigrationFlags.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/migration/MigrationFlags.kt index f47fd63d3..2e9f3aa8d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/migration/MigrationFlags.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/migration/MigrationFlags.kt @@ -4,9 +4,9 @@ import eu.kanade.tachiyomi.R object MigrationFlags { - private const val CHAPTERS = 0b001 - private const val CATEGORIES = 0b010 - private const val TRACK = 0b100 + const val CHAPTERS = 0b001 + const val CATEGORIES = 0b010 + const val TRACK = 0b100 private const val CHAPTERS2 = 0x1 private const val CATEGORIES2 = 0x2 diff --git a/app/src/main/java/exh/smartsearch/SmartSearchEngine.kt b/app/src/main/java/exh/smartsearch/SmartSearchEngine.kt new file mode 100644 index 000000000..26ea3ac60 --- /dev/null +++ b/app/src/main/java/exh/smartsearch/SmartSearchEngine.kt @@ -0,0 +1,176 @@ +package exh.smartsearch + +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.source.CatalogueSource +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.SManga +import exh.ui.smartsearch.SmartSearchPresenter +import exh.util.await +import info.debatty.java.stringsimilarity.NormalizedLevenshtein +import kotlinx.coroutines.* +import rx.schedulers.Schedulers +import uy.kohesive.injekt.injectLazy +import kotlin.coroutines.CoroutineContext + +class SmartSearchEngine(parentContext: CoroutineContext): CoroutineScope { + override val coroutineContext: CoroutineContext = parentContext + Job() + Dispatchers.Default + + private val db: DatabaseHelper by injectLazy() + + suspend fun smartSearch(source: CatalogueSource, title: String): SManga? { + val cleanedTitle = cleanSmartSearchTitle(title) + + val queries = getSmartSearchQueries(cleanedTitle) + + val eligibleManga = supervisorScope { + queries.map { query -> + async(Dispatchers.Default) { + val searchResults = source.fetchSearchManga(1, query, FilterList()).toSingle().await(Schedulers.io()) + + searchResults.mangas.map { + val cleanedMangaTitle = cleanSmartSearchTitle(it.title) + val normalizedDistance = NormalizedLevenshtein().similarity(cleanedTitle, cleanedMangaTitle) + SmartSearchPresenter.SearchEntry(it, normalizedDistance) + }.filter { (_, normalizedDistance) -> + normalizedDistance >= MIN_SMART_ELIGIBLE_THRESHOLD + } + } + }.flatMap { it.await() } + } + + return eligibleManga.maxBy { it.dist }?.manga + } + + suspend fun normalSearch(source: CatalogueSource, title: String): SManga? { + val eligibleManga = supervisorScope { + val searchResults = source.fetchSearchManga(1, title, FilterList()).toSingle().await(Schedulers.io()) + + searchResults.mangas.map { + val normalizedDistance = NormalizedLevenshtein().similarity(title, it.title) + SmartSearchPresenter.SearchEntry(it, normalizedDistance) + }.filter { (_, normalizedDistance) -> + normalizedDistance >= MIN_NORMAL_ELIGIBLE_THRESHOLD + } + } + + return eligibleManga.maxBy { it.dist }?.manga + } + + private fun getSmartSearchQueries(cleanedTitle: String): List { + val splitCleanedTitle = cleanedTitle.split(" ") + val splitSortedByLargest = splitCleanedTitle.sortedByDescending { it.length } + + if(splitCleanedTitle.isEmpty()) { + return emptyList() + } + + // Search cleaned title + // Search two largest words + // Search largest word + // Search first two words + // Search first word + + val searchQueries = listOf( + listOf(cleanedTitle), + splitSortedByLargest.take(2), + splitSortedByLargest.take(1), + splitCleanedTitle.take(2), + splitCleanedTitle.take(1) + ) + + return searchQueries.map { + it.joinToString().trim() + }.distinct() + } + + private fun cleanSmartSearchTitle(title: String): String { + val preTitle = title.toLowerCase() + + // Remove text in brackets + var cleanedTitle = removeTextInBrackets(preTitle, true) + if(cleanedTitle.length <= 5) { // Title is suspiciously short, try parsing it backwards + cleanedTitle = removeTextInBrackets(preTitle, false) + } + + // Strip non-special characters + cleanedTitle = cleanedTitle.replace(titleRegex, " ") + + // Strip splitters and consecutive spaces + cleanedTitle = cleanedTitle.trim().replace(" - ", " ").replace(consecutiveSpacesRegex, " ").trim() + + return cleanedTitle + } + + private fun removeTextInBrackets(text: String, readForward: Boolean): String { + val bracketPairs = listOf( + '(' to ')', + '[' to ']', + '<' to '>', + '{' to '}' + ) + var openingBracketPairs = bracketPairs.mapIndexed { index, (opening, _) -> + opening to index + }.toMap() + var closingBracketPairs = bracketPairs.mapIndexed { index, (_, closing) -> + closing to index + }.toMap() + + // Reverse pairs if reading backwards + if(!readForward) { + val tmp = openingBracketPairs + openingBracketPairs = closingBracketPairs + closingBracketPairs = tmp + } + + val depthPairs = bracketPairs.map { 0 }.toMutableList() + + val result = StringBuilder() + for(c in if(readForward) text else text.reversed()) { + val openingBracketDepthIndex = openingBracketPairs[c] + if(openingBracketDepthIndex != null) { + depthPairs[openingBracketDepthIndex]++ + } else { + val closingBracketDepthIndex = closingBracketPairs[c] + if(closingBracketDepthIndex != null) { + depthPairs[closingBracketDepthIndex]-- + } else { + if(depthPairs.all { it <= 0 }) { + result.append(c) + } else { + // In brackets, do not append to result + } + } + } + } + + return result.toString() + } + + /** + * Returns a manga from the database for the given manga from network. It creates a new entry + * if the manga is not yet in the database. + * + * @param sManga the manga from the source. + * @return a manga from the database. + */ + suspend fun networkToLocalManga(sManga: SManga, sourceId: Long): Manga { + var localManga = db.getManga(sManga.url, sourceId).await() + if (localManga == null) { + val newManga = Manga.create(sManga.url, sManga.title, sourceId) + newManga.copyFrom(sManga) + val result = db.insertManga(newManga).await() + newManga.id = result.insertedId() + localManga = newManga + } + return localManga + } + + companion object { + const val MIN_SMART_ELIGIBLE_THRESHOLD = 0.7 + const val MIN_NORMAL_ELIGIBLE_THRESHOLD = 0.5 + + private val titleRegex = Regex("[^a-zA-Z0-9- ]") + private val consecutiveSpacesRegex = Regex(" +") + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/base/BaseExhController.kt b/app/src/main/java/exh/ui/base/BaseExhController.kt index 05ef7bdf3..2a25f16c9 100644 --- a/app/src/main/java/exh/ui/base/BaseExhController.kt +++ b/app/src/main/java/exh/ui/base/BaseExhController.kt @@ -1,5 +1,6 @@ package exh.ui.base +import android.os.Bundle import android.support.annotation.LayoutRes import android.view.LayoutInflater import android.view.View @@ -11,7 +12,7 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.cancel import kotlin.coroutines.CoroutineContext -abstract class BaseExhController : BaseController(), CoroutineScope { +abstract class BaseExhController(bundle: Bundle? = null) : BaseController(bundle), CoroutineScope { abstract val layoutId: Int @LayoutRes get diff --git a/app/src/main/java/exh/ui/migration/manga/design/MigrationDesignController.kt b/app/src/main/java/exh/ui/migration/manga/design/MigrationDesignController.kt index eeb7f2b90..3c358edb7 100644 --- a/app/src/main/java/exh/ui/migration/manga/design/MigrationDesignController.kt +++ b/app/src/main/java/exh/ui/migration/manga/design/MigrationDesignController.kt @@ -1,5 +1,6 @@ package exh.ui.migration.manga.design +import android.os.Bundle import android.support.v7.widget.LinearLayoutManager import android.view.View import eu.davidea.flexibleadapter.FlexibleAdapter @@ -9,34 +10,40 @@ import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.ui.base.controller.withFadeTransaction +import eu.kanade.tachiyomi.ui.migration.MigrationFlags import exh.ui.base.BaseExhController +import exh.ui.migration.manga.process.MigrationProcedureConfig import exh.ui.migration.manga.process.MigrationProcedureController import kotlinx.android.synthetic.main.eh_migration_design.* import uy.kohesive.injekt.injectLazy // TODO Handle config changes // TODO Select all in library -class MigrationDesignController : BaseExhController(), FlexibleAdapter.OnItemClickListener { +class MigrationDesignController(bundle: Bundle? = null) : BaseExhController(bundle), FlexibleAdapter.OnItemClickListener { private val sourceManager: SourceManager by injectLazy() private val prefs: PreferencesHelper by injectLazy() override val layoutId: Int = R.layout.eh_migration_design - private var adapter: FlexibleAdapter? = null + private var adapter: MigrationSourceAdapter? = null + + private val config: LongArray = args.getLongArray(MANGA_IDS_EXTRA) ?: LongArray(0) override fun getTitle() = "Select target sources" override fun onViewCreated(view: View) { super.onViewCreated(view) - adapter = MigrationSourceAdapter( + val ourAdapter = adapter ?: MigrationSourceAdapter( getEnabledSources().map { MigrationSourceItem(it, true) }, this ) + adapter = ourAdapter recycler.layoutManager = LinearLayoutManager(view.context) recycler.setHasFixedSize(true) - recycler.adapter = adapter - adapter?.isHandleDragEnabled = true + recycler.adapter = ourAdapter + ourAdapter.itemTouchHelperCallback = null // Reset adapter touch adapter to fix drag after rotation + ourAdapter.isHandleDragEnabled = true migration_mode.setOnClickListener { prioritize_chapter_count.toggle() @@ -53,15 +60,41 @@ class MigrationDesignController : BaseExhController(), FlexibleAdapter.OnItemCli updatePrioritizeChapterCount(prioritize_chapter_count.isChecked) begin_migration_btn.setOnClickListener { - router.replaceTopController(MigrationProcedureController().withFadeTransaction()) + var flags = 0 + if(mig_chapters.isChecked) flags = flags or MigrationFlags.CHAPTERS + if(mig_categories.isChecked) flags = flags or MigrationFlags.CATEGORIES + if(mig_categories.isChecked) flags = flags or MigrationFlags.TRACK + + router.replaceTopController(MigrationProcedureController.create( + MigrationProcedureConfig( + config.toList(), + ourAdapter.items.filter { + it.sourceEnabled + }.map { it.source.id }, + useSourceWithMostChapters = prioritize_chapter_count.isChecked, + enableLenientSearch = use_smart_search.isChecked, + migrationFlags = flags + ) + ).withFadeTransaction()) } } + override fun onSaveInstanceState(outState: Bundle) { + super.onSaveInstanceState(outState) + adapter?.onSaveInstanceState(outState) + } + + // TODO Still incorrect, why is this called before onViewCreated? + override fun onRestoreInstanceState(savedInstanceState: Bundle) { + super.onRestoreInstanceState(savedInstanceState) + adapter?.onRestoreInstanceState(savedInstanceState) + } + private fun updatePrioritizeChapterCount(migrationMode: Boolean) { migration_mode.text = if(migrationMode) { - "Use source with most chapters and use the above list to break ties" + "Use the source with the most chapters and use the above list to break ties (slow with many sources or smart search)" } else { - "Use the first source in the list that has at least one chapter of the manga" + "Use the first source in the list that has the manga" } } @@ -88,4 +121,14 @@ class MigrationDesignController : BaseExhController(), FlexibleAdapter.OnItemCli .filterNot { it.id.toString() in hiddenCatalogues } .sortedBy { "(${it.lang}) ${it.name}" } } + + companion object { + private const val MANGA_IDS_EXTRA = "manga_ids" + + fun create(mangaIds: List): MigrationDesignController { + return MigrationDesignController(Bundle().apply { + putLongArray(MANGA_IDS_EXTRA, mangaIds.toLongArray()) + }) + } + } } \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceAdapter.kt b/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceAdapter.kt index 01e0a87f8..1a7eff5ee 100644 --- a/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceAdapter.kt +++ b/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceAdapter.kt @@ -1,10 +1,32 @@ package exh.ui.migration.manga.design +import android.os.Bundle import eu.davidea.flexibleadapter.FlexibleAdapter +import exh.debug.DebugFunctions.sourceManager class MigrationSourceAdapter(val items: List, val controller: MigrationDesignController): FlexibleAdapter( items, controller, true -) \ No newline at end of file +) { + override fun onSaveInstanceState(outState: Bundle) { + super.onSaveInstanceState(outState) + + outState.putParcelableArrayList(SELECTED_SOURCES_KEY, ArrayList(currentItems.map { + it.asParcelable() + })) + } + + override fun onRestoreInstanceState(savedInstanceState: Bundle) { + savedInstanceState.getParcelableArrayList(SELECTED_SOURCES_KEY)?.let { + updateDataSet(it.map { MigrationSourceItem.fromParcelable(sourceManager, it) }) + } + + super.onRestoreInstanceState(savedInstanceState) + } + + companion object { + private const val SELECTED_SOURCES_KEY = "selected_sources" + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceItem.kt b/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceItem.kt index a31d4f552..c5a353ee0 100644 --- a/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceItem.kt +++ b/app/src/main/java/exh/ui/migration/manga/design/MigrationSourceItem.kt @@ -1,12 +1,15 @@ package exh.ui.migration.manga.design +import android.os.Parcelable import android.support.v7.widget.RecyclerView import android.view.View import eu.davidea.flexibleadapter.FlexibleAdapter import eu.davidea.flexibleadapter.items.AbstractFlexibleItem import eu.davidea.flexibleadapter.items.IFlexible import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.online.HttpSource +import kotlinx.android.parcel.Parcelize class MigrationSourceItem(val source: HttpSource, var sourceEnabled: Boolean): AbstractFlexibleItem() { override fun getLayoutRes() = R.layout.eh_source_item @@ -48,4 +51,22 @@ class MigrationSourceItem(val source: HttpSource, var sourceEnabled: Boolean): A override fun hashCode(): Int { return source.id.hashCode() } + + @Parcelize + data class ParcelableSI(val sourceId: Long, val sourceEnabled: Boolean): Parcelable + + fun asParcelable(): ParcelableSI { + return ParcelableSI(source.id, sourceEnabled) + } + + companion object { + fun fromParcelable(sourceManager: SourceManager, si: ParcelableSI): MigrationSourceItem? { + val source = sourceManager.get(si.sourceId) as? HttpSource ?: return null + + return MigrationSourceItem( + source, + si.sourceEnabled + ) + } + } } \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/manga/process/DeactivatableViewPager.kt b/app/src/main/java/exh/ui/migration/manga/process/DeactivatableViewPager.kt new file mode 100644 index 000000000..fb11df2b4 --- /dev/null +++ b/app/src/main/java/exh/ui/migration/manga/process/DeactivatableViewPager.kt @@ -0,0 +1,19 @@ +package exh.ui.migration.manga.process + +import android.content.Context +import android.support.v4.view.ViewPager +import android.util.AttributeSet +import android.view.MotionEvent + +class DeactivatableViewPager: ViewPager { + constructor(context: Context): super(context) + constructor(context: Context, attrs: AttributeSet): super(context, attrs) + + override fun onTouchEvent(event: MotionEvent): Boolean { + return !isEnabled || super.onTouchEvent(event) + } + + override fun onInterceptTouchEvent(event: MotionEvent): Boolean { + return isEnabled && super.onInterceptTouchEvent(event) + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/manga/process/MigratingManga.kt b/app/src/main/java/exh/ui/migration/manga/process/MigratingManga.kt new file mode 100644 index 000000000..94eba088f --- /dev/null +++ b/app/src/main/java/exh/ui/migration/manga/process/MigratingManga.kt @@ -0,0 +1,38 @@ +package exh.ui.migration.manga.process + +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.source.Source +import eu.kanade.tachiyomi.source.SourceManager +import exh.util.DeferredField +import exh.util.await +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.BroadcastChannel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.channels.ConflatedBroadcastChannel +import kotlinx.coroutines.channels.ReceiveChannel +import kotlinx.coroutines.flow.Flow +import kotlin.coroutines.CoroutineContext + +class MigratingManga(private val db: DatabaseHelper, + private val sourceManager: SourceManager, + val mangaId: Long, + parentContext: CoroutineContext) { + val searchResult = DeferredField() + + // + val progress = ConflatedBroadcastChannel(1 to 0) + + val migrationJob = parentContext + SupervisorJob() + Dispatchers.Default + + @Volatile + private var manga: Manga? = null + suspend fun manga(): Manga? { + if(manga == null) manga = db.getManga(mangaId).await() + return manga + } + + suspend fun mangaSource(): Source { + return sourceManager.getOrStub(manga()?.source ?: -1) + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureAdapter.kt b/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureAdapter.kt new file mode 100644 index 000000000..c4babc045 --- /dev/null +++ b/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureAdapter.kt @@ -0,0 +1,180 @@ +package exh.ui.migration.manga.process + +import android.support.v4.view.PagerAdapter +import android.view.View +import android.view.ViewGroup +import com.bumptech.glide.load.engine.DiskCacheStrategy +import com.google.gson.Gson +import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.glide.GlideApp +import eu.kanade.tachiyomi.source.Source +import eu.kanade.tachiyomi.source.SourceManager +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.online.all.MergedSource +import eu.kanade.tachiyomi.ui.base.controller.withFadeTransaction +import eu.kanade.tachiyomi.ui.manga.MangaController +import eu.kanade.tachiyomi.ui.manga.info.MangaInfoController +import eu.kanade.tachiyomi.util.gone +import eu.kanade.tachiyomi.util.inflate +import exh.MERGED_SOURCE_ID +import exh.debug.DebugFunctions.sourceManager +import exh.util.await +import kotlinx.android.synthetic.main.eh_manga_card.view.* +import kotlinx.android.synthetic.main.eh_migration_process_item.view.* +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.collect +import uy.kohesive.injekt.injectLazy +import kotlin.coroutines.CoroutineContext + +class MigrationProcedureAdapter(val controller: MigrationProcedureController, + val migratingManga: List, + override val coroutineContext: CoroutineContext) : PagerAdapter(), CoroutineScope { + private val db: DatabaseHelper by injectLazy() + private val gson: Gson by injectLazy() + private val sourceManager: SourceManager by injectLazy() + + override fun isViewFromObject(p0: View, p1: Any): Boolean { + return p0 == p1 + } + + override fun getCount() = migratingManga.size + + override fun instantiateItem(container: ViewGroup, position: Int): Any { + val item = migratingManga[position] + val view = container.inflate(R.layout.eh_migration_process_item) + container.addView(view) + + view.skip_migration.setOnClickListener { + controller.nextMigration() + } + + view.accept_migration.setOnClickListener { + + } + + val viewTag = ViewTag(coroutineContext) + view.tag = viewTag + view.setupView(viewTag, item) + + return view + } + + fun View.setupView(tag: ViewTag, migratingManga: MigratingManga) { + tag.launch { + val manga = migratingManga.manga() + val source = migratingManga.mangaSource() + if(manga != null) { + withContext(Dispatchers.Main) { + eh_manga_card_from.loading_group.gone() + eh_manga_card_from.attachManga(tag, manga, source) + eh_manga_card_from.setOnClickListener { + controller.router.pushController(MangaController(manga, true).withFadeTransaction()) + } + } + + tag.launch { + migratingManga.progress.asFlow().collect { (max, progress) -> + withContext(Dispatchers.Main) { + eh_manga_card_to.search_progress.let { progressBar -> + progressBar.max = max + progressBar.progress = progress + } + } + } + } + + val searchResult = migratingManga.searchResult.get()?.let { + db.getManga(it).await() + } + val resultSource = searchResult?.source?.let { + sourceManager.get(it) + } + withContext(Dispatchers.Main) { + if(searchResult != null && resultSource != null) { + eh_manga_card_to.loading_group.gone() + eh_manga_card_to.attachManga(tag, searchResult, resultSource) + eh_manga_card_to.setOnClickListener { + controller.router.pushController(MangaController(manga, true).withFadeTransaction()) + } + } else { + eh_manga_card_to.search_progress.gone() + eh_manga_card_to.search_status.text = "Found no manga" + } + } + } + } + } + + fun View.attachManga(tag: ViewTag, manga: Manga, source: Source) { + // TODO Duplicated in MangaInfoController + + GlideApp.with(context) + .load(manga) + .diskCacheStrategy(DiskCacheStrategy.RESOURCE) + .centerCrop() + .into(manga_cover) + + manga_full_title.text = if (manga.title.isBlank()) { + context.getString(R.string.unknown) + } else { + manga.title + } + + manga_artist.text = if (manga.artist.isNullOrBlank()) { + context.getString(R.string.unknown) + } else { + manga.artist + } + + manga_author.text = if (manga.author.isNullOrBlank()) { + context.getString(R.string.unknown) + } else { + manga.author + } + + manga_source.text = if (source.id == MERGED_SOURCE_ID) { + MergedSource.MangaConfig.readFromUrl(gson, manga.url).children.map { + sourceManager.getOrStub(it.source).toString() + }.distinct().joinToString() + } else { + source.toString() + } + + if (source.id == MERGED_SOURCE_ID) { + manga_source_label.text = "Sources" + } else { + manga_source_label.setText(R.string.manga_info_source_label) + } + + manga_status.setText(when (manga.status) { + SManga.ONGOING -> R.string.ongoing + SManga.COMPLETED -> R.string.completed + SManga.LICENSED -> R.string.licensed + else -> R.string.unknown + }) + } + + override fun destroyItem(container: ViewGroup, position: Int, `object`: Any) { + val objectAsView = `object` as View + container.removeView(objectAsView) + (objectAsView.tag as? ViewTag)?.destroy() + } + + class ViewTag(parent: CoroutineContext): CoroutineScope { + /** + * The context of this scope. + * Context is encapsulated by the scope and used for implementation of coroutine builders that are extensions on the scope. + * Accessing this property in general code is not recommended for any purposes except accessing the [Job] instance for advanced usages. + * + * By convention, should contain an instance of a [job][Job] to enforce structured concurrency. + */ + override val coroutineContext = parent + Job() + Dispatchers.Default + + fun destroy() { + cancel() + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureConfig.kt b/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureConfig.kt new file mode 100644 index 000000000..85611c815 --- /dev/null +++ b/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureConfig.kt @@ -0,0 +1,13 @@ +package exh.ui.migration.manga.process + +import android.os.Parcelable +import kotlinx.android.parcel.Parcelize + +@Parcelize +data class MigrationProcedureConfig( + val mangaIds: List, + val targetSourceIds: List, + val useSourceWithMostChapters: Boolean, + val enableLenientSearch: Boolean, + val migrationFlags: Int +): Parcelable \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureController.kt b/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureController.kt index 820b9cb02..badeecb80 100644 --- a/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureController.kt +++ b/app/src/main/java/exh/ui/migration/manga/process/MigrationProcedureController.kt @@ -2,15 +2,43 @@ package exh.ui.migration.manga.process import android.content.pm.ActivityInfo import android.os.Build +import android.os.Bundle import android.view.View +import com.elvishew.xlog.XLog import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.source.CatalogueSource +import eu.kanade.tachiyomi.source.SourceManager +import eu.kanade.tachiyomi.util.toast +import exh.smartsearch.SmartSearchEngine import exh.ui.base.BaseExhController +import exh.util.await +import kotlinx.android.synthetic.main.eh_migration_process.* +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import rx.schedulers.Schedulers +import uy.kohesive.injekt.injectLazy -class MigrationProcedureController : BaseExhController() { +class MigrationProcedureController(bundle: Bundle? = null) : BaseExhController(bundle), CoroutineScope { override val layoutId = R.layout.eh_migration_process private var titleText = "Migrate manga (1/300)" + private var adapter: MigrationProcedureAdapter? = null + + private val config: MigrationProcedureConfig = args.getParcelable(CONFIG_EXTRA) + + private val db: DatabaseHelper by injectLazy() + private val sourceManager: SourceManager by injectLazy() + + private val smartSearchEngine = SmartSearchEngine(coroutineContext) + + private val logger = XLog.tag("MigrationProcedureController") + + private var migrationsJob: Job? = null + private var migratingManga: List? = null + override fun getTitle(): String { return titleText } @@ -20,9 +48,152 @@ class MigrationProcedureController : BaseExhController() { setTitle() activity?.requestedOrientation = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) { - ActivityInfo.SCREEN_ORIENTATION_USER_LANDSCAPE + ActivityInfo.SCREEN_ORIENTATION_USER_PORTRAIT } else { - ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE + ActivityInfo.SCREEN_ORIENTATION_PORTRAIT + } + + val newMigratingManga = migratingManga ?: run { + val new = config.mangaIds.map { + MigratingManga(db, sourceManager, it, coroutineContext) + } + migratingManga = new + new + } + + adapter = MigrationProcedureAdapter(this, newMigratingManga, coroutineContext) + + pager.adapter = adapter + pager.isEnabled = false + + if(migrationsJob == null) { + migrationsJob = launch { + runMigrations(newMigratingManga) + } + } + } + + fun nextMigration() { + adapter?.let { adapter -> + if(pager.currentItem >= adapter.count - 1) { + applicationContext?.toast("All migrations complete!") + router.popCurrentController() + } else { + pager.setCurrentItem(pager.currentItem + 1, true) + titleText = "Migrate manga (${pager.currentItem + 1}/${adapter.count})" + launch(Dispatchers.Main) { + setTitle() + } + } + } + } + + suspend fun runMigrations(mangas: List) { + val sources = config.targetSourceIds.mapNotNull { sourceManager.get(it) as? CatalogueSource } + + for(manga in mangas) { + if(!manga.searchResult.initialized && manga.migrationJob.isActive) { + val mangaObj = manga.manga() + + if(mangaObj == null) { + manga.searchResult.initialize(null) + continue + } + + val mangaSource = manga.mangaSource() + + val result = try { + CoroutineScope(manga.migrationJob).async { + val validSources = sources.filter { + it.id != mangaSource.id + } + if(config.useSourceWithMostChapters) { + val sourceQueue = Channel(Channel.RENDEZVOUS) + launch { + validSources.forEachIndexed { index, catalogueSource -> + sourceQueue.send(catalogueSource) + manga.progress.send(validSources.size to index) + } + sourceQueue.close() + } + + val results = mutableListOf>() + + (1 .. 3).map { + launch { + for(source in sourceQueue) { + try { + supervisorScope { + val searchResult = if (config.enableLenientSearch) { + smartSearchEngine.smartSearch(source, mangaObj.title) + } else { + smartSearchEngine.normalSearch(source, mangaObj.title) + } + + if(searchResult != null) { + val localManga = smartSearchEngine.networkToLocalManga(searchResult, source.id) + val chapters = source.fetchChapterList(localManga).toSingle().await(Schedulers.io()) + results += localManga to chapters.size + } + } + } catch(e: Exception) { + logger.e("Failed to search in source: ${source.id}!", e) + } + } + } + }.forEach { it.join() } + + results.maxBy { it.second }?.first + } else { + validSources.forEachIndexed { index, source -> + val searchResult = try { + supervisorScope { + val searchResult = if (config.enableLenientSearch) { + smartSearchEngine.smartSearch(source, mangaObj.title) + } else { + smartSearchEngine.normalSearch(source, mangaObj.title) + } + + if (searchResult != null) { + smartSearchEngine.networkToLocalManga(searchResult, source.id) + } else null + } + } catch(e: Exception) { + logger.e("Failed to search in source: ${source.id}!", e) + null + } + + manga.progress.send(validSources.size to (index + 1)) + + if(searchResult != null) return@async searchResult + } + + null + } + }.await() + } catch(e: CancellationException) { + // Ignore canceled migrations + continue + } + + if(result != null && result.thumbnail_url == null) { + try { + supervisorScope { + val newManga = sourceManager.getOrStub(result.source) + .fetchMangaDetails(result) + .toSingle() + .await() + result.copyFrom(newManga) + + db.insertManga(result).await() + } + } catch(e: Exception) { + logger.e("Could not load search manga details", e) + } + } + + manga.searchResult.initialize(result?.id) + } } } @@ -31,4 +202,14 @@ class MigrationProcedureController : BaseExhController() { activity?.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED } + + companion object { + const val CONFIG_EXTRA = "config_extra" + + fun create(config: MigrationProcedureConfig): MigrationProcedureController { + return MigrationProcedureController(Bundle().apply { + putParcelable(CONFIG_EXTRA, config) + }) + } + } } \ No newline at end of file diff --git a/app/src/main/java/exh/ui/smartsearch/SmartSearchPresenter.kt b/app/src/main/java/exh/ui/smartsearch/SmartSearchPresenter.kt index 8b2c12cbe..2b09efcb7 100644 --- a/app/src/main/java/exh/ui/smartsearch/SmartSearchPresenter.kt +++ b/app/src/main/java/exh/ui/smartsearch/SmartSearchPresenter.kt @@ -2,39 +2,34 @@ package exh.ui.smartsearch import android.os.Bundle import com.elvishew.xlog.XLog -import eu.kanade.tachiyomi.data.database.DatabaseHelper import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.source.CatalogueSource -import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter import eu.kanade.tachiyomi.ui.catalogue.CatalogueController -import exh.util.await -import info.debatty.java.stringsimilarity.NormalizedLevenshtein +import exh.smartsearch.SmartSearchEngine import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel -import rx.schedulers.Schedulers -import uy.kohesive.injekt.injectLazy class SmartSearchPresenter(private val source: CatalogueSource?, private val config: CatalogueController.SmartSearchConfig?): BasePresenter(), CoroutineScope { - private val db: DatabaseHelper by injectLazy() - private val logger = XLog.tag("SmartSearchPresenter") override val coroutineContext = Job() + Dispatchers.Main val smartSearchChannel = Channel() + private val smartSearchEngine = SmartSearchEngine(coroutineContext) + override fun onCreate(savedState: Bundle?) { super.onCreate(savedState) if(source != null && config != null) { launch(Dispatchers.Default) { val result = try { - val resultManga = smartSearch(source, config) + val resultManga = smartSearchEngine.smartSearch(source, config.origTitle) if (resultManga != null) { - val localManga = networkToLocalManga(resultManga, source.id) + val localManga = smartSearchEngine.networkToLocalManga(resultManga, source.id) SearchResults.Found(localManga) } else { SearchResults.NotFound @@ -53,138 +48,6 @@ class SmartSearchPresenter(private val source: CatalogueSource?, private val con } } - private suspend fun smartSearch(source: CatalogueSource, config: CatalogueController.SmartSearchConfig): SManga? { - val cleanedTitle = cleanSmartSearchTitle(config.origTitle) - - val queries = getSmartSearchQueries(cleanedTitle) - - val eligibleManga = supervisorScope { - queries.map { query -> - async(Dispatchers.Default) { - val searchResults = source.fetchSearchManga(1, query, FilterList()).toSingle().await(Schedulers.io()) - - searchResults.mangas.map { - val cleanedMangaTitle = cleanSmartSearchTitle(it.title) - val normalizedDistance = NormalizedLevenshtein().similarity(cleanedTitle, cleanedMangaTitle) - SearchEntry(it, normalizedDistance) - }.filter { (_, normalizedDistance) -> - normalizedDistance >= MIN_ELIGIBLE_THRESHOLD - } - } - }.flatMap { it.await() } - } - - return eligibleManga.maxBy { it.dist }?.manga - } - - private fun getSmartSearchQueries(cleanedTitle: String): List { - val splitCleanedTitle = cleanedTitle.split(" ") - val splitSortedByLargest = splitCleanedTitle.sortedByDescending { it.length } - - if(splitCleanedTitle.isEmpty()) { - return emptyList() - } - - // Search cleaned title - // Search two largest words - // Search largest word - // Search first two words - // Search first word - - val searchQueries = listOf( - listOf(cleanedTitle), - splitSortedByLargest.take(2), - splitSortedByLargest.take(1), - splitCleanedTitle.take(2), - splitCleanedTitle.take(1) - ) - - return searchQueries.map { - it.joinToString().trim() - }.distinct() - } - - private fun cleanSmartSearchTitle(title: String): String { - val preTitle = title.toLowerCase() - - // Remove text in brackets - var cleanedTitle = removeTextInBrackets(preTitle, true) - if(cleanedTitle.length <= 5) { // Title is suspiciously short, try parsing it backwards - cleanedTitle = removeTextInBrackets(preTitle, false) - } - - // Strip non-special characters - cleanedTitle = cleanedTitle.replace(titleRegex, " ") - - // Strip splitters and consecutive spaces - cleanedTitle = cleanedTitle.trim().replace(" - ", " ").replace(consecutiveSpacesRegex, " ").trim() - - return cleanedTitle - } - - private fun removeTextInBrackets(text: String, readForward: Boolean): String { - val bracketPairs = listOf( - '(' to ')', - '[' to ']', - '<' to '>', - '{' to '}' - ) - var openingBracketPairs = bracketPairs.mapIndexed { index, (opening, _) -> - opening to index - }.toMap() - var closingBracketPairs = bracketPairs.mapIndexed { index, (_, closing) -> - closing to index - }.toMap() - - // Reverse pairs if reading backwards - if(!readForward) { - val tmp = openingBracketPairs - openingBracketPairs = closingBracketPairs - closingBracketPairs = tmp - } - - val depthPairs = bracketPairs.map { 0 }.toMutableList() - - val result = StringBuilder() - for(c in if(readForward) text else text.reversed()) { - val openingBracketDepthIndex = openingBracketPairs[c] - if(openingBracketDepthIndex != null) { - depthPairs[openingBracketDepthIndex]++ - } else { - val closingBracketDepthIndex = closingBracketPairs[c] - if(closingBracketDepthIndex != null) { - depthPairs[closingBracketDepthIndex]-- - } else { - if(depthPairs.all { it <= 0 }) { - result.append(c) - } else { - // In brackets, do not append to result - } - } - } - } - - return result.toString() - } - - /** - * Returns a manga from the database for the given manga from network. It creates a new entry - * if the manga is not yet in the database. - * - * @param sManga the manga from the source. - * @return a manga from the database. - */ - private fun networkToLocalManga(sManga: SManga, sourceId: Long): Manga { - var localManga = db.getManga(sManga.url, sourceId).executeAsBlocking() - if (localManga == null) { - val newManga = Manga.create(sManga.url, sManga.title, sourceId) - newManga.copyFrom(sManga) - val result = db.insertManga(newManga).executeAsBlocking() - newManga.id = result.insertedId() - localManga = newManga - } - return localManga - } override fun onDestroy() { super.onDestroy() @@ -199,11 +62,4 @@ class SmartSearchPresenter(private val source: CatalogueSource?, private val con object NotFound: SearchResults() object Error: SearchResults() } - - companion object { - const val MIN_ELIGIBLE_THRESHOLD = 0.7 - - private val titleRegex = Regex("[^a-zA-Z0-9- ]") - private val consecutiveSpacesRegex = Regex(" +") - } } \ No newline at end of file diff --git a/app/src/main/java/exh/util/DeferredField.kt b/app/src/main/java/exh/util/DeferredField.kt new file mode 100644 index 000000000..8137fcb79 --- /dev/null +++ b/app/src/main/java/exh/util/DeferredField.kt @@ -0,0 +1,46 @@ +package exh.util + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * Field that can be initialized later. Users can suspend while waiting for the field to initialize. + * + * @author nulldev + */ +class DeferredField { + @Volatile + private var content: T? = null + + @Volatile + var initialized = false + private set + + private val mutex = Mutex(true) + + /** + * Initialize the field + */ + fun initialize(content: T) { + // Fast-path new listeners + this.content = content + initialized = true + + // Notify current listeners + mutex.unlock() + } + + /** + * Will only suspend if !initialized. + */ + suspend fun get(): T { + // Check if field is initialized and return immediately if it is + if(initialized) return content as T + + // Wait for field to initialize + mutex.withLock {} + + // Field is initialized, return value + return content!! + } +} \ No newline at end of file diff --git a/app/src/main/res/layout/eh_manga_card.xml b/app/src/main/res/layout/eh_manga_card.xml index 796c841cc..eb83d4c11 100644 --- a/app/src/main/res/layout/eh_manga_card.xml +++ b/app/src/main/res/layout/eh_manga_card.xml @@ -1,13 +1,17 @@ + android:foreground="?android:attr/selectableItemBackground" + android:clickable="true" + android:focusable="true"> + android:layout_height="wrap_content" + android:clickable="false"> - + app:layout_constraintTop_toTopOf="@+id/manga_cover"> - + - + - - app:layout_constraintLeft_toLeftOf="parent" - app:layout_constraintTop_toBottomOf="@+id/manga_full_title" /> + - + - + - + - + - + - + - + - + - + - - - - - - - + \ No newline at end of file diff --git a/app/src/main/res/layout/eh_migration_design.xml b/app/src/main/res/layout/eh_migration_design.xml index bc0de101d..e8bdf43c8 100644 --- a/app/src/main/res/layout/eh_migration_design.xml +++ b/app/src/main/res/layout/eh_migration_design.xml @@ -1,22 +1,66 @@ - + android:layout_height="match_parent" + android:animateLayoutChanges="true"> + tools:listitem="@layout/eh_source_item"> + + + + + + + + + + @@ -70,7 +114,7 @@ android:layout_marginRight="8dp" android:layout_marginBottom="8dp" android:gravity="start|center_vertical" - android:text="Use lenient search algorithm" + android:text="Use smart search algorithm" app:layout_constraintBottom_toTopOf="@+id/begin_migration_btn" app:layout_constraintEnd_toEndOf="parent" app:layout_constraintStart_toEndOf="@+id/prioritize_chapter_count" /> diff --git a/app/src/main/res/layout/eh_migration_process.xml b/app/src/main/res/layout/eh_migration_process.xml index fffea39ef..a6027ef9d 100644 --- a/app/src/main/res/layout/eh_migration_process.xml +++ b/app/src/main/res/layout/eh_migration_process.xml @@ -7,18 +7,10 @@ android:layout_height="match_parent" android:background="?attr/colorPrimary" > - - - - - - - - -