Completed most of auto-migration UI

This commit is contained in:
NerdNumber9 2019-07-31 03:39:51 -04:00
parent f811cc5c87
commit 5b3e72db54
18 changed files with 976 additions and 339 deletions

View File

@ -437,8 +437,10 @@ class LibraryController(
R.id.action_move_to_category -> showChangeMangaCategoriesDialog()
R.id.action_delete -> showDeleteMangaDialog()
R.id.action_auto_source_migration -> {
router.pushController(MigrationDesignController.create(
selectedMangas.mapNotNull { it.id }
).withFadeTransaction())
destroyActionModeIfNeeded()
router.pushController(MigrationDesignController().withFadeTransaction())
}
else -> return false
}

View File

@ -252,6 +252,8 @@ class MangaInfoController : NucleusController<MangaInfoPresenter>(),
private fun setMangaInfo(manga: Manga, source: Source?) {
val view = view ?: return
// TODO Duplicated in MigrationProcedureAdapter
//update full title TextView.
manga_full_title.text = if (manga.title.isBlank()) {
view.context.getString(R.string.unknown)

View File

@ -4,9 +4,9 @@ import eu.kanade.tachiyomi.R
object MigrationFlags {
private const val CHAPTERS = 0b001
private const val CATEGORIES = 0b010
private const val TRACK = 0b100
const val CHAPTERS = 0b001
const val CATEGORIES = 0b010
const val TRACK = 0b100
private const val CHAPTERS2 = 0x1
private const val CATEGORIES2 = 0x2

View File

@ -0,0 +1,176 @@
package exh.smartsearch
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.source.CatalogueSource
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.SManga
import exh.ui.smartsearch.SmartSearchPresenter
import exh.util.await
import info.debatty.java.stringsimilarity.NormalizedLevenshtein
import kotlinx.coroutines.*
import rx.schedulers.Schedulers
import uy.kohesive.injekt.injectLazy
import kotlin.coroutines.CoroutineContext
class SmartSearchEngine(parentContext: CoroutineContext): CoroutineScope {
override val coroutineContext: CoroutineContext = parentContext + Job() + Dispatchers.Default
private val db: DatabaseHelper by injectLazy()
suspend fun smartSearch(source: CatalogueSource, title: String): SManga? {
val cleanedTitle = cleanSmartSearchTitle(title)
val queries = getSmartSearchQueries(cleanedTitle)
val eligibleManga = supervisorScope {
queries.map { query ->
async(Dispatchers.Default) {
val searchResults = source.fetchSearchManga(1, query, FilterList()).toSingle().await(Schedulers.io())
searchResults.mangas.map {
val cleanedMangaTitle = cleanSmartSearchTitle(it.title)
val normalizedDistance = NormalizedLevenshtein().similarity(cleanedTitle, cleanedMangaTitle)
SmartSearchPresenter.SearchEntry(it, normalizedDistance)
}.filter { (_, normalizedDistance) ->
normalizedDistance >= MIN_SMART_ELIGIBLE_THRESHOLD
}
}
}.flatMap { it.await() }
}
return eligibleManga.maxBy { it.dist }?.manga
}
suspend fun normalSearch(source: CatalogueSource, title: String): SManga? {
val eligibleManga = supervisorScope {
val searchResults = source.fetchSearchManga(1, title, FilterList()).toSingle().await(Schedulers.io())
searchResults.mangas.map {
val normalizedDistance = NormalizedLevenshtein().similarity(title, it.title)
SmartSearchPresenter.SearchEntry(it, normalizedDistance)
}.filter { (_, normalizedDistance) ->
normalizedDistance >= MIN_NORMAL_ELIGIBLE_THRESHOLD
}
}
return eligibleManga.maxBy { it.dist }?.manga
}
private fun getSmartSearchQueries(cleanedTitle: String): List<String> {
val splitCleanedTitle = cleanedTitle.split(" ")
val splitSortedByLargest = splitCleanedTitle.sortedByDescending { it.length }
if(splitCleanedTitle.isEmpty()) {
return emptyList()
}
// Search cleaned title
// Search two largest words
// Search largest word
// Search first two words
// Search first word
val searchQueries = listOf(
listOf(cleanedTitle),
splitSortedByLargest.take(2),
splitSortedByLargest.take(1),
splitCleanedTitle.take(2),
splitCleanedTitle.take(1)
)
return searchQueries.map {
it.joinToString().trim()
}.distinct()
}
private fun cleanSmartSearchTitle(title: String): String {
val preTitle = title.toLowerCase()
// Remove text in brackets
var cleanedTitle = removeTextInBrackets(preTitle, true)
if(cleanedTitle.length <= 5) { // Title is suspiciously short, try parsing it backwards
cleanedTitle = removeTextInBrackets(preTitle, false)
}
// Strip non-special characters
cleanedTitle = cleanedTitle.replace(titleRegex, " ")
// Strip splitters and consecutive spaces
cleanedTitle = cleanedTitle.trim().replace(" - ", " ").replace(consecutiveSpacesRegex, " ").trim()
return cleanedTitle
}
private fun removeTextInBrackets(text: String, readForward: Boolean): String {
val bracketPairs = listOf(
'(' to ')',
'[' to ']',
'<' to '>',
'{' to '}'
)
var openingBracketPairs = bracketPairs.mapIndexed { index, (opening, _) ->
opening to index
}.toMap()
var closingBracketPairs = bracketPairs.mapIndexed { index, (_, closing) ->
closing to index
}.toMap()
// Reverse pairs if reading backwards
if(!readForward) {
val tmp = openingBracketPairs
openingBracketPairs = closingBracketPairs
closingBracketPairs = tmp
}
val depthPairs = bracketPairs.map { 0 }.toMutableList()
val result = StringBuilder()
for(c in if(readForward) text else text.reversed()) {
val openingBracketDepthIndex = openingBracketPairs[c]
if(openingBracketDepthIndex != null) {
depthPairs[openingBracketDepthIndex]++
} else {
val closingBracketDepthIndex = closingBracketPairs[c]
if(closingBracketDepthIndex != null) {
depthPairs[closingBracketDepthIndex]--
} else {
if(depthPairs.all { it <= 0 }) {
result.append(c)
} else {
// In brackets, do not append to result
}
}
}
}
return result.toString()
}
/**
* Returns a manga from the database for the given manga from network. It creates a new entry
* if the manga is not yet in the database.
*
* @param sManga the manga from the source.
* @return a manga from the database.
*/
suspend fun networkToLocalManga(sManga: SManga, sourceId: Long): Manga {
var localManga = db.getManga(sManga.url, sourceId).await()
if (localManga == null) {
val newManga = Manga.create(sManga.url, sManga.title, sourceId)
newManga.copyFrom(sManga)
val result = db.insertManga(newManga).await()
newManga.id = result.insertedId()
localManga = newManga
}
return localManga
}
companion object {
const val MIN_SMART_ELIGIBLE_THRESHOLD = 0.7
const val MIN_NORMAL_ELIGIBLE_THRESHOLD = 0.5
private val titleRegex = Regex("[^a-zA-Z0-9- ]")
private val consecutiveSpacesRegex = Regex(" +")
}
}

View File

@ -1,5 +1,6 @@
package exh.ui.base
import android.os.Bundle
import android.support.annotation.LayoutRes
import android.view.LayoutInflater
import android.view.View
@ -11,7 +12,7 @@ import kotlinx.coroutines.Job
import kotlinx.coroutines.cancel
import kotlin.coroutines.CoroutineContext
abstract class BaseExhController : BaseController(), CoroutineScope {
abstract class BaseExhController(bundle: Bundle? = null) : BaseController(bundle), CoroutineScope {
abstract val layoutId: Int
@LayoutRes get

View File

@ -1,5 +1,6 @@
package exh.ui.migration.manga.design
import android.os.Bundle
import android.support.v7.widget.LinearLayoutManager
import android.view.View
import eu.davidea.flexibleadapter.FlexibleAdapter
@ -9,34 +10,40 @@ import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.ui.base.controller.withFadeTransaction
import eu.kanade.tachiyomi.ui.migration.MigrationFlags
import exh.ui.base.BaseExhController
import exh.ui.migration.manga.process.MigrationProcedureConfig
import exh.ui.migration.manga.process.MigrationProcedureController
import kotlinx.android.synthetic.main.eh_migration_design.*
import uy.kohesive.injekt.injectLazy
// TODO Handle config changes
// TODO Select all in library
class MigrationDesignController : BaseExhController(), FlexibleAdapter.OnItemClickListener {
class MigrationDesignController(bundle: Bundle? = null) : BaseExhController(bundle), FlexibleAdapter.OnItemClickListener {
private val sourceManager: SourceManager by injectLazy()
private val prefs: PreferencesHelper by injectLazy()
override val layoutId: Int = R.layout.eh_migration_design
private var adapter: FlexibleAdapter<MigrationSourceItem>? = null
private var adapter: MigrationSourceAdapter? = null
private val config: LongArray = args.getLongArray(MANGA_IDS_EXTRA) ?: LongArray(0)
override fun getTitle() = "Select target sources"
override fun onViewCreated(view: View) {
super.onViewCreated(view)
adapter = MigrationSourceAdapter(
val ourAdapter = adapter ?: MigrationSourceAdapter(
getEnabledSources().map { MigrationSourceItem(it, true) },
this
)
adapter = ourAdapter
recycler.layoutManager = LinearLayoutManager(view.context)
recycler.setHasFixedSize(true)
recycler.adapter = adapter
adapter?.isHandleDragEnabled = true
recycler.adapter = ourAdapter
ourAdapter.itemTouchHelperCallback = null // Reset adapter touch adapter to fix drag after rotation
ourAdapter.isHandleDragEnabled = true
migration_mode.setOnClickListener {
prioritize_chapter_count.toggle()
@ -53,15 +60,41 @@ class MigrationDesignController : BaseExhController(), FlexibleAdapter.OnItemCli
updatePrioritizeChapterCount(prioritize_chapter_count.isChecked)
begin_migration_btn.setOnClickListener {
router.replaceTopController(MigrationProcedureController().withFadeTransaction())
var flags = 0
if(mig_chapters.isChecked) flags = flags or MigrationFlags.CHAPTERS
if(mig_categories.isChecked) flags = flags or MigrationFlags.CATEGORIES
if(mig_categories.isChecked) flags = flags or MigrationFlags.TRACK
router.replaceTopController(MigrationProcedureController.create(
MigrationProcedureConfig(
config.toList(),
ourAdapter.items.filter {
it.sourceEnabled
}.map { it.source.id },
useSourceWithMostChapters = prioritize_chapter_count.isChecked,
enableLenientSearch = use_smart_search.isChecked,
migrationFlags = flags
)
).withFadeTransaction())
}
}
override fun onSaveInstanceState(outState: Bundle) {
super.onSaveInstanceState(outState)
adapter?.onSaveInstanceState(outState)
}
// TODO Still incorrect, why is this called before onViewCreated?
override fun onRestoreInstanceState(savedInstanceState: Bundle) {
super.onRestoreInstanceState(savedInstanceState)
adapter?.onRestoreInstanceState(savedInstanceState)
}
private fun updatePrioritizeChapterCount(migrationMode: Boolean) {
migration_mode.text = if(migrationMode) {
"Use source with most chapters and use the above list to break ties"
"Use the source with the most chapters and use the above list to break ties (slow with many sources or smart search)"
} else {
"Use the first source in the list that has at least one chapter of the manga"
"Use the first source in the list that has the manga"
}
}
@ -88,4 +121,14 @@ class MigrationDesignController : BaseExhController(), FlexibleAdapter.OnItemCli
.filterNot { it.id.toString() in hiddenCatalogues }
.sortedBy { "(${it.lang}) ${it.name}" }
}
companion object {
private const val MANGA_IDS_EXTRA = "manga_ids"
fun create(mangaIds: List<Long>): MigrationDesignController {
return MigrationDesignController(Bundle().apply {
putLongArray(MANGA_IDS_EXTRA, mangaIds.toLongArray())
})
}
}
}

View File

@ -1,10 +1,32 @@
package exh.ui.migration.manga.design
import android.os.Bundle
import eu.davidea.flexibleadapter.FlexibleAdapter
import exh.debug.DebugFunctions.sourceManager
class MigrationSourceAdapter(val items: List<MigrationSourceItem>,
val controller: MigrationDesignController): FlexibleAdapter<MigrationSourceItem>(
items,
controller,
true
)
) {
override fun onSaveInstanceState(outState: Bundle) {
super.onSaveInstanceState(outState)
outState.putParcelableArrayList(SELECTED_SOURCES_KEY, ArrayList(currentItems.map {
it.asParcelable()
}))
}
override fun onRestoreInstanceState(savedInstanceState: Bundle) {
savedInstanceState.getParcelableArrayList<MigrationSourceItem.ParcelableSI>(SELECTED_SOURCES_KEY)?.let {
updateDataSet(it.map { MigrationSourceItem.fromParcelable(sourceManager, it) })
}
super.onRestoreInstanceState(savedInstanceState)
}
companion object {
private const val SELECTED_SOURCES_KEY = "selected_sources"
}
}

View File

@ -1,12 +1,15 @@
package exh.ui.migration.manga.design
import android.os.Parcelable
import android.support.v7.widget.RecyclerView
import android.view.View
import eu.davidea.flexibleadapter.FlexibleAdapter
import eu.davidea.flexibleadapter.items.AbstractFlexibleItem
import eu.davidea.flexibleadapter.items.IFlexible
import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.online.HttpSource
import kotlinx.android.parcel.Parcelize
class MigrationSourceItem(val source: HttpSource, var sourceEnabled: Boolean): AbstractFlexibleItem<MigrationSourceHolder>() {
override fun getLayoutRes() = R.layout.eh_source_item
@ -48,4 +51,22 @@ class MigrationSourceItem(val source: HttpSource, var sourceEnabled: Boolean): A
override fun hashCode(): Int {
return source.id.hashCode()
}
@Parcelize
data class ParcelableSI(val sourceId: Long, val sourceEnabled: Boolean): Parcelable
fun asParcelable(): ParcelableSI {
return ParcelableSI(source.id, sourceEnabled)
}
companion object {
fun fromParcelable(sourceManager: SourceManager, si: ParcelableSI): MigrationSourceItem? {
val source = sourceManager.get(si.sourceId) as? HttpSource ?: return null
return MigrationSourceItem(
source,
si.sourceEnabled
)
}
}
}

View File

@ -0,0 +1,19 @@
package exh.ui.migration.manga.process
import android.content.Context
import android.support.v4.view.ViewPager
import android.util.AttributeSet
import android.view.MotionEvent
class DeactivatableViewPager: ViewPager {
constructor(context: Context): super(context)
constructor(context: Context, attrs: AttributeSet): super(context, attrs)
override fun onTouchEvent(event: MotionEvent): Boolean {
return !isEnabled || super.onTouchEvent(event)
}
override fun onInterceptTouchEvent(event: MotionEvent): Boolean {
return isEnabled && super.onInterceptTouchEvent(event)
}
}

View File

@ -0,0 +1,38 @@
package exh.ui.migration.manga.process
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.SourceManager
import exh.util.DeferredField
import exh.util.await
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.BroadcastChannel
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.channels.ConflatedBroadcastChannel
import kotlinx.coroutines.channels.ReceiveChannel
import kotlinx.coroutines.flow.Flow
import kotlin.coroutines.CoroutineContext
class MigratingManga(private val db: DatabaseHelper,
private val sourceManager: SourceManager,
val mangaId: Long,
parentContext: CoroutineContext) {
val searchResult = DeferredField<Long?>()
// <MAX, PROGRESS>
val progress = ConflatedBroadcastChannel(1 to 0)
val migrationJob = parentContext + SupervisorJob() + Dispatchers.Default
@Volatile
private var manga: Manga? = null
suspend fun manga(): Manga? {
if(manga == null) manga = db.getManga(mangaId).await()
return manga
}
suspend fun mangaSource(): Source {
return sourceManager.getOrStub(manga()?.source ?: -1)
}
}

View File

@ -0,0 +1,180 @@
package exh.ui.migration.manga.process
import android.support.v4.view.PagerAdapter
import android.view.View
import android.view.ViewGroup
import com.bumptech.glide.load.engine.DiskCacheStrategy
import com.google.gson.Gson
import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.data.glide.GlideApp
import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.all.MergedSource
import eu.kanade.tachiyomi.ui.base.controller.withFadeTransaction
import eu.kanade.tachiyomi.ui.manga.MangaController
import eu.kanade.tachiyomi.ui.manga.info.MangaInfoController
import eu.kanade.tachiyomi.util.gone
import eu.kanade.tachiyomi.util.inflate
import exh.MERGED_SOURCE_ID
import exh.debug.DebugFunctions.sourceManager
import exh.util.await
import kotlinx.android.synthetic.main.eh_manga_card.view.*
import kotlinx.android.synthetic.main.eh_migration_process_item.view.*
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.collect
import uy.kohesive.injekt.injectLazy
import kotlin.coroutines.CoroutineContext
class MigrationProcedureAdapter(val controller: MigrationProcedureController,
val migratingManga: List<MigratingManga>,
override val coroutineContext: CoroutineContext) : PagerAdapter(), CoroutineScope {
private val db: DatabaseHelper by injectLazy()
private val gson: Gson by injectLazy()
private val sourceManager: SourceManager by injectLazy()
override fun isViewFromObject(p0: View, p1: Any): Boolean {
return p0 == p1
}
override fun getCount() = migratingManga.size
override fun instantiateItem(container: ViewGroup, position: Int): Any {
val item = migratingManga[position]
val view = container.inflate(R.layout.eh_migration_process_item)
container.addView(view)
view.skip_migration.setOnClickListener {
controller.nextMigration()
}
view.accept_migration.setOnClickListener {
}
val viewTag = ViewTag(coroutineContext)
view.tag = viewTag
view.setupView(viewTag, item)
return view
}
fun View.setupView(tag: ViewTag, migratingManga: MigratingManga) {
tag.launch {
val manga = migratingManga.manga()
val source = migratingManga.mangaSource()
if(manga != null) {
withContext(Dispatchers.Main) {
eh_manga_card_from.loading_group.gone()
eh_manga_card_from.attachManga(tag, manga, source)
eh_manga_card_from.setOnClickListener {
controller.router.pushController(MangaController(manga, true).withFadeTransaction())
}
}
tag.launch {
migratingManga.progress.asFlow().collect { (max, progress) ->
withContext(Dispatchers.Main) {
eh_manga_card_to.search_progress.let { progressBar ->
progressBar.max = max
progressBar.progress = progress
}
}
}
}
val searchResult = migratingManga.searchResult.get()?.let {
db.getManga(it).await()
}
val resultSource = searchResult?.source?.let {
sourceManager.get(it)
}
withContext(Dispatchers.Main) {
if(searchResult != null && resultSource != null) {
eh_manga_card_to.loading_group.gone()
eh_manga_card_to.attachManga(tag, searchResult, resultSource)
eh_manga_card_to.setOnClickListener {
controller.router.pushController(MangaController(manga, true).withFadeTransaction())
}
} else {
eh_manga_card_to.search_progress.gone()
eh_manga_card_to.search_status.text = "Found no manga"
}
}
}
}
}
fun View.attachManga(tag: ViewTag, manga: Manga, source: Source) {
// TODO Duplicated in MangaInfoController
GlideApp.with(context)
.load(manga)
.diskCacheStrategy(DiskCacheStrategy.RESOURCE)
.centerCrop()
.into(manga_cover)
manga_full_title.text = if (manga.title.isBlank()) {
context.getString(R.string.unknown)
} else {
manga.title
}
manga_artist.text = if (manga.artist.isNullOrBlank()) {
context.getString(R.string.unknown)
} else {
manga.artist
}
manga_author.text = if (manga.author.isNullOrBlank()) {
context.getString(R.string.unknown)
} else {
manga.author
}
manga_source.text = if (source.id == MERGED_SOURCE_ID) {
MergedSource.MangaConfig.readFromUrl(gson, manga.url).children.map {
sourceManager.getOrStub(it.source).toString()
}.distinct().joinToString()
} else {
source.toString()
}
if (source.id == MERGED_SOURCE_ID) {
manga_source_label.text = "Sources"
} else {
manga_source_label.setText(R.string.manga_info_source_label)
}
manga_status.setText(when (manga.status) {
SManga.ONGOING -> R.string.ongoing
SManga.COMPLETED -> R.string.completed
SManga.LICENSED -> R.string.licensed
else -> R.string.unknown
})
}
override fun destroyItem(container: ViewGroup, position: Int, `object`: Any) {
val objectAsView = `object` as View
container.removeView(objectAsView)
(objectAsView.tag as? ViewTag)?.destroy()
}
class ViewTag(parent: CoroutineContext): CoroutineScope {
/**
* The context of this scope.
* Context is encapsulated by the scope and used for implementation of coroutine builders that are extensions on the scope.
* Accessing this property in general code is not recommended for any purposes except accessing the [Job] instance for advanced usages.
*
* By convention, should contain an instance of a [job][Job] to enforce structured concurrency.
*/
override val coroutineContext = parent + Job() + Dispatchers.Default
fun destroy() {
cancel()
}
}
}

View File

@ -0,0 +1,13 @@
package exh.ui.migration.manga.process
import android.os.Parcelable
import kotlinx.android.parcel.Parcelize
@Parcelize
data class MigrationProcedureConfig(
val mangaIds: List<Long>,
val targetSourceIds: List<Long>,
val useSourceWithMostChapters: Boolean,
val enableLenientSearch: Boolean,
val migrationFlags: Int
): Parcelable

View File

@ -2,15 +2,43 @@ package exh.ui.migration.manga.process
import android.content.pm.ActivityInfo
import android.os.Build
import android.os.Bundle
import android.view.View
import com.elvishew.xlog.XLog
import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.source.CatalogueSource
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.util.toast
import exh.smartsearch.SmartSearchEngine
import exh.ui.base.BaseExhController
import exh.util.await
import kotlinx.android.synthetic.main.eh_migration_process.*
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import rx.schedulers.Schedulers
import uy.kohesive.injekt.injectLazy
class MigrationProcedureController : BaseExhController() {
class MigrationProcedureController(bundle: Bundle? = null) : BaseExhController(bundle), CoroutineScope {
override val layoutId = R.layout.eh_migration_process
private var titleText = "Migrate manga (1/300)"
private var adapter: MigrationProcedureAdapter? = null
private val config: MigrationProcedureConfig = args.getParcelable(CONFIG_EXTRA)
private val db: DatabaseHelper by injectLazy()
private val sourceManager: SourceManager by injectLazy()
private val smartSearchEngine = SmartSearchEngine(coroutineContext)
private val logger = XLog.tag("MigrationProcedureController")
private var migrationsJob: Job? = null
private var migratingManga: List<MigratingManga>? = null
override fun getTitle(): String {
return titleText
}
@ -20,9 +48,152 @@ class MigrationProcedureController : BaseExhController() {
setTitle()
activity?.requestedOrientation = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) {
ActivityInfo.SCREEN_ORIENTATION_USER_LANDSCAPE
ActivityInfo.SCREEN_ORIENTATION_USER_PORTRAIT
} else {
ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE
ActivityInfo.SCREEN_ORIENTATION_PORTRAIT
}
val newMigratingManga = migratingManga ?: run {
val new = config.mangaIds.map {
MigratingManga(db, sourceManager, it, coroutineContext)
}
migratingManga = new
new
}
adapter = MigrationProcedureAdapter(this, newMigratingManga, coroutineContext)
pager.adapter = adapter
pager.isEnabled = false
if(migrationsJob == null) {
migrationsJob = launch {
runMigrations(newMigratingManga)
}
}
}
fun nextMigration() {
adapter?.let { adapter ->
if(pager.currentItem >= adapter.count - 1) {
applicationContext?.toast("All migrations complete!")
router.popCurrentController()
} else {
pager.setCurrentItem(pager.currentItem + 1, true)
titleText = "Migrate manga (${pager.currentItem + 1}/${adapter.count})"
launch(Dispatchers.Main) {
setTitle()
}
}
}
}
suspend fun runMigrations(mangas: List<MigratingManga>) {
val sources = config.targetSourceIds.mapNotNull { sourceManager.get(it) as? CatalogueSource }
for(manga in mangas) {
if(!manga.searchResult.initialized && manga.migrationJob.isActive) {
val mangaObj = manga.manga()
if(mangaObj == null) {
manga.searchResult.initialize(null)
continue
}
val mangaSource = manga.mangaSource()
val result = try {
CoroutineScope(manga.migrationJob).async {
val validSources = sources.filter {
it.id != mangaSource.id
}
if(config.useSourceWithMostChapters) {
val sourceQueue = Channel<CatalogueSource>(Channel.RENDEZVOUS)
launch {
validSources.forEachIndexed { index, catalogueSource ->
sourceQueue.send(catalogueSource)
manga.progress.send(validSources.size to index)
}
sourceQueue.close()
}
val results = mutableListOf<Pair<Manga, Int>>()
(1 .. 3).map {
launch {
for(source in sourceQueue) {
try {
supervisorScope {
val searchResult = if (config.enableLenientSearch) {
smartSearchEngine.smartSearch(source, mangaObj.title)
} else {
smartSearchEngine.normalSearch(source, mangaObj.title)
}
if(searchResult != null) {
val localManga = smartSearchEngine.networkToLocalManga(searchResult, source.id)
val chapters = source.fetchChapterList(localManga).toSingle().await(Schedulers.io())
results += localManga to chapters.size
}
}
} catch(e: Exception) {
logger.e("Failed to search in source: ${source.id}!", e)
}
}
}
}.forEach { it.join() }
results.maxBy { it.second }?.first
} else {
validSources.forEachIndexed { index, source ->
val searchResult = try {
supervisorScope {
val searchResult = if (config.enableLenientSearch) {
smartSearchEngine.smartSearch(source, mangaObj.title)
} else {
smartSearchEngine.normalSearch(source, mangaObj.title)
}
if (searchResult != null) {
smartSearchEngine.networkToLocalManga(searchResult, source.id)
} else null
}
} catch(e: Exception) {
logger.e("Failed to search in source: ${source.id}!", e)
null
}
manga.progress.send(validSources.size to (index + 1))
if(searchResult != null) return@async searchResult
}
null
}
}.await()
} catch(e: CancellationException) {
// Ignore canceled migrations
continue
}
if(result != null && result.thumbnail_url == null) {
try {
supervisorScope {
val newManga = sourceManager.getOrStub(result.source)
.fetchMangaDetails(result)
.toSingle()
.await()
result.copyFrom(newManga)
db.insertManga(result).await()
}
} catch(e: Exception) {
logger.e("Could not load search manga details", e)
}
}
manga.searchResult.initialize(result?.id)
}
}
}
@ -31,4 +202,14 @@ class MigrationProcedureController : BaseExhController() {
activity?.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED
}
companion object {
const val CONFIG_EXTRA = "config_extra"
fun create(config: MigrationProcedureConfig): MigrationProcedureController {
return MigrationProcedureController(Bundle().apply {
putParcelable(CONFIG_EXTRA, config)
})
}
}
}

View File

@ -2,39 +2,34 @@ package exh.ui.smartsearch
import android.os.Bundle
import com.elvishew.xlog.XLog
import eu.kanade.tachiyomi.data.database.DatabaseHelper
import eu.kanade.tachiyomi.data.database.models.Manga
import eu.kanade.tachiyomi.source.CatalogueSource
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.ui.base.presenter.BasePresenter
import eu.kanade.tachiyomi.ui.catalogue.CatalogueController
import exh.util.await
import info.debatty.java.stringsimilarity.NormalizedLevenshtein
import exh.smartsearch.SmartSearchEngine
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import rx.schedulers.Schedulers
import uy.kohesive.injekt.injectLazy
class SmartSearchPresenter(private val source: CatalogueSource?, private val config: CatalogueController.SmartSearchConfig?):
BasePresenter<SmartSearchController>(), CoroutineScope {
private val db: DatabaseHelper by injectLazy()
private val logger = XLog.tag("SmartSearchPresenter")
override val coroutineContext = Job() + Dispatchers.Main
val smartSearchChannel = Channel<SearchResults>()
private val smartSearchEngine = SmartSearchEngine(coroutineContext)
override fun onCreate(savedState: Bundle?) {
super.onCreate(savedState)
if(source != null && config != null) {
launch(Dispatchers.Default) {
val result = try {
val resultManga = smartSearch(source, config)
val resultManga = smartSearchEngine.smartSearch(source, config.origTitle)
if (resultManga != null) {
val localManga = networkToLocalManga(resultManga, source.id)
val localManga = smartSearchEngine.networkToLocalManga(resultManga, source.id)
SearchResults.Found(localManga)
} else {
SearchResults.NotFound
@ -53,138 +48,6 @@ class SmartSearchPresenter(private val source: CatalogueSource?, private val con
}
}
private suspend fun smartSearch(source: CatalogueSource, config: CatalogueController.SmartSearchConfig): SManga? {
val cleanedTitle = cleanSmartSearchTitle(config.origTitle)
val queries = getSmartSearchQueries(cleanedTitle)
val eligibleManga = supervisorScope {
queries.map { query ->
async(Dispatchers.Default) {
val searchResults = source.fetchSearchManga(1, query, FilterList()).toSingle().await(Schedulers.io())
searchResults.mangas.map {
val cleanedMangaTitle = cleanSmartSearchTitle(it.title)
val normalizedDistance = NormalizedLevenshtein().similarity(cleanedTitle, cleanedMangaTitle)
SearchEntry(it, normalizedDistance)
}.filter { (_, normalizedDistance) ->
normalizedDistance >= MIN_ELIGIBLE_THRESHOLD
}
}
}.flatMap { it.await() }
}
return eligibleManga.maxBy { it.dist }?.manga
}
private fun getSmartSearchQueries(cleanedTitle: String): List<String> {
val splitCleanedTitle = cleanedTitle.split(" ")
val splitSortedByLargest = splitCleanedTitle.sortedByDescending { it.length }
if(splitCleanedTitle.isEmpty()) {
return emptyList()
}
// Search cleaned title
// Search two largest words
// Search largest word
// Search first two words
// Search first word
val searchQueries = listOf(
listOf(cleanedTitle),
splitSortedByLargest.take(2),
splitSortedByLargest.take(1),
splitCleanedTitle.take(2),
splitCleanedTitle.take(1)
)
return searchQueries.map {
it.joinToString().trim()
}.distinct()
}
private fun cleanSmartSearchTitle(title: String): String {
val preTitle = title.toLowerCase()
// Remove text in brackets
var cleanedTitle = removeTextInBrackets(preTitle, true)
if(cleanedTitle.length <= 5) { // Title is suspiciously short, try parsing it backwards
cleanedTitle = removeTextInBrackets(preTitle, false)
}
// Strip non-special characters
cleanedTitle = cleanedTitle.replace(titleRegex, " ")
// Strip splitters and consecutive spaces
cleanedTitle = cleanedTitle.trim().replace(" - ", " ").replace(consecutiveSpacesRegex, " ").trim()
return cleanedTitle
}
private fun removeTextInBrackets(text: String, readForward: Boolean): String {
val bracketPairs = listOf(
'(' to ')',
'[' to ']',
'<' to '>',
'{' to '}'
)
var openingBracketPairs = bracketPairs.mapIndexed { index, (opening, _) ->
opening to index
}.toMap()
var closingBracketPairs = bracketPairs.mapIndexed { index, (_, closing) ->
closing to index
}.toMap()
// Reverse pairs if reading backwards
if(!readForward) {
val tmp = openingBracketPairs
openingBracketPairs = closingBracketPairs
closingBracketPairs = tmp
}
val depthPairs = bracketPairs.map { 0 }.toMutableList()
val result = StringBuilder()
for(c in if(readForward) text else text.reversed()) {
val openingBracketDepthIndex = openingBracketPairs[c]
if(openingBracketDepthIndex != null) {
depthPairs[openingBracketDepthIndex]++
} else {
val closingBracketDepthIndex = closingBracketPairs[c]
if(closingBracketDepthIndex != null) {
depthPairs[closingBracketDepthIndex]--
} else {
if(depthPairs.all { it <= 0 }) {
result.append(c)
} else {
// In brackets, do not append to result
}
}
}
}
return result.toString()
}
/**
* Returns a manga from the database for the given manga from network. It creates a new entry
* if the manga is not yet in the database.
*
* @param sManga the manga from the source.
* @return a manga from the database.
*/
private fun networkToLocalManga(sManga: SManga, sourceId: Long): Manga {
var localManga = db.getManga(sManga.url, sourceId).executeAsBlocking()
if (localManga == null) {
val newManga = Manga.create(sManga.url, sManga.title, sourceId)
newManga.copyFrom(sManga)
val result = db.insertManga(newManga).executeAsBlocking()
newManga.id = result.insertedId()
localManga = newManga
}
return localManga
}
override fun onDestroy() {
super.onDestroy()
@ -199,11 +62,4 @@ class SmartSearchPresenter(private val source: CatalogueSource?, private val con
object NotFound: SearchResults()
object Error: SearchResults()
}
companion object {
const val MIN_ELIGIBLE_THRESHOLD = 0.7
private val titleRegex = Regex("[^a-zA-Z0-9- ]")
private val consecutiveSpacesRegex = Regex(" +")
}
}

View File

@ -0,0 +1,46 @@
package exh.util
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
/**
* Field that can be initialized later. Users can suspend while waiting for the field to initialize.
*
* @author nulldev
*/
class DeferredField<T> {
@Volatile
private var content: T? = null
@Volatile
var initialized = false
private set
private val mutex = Mutex(true)
/**
* Initialize the field
*/
fun initialize(content: T) {
// Fast-path new listeners
this.content = content
initialized = true
// Notify current listeners
mutex.unlock()
}
/**
* Will only suspend if !initialized.
*/
suspend fun get(): T {
// Check if field is initialized and return immediately if it is
if(initialized) return content as T
// Wait for field to initialize
mutex.withLock {}
// Field is initialized, return value
return content!!
}
}

View File

@ -1,13 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<android.support.v7.widget.CardView xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="wrap_content"
xmlns:tools="http://schemas.android.com/tools">
android:foreground="?android:attr/selectableItemBackground"
android:clickable="true"
android:focusable="true">
<android.support.constraint.ConstraintLayout
android:layout_width="match_parent"
android:layout_height="wrap_content">
android:layout_height="wrap_content"
android:clickable="false">
<ImageView
android:id="@+id/manga_cover"
@ -17,34 +21,30 @@
android:layout_marginTop="16dp"
android:layout_marginBottom="16dp"
android:contentDescription="@string/description_cover"
app:layout_constraintWidth_min="100dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintDimensionRatio="h,3:2"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintWidth_min="100dp"
tools:background="@color/material_grey_700" />
<android.support.v4.widget.NestedScrollView
android:id="@+id/info_scrollview"
<android.support.constraint.ConstraintLayout
android:id="@+id/card_scroll_content"
android:layout_width="0dp"
android:layout_height="0dp"
android:layout_marginLeft="8dp"
android:layout_marginLeft="16dp"
android:layout_marginRight="16dp"
app:layout_constraintBottom_toBottomOf="@+id/manga_cover"
app:layout_constraintLeft_toRightOf="@+id/manga_cover"
app:layout_constraintRight_toRightOf="parent"
app:layout_constraintTop_toTopOf="@+id/manga_cover"
tools:layout_height="200dp">
<android.support.constraint.ConstraintLayout
android:layout_width="match_parent"
android:layout_height="wrap_content">
app:layout_constraintTop_toTopOf="@+id/manga_cover">
<TextView
android:id="@+id/manga_full_title"
style="@style/TextAppearance.Medium.Title"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:clickable="false"
android:maxLines="2"
android:text="@string/manga_info_full_title_label"
android:textIsSelectable="false"
@ -60,9 +60,9 @@
style="@style/TextAppearance.Medium.Body2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:clickable="false"
android:text="@string/manga_info_author_label"
android:textIsSelectable="false"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toBottomOf="@+id/manga_full_title" />
@ -72,6 +72,7 @@
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginLeft="8dp"
android:clickable="false"
android:ellipsize="end"
android:maxLines="1"
android:textIsSelectable="false"
@ -84,6 +85,7 @@
style="@style/TextAppearance.Medium.Body2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:clickable="false"
android:text="@string/manga_info_artist_label"
android:textIsSelectable="false"
app:layout_constraintLeft_toLeftOf="parent"
@ -95,6 +97,7 @@
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginLeft="8dp"
android:clickable="false"
android:ellipsize="end"
android:maxLines="1"
android:textIsSelectable="false"
@ -102,61 +105,16 @@
app:layout_constraintLeft_toRightOf="@+id/manga_artist_label"
app:layout_constraintRight_toRightOf="parent" />
<TextView
android:id="@+id/manga_chapters_label"
style="@style/TextAppearance.Medium.Body2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/manga_info_last_chapter_label"
android:textIsSelectable="false"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toBottomOf="@+id/manga_artist_label" />
<TextView
android:id="@+id/manga_chapters"
style="@style/TextAppearance.Regular.Body1.Secondary"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginLeft="8dp"
android:ellipsize="end"
android:maxLines="1"
android:textIsSelectable="false"
app:layout_constraintBaseline_toBaselineOf="@+id/manga_chapters_label"
app:layout_constraintLeft_toRightOf="@+id/manga_chapters_label"
app:layout_constraintRight_toRightOf="parent" />
<TextView
android:id="@+id/manga_last_update_label"
style="@style/TextAppearance.Medium.Body2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="@string/manga_info_latest_data_label"
android:textIsSelectable="false"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toBottomOf="@+id/manga_chapters_label" />
<TextView
android:id="@+id/manga_last_update"
style="@style/TextAppearance.Regular.Body1.Secondary"
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginLeft="8dp"
android:ellipsize="end"
android:maxLines="1"
android:textIsSelectable="false"
app:layout_constraintBaseline_toBaselineOf="@+id/manga_last_update_label"
app:layout_constraintLeft_toRightOf="@+id/manga_last_update_label"
app:layout_constraintRight_toRightOf="parent" />
<TextView
android:id="@+id/manga_status_label"
style="@style/TextAppearance.Medium.Body2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:clickable="false"
android:text="@string/manga_info_status_label"
android:textIsSelectable="false"
app:layout_constraintLeft_toLeftOf="parent"
app:layout_constraintTop_toBottomOf="@+id/manga_last_update_label" />
app:layout_constraintTop_toBottomOf="@+id/manga_artist_label" />
<TextView
android:id="@+id/manga_status"
@ -164,6 +122,7 @@
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginLeft="8dp"
android:clickable="false"
android:ellipsize="end"
android:maxLines="1"
android:textIsSelectable="false"
@ -176,6 +135,7 @@
style="@style/TextAppearance.Medium.Body2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:clickable="false"
android:text="@string/manga_info_source_label"
android:textIsSelectable="false"
app:layout_constraintLeft_toLeftOf="parent"
@ -187,6 +147,7 @@
android:layout_width="0dp"
android:layout_height="wrap_content"
android:layout_marginLeft="8dp"
android:clickable="false"
android:textIsSelectable="false"
app:layout_constraintLeft_toRightOf="@+id/manga_source_label"
app:layout_constraintRight_toRightOf="parent"
@ -194,6 +155,46 @@
</android.support.constraint.ConstraintLayout>
</android.support.v4.widget.NestedScrollView>
<View
android:id="@+id/card_shim"
android:layout_width="0dp"
android:layout_height="0dp"
android:background="#E6FFFFFF"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toTopOf="parent" />
<TextView
android:id="@+id/search_status"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:layout_marginLeft="8dp"
android:layout_marginTop="8dp"
android:layout_marginEnd="8dp"
android:layout_marginRight="8dp"
android:layout_marginBottom="8dp"
android:text="Searching..."
android:textAppearance="@style/TextAppearance.AppCompat.Large"
app:layout_constraintBottom_toTopOf="@+id/search_progress"
app:layout_constraintEnd_toEndOf="@+id/card_shim"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toTopOf="@+id/card_shim" />
<ProgressBar
android:id="@+id/search_progress"
style="?android:attr/progressBarStyleHorizontal"
android:layout_width="0dp"
android:layout_height="wrap_content"
app:layout_constraintBottom_toBottomOf="@+id/card_shim"
app:layout_constraintEnd_toEndOf="@+id/card_shim"
app:layout_constraintStart_toStartOf="parent" />
<android.support.constraint.Group
android:id="@+id/loading_group"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
app:constraint_referenced_ids="card_shim,search_status,search_progress" />
</android.support.constraint.ConstraintLayout>
</android.support.v7.widget.CardView>

View File

@ -1,22 +1,66 @@
<?xml version="1.0" encoding="utf-8"?>
<android.support.constraint.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android"
<android.support.constraint.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent">
android:layout_height="match_parent"
android:animateLayoutChanges="true">
<android.support.v7.widget.RecyclerView
android:id="@+id/recycler"
android:layout_width="0dp"
android:layout_height="0dp"
android:layout_marginBottom="8dp"
app:layout_constraintBottom_toTopOf="@+id/textView"
app:layout_constraintBottom_toTopOf="@+id/textView2"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintHorizontal_bias="0.0"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toTopOf="parent"
tools:listitem="@layout/eh_source_item" />
tools:listitem="@layout/eh_source_item">
</android.support.v7.widget.RecyclerView>
<TextView
android:id="@+id/textView2"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="8dp"
android:text="Data to include in migration"
android:textAppearance="@style/TextAppearance.Medium.Body2"
app:layout_constraintBottom_toTopOf="@+id/mig_chapters"
app:layout_constraintStart_toStartOf="@+id/textView" />
<CheckBox
android:id="@+id/mig_chapters"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="8dp"
android:text="@string/chapters"
android:checked="true"
app:layout_constraintBottom_toTopOf="@+id/textView"
app:layout_constraintStart_toStartOf="@+id/textView2" />
<CheckBox
android:id="@+id/mig_categories"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:layout_marginLeft="8dp"
android:text="@string/categories"
android:checked="true"
app:layout_constraintBottom_toBottomOf="@+id/mig_chapters"
app:layout_constraintStart_toEndOf="@+id/mig_chapters" />
<CheckBox
android:id="@+id/mig_tracking"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginStart="8dp"
android:layout_marginLeft="8dp"
android:text="@string/track"
android:checked="true"
app:layout_constraintBottom_toBottomOf="@+id/mig_categories"
app:layout_constraintStart_toEndOf="@+id/mig_categories" />
<TextView
android:id="@+id/textView"
@ -25,7 +69,7 @@
android:layout_marginStart="8dp"
android:layout_marginLeft="8dp"
android:layout_marginBottom="8dp"
android:text="Migration mode"
android:text="Search options"
android:textAppearance="@style/TextAppearance.Medium.Body2"
app:layout_constraintBottom_toTopOf="@+id/prioritize_chapter_count"
app:layout_constraintStart_toStartOf="parent" />
@ -70,7 +114,7 @@
android:layout_marginRight="8dp"
android:layout_marginBottom="8dp"
android:gravity="start|center_vertical"
android:text="Use lenient search algorithm"
android:text="Use smart search algorithm"
app:layout_constraintBottom_toTopOf="@+id/begin_migration_btn"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toEndOf="@+id/prioritize_chapter_count" />

View File

@ -7,18 +7,10 @@
android:layout_height="match_parent"
android:background="?attr/colorPrimary" >
<!--<android.support.v4.view.ViewPager-->
<!--android:id="@+id/pager"-->
<!--android:layout_width="match_parent"-->
<!--android:layout_height="match_parent"-->
<!--app:layout_constraintBottom_toBottomOf="parent"-->
<!--app:layout_constraintEnd_toEndOf="parent"-->
<!--app:layout_constraintStart_toStartOf="parent"-->
<!--app:layout_constraintTop_toTopOf="parent" />-->
<include
layout="@layout/eh_migration_process_item"
android:layout_width="0dp"
android:layout_height="0dp"
<exh.ui.migration.manga.process.DeactivatableViewPager
android:id="@+id/pager"
android:layout_width="match_parent"
android:layout_height="match_parent"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"