Merge branch 'master' of https://github.com/inorichi/tachiyomi

# Conflicts:
#	README.md
#	app/build.gradle
#	app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt
#	app/src/main/java/eu/kanade/tachiyomi/data/source/SourceManager.kt
#	app/src/main/java/eu/kanade/tachiyomi/source/model/Page.kt
#	app/src/main/java/eu/kanade/tachiyomi/ui/backup/BackupPresenter.kt
#	app/src/main/java/eu/kanade/tachiyomi/ui/main/ChangelogDialogFragment.kt
#	app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt
#	app/src/main/res/raw/changelog_release.xml
#	app/src/main/res/values/arrays.xml
#	app/src/main/res/values/strings.xml

app/src/main/AndroidManifest.xml

@@ -130,6 +130,14 @@
                     android:host="g.e-hentai.org"
                     android:pathPrefix="/g/"
                     android:scheme="https"/>
+                <data
+                    android:host="e-hentai.org"
+                    android:pathPrefix="/g/"
+                    android:scheme="http"/>
+                <data
+                    android:host="e-hentai.org"
+                    android:pathPrefix="/g/"
+                    android:scheme="https"/>
                 <data
                     android:host="exhentai.org"
                     android:pathPrefix="/g/"

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt

@@ -170,6 +170,8 @@ class PreferencesHelper(val context: Context) {
 
     fun hasPerformedURLMigration() = rxPrefs.getBoolean("performed_url_migration", false)
 
+    fun hasPerformedSourceMigration() = rxPrefs.getBoolean("performed_source_migration", false)
+
     //EH Cookies
     fun memberIdVal() = rxPrefs.getString("eh_ipb_member_id", null)
     fun passHashVal() = rxPrefs.getString("eh_ipb_pass_hash", null)

app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt

@@ -7,6 +7,10 @@ import android.content.pm.PackageManager
 import android.os.Environment
 import dalvik.system.PathClassLoader
 import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import eu.kanade.tachiyomi.data.preference.getOrDefault
+import eu.kanade.tachiyomi.source.online.all.EHentai
+import eu.kanade.tachiyomi.source.online.all.EHentaiMetadata
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.source.online.YamlHttpSource
 import eu.kanade.tachiyomi.source.online.english.*
@@ -15,16 +19,38 @@ import eu.kanade.tachiyomi.source.online.russian.Mangachan
 import eu.kanade.tachiyomi.source.online.russian.Mintmanga
 import eu.kanade.tachiyomi.source.online.russian.Readmanga
 import eu.kanade.tachiyomi.util.hasPermission
+import exh.EH_METADATA_SOURCE_ID
+import exh.EH_SOURCE_ID
+import exh.EXH_METADATA_SOURCE_ID
+import exh.EXH_SOURCE_ID
 import org.yaml.snakeyaml.Yaml
+import rx.functions.Action1
 import timber.log.Timber
+import uy.kohesive.injekt.injectLazy
 import java.io.File
 
 open class SourceManager(private val context: Context) {
 
+    private val prefs: PreferencesHelper by injectLazy()
+
     private val sourcesMap = mutableMapOf<Long, Source>()
 
     init {
         createSources()
+
+        //Rebuild sources when settings change
+        val action: Action1<Any> = Action1 {
+            sourcesMap.clear()
+            createSources()
+        }
+        prefs.enableExhentai().asObservable().subscribe(action)
+        prefs.imageQuality().asObservable().subscribe (action)
+        prefs.useHentaiAtHome().asObservable().subscribe(action)
+        prefs.useJapaneseTitle().asObservable().subscribe {
+            action.call(null)
+        }
+        prefs.ehSearchSize().asObservable().subscribe (action)
+        prefs.thumbnailRows().asObservable().subscribe(action)
     }
 
     open fun get(sourceKey: Long): Source? {
@@ -39,6 +65,8 @@ open class SourceManager(private val context: Context) {
         createExtensionSources().forEach { registerSource(it) }
         createYamlSources().forEach { registerSource(it) }
         createInternalSources().forEach { registerSource(it) }
+        //EH
+        createEHSources().forEach { registerSource(it) }
     }
 
     private fun registerSource(source: Source, overwrite: Boolean = false) {
@@ -61,6 +89,19 @@ open class SourceManager(private val context: Context) {
             WieManga()
     )
 
+    private fun createEHSources(): List<Source> {
+        //TODO Fix and hook up to createSources...
+        val exSrcs = mutableListOf(
+                EHentai(EH_SOURCE_ID, false, context),
+                EHentaiMetadata(EH_METADATA_SOURCE_ID, false, context)
+        )
+        if(prefs.enableExhentai().getOrDefault()) {
+            exSrcs += EHentai(EXH_SOURCE_ID, true, context)
+            exSrcs += EHentaiMetadata(EXH_METADATA_SOURCE_ID, true, context)
+        }
+        return exSrcs
+    }
+
     private fun createYamlSources(): List<Source> {
         val sources = mutableListOf<Source>()
 

app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt

@@ -2,20 +2,17 @@ package eu.kanade.tachiyomi.source.online.all
 
 import android.content.Context
 import android.net.Uri
-import eu.kanade.tachiyomi.data.database.models.Chapter
 import eu.kanade.tachiyomi.data.database.models.Manga
-import eu.kanade.tachiyomi.data.network.GET
-import eu.kanade.tachiyomi.data.network.asObservableSuccess
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
-import eu.kanade.tachiyomi.data.source.model.MangasPage
+import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.data.source.model.Page
+import eu.kanade.tachiyomi.network.asObservableSuccess
-import eu.kanade.tachiyomi.data.source.online.OnlineSource
+import eu.kanade.tachiyomi.source.model.*
+import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
 import exh.metadata.*
 import exh.metadata.models.ExGalleryMetadata
 import exh.metadata.models.Tag
-import exh.plusAssign
 import okhttp3.Response
 import org.jsoup.nodes.Element
 import rx.Observable
@@ -23,10 +20,11 @@ import uy.kohesive.injekt.injectLazy
 import java.net.URLEncoder
 import java.util.*
 import exh.ui.login.LoginActivity
+import okhttp3.Request
 
-class EHentai(override val id: Int,
+class EHentai(override val id: Long,
               val exh: Boolean,
-              val context: Context) : OnlineSource() {
+              val context: Context) : HttpSource() {
 
     val schema: String
         get() = if(prefs.secureEXH().getOrDefault())
@@ -55,7 +53,7 @@ class EHentai(override val id: Int,
     /**
      * Parse a list of galleries
      */
-    fun genericMangaParse(response: Response, page: MangasPage? = null)
+    fun genericMangaParse(response: Response)
             = with(response.asJsoup()) {
         //Parse mangas
         val parsedMangas = select(".gtr0,.gtr1").map {
@@ -81,32 +79,27 @@ class EHentai(override val id: Int,
 
         }
         //Add to page if required
-            page?.let { page ->
+        val hasNextPage = select("a[onclick=return false]").last()?.let {
-                page.mangas += parsedMangas.map { it.manga }
+            it.text() == ">"
-                select("a[onclick=return false]").last()?.let {
+        } ?: false
-                    if(it.text() == ">") page.nextPageUrl = it.attr("href")
+        MangasPage(parsedMangas.map { it.manga }, hasNextPage)
-                }
-            }
-        //Return parsed mangas anyways
-        parsedMangas
     }
 
-    override fun fetchChapterList(manga: Manga): Observable<List<Chapter>>
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>>
-            = Observable.just(listOf(Chapter.create().apply {
+            = Observable.just(listOf(SChapter.create().apply {
-        manga_id = manga.id
         url = manga.url
         name = "Chapter"
         chapter_number = 1f
     }))
 
-    override fun fetchPageListFromNetwork(chapter: Chapter)
+    override fun fetchPageList(chapter: SChapter)
-            = fetchChapterPage(chapter, "$baseUrl${chapter.url}").map {
+            = fetchChapterPage(chapter, "$baseUrl/${chapter.url}").map {
         it.mapIndexed { i, s ->
             Page(i, s)
         }
     }!!
 
-    private fun fetchChapterPage(chapter: Chapter, np: String,
+    private fun fetchChapterPage(chapter: SChapter, np: String,
                                  pastUrls: List<String> = emptyList()): Observable<List<String>> {
         val urls = ArrayList(pastUrls)
         return chapterPageCall(np).flatMap {
@@ -134,60 +127,46 @@ class EHentai(override val id: Int,
         return if (it.text() == ">") it.attr("href") else null
     }
 
-    private fun buildGenreString(filters: List<OnlineSource.Filter>): String {
+    override fun popularMangaRequest(page: Int) = if(exh)
-        val genreString = StringBuilder()
+        latestUpdatesRequest(page)
-        for (genre in GENRE_LIST) {
-            genreString += "&f_"
-            genreString += genre
-            genreString += "="
-            genreString += if (filters.isEmpty()
-                    || !filters
-                    .map { it.id }
-                    .find { it == genre }
-                    .isNullOrEmpty())
-                "1"
-            else
-                "0"
-        }
-        return genreString.toString()
-    }
-
-    override fun popularMangaInitialUrl() = if(exh)
-        latestUpdatesInitialUrl()
     else
-        "$baseUrl/toplist.php?tl=15"
+        exGet("$baseUrl/toplist.php?tl=15", page)
 
-    override fun popularMangaParse(response: Response, page: MangasPage) {
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        genericMangaParse(response, page)
+        val uri = Uri.parse("$baseUrl$QUERY_PREFIX").buildUpon()
+        uri.appendQueryParameter("f_search", query)
+        filters.forEach {
+            if(it is UriFilter) it.addToUri(uri)
+        }
+        return exGet(uri.toString(), page)
     }
 
-    override fun searchMangaInitialUrl(query: String, filters: List<Filter>)
+    override fun latestUpdatesRequest(page: Int) = exGet(baseUrl, page)
-            = "$baseUrl$QUERY_PREFIX${buildGenreString(filters)}&f_search=${URLEncoder.encode(query, "UTF-8")}"
 
-    override fun searchMangaParse(response: Response, page: MangasPage, query: String, filters: List<Filter>) {
+    override fun popularMangaParse(response: Response) = genericMangaParse(response)
-        genericMangaParse(response, page)
+    override fun searchMangaParse(response: Response) = genericMangaParse(response)
-    }
+    override fun latestUpdatesParse(response: Response) = genericMangaParse(response)
 
-    override fun latestUpdatesInitialUrl() = baseUrl
+    fun exGet(url: String, page: Int? = null)
-
+        = GET(page?.let {
-    override fun latestUpdatesParse(response: Response, page: MangasPage) {
+            addParam(url, "page", Integer.toString(page - 1))
-        genericMangaParse(response, page)
+        } ?: url, headers)
-    }
 
     /**
      * Parse gallery page to metadata model
      */
-    override fun mangaDetailsParse(response: Response, manga: Manga) = with(response.asJsoup()) {
+    override fun mangaDetailsParse(response: Response) = with(response.asJsoup()) {
         val metdata = ExGalleryMetadata()
         with(metdata) {
-            url = manga.url
+            val manga = SManga.create()
+            url = response.request().url().toString()
             exh = this@EHentai.exh
             title = select("#gn").text().nullIfBlank()?.trim()
             altTitle = select("#gj").text().nullIfBlank()?.trim()
 
             thumbnailUrl = select("#gd1 img").attr("src").nullIfBlank()?.trim()
 
-            genre = select(".ic").attr("alt").nullIfBlank()?.trim()
+            genre = select(".ic").parents().attr("href").nullIfBlank()?.trim()?.substringAfterLast('/')
 
             uploader = select("#gdn").text().nullIfBlank()?.trim()
 
@@ -252,41 +231,25 @@ class EHentai(override val id: Int,
 
             //Copy metadata to manga
             copyTo(manga)
+
+            manga
         }
     }
 
-    override fun chapterListParse(response: Response, chapters: MutableList<Chapter>) {
+    override fun chapterListParse(response: Response)
-        throw UnsupportedOperationException()
+            = throw UnsupportedOperationException("Unused method was called somehow!")
-    }
 
-    override fun pageListParse(response: Response, pages: MutableList<Page>) {
+    override fun pageListParse(response: Response)
-        throw UnsupportedOperationException()
+            = throw UnsupportedOperationException("Unused method was called somehow!")
-    }
 
     override fun imageUrlParse(response: Response): String {
-        throw UnsupportedOperationException()
-    }
-
-    //Copy and paste from OnlineSource as we need the page argument
-    override public fun fetchImageUrl(page: Page): Observable<Page> {
-        page.status = Page.LOAD_PAGE
-        return client
-                .newCall(imageUrlRequest(page))
-                .asObservableSuccess()
-                .map { imageUrlParse(it, page) }
-                .doOnError { page.status = Page.ERROR }
-                .onErrorReturn { null }
-                .doOnNext { page.imageUrl = it }
-                .map { page }
-    }
-
-    fun imageUrlParse(response: Response, page: Page): String {
         with(response.asJsoup()) {
             val currentImage = select("img[onerror]").attr("src")
+            //TODO This doesn't work currently. Find a better way to do this
             //Each press of the retry button will choose another server
-            select("#loadfail").attr("onclick").nullIfBlank()?.let {
+//            select("#loadfail").attr("onclick").nullIfBlank()?.let {
-                page.url = addParam(page.url, "nl", it.substring(it.indexOf('\'') + 1 .. it.lastIndexOf('\'') - 1))
+//                page.url = addParam(page.url, "nl", it.substring(it.indexOf('\'') + 1 .. it.lastIndexOf('\'') - 1))
-            }
+//            }
             return currentImage
         }
     }
@@ -366,8 +329,70 @@ class EHentai(override val id: Int,
             }.build()!!
 
     //Filters
-    val generatedFilters = GENRE_LIST.map { Filter(it, it) }
+    override fun getFilterList() = FilterList(
-    override fun getFilterList() = generatedFilters
+            GenreGroup(),
+            AdvancedGroup()
+    )
+    private interface UriFilter {
+        fun addToUri(builder: Uri.Builder)
+    }
+
+    class GenreOption(name: String, val genreId: String): Filter.CheckBox(name, false), UriFilter {
+        override fun addToUri(builder: Uri.Builder) {
+            builder.appendQueryParameter("f_" + genreId, if(state) "1" else "0")
+        }
+    }
+    class GenreGroup : UriGroup<GenreOption>("Genres", listOf(
+            GenreOption("Dōjinshi", "doujinshi"),
+            GenreOption("Manga", "manga"),
+            GenreOption("Artist CG", "artistcg"),
+            GenreOption("Game CG", "gamecg"),
+            GenreOption("Western", "western"),
+            GenreOption("Non-H", "non-h"),
+            GenreOption("Image Set", "imageset"),
+            GenreOption("Cosplay", "cosplay"),
+            GenreOption("Asian Porn", "asianporn"),
+            GenreOption("Misc", "misc")
+    ))
+
+    class AdvancedOption(name: String, val param: String, defValue: Boolean = false): Filter.CheckBox(name, defValue), UriFilter {
+        override fun addToUri(builder: Uri.Builder) {
+            if(state)
+                builder.appendQueryParameter(param, "on")
+        }
+    }
+    class RatingOption : Filter.Select<String>("Minimum Rating", arrayOf(
+            "Any",
+            "2 stars",
+            "3 stars",
+            "4 stars",
+            "5 stars"
+    )), UriFilter {
+        override fun addToUri(builder: Uri.Builder) {
+            if(state > 0) builder.appendQueryParameter("f_srdd", Integer.toString(state + 1))
+        }
+    }
+
+    //Explicit type arg for listOf() to workaround this: KT-16570
+    class AdvancedGroup : UriGroup<Filter<*>>("Advanced Options", listOf<Filter<*>>(
+            AdvancedOption("Search Gallery Name", "f_sname", true),
+            AdvancedOption("Search Gallery Tags", "f_stags", true),
+            AdvancedOption("Search Gallery Description", "f_sdesc"),
+            AdvancedOption("Search Torrent Filenames", "f_storr"),
+            AdvancedOption("Only Show Galleries With Torrents", "f_sto"),
+            AdvancedOption("Search Low-Power Tags", "f_sdt1"),
+            AdvancedOption("Search Downvoted Tags", "f_sdt2"),
+            AdvancedOption("Show Expunged Galleries", "f_sh"),
+            RatingOption()
+    ))
+
+    open class UriGroup<V>(name: String, state: List<V>) : Filter.Group<V>(name, state), UriFilter {
+        override fun addToUri(builder: Uri.Builder) {
+            state.forEach {
+                if(it is UriFilter) it.addToUri(builder)
+            }
+        }
+    }
 
     override val name = if(exh)
         "ExHentai"
@@ -376,7 +401,6 @@ class EHentai(override val id: Int,
 
     companion object {
         val QUERY_PREFIX = "?f_apply=Apply+Filter"
-        val GENRE_LIST = arrayOf("doujinshi", "manga", "artistcg", "gamecg", "western", "non-h", "imageset", "cosplay", "asianporn", "misc")
         val TR_SUFFIX = "TR"
     }
 }

app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentaiMetadata.kt

@@ -3,9 +3,8 @@ package eu.kanade.tachiyomi.source.online.all
 import android.content.Context
 import eu.kanade.tachiyomi.data.database.models.Chapter
 import eu.kanade.tachiyomi.data.database.models.Manga
-import eu.kanade.tachiyomi.data.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.*
-import eu.kanade.tachiyomi.data.source.model.Page
+import eu.kanade.tachiyomi.source.online.HttpSource
-import eu.kanade.tachiyomi.data.source.online.OnlineSource
 import exh.metadata.MetadataHelper
 import exh.metadata.copyTo
 import exh.metadata.models.ExGalleryMetadata
@@ -15,11 +14,35 @@ import rx.Observable
 
 /**
  * Offline metadata store source
+ *
+ * TODO This no longer fakes an online source because of technical reasons.
+ * If we still want offline search, we must find out a way to rearchitecture the source system so it supports
+ * online source faking again.
  */
 
-class EHentaiMetadata(override val id: Int,
+class EHentaiMetadata(override val id: Long,
                       val exh: Boolean,
-                      val context: Context) : OnlineSource() {
+                      val context: Context) : HttpSource() {
+    override fun popularMangaRequest(page: Int)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun popularMangaParse(response: Response)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun searchMangaParse(response: Response)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun latestUpdatesRequest(page: Int)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun latestUpdatesParse(response: Response)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun mangaDetailsParse(response: Response)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun chapterListParse(response: Response)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun pageListParse(response: Response)
+            = throw UnsupportedOperationException("Unused method called!")
+    override fun imageUrlParse(response: Response)
+            = throw UnsupportedOperationException("Unused method called!")
 
     val metadataHelper = MetadataHelper()
 
@@ -34,55 +57,14 @@ class EHentaiMetadata(override val id: Int,
     override val supportsLatest: Boolean
         get() = true
 
-    override fun popularMangaInitialUrl(): String {
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>>
-        throw UnsupportedOperationException()
-    }
-
-    override fun popularMangaParse(response: Response, page: MangasPage) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun searchMangaInitialUrl(query: String, filters: List<Filter>): String {
-        throw UnsupportedOperationException()
-    }
-
-    override fun searchMangaParse(response: Response, page: MangasPage, query: String, filters: List<Filter>) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun latestUpdatesInitialUrl(): String {
-        throw UnsupportedOperationException()
-    }
-
-    override fun latestUpdatesParse(response: Response, page: MangasPage) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun mangaDetailsParse(response: Response, manga: Manga) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun chapterListParse(response: Response, chapters: MutableList<Chapter>) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun pageListParse(response: Response, pages: MutableList<Page>) {
-        throw UnsupportedOperationException()
-    }
-
-    override fun imageUrlParse(response: Response): String {
-        throw UnsupportedOperationException()
-    }
-
-    override fun fetchChapterList(manga: Manga): Observable<List<Chapter>>
             = Observable.just(listOf(Chapter.create().apply {
-        manga_id = manga.id
         url = manga.url
         name = "ONLINE - Chapter"
         chapter_number = 1f
     }))
 
-    override fun fetchPageListFromNetwork(chapter: Chapter) = internalEx.fetchPageListFromNetwork(chapter)
+    override fun fetchPageList(chapter: SChapter) = internalEx.fetchPageList(chapter)
 
     override fun fetchImageUrl(page: Page) = internalEx.fetchImageUrl(page)
 
@@ -98,38 +80,42 @@ class EHentaiMetadata(override val id: Int,
         it.datePosted ?: 0
     }
 
-    override fun fetchPopularManga(page: MangasPage)
+    override fun fetchPopularManga(page: Int)
         = Observable.fromCallable {
-            page.mangas.addAll(metadataHelper.getAllGalleries().sortedByDescending {
+            MangasPage(metadataHelper.getAllGalleries().sortedByDescending {
                 it.ratingCount ?: 0
-            }.mapToManga())
+            }.mapToManga(), false)
-            page
         }!!
 
-    override fun fetchSearchManga(page: MangasPage, query: String, filters: List<Filter>)
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList)
     = Observable.fromCallable {
+        val genreGroup = filters.find {
+            it is EHentai.GenreGroup
+        }!! as EHentai.GenreGroup
+        val disableGenreFilter = genreGroup.state.find(EHentai.GenreOption::state) == null
+
         val parsed = searchEngine.parseQuery(query)
-        page.mangas.addAll(sortedByTimeGalleries().filter { manga ->
+        MangasPage(sortedByTimeGalleries().filter { manga ->
-            filters.isEmpty() || filters.filter { it.id == manga.genre }.isNotEmpty()
+            disableGenreFilter || genreGroup.state.find {
+                it.state && it.genreId == manga.genre
+            } != null
         }.filter {
             searchEngine.matches(it, parsed)
-        }.mapToManga())
+        }.mapToManga(), false)
-        page
     }!!
 
-    override fun fetchLatestUpdates(page: MangasPage)
+    override fun fetchLatestUpdates(page: Int)
     = Observable.fromCallable {
-        page.mangas.addAll(sortedByTimeGalleries().mapToManga())
+        MangasPage(sortedByTimeGalleries().mapToManga(), false)
-        page
     }!!
 
-    override fun fetchMangaDetails(manga: Manga) = Observable.fromCallable {
+    override fun fetchMangaDetails(manga: SManga) = Observable.fromCallable {
         //Hack to convert the gallery into an online gallery when favoriting it or reading it
         metadataHelper.fetchMetadata(manga.url, exh)?.copyTo(manga)
         manga
     }!!
 
-    override fun getFilterList() = internalEx.getFilterList()
+    override fun getFilterList() = FilterList(EHentai.GenreGroup())
 
     override val name: String
         get() = if(exh) {

app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryAdapter.kt

@@ -9,17 +9,17 @@ import android.widget.FrameLayout
 import eu.davidea.flexibleadapter4.FlexibleAdapter
 import eu.kanade.tachiyomi.R
 import eu.kanade.tachiyomi.data.database.models.Manga
-import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.source.SourceManager
-import eu.kanade.tachiyomi.data.source.online.all.EHentai
+import eu.kanade.tachiyomi.source.online.all.EHentai
-import eu.kanade.tachiyomi.data.source.online.all.EHentaiMetadata
+import eu.kanade.tachiyomi.source.online.all.EHentaiMetadata
 import eu.kanade.tachiyomi.util.inflate
 import eu.kanade.tachiyomi.widget.AutofitRecyclerView
+import exh.isLewdSource
 import exh.metadata.MetadataHelper
 import exh.search.SearchEngine
 import kotlinx.android.synthetic.main.item_catalogue_grid.view.*
 import uy.kohesive.injekt.injectLazy
 import java.util.*
-import kotlin.concurrent.thread
 
 /**
  * Adapter storing a list of manga in a certain category.
@@ -95,7 +95,7 @@ class LibraryCategoryAdapter(val fragment: LibraryCategoryView) :
      * @return true if the manga should be included, false otherwise.
      */
     override fun filterObject(manga: Manga, query: String): Boolean = with(manga) {
-        if(manga.source > 100) {
+        if(!isLewdSource(manga.source)) {
             //Regular searching for normal manga
             title.toLowerCase().contains(query) ||
                     author != null && author!!.toLowerCase().contains(query)

app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt

@@ -20,6 +20,7 @@ import eu.kanade.tachiyomi.ui.recently_read.RecentlyReadFragment
 import eu.kanade.tachiyomi.ui.setting.SettingsActivity
 import exh.ui.batchadd.BatchAddFragment
 import exh.ui.migration.LibraryMigrationManager
+import exh.ui.migration.SourceMigrator
 import exh.ui.migration.UrlMigrator
 import kotlinx.android.synthetic.main.activity_main.*
 import kotlinx.android.synthetic.main.toolbar.*
@@ -89,25 +90,31 @@ class MainActivity : BaseActivity() {
         }
 
         if (savedState == null) {
-            // Set start screen
+            //Perform source migration
-            setSelectedDrawerItem(startScreenId)
+            SourceMigrator().tryMigrationWithDialog(this, {
 
-            // Show changelog if needed
+                // Set start screen
-            ChangelogDialogFragment.show(this, preferences, supportFragmentManager)
+                try {
+                    setSelectedDrawerItem(startScreenId)
+                } catch(e: Exception) {}
 
-            // Migrate library if needed
+                // Show changelog if needed
-            LibraryMigrationManager(this, dismissQueue).askMigrationIfNecessary()
+                ChangelogDialogFragment.show(this, preferences, supportFragmentManager)
 
-            //Last part of migration requires finishing this activity
+                // Migrate library if needed
-            finishSubscription?.unsubscribe()
+                LibraryMigrationManager(this, dismissQueue).askMigrationIfNecessary()
-            preferences.finishMainActivity().set(false)
-            finishSubscription = preferences.finishMainActivity().asObservable().subscribe {
-                if(it)
-                    finish()
-            }
 
-            //Migrate URLs if necessary
+                //Last part of migration requires finishing this activity
-            UrlMigrator().tryMigration()
+                finishSubscription?.unsubscribe()
+                preferences.finishMainActivity().set(false)
+                finishSubscription = preferences.finishMainActivity().asObservable().subscribe {
+                    if (it)
+                        finish()
+                }
+
+                //Migrate URLs if necessary
+                UrlMigrator().tryMigration()
+            })
         }
     }
 

app/src/main/java/exh/EHSourceHelpers.kt

@@ -1,5 +1,17 @@
 package exh
 
 /**
- * Created by nulldev on 2/28/17.
+ * Source helpers
  */
+
+val LEWD_SOURCE_SERIES = 6900L
+val EH_SOURCE_ID = LEWD_SOURCE_SERIES + 1
+val EXH_SOURCE_ID = LEWD_SOURCE_SERIES + 2
+val EH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 3
+val EXH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 4
+
+fun isLewdSource(source: Long) = source >= 6900
+        && source <= 6999
+
+fun isExSource(source: Long) = source == EXH_SOURCE_ID
+        || source == EXH_METADATA_SOURCE_ID

app/src/main/java/exh/GalleryAdder.kt

@@ -2,14 +2,15 @@ package exh
 
 import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.database.models.Manga
-import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.source.SourceManager
-import eu.kanade.tachiyomi.util.UrlUtil
 import eu.kanade.tachiyomi.util.syncChaptersWithSource
 import exh.metadata.MetadataHelper
 import exh.metadata.copyTo
 import timber.log.Timber
 import uy.kohesive.injekt.injectLazy
 import java.net.MalformedURLException
+import java.net.URI
+import java.net.URISyntaxException
 import java.net.URL
 
 class GalleryAdder {
@@ -22,19 +23,20 @@ class GalleryAdder {
 
     fun addGallery(url: String, fav: Boolean = false): Manga {
         val source = when(URL(url).host) {
-            "g.e-hentai.org", "e-hentai.org" -> 1
+            "g.e-hentai.org", "e-hentai.org" -> EH_SOURCE_ID
-            "exhentai.org" -> 2
+            "exhentai.org" -> EXH_SOURCE_ID
             else -> throw MalformedURLException("Not a valid gallery URL!")
         }
 
         val sourceObj = sourceManager.get(source)
                 ?: throw IllegalStateException("Could not find EH source!")
 
-        val pathOnlyUrl = UrlUtil.getPath(url)
+        val pathOnlyUrl = getUrlWithoutDomain(url)
 
         //Use manga in DB if possible, otherwise, make a new manga
         val manga = db.getManga(pathOnlyUrl, source).executeAsBlocking()
-                ?: Manga.create(pathOnlyUrl, source).apply {
+                ?: Manga.create(source).apply {
+            this.url = pathOnlyUrl
             title = url
         }
 
@@ -42,7 +44,7 @@ class GalleryAdder {
         manga.copyFrom(sourceObj.fetchMangaDetails(manga).toBlocking().first())
 
         //Apply metadata
-        metadataHelper.fetchMetadata(url, source == 2)?.copyTo(manga)
+        metadataHelper.fetchMetadata(url, isExSource(source))?.copyTo(manga)
 
         if(fav) manga.favorite = true
 
@@ -61,4 +63,18 @@ class GalleryAdder {
 
         return manga
     }
+
+    private fun getUrlWithoutDomain(orig: String): String {
+        try {
+            val uri = URI(orig)
+            var out = uri.path
+            if (uri.query != null)
+                out += "?" + uri.query
+            if (uri.fragment != null)
+                out += "#" + uri.fragment
+            return out
+        } catch (e: URISyntaxException) {
+            return orig
+        }
+    }
 }

app/src/main/java/exh/metadata/MetdataCopier.kt

@@ -1,9 +1,8 @@
 package exh.metadata
 
-import eu.kanade.tachiyomi.data.database.models.Manga
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
-import eu.kanade.tachiyomi.util.UrlUtil
+import eu.kanade.tachiyomi.source.model.SManga
 import exh.metadata.models.ExGalleryMetadata
 import exh.metadata.models.Tag
 import exh.plusAssign
@@ -28,16 +27,18 @@ val EX_DATE_FORMAT = SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US)
 
 private val prefs: PreferencesHelper by injectLazy()
 
-fun ExGalleryMetadata.copyTo(manga: Manga) {
+fun ExGalleryMetadata.copyTo(manga: SManga) {
-    exh?.let {
+    //TODO Find some way to do this with SManga
+    /*exh?.let {
         manga.source = if(it)
             2
         else
             1
-    }
+    }*/
     url?.let { manga.url = it }
     thumbnailUrl?.let { manga.thumbnail_url = it }
 
+    //No title bug?
     val titleObj = if(prefs.useJapaneseTitle().getOrDefault())
         altTitle ?: title
     else
@@ -57,12 +58,12 @@ fun ExGalleryMetadata.copyTo(manga: Manga) {
 
     //Try to automatically identify if it is ongoing, we try not to be too lenient here to avoid making mistakes
     //We default to completed
-    manga.status = Manga.COMPLETED
+    manga.status = SManga.COMPLETED
     title?.let { t ->
         ONGOING_SUFFIX.find {
             t.endsWith(it, ignoreCase = true)
         }?.let {
-            manga.status = Manga.ONGOING
+            manga.status = SManga.ONGOING
         }
     }
 

app/src/main/java/exh/ui/migration/MetadataFetchDialog.kt

@@ -7,8 +7,9 @@ import eu.kanade.tachiyomi.R
 import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
-import eu.kanade.tachiyomi.data.source.SourceManager
+import eu.kanade.tachiyomi.source.SourceManager
-import eu.kanade.tachiyomi.data.source.online.all.EHentai
+import eu.kanade.tachiyomi.source.online.all.EHentai
+import exh.isExSource
 import exh.metadata.MetadataHelper
 import exh.metadata.copyTo
 import timber.log.Timber
@@ -44,7 +45,7 @@ class MetadataFetchDialog {
                     .executeAsBlocking()
                     .filter {
                         it.source <= 2
-                        && !metadataHelper.hasMetadata(it.url, it.source == 2)
+                        && !metadataHelper.hasMetadata(it.url, isExSource(it.source))
                     }
 
             context.runOnUiThread {
@@ -91,7 +92,7 @@ class MetadataFetchDialog {
         db.getLibraryMangas().asRxSingle().subscribe {
             //Not logged in but have ExHentai galleries
             if(!preferenceHelper.enableExhentai().getOrDefault()) {
-                it.find { it.source == 2 }?.let {
+                it.find { isExSource(it.source) }?.let {
                     extra = "<b><font color='red'>If you use ExHentai, please log in first before fetching your library metadata!</font></b><br><br>"
                 }
             }

app/src/main/java/exh/ui/migration/MigrationCompletionActivity.kt

@@ -56,6 +56,9 @@ class MigrationCompletionActivity : BaseActivity() {
                     //Migrate urls
                     UrlMigrator().perform()
 
+                    //Migrate source IDs
+                    SourceMigrator().perform()
+
                     //Go back to MainActivity
                     //Set final steps
                     preferenceManager.migrationStatus().set(MigrationStatus.FINALIZE_MIGRATION)

app/src/main/java/exh/ui/migration/SourceMigrator.kt

@@ -1,5 +1,74 @@
 package exh.ui.migration
 
-/**
+import android.app.Activity
- * Created by nulldev on 2/28/17.
+import com.afollestad.materialdialogs.MaterialDialog
- */
+import eu.kanade.tachiyomi.data.backup.BackupManager
+import eu.kanade.tachiyomi.data.database.DatabaseHelper
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import eu.kanade.tachiyomi.data.preference.getOrDefault
+import exh.LEWD_SOURCE_SERIES
+import timber.log.Timber
+import uy.kohesive.injekt.injectLazy
+import java.io.File
+import kotlin.concurrent.thread
+
+class SourceMigrator {
+
+    val db: DatabaseHelper by injectLazy()
+
+    val prefs: PreferencesHelper by injectLazy()
+
+    val backupManager by lazy {
+        BackupManager(db)
+    }
+
+    fun perform() {
+        db.insertMangas(db.getMangas().executeAsBlocking().map {
+            if(it.source < 100) {
+                if(it.url.trim('/').startsWith("g/")) {
+                    //EH source, move ID
+                    it.source += LEWD_SOURCE_SERIES
+                }
+            } else if(it.source < 200) {
+                //Regular source, move ID down
+                it.source -= 100
+            }
+            it
+        }).executeAsBlocking()
+    }
+
+    fun tryMigrationWithDialog(context: Activity, callback: () -> Unit) {
+        if(!prefs.hasPerformedSourceMigration().getOrDefault()) {
+            val dialog = MaterialDialog.Builder(context)
+                    .title("Migrating galleries")
+                    .progress(true, 0)
+                    .cancelable(false)
+                    .show()
+
+            thread {
+                try {
+                    context.runOnUiThread {
+                        dialog.setContent("Backing up library...")
+                    }
+                    backupManager.backupToFile(File(context.filesDir, "teh-source-migration-bck.json"))
+                    context.runOnUiThread {
+                        dialog.setContent("Performing migration...")
+                    }
+                    perform()
+                    context.runOnUiThread {
+                        dialog.setContent("Completing migration...")
+                    }
+                    prefs.hasPerformedSourceMigration().set(true)
+                    dialog.dismiss()
+                } catch(e: Exception) {
+                    Timber.e(e, "Error migrating source IDs!")
+                }
+                context.runOnUiThread {
+                    callback()
+                }
+            }
+        } else {
+            callback()
+        }
+    }
+}

app/src/main/java/exh/ui/migration/UrlMigrator.kt

@@ -4,6 +4,8 @@ import eu.kanade.tachiyomi.data.database.DatabaseHelper
 import eu.kanade.tachiyomi.data.database.models.Manga
 import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.data.preference.getOrDefault
+import exh.isExSource
+import exh.isLewdSource
 import exh.metadata.MetadataHelper
 import uy.kohesive.injekt.injectLazy
 
@@ -21,8 +23,7 @@ class UrlMigrator {
 
             //Find all EX mangas
             val qualifyingMangas = dbMangas.asSequence().filter {
-                it.source > 0
+                isLewdSource(it.source)
-                        && it.source <= 4
             }
 
             val possibleDups = mutableListOf<Manga>()
@@ -42,8 +43,7 @@ class UrlMigrator {
                 //Build fixed URL
                 val urlWithSlash = "/" + it.url
                 //Fix metadata if required
-                val metadata = metadataHelper.fetchMetadata(it.url, it.source == 2
+                val metadata = metadataHelper.fetchMetadata(it.url, isExSource(it.source))
-                        || it.source == 4)
                 metadata?.url?.let {
                     if(it.startsWith("g/")) { //Check if metadata URL has no slash
                         metadata.url = urlWithSlash //Fix it