diff --git a/.gitignore b/.gitignore index 4169827c4..5d311578b 100755 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ */build /mainframer /.mainframer -*.apk \ No newline at end of file +*.apk +TODO.md +CHANGELOG.md \ No newline at end of file diff --git a/app/build.gradle b/app/build.gradle index 024e6218f..b6357730c 100755 --- a/app/build.gradle +++ b/app/build.gradle @@ -241,6 +241,7 @@ dependencies { testImplementation "org.robolectric:shadows-play-services:$robolectric_version" implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + implementation "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version" final coroutines_version = '0.22.2' implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutines_version" @@ -262,6 +263,11 @@ dependencies { // Debug network interceptor (EH) devImplementation "com.squareup.okhttp3:logging-interceptor:3.10.0" + // Serialization + implementation ("com.fasterxml.jackson.module:jackson-module-kotlin:2.9.+") { + exclude group: 'org.jetbrains.kotlin', module: 'kotlin-reflect' + } + // Firebase (EH) implementation 'com.google.firebase:firebase-perf:16.0.0' implementation 'com.crashlytics.sdk.android:crashlytics:2.9.4' diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index 39999f474..4e0044461 100755 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -93,6 +93,10 @@ # [EH] -keep class exh.** { *; } +-dontwarn com.fasterxml.jackson.databind.ext.DOMSerializer +-dontwarn com.fasterxml.jackson.databind.ext.Java7SupportImpl +-dontwarn com.fasterxml.jackson.module.kotlin.KotlinNamesAnnotationIntrospector$hasCreatorAnnotation$1 +-dontwarn com.fasterxml.jackson.module.kotlin.KotlinValueInstantiator # Realm -dontnote rx.internal.util.PlatformDependent diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index fe978995c..aee13184b 100755 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -100,7 +100,7 @@ + android:theme="@style/Theme.EHActivity"> @@ -191,7 +191,8 @@ + android:launchMode="singleInstance" + android:theme="@style/Theme.EHActivity" /> diff --git a/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt b/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt index 3a4694718..48eb2f061 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/Migrations.kt @@ -8,6 +8,8 @@ import java.io.File object Migrations { + // TODO NATIVE TACHIYOMI MIGRATIONS ARE FUCKED UP DUE TO DIFFERING VERSION NUMBERS + /** * Performs a migration when the application is updated. * diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/DatabaseHelper.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/DatabaseHelper.kt index 6148e49ae..fb3803cfa 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/DatabaseHelper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/DatabaseHelper.kt @@ -6,18 +6,28 @@ import com.pushtorefresh.storio.sqlite.impl.DefaultStorIOSQLite import eu.kanade.tachiyomi.data.database.mappers.* import eu.kanade.tachiyomi.data.database.models.* import eu.kanade.tachiyomi.data.database.queries.* +import exh.metadata.sql.mappers.SearchMetadataTypeMapping +import exh.metadata.sql.mappers.SearchTagTypeMapping +import exh.metadata.sql.mappers.SearchTitleTypeMapping +import exh.metadata.sql.models.SearchMetadata +import exh.metadata.sql.models.SearchTag +import exh.metadata.sql.models.SearchTitle +import exh.metadata.sql.queries.SearchMetadataQueries +import exh.metadata.sql.queries.SearchTagQueries +import exh.metadata.sql.queries.SearchTitleQueries import io.requery.android.database.sqlite.RequerySQLiteOpenHelperFactory /** * This class provides operations to manage the database through its interfaces. */ open class DatabaseHelper(context: Context) -: MangaQueries, ChapterQueries, TrackQueries, CategoryQueries, MangaCategoryQueries, HistoryQueries { - + : MangaQueries, ChapterQueries, TrackQueries, CategoryQueries, MangaCategoryQueries, HistoryQueries, + /* EXH --> */ SearchMetadataQueries, SearchTagQueries, SearchTitleQueries /* EXH <-- */ +{ private val configuration = SupportSQLiteOpenHelper.Configuration.builder(context) - .name(DbOpenCallback.DATABASE_NAME) - .callback(DbOpenCallback()) - .build() + .name(DbOpenCallback.DATABASE_NAME) + .callback(DbOpenCallback()) + .build() override val db = DefaultStorIOSQLite.builder() .sqliteOpenHelper(RequerySQLiteOpenHelperFactory().create(configuration)) @@ -27,6 +37,11 @@ open class DatabaseHelper(context: Context) .addTypeMapping(Category::class.java, CategoryTypeMapping()) .addTypeMapping(MangaCategory::class.java, MangaCategoryTypeMapping()) .addTypeMapping(History::class.java, HistoryTypeMapping()) + // EXH --> + .addTypeMapping(SearchMetadata::class.java, SearchMetadataTypeMapping()) + .addTypeMapping(SearchTag::class.java, SearchTagTypeMapping()) + .addTypeMapping(SearchTitle::class.java, SearchTitleTypeMapping()) + // EXH <-- .build() inline fun inTransaction(block: () -> Unit) = db.inTransaction(block) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt index cf65b6a1d..a2c532bbf 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt @@ -2,10 +2,10 @@ package eu.kanade.tachiyomi.data.database import android.arch.persistence.db.SupportSQLiteDatabase import android.arch.persistence.db.SupportSQLiteOpenHelper -import android.content.Context -import android.database.sqlite.SQLiteDatabase -import android.database.sqlite.SQLiteOpenHelper import eu.kanade.tachiyomi.data.database.tables.* +import exh.metadata.sql.tables.SearchMetadataTable +import exh.metadata.sql.tables.SearchTagTable +import exh.metadata.sql.tables.SearchTitleTable class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) { @@ -18,7 +18,7 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) { /** * Version of the database. */ - const val DATABASE_VERSION = 8 + const val DATABASE_VERSION = 9 // [EXH] } override fun onCreate(db: SupportSQLiteDatabase) = with(db) { @@ -28,6 +28,11 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) { execSQL(CategoryTable.createTableQuery) execSQL(MangaCategoryTable.createTableQuery) execSQL(HistoryTable.createTableQuery) + // EXH --> + execSQL(SearchMetadataTable.createTableQuery) + execSQL(SearchTagTable.createTableQuery) + execSQL(SearchTitleTable.createTableQuery) + // EXH <-- // DB indexes execSQL(MangaTable.createUrlIndexQuery) @@ -35,6 +40,14 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) { execSQL(ChapterTable.createMangaIdIndexQuery) execSQL(ChapterTable.createUnreadChaptersIndexQuery) execSQL(HistoryTable.createChapterIdIndexQuery) + // EXH --> + db.execSQL(SearchMetadataTable.createUploaderIndexQuery) + db.execSQL(SearchMetadataTable.createIndexedExtraIndexQuery) + db.execSQL(SearchTagTable.createMangaIdIndexQuery) + db.execSQL(SearchTagTable.createNamespaceNameIndexQuery) + db.execSQL(SearchTitleTable.createMangaIdIndexQuery) + db.execSQL(SearchTitleTable.createTitleIndexQuery) + // EXH <-- } override fun onUpgrade(db: SupportSQLiteDatabase, oldVersion: Int, newVersion: Int) { @@ -67,6 +80,21 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) { db.execSQL(MangaTable.createLibraryIndexQuery) db.execSQL(ChapterTable.createUnreadChaptersIndexQuery) } + // EXH --> + if (oldVersion < 9) { + db.execSQL(SearchMetadataTable.createTableQuery) + db.execSQL(SearchTagTable.createTableQuery) + db.execSQL(SearchTitleTable.createTableQuery) + + db.execSQL(SearchMetadataTable.createUploaderIndexQuery) + db.execSQL(SearchMetadataTable.createIndexedExtraIndexQuery) + db.execSQL(SearchTagTable.createMangaIdIndexQuery) + db.execSQL(SearchTagTable.createNamespaceNameIndexQuery) + db.execSQL(SearchTitleTable.createMangaIdIndexQuery) + db.execSQL(SearchTitleTable.createTitleIndexQuery) + } + // Remember to increment any Tachiyomi database upgrades after this + // EXH <-- } override fun onConfigure(db: SupportSQLiteDatabase) { diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaUrlPutResolver.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaUrlPutResolver.kt new file mode 100644 index 000000000..ca4d7fc0e --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaUrlPutResolver.kt @@ -0,0 +1,34 @@ +package eu.kanade.tachiyomi.data.database.resolvers + +import android.content.ContentValues +import com.pushtorefresh.storio.sqlite.StorIOSQLite +import com.pushtorefresh.storio.sqlite.operations.put.PutResolver +import com.pushtorefresh.storio.sqlite.operations.put.PutResult +import com.pushtorefresh.storio.sqlite.queries.UpdateQuery +import eu.kanade.tachiyomi.data.database.inTransactionReturn +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.database.tables.MangaTable + +// [EXH] +class MangaUrlPutResolver : PutResolver() { + + override fun performPut(db: StorIOSQLite, manga: Manga) = db.inTransactionReturn { + val updateQuery = mapToUpdateQuery(manga) + val contentValues = mapToContentValues(manga) + + val numberOfRowsUpdated = db.lowLevel().update(updateQuery, contentValues) + PutResult.newUpdateResult(numberOfRowsUpdated, updateQuery.table()) + } + + fun mapToUpdateQuery(manga: Manga) = UpdateQuery.builder() + .table(MangaTable.TABLE) + .where("${MangaTable.COL_ID} = ?") + .whereArgs(manga.id) + .build() + + fun mapToContentValues(manga: Manga) = ContentValues(1).apply { + put(MangaTable.COL_URL, manga.url) + } + +} + diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt index 4c3450517..7189a0e22 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt @@ -157,15 +157,7 @@ object PreferenceKeys { const val eh_ts_aspNetCookie = "eh_ts_aspNetCookie" - const val eh_showSettingsUploadWarning = "eh_showSettingsUploadWarning1" - - const val eh_hl_earlyRefresh = "eh_hl_early_refresh" - - const val eh_hl_refreshFrequency = "eh_hl_refresh_frequency" - - const val eh_hl_lastRefresh = "eh_hl_last_refresh" - - const val eh_hl_lastRealmIndex = "eh_hl_lastRealmIndex" + const val eh_showSettingsUploadWarning = "eh_showSettingsUploadWarning2" const val eh_expandFilters = "eh_expand_filters" @@ -182,4 +174,8 @@ object PreferenceKeys { const val eh_preserveReadingPosition = "eh_preserve_reading_position" const val eh_incogWebview = "eh_incognito_webview" + + const val eh_autoSolveCaptchas = "eh_autosolve_captchas" + + const val eh_delegateSources = "eh_delegate_sources" } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt index 647f0268c..25d21c1d6 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt @@ -189,16 +189,16 @@ class PreferencesHelper(val context: Context) { fun thumbnailRows() = rxPrefs.getString("ex_thumb_rows", "tr_2") - fun migrateLibraryAsked2() = rxPrefs.getBoolean("ex_migrate_library2", false) + fun migrateLibraryAsked() = rxPrefs.getBoolean("ex_migrate_library3", false) fun migrationStatus() = rxPrefs.getInteger("migration_status", MigrationStatus.NOT_INITIALIZED) fun hasPerformedURLMigration() = rxPrefs.getBoolean("performed_url_migration", false) //EH Cookies - fun memberIdVal() = rxPrefs.getString("eh_ipb_member_id", null) - fun passHashVal() = rxPrefs.getString("eh_ipb_pass_hash", null) - fun igneousVal() = rxPrefs.getString("eh_igneous", null) + fun memberIdVal() = rxPrefs.getString("eh_ipb_member_id", "") + fun passHashVal() = rxPrefs.getString("eh_ipb_pass_hash", "") + fun igneousVal() = rxPrefs.getString("eh_igneous", "") fun eh_ehSettingsProfile() = rxPrefs.getInteger(Keys.eh_ehSettingsProfile, -1) fun eh_exhSettingsProfile() = rxPrefs.getInteger(Keys.eh_exhSettingsProfile, -1) fun eh_settingsKey() = rxPrefs.getString(Keys.eh_settingsKey, "") @@ -228,16 +228,6 @@ class PreferencesHelper(val context: Context) { fun eh_showSettingsUploadWarning() = rxPrefs.getBoolean(Keys.eh_showSettingsUploadWarning, true) - // Default is 24h, refresh daily - fun eh_hl_earlyRefresh() = rxPrefs.getBoolean(Keys.eh_hl_earlyRefresh, false) - - fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24") - - fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L) - - fun eh_hl_lastRealmIndex() = rxPrefs.getInteger(Keys.eh_hl_lastRealmIndex, -1) - // <-- EH - fun eh_expandFilters() = rxPrefs.getBoolean(Keys.eh_expandFilters, false) fun eh_readerThreads() = rxPrefs.getInteger(Keys.eh_readerThreads, 2) @@ -253,4 +243,12 @@ class PreferencesHelper(val context: Context) { fun eh_incogWebview() = rxPrefs.getBoolean(Keys.eh_incogWebview, false) fun eh_askCategoryOnLongPress() = rxPrefs.getBoolean(Keys.eh_askCategoryOnLongPress, false) + + fun eh_autoSolveCaptchas() = rxPrefs.getBoolean(Keys.eh_autoSolveCaptchas, false) + + fun eh_delegateSources() = rxPrefs.getBoolean(Keys.eh_delegateSources, true) + + fun eh_lastVersionCode() = rxPrefs.getInteger("eh_last_version_code", 0) + + fun eh_savedSearches() = rxPrefs.getString("eh_saved_searches", "") } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt b/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt index e07e58c1a..ffec9db8d 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt @@ -22,8 +22,12 @@ import exh.EH_SOURCE_ID import exh.EXH_SOURCE_ID import exh.PERV_EDEN_EN_SOURCE_ID import exh.PERV_EDEN_IT_SOURCE_ID -import exh.metadata.models.PervEdenLang +import exh.metadata.metadata.PervEdenLang +import exh.source.DelegatedHttpSource +import exh.source.EnhancedHttpSource +import timber.log.Timber import uy.kohesive.injekt.injectLazy +import kotlin.reflect.KClass open class SourceManager(private val context: Context) { @@ -66,8 +70,17 @@ open class SourceManager(private val context: Context) { fun getCatalogueSources() = sourcesMap.values.filterIsInstance() internal fun registerSource(source: Source, overwrite: Boolean = false) { + val sourceQName = source::class.qualifiedName + val delegate = DELEGATED_SOURCES[sourceQName] + val newSource = if(source is HttpSource && delegate != null) { + Timber.d("[EXH] Delegating source: %s -> %s!", sourceQName, delegate.newSourceClass.qualifiedName) + EnhancedHttpSource( + source, + delegate.newSourceClass.constructors.find { it.parameters.size == 1 }!!.call(source) + ) + } else source if (overwrite || !sourcesMap.containsKey(source.id)) { - sourcesMap[source.id] = source + sourcesMap[source.id] = newSource } } @@ -99,9 +112,8 @@ open class SourceManager(private val context: Context) { exSrcs += PervEden(PERV_EDEN_EN_SOURCE_ID, PervEdenLang.en) exSrcs += PervEden(PERV_EDEN_IT_SOURCE_ID, PervEdenLang.it) exSrcs += NHentai(context) - exSrcs += HentaiCafe() exSrcs += Tsumino(context) - exSrcs += Hitomi(context) + exSrcs += Hitomi() return exSrcs } @@ -130,4 +142,20 @@ open class SourceManager(private val context: Context) { return Exception(context.getString(R.string.source_not_installed, id.toString())) } } + + companion object { + val DELEGATED_SOURCES = listOf( + DelegatedSource( + "Hentai Cafe", + 260868874183818481, + "eu.kanade.tachiyomi.extension.all.foolslide.HentaiCafe", + HentaiCafe::class + ) + ).associateBy { it.originalSourcePackageName } + + data class DelegatedSource(val sourceName: String, + val sourceId: Long, + val originalSourcePackageName: String, + val newSourceClass: KClass) + } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt index 5a2118842..c26041431 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt @@ -1,54 +1,102 @@ package eu.kanade.tachiyomi.source.online +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Chapter +import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.source.CatalogueSource +import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga -import exh.metadata.models.GalleryQuery -import exh.metadata.models.SearchableGalleryMetadata -import exh.util.createUUIDObj -import exh.util.defRealm -import exh.util.realmTrans -import rx.Observable +import exh.metadata.metadata.base.RaisedSearchMetadata +import exh.metadata.metadata.base.getFlatMetadataForManga +import exh.metadata.metadata.base.insertFlatMetadata +import rx.Completable +import rx.Single +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import kotlin.reflect.KClass /** * LEWD! */ -interface LewdSource : CatalogueSource { - fun queryAll(): GalleryQuery +interface LewdSource : CatalogueSource { + val db: DatabaseHelper get() = Injekt.get() - fun queryFromUrl(url: String): GalleryQuery + /** + * The class of the metadata used by this source + */ + val metaClass: KClass - val metaParser: M.(I) -> Unit + /** + * Parse the supplied input into the supplied metadata object + */ + fun parseIntoMetadata(metadata: M, input: I) - fun parseToManga(query: GalleryQuery, input: I): SManga - = realmTrans { realm -> - val meta = realm.copyFromRealm(query.query(realm).findFirst() - ?: realm.createUUIDObj(queryAll().clazz.java)) + /** + * Use reflection to create a new instance of metadata + */ + private fun newMetaInstance() = metaClass.constructors.find { + it.parameters.isEmpty() + }?.call() ?: error("Could not find no-args constructor for meta class: ${metaClass.qualifiedName}!") - metaParser(meta, input) - - realm.copyToRealmOrUpdate(meta) - - SManga.create().apply { - meta.copyTo(this) - } - } - - fun lazyLoadMeta(query: GalleryQuery, parserInput: Observable): Observable { - return defRealm { realm -> - val possibleOutput = query.query(realm).findFirst() - - if(possibleOutput == null) - parserInput.map { - realmTrans { realm -> - val meta = realm.createUUIDObj(queryAll().clazz.java) - - metaParser(meta, it) - - realm.copyFromRealm(meta) + /** + * Parses metadata from the input and then copies it into the manga + * + * Will also save the metadata to the DB if possible + */ + fun parseToManga(manga: SManga, input: I): Completable { + val mangaId = (manga as? Manga)?.id + val metaObservable = if(mangaId != null) { + db.getFlatMetadataForManga(mangaId).asRxSingle() + .map { + if(it != null) it.raise(metaClass) + else newMetaInstance() } - } - else - Observable.just(realm.copyFromRealm(possibleOutput)) + } else { + Single.just(newMetaInstance()) + } + + return metaObservable.map { + parseIntoMetadata(it, input) + it.copyTo(manga) + it + }.flatMapCompletable { + if(mangaId != null) { + it.mangaId = mangaId + db.insertFlatMetadata(it.flatten()) + } else Completable.complete() } } + + /** + * Try to first get the metadata from the DB. If the metadata is not in the DB, calls the input + * producer and parses the metadata from the input + * + * If the metadata needs to be parsed from the input producer, the resulting parsed metadata will + * also be saved to the DB. + */ + fun getOrLoadMetadata(mangaId: Long?, inputProducer: () -> Single): Single { + val metaObservable = if(mangaId != null) { + db.getFlatMetadataForManga(mangaId).asRxSingle() + .map { + it?.raise(metaClass) + } + } else Single.just(null) + + return metaObservable.flatMap { existingMeta -> + if(existingMeta == null) { + inputProducer().flatMap { input -> + val newMeta = newMetaInstance() + parseIntoMetadata(newMeta, input) + val newMetaSingle = Single.just(newMeta) + if(mangaId != null) { + newMeta.mangaId = mangaId + db.insertFlatMetadata(newMeta.flatten()).andThen(newMetaSingle) + } else newMetaSingle + } + } else Single.just(existingMeta) + } + } + + val SManga.id get() = (this as? Manga)?.id + val SChapter.mangaId get() = (this as? Chapter)?.manga_id } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt index 56d966305..cb759e28a 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/EHentai.kt @@ -12,8 +12,12 @@ import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.LewdSource import eu.kanade.tachiyomi.util.asJsoup import exh.metadata.EX_DATE_FORMAT +import exh.metadata.metadata.EHentaiSearchMetadata +import exh.metadata.metadata.EHentaiSearchMetadata.Companion.EH_GENRE_NAMESPACE +import exh.metadata.metadata.EHentaiSearchMetadata.Companion.TAG_TYPE_LIGHT +import exh.metadata.metadata.EHentaiSearchMetadata.Companion.TAG_TYPE_NORMAL +import exh.metadata.metadata.base.RaisedSearchMetadata.Companion.TAG_TYPE_VIRTUAL import exh.metadata.models.ExGalleryMetadata -import exh.metadata.models.Tag import exh.metadata.nullIfBlank import exh.metadata.parseHumanReadableByteCount import exh.ui.login.LoginController @@ -28,10 +32,12 @@ import rx.Observable import uy.kohesive.injekt.injectLazy import java.net.URLEncoder import java.util.* +import exh.metadata.metadata.base.RaisedTag class EHentai(override val id: Long, val exh: Boolean, - val context: Context) : HttpSource(), LewdSource { + val context: Context) : HttpSource(), LewdSource { + override val metaClass = EHentaiSearchMetadata::class val schema: String get() = if(prefs.secureEXH().getOrDefault()) @@ -60,26 +66,29 @@ class EHentai(override val id: Long, fun extendedGenericMangaParse(doc: Document) = with(doc) { - //Parse mangas - val parsedMangas = select(".gtr0,.gtr1").map { + // Parse mangas (supports compact + extended layout) + val parsedMangas = select(".itg > tbody > tr").filter { + // Do not parse header and ads + it.selectFirst("th") == null && it.selectFirst(".itd") == null + }.map { + val thumbnailElement = it.selectFirst(".gl1e img, .gl2c .glthumb img") + val column2 = it.selectFirst(".gl3e, .gl2c") + val linkElement = it.selectFirst(".gl3c > a, .gl2e > div > a") + + val favElement = column2.children().find { it.attr("style").startsWith("border-color") } + ParsedManga( - fav = parseFavoritesStyle(it.select(".itd .it3 > .i[id]").first()?.attr("style")), + fav = FAVORITES_BORDER_HEX_COLORS.indexOf( + favElement?.attr("style")?.substring(14, 17) + ), manga = Manga.create(id).apply { //Get title - it.select(".itd .it5 a").first()?.apply { - title = text() - url = ExGalleryMetadata.normalizeUrl(attr("href")) - } + title = thumbnailElement.attr("title") + url = EHentaiSearchMetadata.normalizeUrl(linkElement.attr("href")) //Get image - it.select(".itd .it2").first()?.apply { - children().first()?.let { - thumbnail_url = it.attr("src") - } ?: let { - text().split("~").apply { - thumbnail_url = "http://${this[1]}/${this[2]}" - } - } - } + thumbnail_url = thumbnailElement.attr("src") + + // TODO Parse genre + uploader + tags }) } @@ -97,14 +106,6 @@ class EHentai(override val id: Long, Pair(parsedMangas, hasNextPage) } - fun parseFavoritesStyle(style: String?): Int { - val offset = style?.substringAfterLast("background-position:0px ") - ?.removeSuffix("px; cursor:pointer") - ?.toIntOrNull() ?: return -1 - - return (offset + 2)/-19 - } - /** * Parse a list of galleries */ @@ -158,17 +159,17 @@ class EHentai(override val id: Long, override fun popularMangaRequest(page: Int) = if(exh) latestUpdatesRequest(page) else - exGet("$baseUrl/toplist.php?tl=15", page) + exGet("$baseUrl/toplist.php?tl=15&p=${page - 1}", null) // Custom page logic for toplists //Support direct URL importing override fun fetchSearchManga(page: Int, query: String, filters: FilterList) = - urlImportFetchSearchManga(query, { + urlImportFetchSearchManga(query) { searchMangaRequestObservable(page, query, filters).flatMap { client.newCall(it).asObservableSuccess() } .map { response -> searchMangaParse(response) } - }) + } private fun searchMangaRequestObservable(page: Int, query: String, filters: FilterList): Observable { val uri = Uri.parse("$baseUrl$QUERY_PREFIX").buildUpon() @@ -229,82 +230,112 @@ class EHentai(override val id: Long, }!! /** - * Parse gallery page to metadata model + * Returns an observable with the updated details for a manga. Normally it's not needed to + * override this method. + * + * @param manga the manga to be updated. */ - override fun mangaDetailsParse(response: Response): SManga { - return parseToManga(queryFromUrl(response.request().url().toString()), response) + override fun fetchMangaDetails(manga: SManga): Observable { + return client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .flatMap { + parseToManga(manga, it).andThen(Observable.just(manga.apply { + initialized = true + })) + } } - override val metaParser: ExGalleryMetadata.(Response) -> Unit = { response -> - with(response.asJsoup()) { - url = response.request().url().encodedPath()!! - gId = ExGalleryMetadata.galleryId(url!!) - gToken = ExGalleryMetadata.galleryToken(url!!) + /** + * Parse gallery page to metadata model + */ + override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException() - exh = this@EHentai.exh - title = select("#gn").text().nullIfBlank()?.trim() + override fun parseIntoMetadata(metadata: EHentaiSearchMetadata, input: Response) { + with(metadata) { + with(input.asJsoup()) { + val url = input.request().url().encodedPath()!! + gId = ExGalleryMetadata.galleryId(url) + gToken = ExGalleryMetadata.galleryToken(url) - altTitle = select("#gj").text().nullIfBlank()?.trim() + exh = this@EHentai.exh + title = select("#gn").text().nullIfBlank()?.trim() - thumbnailUrl = select("#gd1 div").attr("style").nullIfBlank()?.let { - it.substring(it.indexOf('(') + 1 until it.lastIndexOf(')')) - } - genre = select(".ic").parents().attr("href").nullIfBlank()?.trim()?.substringAfterLast('/') + altTitle = select("#gj").text().nullIfBlank()?.trim() - uploader = select("#gdn").text().nullIfBlank()?.trim() - - //Parse the table - select("#gdd tr").forEach { - it.select(".gdt1") - .text() + thumbnailUrl = select("#gd1 div").attr("style").nullIfBlank()?.let { + it.substring(it.indexOf('(') + 1 until it.lastIndexOf(')')) + } + genre = select(".cs") + .attr("onclick") .nullIfBlank() ?.trim() - ?.let { left -> - it.select(".gdt2") - .text() - .nullIfBlank() - ?.trim() - ?.let { right -> - ignore { - when (left.removeSuffix(":") - .toLowerCase()) { - "posted" -> datePosted = EX_DATE_FORMAT.parse(right).time - "visible" -> visible = right.nullIfBlank() - "language" -> { - language = right.removeSuffix(TR_SUFFIX).trim().nullIfBlank() - translated = right.endsWith(TR_SUFFIX, true) - } - "file size" -> size = parseHumanReadableByteCount(right)?.toLong() - "length" -> length = right.removeSuffix("pages").trim().nullIfBlank()?.toInt() - "favorited" -> favorites = right.removeSuffix("times").trim().nullIfBlank()?.toInt() - } - } - } + ?.substringAfterLast('/') + ?.removeSuffix("'") + + uploader = select("#gdn").text().nullIfBlank()?.trim() + + //Parse the table + select("#gdd tr").forEach { + val left = it.select(".gdt1").text().nullIfBlank()?.trim() + val rightElement = it.selectFirst(".gdt2") + val right = rightElement.text().nullIfBlank()?.trim() + if(left != null && right != null) { + ignore { + when (left.removeSuffix(":") + .toLowerCase()) { + "posted" -> datePosted = EX_DATE_FORMAT.parse(right).time + // Example gallery with parent: https://e-hentai.org/g/1390451/7f181c2426/ + "parent" -> parent = if (!right.equals("None", true)) { + rightElement.child(0).attr("href") + } else null + "visible" -> visible = right.nullIfBlank() + "language" -> { + language = right.removeSuffix(TR_SUFFIX).trim().nullIfBlank() + translated = right.endsWith(TR_SUFFIX, true) + } + "file size" -> size = parseHumanReadableByteCount(right)?.toLong() + "length" -> length = right.removeSuffix("pages").trim().nullIfBlank()?.toInt() + "favorited" -> favorites = right.removeSuffix("times").trim().nullIfBlank()?.toInt() + } } - } + } + } - //Parse ratings - ignore { - averageRating = select("#rating_label") - .text() - .removePrefix("Average:") - .trim() - .nullIfBlank() - ?.toDouble() - ratingCount = select("#rating_count") - .text() - .trim() - .nullIfBlank() - ?.toInt() - } + //Parse ratings + ignore { + averageRating = select("#rating_label") + .text() + .removePrefix("Average:") + .trim() + .nullIfBlank() + ?.toDouble() + ratingCount = select("#rating_count") + .text() + .trim() + .nullIfBlank() + ?.toInt() + } - //Parse tags - tags.clear() - select("#taglist tr").forEach { - val namespace = it.select(".tc").text().removeSuffix(":") - tags.addAll(it.select("div").map { - Tag(namespace, it.text().trim(), it.hasClass("gtl")) - }) + //Parse tags + tags.clear() + select("#taglist tr").forEach { + val namespace = it.select(".tc").text().removeSuffix(":") + tags.addAll(it.select("div").map { element -> + RaisedTag( + namespace, + element.text().trim(), + if(element.hasClass("gtl")) + TAG_TYPE_LIGHT + else + TAG_TYPE_NORMAL + ) + }) + } + + // Add genre as virtual tag + genre?.let { + tags.add(RaisedTag(EH_GENRE_NAMESPACE, it, TAG_TYPE_VIRTUAL)) + } } } } @@ -392,8 +423,11 @@ class EHentai(override val id: Long, cookies["hath_perks"] = hathPerksCookie } - //Session-less list display mode (for users without ExHentai) - cookies["sl"] = "dm_0" + // Session-less extended display mode (for users without ExHentai) + cookies["sl"] = "dm_2" + + // Ignore all content warnings + cookies["nw"] = "1" return cookies } @@ -431,23 +465,26 @@ class EHentai(override val id: Long, ReverseFilter() ) - class GenreOption(name: String, val genreId: String): Filter.CheckBox(name, false), UriFilter { + class GenreOption(name: String, val genreId: Int): Filter.CheckBox(name, false) + class GenreGroup : Filter.Group("Genres", listOf( + GenreOption("Dōjinshi", 2), + GenreOption("Manga", 4), + GenreOption("Artist CG", 8), + GenreOption("Game CG", 16), + GenreOption("Western", 512), + GenreOption("Non-H", 256), + GenreOption("Image Set", 32), + GenreOption("Cosplay", 64), + GenreOption("Asian Porn", 128), + GenreOption("Misc", 1) + )), UriFilter { override fun addToUri(builder: Uri.Builder) { - builder.appendQueryParameter("f_" + genreId, if(state) "1" else "0") + val bits = state.fold(0) { acc, genre -> + if(!genre.state) acc + genre.genreId else acc + } + builder.appendQueryParameter("f_cats", bits.toString()) } } - class GenreGroup : UriGroup("Genres", listOf( - GenreOption("Dōjinshi", "doujinshi"), - GenreOption("Manga", "manga"), - GenreOption("Artist CG", "artistcg"), - GenreOption("Game CG", "gamecg"), - GenreOption("Western", "western"), - GenreOption("Non-H", "non-h"), - GenreOption("Image Set", "imageset"), - GenreOption("Cosplay", "cosplay"), - GenreOption("Asian Porn", "asianporn"), - GenreOption("Misc", "misc") - )) class AdvancedOption(name: String, val param: String, defValue: Boolean = false): Filter.CheckBox(name, defValue), UriFilter { override fun addToUri(builder: Uri.Builder) { @@ -486,13 +523,23 @@ class EHentai(override val id: Long, else "E-Hentai" - override fun queryAll() = ExGalleryMetadata.EmptyQuery() - override fun queryFromUrl(url: String) = ExGalleryMetadata.UrlQuery(url, exh) - companion object { - val QUERY_PREFIX = "?f_apply=Apply+Filter" - val TR_SUFFIX = "TR" - val REVERSE_PARAM = "TEH_REVERSE" + private const val QUERY_PREFIX = "?f_apply=Apply+Filter" + private const val TR_SUFFIX = "TR" + private const val REVERSE_PARAM = "TEH_REVERSE" + + private val FAVORITES_BORDER_HEX_COLORS = listOf( + "000", + "f00", + "fa0", + "dd0", + "080", + "9f4", + "4bf", + "00f", + "508", + "e8e" + ) fun buildCookies(cookies: Map) = cookies.entries.joinToString(separator = "; ") { diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt index 7c64a9ace..37772ef9c 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt @@ -1,15 +1,10 @@ package eu.kanade.tachiyomi.source.online.all -import android.content.Context import android.os.Build -import android.os.HandlerThread -import com.github.salomonbrys.kotson.* -import com.google.gson.JsonObject +import com.github.salomonbrys.kotson.array +import com.github.salomonbrys.kotson.get +import com.github.salomonbrys.kotson.string import com.google.gson.JsonParser -import com.google.gson.stream.JsonReader -import com.squareup.duktape.Duktape -import eu.kanade.tachiyomi.data.preference.PreferencesHelper -import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.model.* @@ -17,734 +12,379 @@ import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.LewdSource import eu.kanade.tachiyomi.util.asJsoup import exh.HITOMI_SOURCE_ID -import exh.metadata.EMULATED_TAG_NAMESPACE -import exh.metadata.models.HitomiGalleryMetadata -import exh.metadata.models.HitomiGalleryMetadata.Companion.BASE_URL -import exh.metadata.models.HitomiGalleryMetadata.Companion.LTN_BASE_URL -import exh.metadata.models.HitomiGalleryMetadata.Companion.hlIdFromUrl -import exh.metadata.models.HitomiPage -import exh.metadata.models.HitomiSkeletonGalleryMetadata -import exh.metadata.models.Tag -import exh.metadata.nullIfBlank -import exh.search.SearchEngine -import exh.util.* -import io.realm.Realm -import io.realm.RealmConfiguration -import io.realm.RealmResults +import exh.hitomi.HitomiNozomi +import exh.metadata.metadata.HitomiSearchMetadata +import exh.metadata.metadata.HitomiSearchMetadata.Companion.BASE_URL +import exh.metadata.metadata.HitomiSearchMetadata.Companion.LTN_BASE_URL +import exh.metadata.metadata.HitomiSearchMetadata.Companion.TAG_TYPE_DEFAULT +import exh.metadata.metadata.base.RaisedSearchMetadata.Companion.TAG_TYPE_VIRTUAL +import exh.metadata.metadata.base.RaisedTag +import exh.util.urlImportFetchSearchManga import okhttp3.Request import okhttp3.Response -import org.jsoup.nodes.Element +import org.jsoup.nodes.Document +import org.vepta.vdm.ByteCursor import rx.Observable -import rx.Scheduler -import rx.android.schedulers.AndroidSchedulers +import rx.Single import rx.schedulers.Schedulers -import rx.subjects.AsyncSubject -import timber.log.Timber -import uy.kohesive.injekt.injectLazy -import java.nio.BufferUnderflowException -import java.nio.ByteBuffer import java.text.SimpleDateFormat import java.util.* -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.locks.ReentrantLock -import kotlin.concurrent.thread /** - * WTF is going on in this class? + * Man, I hate this source :( */ -class Hitomi(private val context: Context) - :HttpSource(), LewdSource { - private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) { JsonParser() } - private val searchEngine by lazy { SearchEngine() } - private val prefs: PreferencesHelper by injectLazy() - - private val queryCache = mutableMapOf>() - private val queryWorkQueue = LinkedBlockingQueue>>>() - private var searchWorker: Thread? = null - - private var parseToMangaScheduler: Scheduler? = null - - override fun queryAll() = HitomiGalleryMetadata.EmptyQuery() - override fun queryFromUrl(url: String) = HitomiGalleryMetadata.UrlQuery(url) - - override val metaParser: HitomiGalleryMetadata.(HitomiSkeletonGalleryMetadata) -> Unit = { - hlId = it.hlId - thumbnailUrl = it.thumbnailUrl - artist = it.artist - group = it.group - type = it.type - language = it.language - languageSimple = it.languageSimple - series.clear() - series.addAll(it.series) - characters.clear() - characters.addAll(it.characters) - buyLink = it.buyLink - uploadDate = it.uploadDate - tags.clear() - tags.addAll(it.tags) - title = it.title - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Unused method called!") - - override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!") - - override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException("Unused method called!") - - /** >>> PARSE TO MANGA SCHEDULER <<< **/ - /* - Realm becomes very, very slow after you start opening and closing Realms rapidly. - By keeping a global Realm open at all times, we can migitate this. - Realms are per-thread so we create our own RxJava scheduler to schedule realm-heavy - operations on. - */ - @Synchronized - private fun startParseToMangaScheduler() { - if(parseToMangaScheduler != null) return - - val thread = object : HandlerThread("parse-to-manga-thread") { - override fun onLooperPrepared() { - // Open permanent Realm instance on this thread! - Realm.getDefaultInstance() - } - } - - thread.start() - parseToMangaScheduler = AndroidSchedulers.from(thread.looper) - } - - private fun parseToMangaScheduler(): Scheduler { - startParseToMangaScheduler() - return parseToMangaScheduler!! - } - - /** >>> SEARCH WORKER <<< **/ - /* - Running RealmResults.size on a new RealmResults object is very, very slow. - By caching our RealmResults in memory, we avoid creating many new RealmResults objects, - thus speeding up RealmResults.size. - - Realms are per-thread and RealmResults are bound to Realms. Therefore we create a - permanent thread that will open a permanent realm and wait for requests to load RealmResults. - */ - - @Synchronized - private fun startSearchWorker() { - if(searchWorker != null) return - - searchWorker = thread { - ensureCacheLoaded().toBlocking().first() - - val realms = arrayOf(getCacheRealm(0), getCacheRealm(1)) - - Timber.d("[SW] New search worker thread started!") - while (true) { - val realm = realms[prefs.eh_hl_lastRealmIndex().getOrDefault()] - - Timber.d("[SW] Waiting for next query!") - val next = queryWorkQueue.take() - Timber.d("[SW] Found new query (page ${next.second}): ${next.first}") - - if(queryCache[next.first] == null) { - val first = realm.where(HitomiSkeletonGalleryMetadata::class.java).findFirst() - - if (first == null) { - next.third.onNext(emptyList()) - next.third.onCompleted() - continue - } - - val parsed = searchEngine.parseQuery(next.first) - val filtered = searchEngine.filterResults(realm.where(HitomiSkeletonGalleryMetadata::class.java), - parsed, - first.titleFields).findAll() - - queryCache[next.first] = filtered - } - - val filtered = queryCache[next.first]!! - - val beginIndex = (next.second - 1) * PAGE_SIZE - if (beginIndex > filtered.lastIndex) { - next.third.onNext(emptyList()) - next.third.onCompleted() - continue - } - - // Chunk into pages of 100 - val res = realm.copyFromRealm(filtered.subList(beginIndex, - Math.min(next.second * PAGE_SIZE, filtered.size))) - - next.third.onNext(res) - next.third.onCompleted() - } - } - } - - private fun trySearch(page: Int, query: String): Observable> { - startSearchWorker() - - val subject = AsyncSubject.create>() - queryWorkQueue.clear() - queryWorkQueue.add(Triple(query, page, subject)) - return subject - } - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - return urlImportFetchSearchManga(query) { - trySearch(page, query).map { - val res = it.map { - SManga.create().apply { - setUrlWithoutDomain(it.url!!) - - title = it.title!! - - it.thumbnailUrl?.let { - thumbnail_url = it - } - } - } - - MangasPage(res, it.isNotEmpty()) - } - } - } - - override fun fetchMangaDetails(manga: SManga): Observable { - return lazyLoadMetaPages(HitomiGalleryMetadata.hlIdFromUrl(manga.url), true) - .map { - val newManga = parseToManga(queryFromUrl(manga.url), it.first) - manga.copyFrom(newManga) - // Forcibly copy title as copyFrom does not - manga.title = newManga.title - - manga - } - .subscribeOn(parseToMangaScheduler()) - } - - override fun fetchChapterList(manga: SManga): Observable> { - return lazyLoadMeta(queryFromUrl(manga.url), - lazyLoadMetaPages(hlIdFromUrl(manga.url), false).map { it.first } - ).map { - listOf(SChapter.create().apply { - url = readerUrl(it.hlId!!) - - name = "Chapter" - - chapter_number = 1f - - it.uploadDate?.let { - date_upload = it - } - }) - } - } - - override fun fetchPageList(chapter: SChapter): Observable> { - val hlId = chapter.url.substringAfterLast('/').removeSuffix(".html") - return lazyLoadMetaPages(hlId, false).map { (_, it) -> - it.mapIndexed { index, s -> - Page(index, s, s) - } - } - } - - override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Unused method called!") - - override fun pageListParse(response: Response) = throw UnsupportedOperationException("Unused method called!") - - override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Unused method called!") - - override val name = "hitomi.la (very slow search)" - - override val baseUrl = BASE_URL - - override val lang = "all" +class Hitomi : HttpSource(), LewdSource { + private val jsonParser by lazy { JsonParser() } override val id = HITOMI_SOURCE_ID + /** + * Whether the source has support for latest updates. + */ override val supportsLatest = true + /** + * Name of the source. + */ + override val name = "hitomi.la" + /** + * The class of the metadata used by this source + */ + override val metaClass = HitomiSearchMetadata::class - private val cacheLocks = arrayOf(ReentrantLock(), ReentrantLock()) - - override fun popularMangaRequest(page: Int) = GET("$LTN_BASE_URL/popular-all.nozomi") - - override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!") - override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException("Unused method called!") - - override fun latestUpdatesRequest(page: Int) = GET("$LTN_BASE_URL/index-all.nozomi") - - fun readerUrl(hlId: String) = "$BASE_URL/reader/$hlId.html" - - private fun lazyLoadMetaPages(hlId: String, forceReload: Boolean): - Observable>> { - val pages = defRealm { realm -> - val rres = realm.where(HitomiPage::class.java) - .equalTo(HitomiPage::gallery.name, hlId) - .sort(HitomiPage::index.name) - .findAll() - - if (rres.isNotEmpty()) - rres.map(HitomiPage::url) - else null - } - - val meta = getAvailableCacheRealm()?.use { - val res = it.where(HitomiSkeletonGalleryMetadata::class.java) - .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId) - .findFirst() - - // Force reload if no thumbnail - if(res?.thumbnailUrl == null) null else res - } - - if(pages != null && meta != null && !forceReload) { - return Observable.just(meta to pages) - } - - val loc = "$BASE_URL/galleries/$hlId.html" - val req = GET(loc) - - return client.newCall(req).asObservableSuccess().map { response -> - val doc = response.asJsoup() - - Duktape.create().use { duck -> - val thumbs = doc.getElementsByTag("script").find { - it.html().startsWith("var thumbnails") - } - - val parsedThumbs = jsonParser.parse(thumbs!!.html() - .removePrefix("var thumbnails = ") - .removeSuffix(";")).array - - // Get pages (drop last element as its always null) - val newPages = parsedThumbs.take(parsedThumbs.size() - 1).mapIndexed { index, item -> - val itemName = item.string - .substringAfterLast('/') - .removeSuffix(".jpg") - - val url = "//a.hitomi.la/galleries/$hlId/$itemName" - - val resolved = resolveImage(duck, url) - HitomiPage().apply { - gallery = hlId - this.index = index - this.url = resolved - } - } - - // Parse meta - val galleryParent = doc.select(".gallery") - - val newMeta = HitomiSkeletonGalleryMetadata().apply { - url = loc - - title = galleryParent.select("h1 > a").text() - - artist = galleryParent.select("h2 > .comma-list > li").joinToString { it.text() }.nullIfBlank() - - thumbnailUrl = "https:" + doc.select(".cover img").attr("src") - - uploadDate = DATE_FORMAT.parse(doc.select(".date").text()).time - - galleryParent.select(".gallery-info tr").forEach { element -> - val content = element.child(1) - - when(element.child(0).text().toLowerCase()) { - "group" -> group = content.text().trim() - "type" -> type = content.text().trim() - "language" -> { - language = content.text().trim() - languageSimple = content.select("a") - .attr("href") - .split("-").getOrNull(1) ?: "speechless" - } - "series" -> { - series.clear() - series.addAll(content.select("li").map(Element::text)) - } - "characters" -> { - characters.clear() - characters.addAll(content.select("li").map(Element::text)) - } - "tags" -> { - tags.clear() - tags.addAll(content.select("li").map { - val txt = it.text() - - val ns: String - val name: String - - when { - txt.endsWith(CHAR_MALE) -> { - ns = "male" - name = txt.removeSuffix(CHAR_MALE).trim() - } - txt.endsWith(CHAR_FEMALE) -> { - ns = "female" - name = txt.removeSuffix(CHAR_FEMALE).trim() - } - else -> { - ns = EMULATED_TAG_NAMESPACE - name = txt.trim() - } - } - - Tag(ns, name) - }) - } - } - } - - // Inject pseudo tags - fun String?.nullNaTag(name: String) { - if(this == null || this == NOT_AVAILABLE) return - - tags.add(Tag(name, this)) - } - - group.nullNaTag("group") - artist.nullNaTag("artist") - languageSimple.nullNaTag("language") - series.forEach { - it.nullNaTag("parody") - } - characters.forEach { - it.nullNaTag("character") - } - type.nullNaTag("category") - } - - realmTrans { - // Delete old pages - it.where(HitomiPage::class.java) - .equalTo(HitomiPage::gallery.name, hlId) - .findAll().deleteAllFromRealm() - - // Add new pages - it.insert(newPages) - } - - (0 .. 1).map { getCacheRealm(it) }.forEach { realm -> - realm.useTrans { - // Delete old meta - it.where(HitomiSkeletonGalleryMetadata::class.java) - .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId) - .findAll().deleteAllFromRealm() - - // Add new meta - it.insert(newMeta) - } - } - - newMeta to newPages.map(HitomiPage::url) - } + private var cachedTagIndexVersion: Long? = null + private var tagIndexVersionCacheTime: Long = 0 + private fun tagIndexVersion(): Single { + val sCachedTagIndexVersion = cachedTagIndexVersion + return if(sCachedTagIndexVersion == null + || tagIndexVersionCacheTime + INDEX_VERSION_CACHE_TIME_MS < System.currentTimeMillis()) { + HitomiNozomi.getIndexVersion(client, "tagindex").subscribeOn(Schedulers.io()).doOnNext { + cachedTagIndexVersion = it + tagIndexVersionCacheTime = System.currentTimeMillis() + }.toSingle() + } else { + Single.just(sCachedTagIndexVersion) } } - private fun fetchAndResolveRequest(page: Int, request: Request): Observable { - //Begin pre-loading cache - ensureCacheLoaded(false).subscribeOn(Schedulers.computation()).subscribe() - - return client.newCall(request) - .asObservableSuccess() - .map { response -> - val buffer = ByteBuffer.wrap(response.body()!!.bytes()) - - val out = mutableListOf() - - try { - while(true) { - out += SManga.create().apply { - setUrlWithoutDomain("$BASE_URL/galleries/${buffer.int}.html") - - title = "Loading..." - } - } - } catch(e: BufferUnderflowException) {} - - val offset = PAGE_SIZE * (page - 1) - val endIndex = Math.min(offset + PAGE_SIZE, out.size) - - MangasPage(out.subList(offset, endIndex), - endIndex < out.size) - } - - } - - override fun fetchPopularManga(page: Int) - = fetchAndResolveRequest(page, popularMangaRequest(page)) - override fun fetchLatestUpdates(page: Int) - = fetchAndResolveRequest(page, latestUpdatesRequest(page)) - - private fun shouldRefreshGalleryFiles(): Boolean { - val timeDiff = System.currentTimeMillis() - prefs.eh_hl_lastRefresh().getOrDefault() - return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L - } - - private inline fun lockCache(index: Int, block: () -> T): T { - cacheLocks[index].lock() - try { - return block() - } finally { - cacheLocks[index].unlock() + private var cachedGalleryIndexVersion: Long? = null + private var galleryIndexVersionCacheTime: Long = 0 + private fun galleryIndexVersion(): Single { + val sCachedGalleryIndexVersion = cachedGalleryIndexVersion + return if(sCachedGalleryIndexVersion == null + || galleryIndexVersionCacheTime + INDEX_VERSION_CACHE_TIME_MS < System.currentTimeMillis()) { + HitomiNozomi.getIndexVersion(client, "galleriesindex").subscribeOn(Schedulers.io()).doOnNext { + cachedGalleryIndexVersion = it + galleryIndexVersionCacheTime = System.currentTimeMillis() + }.toSingle() + } else { + Single.just(sCachedGalleryIndexVersion) } } - private fun loadGalleryMetadata(url: String): Observable { - val mid = HitomiGalleryMetadata.hlIdFromUrl(url) - - return ensureCacheLoaded().map { - getAvailableCacheRealm()?.use { realm -> - findCacheMetadataById(realm, mid) - } - } - } - - private fun findCacheMetadataById(realm: Realm, hlId: String): HitomiSkeletonGalleryMetadata? { - return realm.where(HitomiSkeletonGalleryMetadata::class.java) - .equalTo(HitomiSkeletonGalleryMetadata::hlId.name, hlId) - .findFirst()?.let { realm.copyFromRealm(it) } - } - - fun ensureCacheLoaded(blocking: Boolean = true): Observable { - return Observable.fromCallable { - if(prefs.eh_hl_lastRealmIndex().getOrDefault() >= 0) { return@fromCallable Any() } - - val nextRealmIndex = when(prefs.eh_hl_lastRealmIndex().getOrDefault()) { - 0 -> 1 - 1 -> 0 - else -> 0 - } - - if(!blocking && cacheLocks[nextRealmIndex].isLocked) return@fromCallable Any() - - lockCache(nextRealmIndex) { - val shouldRefresh = shouldRefreshGalleryFiles() - getCacheRealm(nextRealmIndex).useTrans { realm -> - if (!realm.isEmpty && !shouldRefresh) - return@fromCallable Any() - - realm.deleteAll() - } - - val cores = Runtime.getRuntime().availableProcessors() - Timber.d("Starting $cores threads to parse hitomi.la gallery data...") - - val workQueue = ConcurrentLinkedQueue((0 until GALLERY_CHUNK_COUNT).toList()) - val threads = mutableListOf() - - for(threadIndex in 1 .. cores) { - threads += thread { - getCacheRealm(nextRealmIndex).use { realm -> - while (true) { - val i = workQueue.poll() ?: break - - Timber.d("[$threadIndex] Downloading + parsing hitomi.la gallery data ${i + 1}/$GALLERY_CHUNK_COUNT...") - - val url = "https://ltn.hitomi.la/galleries$i.json" - - val resp = client.newCall(GET(url)).execute().body()!! - - val out = mutableListOf() - - JsonReader(resp.charStream()).use { reader -> - reader.beginArray() - - while (reader.hasNext()) { - val gallery = HitomiGallery.fromJson(reader.nextJsonObject()) - val meta = HitomiSkeletonGalleryMetadata() - gallery.addToGalleryMeta(meta) - - out.add(meta) - } - } - - Timber.d("[$threadIndex] Saving hitomi.la gallery data ${i + 1}/$GALLERY_CHUNK_COUNT...") - - realm.trans { - realm.insert(out) - } - } - } - } - } - - threads.forEach(Thread::join) - - // Update refresh time - prefs.eh_hl_lastRefresh().set(System.currentTimeMillis()) - - // Update last refreshed realm - prefs.eh_hl_lastRealmIndex().set(nextRealmIndex) - - Timber.d("Successfully refreshed realm #$nextRealmIndex!") - } - - return@fromCallable Any() - } - } - - private fun resolveImage(duktape: Duktape, url: String): String { - return "https:" + duktape.evaluate(IMAGE_RESOLVER.replace(IMAGE_RESOLVER_URL_VAR, url)) as String - } - - private fun HitomiGallery.addToGalleryMeta(meta: HitomiSkeletonGalleryMetadata) { - with(meta) { - hlId = id.toString() - title = name - // Intentionally avoid setting thumbnails - // We need another request to get them anyways - artist = artists.firstOrNull() - group = groups.firstOrNull() - type = this@addToGalleryMeta.type - languageSimple = language - series.clear() - series.addAll(parodies) - characters.clear() - characters.addAll(this@addToGalleryMeta.characters) + /** + * Parse the supplied input into the supplied metadata object + */ + override fun parseIntoMetadata(metadata: HitomiSearchMetadata, input: Document) { + with(metadata) { + url = input.location() tags.clear() - this@addToGalleryMeta.tags.mapTo(tags) { Tag(it.key, it.value) } - } - } - private fun getAndLockAvailableCacheRealm(block: (Realm) -> T): T? { - val index = prefs.eh_hl_lastRealmIndex().getOrDefault() + thumbnailUrl = "https:" + input.selectFirst(".cover img").attr("src") - return if(index >= 0) { - val cache = getCacheRealm(index) - lockCache(index) { - block(cache) + val galleryElement = input.selectFirst(".gallery") + + title = galleryElement.selectFirst("h1").text() + artists = galleryElement.select("h2 a").map { it.text() } + tags += artists.map { RaisedTag("artist", it, TAG_TYPE_VIRTUAL) } + + input.select(".gallery-info tr").forEach { + val content = it.child(1) + when(it.child(0).text().toLowerCase()) { + "group" -> { + group = content.text() + tags += RaisedTag("group", group!!, TAG_TYPE_VIRTUAL) + } + "type" -> { + type = content.text() + tags += RaisedTag("type", type!!, TAG_TYPE_VIRTUAL) + } + "series" -> { + series = content.select("a").map { it.text() } + tags += series.map { + RaisedTag("series", it, TAG_TYPE_VIRTUAL) + } + } + "language" -> { + language = content.selectFirst("a")?.attr("href")?.split('-')?.get(1) + language?.let { + tags += RaisedTag("language", it, TAG_TYPE_VIRTUAL) + } + } + "characters" -> { + characters = content.select("a").map { it.text() } + tags += characters.map { RaisedTag("character", it, TAG_TYPE_DEFAULT) } + } + "tags" -> { + tags += content.select("a").map { + val ns = if(it.attr("href").startsWith("/tag/male")) "male" else "female" + RaisedTag(ns, it.text().dropLast(2), TAG_TYPE_DEFAULT) + } + } + } } - } else { - null + + uploadDate = DATE_FORMAT.parse(input.selectFirst(".gallery-info .date").text()).time } } - private fun getAvailableCacheRealm(): Realm? { - val index = prefs.eh_hl_lastRealmIndex().getOrDefault() + override val lang = "all" - return if(index >= 0) { - getCacheRealm(index) - } else { - null + /** + * Base url of the website without the trailing slash, like: http://mysite.com + */ + override val baseUrl = BASE_URL + + /** + * Returns the request for the popular manga given the page. + * + * @param page the page number to retrieve. + */ + override fun popularMangaRequest(page: Int) = HitomiNozomi.rangedGet( + "$LTN_BASE_URL/popular-all.nozomi", + 100L * (page - 1), + 99L + 100 * (page - 1) + ) + + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun popularMangaParse(response: Response) = throw UnsupportedOperationException() + + /** + * Returns the request for the search manga given the page. + * + * @param page the page number to retrieve. + * @param query the search query. + * @param filters the list of filters to apply. + */ + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) + = throw UnsupportedOperationException() + + override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { + return urlImportFetchSearchManga(query) { + val splitQuery = query.split(" ") + + val positive = splitQuery.filter { !it.startsWith('-') }.toMutableList() + val negative = (splitQuery - positive).map { it.removePrefix("-") } + + // TODO Cache the results coming out of HitomiNozomi + val hn = Single.zip(tagIndexVersion(), galleryIndexVersion()) { tv, gv -> tv to gv } + .map { HitomiNozomi(client, it.first, it.second) } + + var base = if(positive.isEmpty()) { + hn.flatMap { n -> n.getGalleryIdsFromNozomi(null, "index", "all").map { n to it.toSet() } } + } else { + val q = positive.removeAt(0) + hn.flatMap { n -> n.getGalleryIdsForQuery(q).map { n to it.toSet() } } + } + + base = positive.fold(base) { acc, q -> + acc.flatMap { (nozomi, mangas) -> + nozomi.getGalleryIdsForQuery(q).map { + nozomi to mangas.intersect(it) + } + } + } + + base = negative.fold(base) { acc, q -> + acc.flatMap { (nozomi, mangas) -> + nozomi.getGalleryIdsForQuery(q).map { + nozomi to (mangas - it) + } + } + } + + base.flatMap { (_, ids) -> + val chunks = ids.chunked(PAGE_SIZE) + + nozomiIdsToMangas(chunks[page - 1]).map { mangas -> + MangasPage(mangas, page < chunks.size) + } + }.toObservable() } } - private fun getCacheRealm(index: Int) = Realm.getInstance(getRealmConfig(index)) + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun searchMangaParse(response: Response) = throw UnsupportedOperationException() - private fun getRealmConfig(index: Int) = RealmConfiguration.Builder() - .name("hitomi-cache-$index") - .deleteRealmIfMigrationNeeded() - .build() + /** + * Returns the request for latest manga given the page. + * + * @param page the page number to retrieve. + */ + override fun latestUpdatesRequest(page: Int) = HitomiNozomi.rangedGet( + "$LTN_BASE_URL/index-all.nozomi", + 100L * (page - 1), + 99L + 100 * (page - 1) + ) - fun forceEnsureCacheLoaded(): Boolean { - // Lock all caches - if(!cacheLocks[0].tryLock() || !cacheLocks[1].tryLock()) { - if(cacheLocks[0].isHeldByCurrentThread) - cacheLocks[0].unlock() - if(cacheLocks[1].isHeldByCurrentThread) - cacheLocks[1].unlock() + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException() - return false - } - - try { - prefs.eh_hl_lastRealmIndex().set(-1) - prefs.eh_hl_lastRefresh().set(0) - ensureCacheLoaded(false).subscribeOn(Schedulers.computation()).subscribe() - } finally { - cacheLocks[0].unlock() - cacheLocks[1].unlock() - } - - return true + override fun fetchPopularManga(page: Int): Observable { + return client.newCall(popularMangaRequest(page)) + .asObservableSuccess() + .flatMap { responseToMangas(it) } } + override fun fetchLatestUpdates(page: Int): Observable { + return client.newCall(latestUpdatesRequest(page)) + .asObservableSuccess() + .flatMap { responseToMangas(it) } + } + + fun responseToMangas(response: Response): Observable { + val range = response.header("Content-Range")!! + val total = range.substringAfter('/').toLong() + val end = range.substringBefore('/').substringAfter('-').toLong() + val body = response.body()!! + return parseNozomiPage(body.bytes()) + .map { + MangasPage(it, end < total - 1) + } + } + + private fun parseNozomiPage(array: ByteArray): Observable> { + val cursor = ByteCursor(array) + val ids = (1 .. array.size / 4).map { + cursor.nextInt() + } + + return nozomiIdsToMangas(ids).toObservable() + } + + private fun nozomiIdsToMangas(ids: List): Single> { + return Single.zip(ids.map { + client.newCall(GET("$LTN_BASE_URL/galleryblock/$it.html")) + .asObservableSuccess() + .subscribeOn(Schedulers.io()) // Perform all these requests in parallel + .map { parseGalleryBlock(it) } + .toSingle() + }) { it.map { m -> m as SManga } } + } + + private fun parseGalleryBlock(response: Response): SManga { + val doc = response.asJsoup() + return SManga.create().apply { + val titleElement = doc.selectFirst("h1") + title = titleElement.text() + // TODO High/low quality thumbnail toggle +// thumbnail_url = "https:" + doc.selectFirst("img").attr("data-srcset").substringBefore(' ') + thumbnail_url = "https:" + doc.selectFirst("img").attr("data-src") + url = titleElement.child(0).attr("href") + + // TODO Parse tags and stuff + } + } + + + /** + * Returns an observable with the updated details for a manga. Normally it's not needed to + * override this method. + * + * @param manga the manga to be updated. + */ + override fun fetchMangaDetails(manga: SManga): Observable { + return client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .flatMap { + parseToManga(manga, it.asJsoup()).andThen(Observable.just(manga.apply { + initialized = true + })) + } + } + + override fun fetchChapterList(manga: SManga): Observable> { + return Observable.just( + listOf( + SChapter.create().apply { + url = manga.url + name = "Chapter" + chapter_number = 0.0f + } + ) + ) + } + + override fun pageListRequest(chapter: SChapter): Request { + return GET("$LTN_BASE_URL/galleries/${HitomiSearchMetadata.hlIdFromUrl(chapter.url)}.js") + } + + /** + * Parses the response from the site and returns the details of a manga. + * + * @param response the response from the site. + */ + override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException() + + /** + * Parses the response from the site and returns a list of chapters. + * + * @param response the response from the site. + */ + override fun chapterListParse(response: Response) = throw UnsupportedOperationException() + + /** + * Parses the response from the site and returns a list of pages. + * + * @param response the response from the site. + */ + override fun pageListParse(response: Response): List { + val hlId = response.request().url().pathSegments().last().removeSuffix(".js").toLong() + val str = response.body()!!.string() + val json = jsonParser.parse(str.removePrefix("var galleryinfo =")) + return json.array.mapIndexed { index, jsonElement -> + Page( + index, + "", + "https://${subdomainFromGalleryId(hlId)}a.hitomi.la/galleries/$hlId/${jsonElement["name"].string}" + ) + } + } + + private fun subdomainFromGalleryId(id: Long): Char { + return (97 + id.rem(NUMBER_OF_FRONTENDS)).toChar() + } + + /** + * Parses the response from the site and returns the absolute url to the source image. + * + * @param response the response from the site. + */ + override fun imageUrlParse(response: Response) = throw UnsupportedOperationException() + companion object { + private val INDEX_VERSION_CACHE_TIME_MS = 1000 * 60 * 10 private val PAGE_SIZE = 25 - private val CHAR_MALE = "♂" - private val CHAR_FEMALE = "♀" - private val GALLERY_CHUNK_COUNT = 20 - private val IMAGE_RESOLVER_URL_VAR = "%IMAGE_URL%" - private val NOT_AVAILABLE = "N/A" + private val NUMBER_OF_FRONTENDS = 2 + private val DATE_FORMAT by lazy { if(Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) SimpleDateFormat("yyyy-MM-dd HH:mm:ssX", Locale.US) else - SimpleDateFormat("yyyy-MM-dd HH:mm:ss'-05'", Locale.US) + SimpleDateFormat("yyyy-MM-dd HH:mm:ss'-05'", Locale.US) } - private val IMAGE_RESOLVER = """ - (function() { -var adapose = false; // Currently not sure what this does, it switches out frontend URL when we right click??? -var number_of_frontends = 2; -function subdomain_from_galleryid(g) { - if (adapose) { - return '0'; - } - return String.fromCharCode(97 + (g % number_of_frontends)); -} -function subdomain_from_url(url, base) { - var retval = 'a'; - if (base) { - retval = base; - } - - var r = /\/(\d+)\//; - var m = r.exec(url); - var g; - if (m) { - g = parseInt(m[1]); - } - if (g) { - retval = subdomain_from_galleryid(g) + retval; - } - - return retval; -} -function url_from_url(url, base) { - return url.replace(/\/\/..?\.hitomi\.la\//, '//'+subdomain_from_url(url, base)+'.hitomi.la/'); -} - -return url_from_url('$IMAGE_RESOLVER_URL_VAR'); -})(); - """.trimIndent() } -} -data class HitomiGallery(val artists: List, - val parodies: List, - val id: Int, - val name: String, - val groups: List, - val tags: Map, - val characters: List, - val type: String, - val language: String?) { - companion object { - fun fromJson(obj: JsonObject): HitomiGallery - = HitomiGallery( - obj.mapNullStringList("a"), - obj.mapNullStringList("p"), - obj["id"].int, - obj["n"].string, - obj.mapNullStringList("g"), - obj["t"]?.nullArray?.associate { - val str = it.string - if(str.contains(":")) - str.substringBefore(':') to str.substringAfter(':') - else - EMULATED_TAG_NAMESPACE to str - } ?: emptyMap(), - obj.mapNullStringList("c"), - obj["type"].string, - obj["l"].nullString) - - private fun JsonObject.mapNullStringList(key: String) - = this[key]?.nullArray?.map { it.string } ?: emptyList() - } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/NHentai.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/NHentai.kt index 19385ebd5..6a3a9b838 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/NHentai.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/NHentai.kt @@ -5,7 +5,6 @@ import android.net.Uri import com.github.salomonbrys.kotson.* import com.google.gson.JsonElement import com.google.gson.JsonNull -import com.google.gson.JsonObject import com.google.gson.JsonParser import eu.kanade.tachiyomi.BuildConfig import eu.kanade.tachiyomi.R @@ -14,55 +13,93 @@ import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.model.* import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.LewdSource +import eu.kanade.tachiyomi.util.asJsoup import exh.NHENTAI_SOURCE_ID +import exh.metadata.metadata.NHentaiSearchMetadata +import exh.metadata.metadata.NHentaiSearchMetadata.Companion.TAG_TYPE_DEFAULT +import exh.metadata.metadata.base.RaisedTag import exh.metadata.models.NHentaiMetadata -import exh.metadata.models.PageImageType -import exh.metadata.models.Tag import exh.util.* import okhttp3.Request import okhttp3.Response import rx.Observable -import timber.log.Timber /** * NHentai source */ -class NHentai(context: Context) : HttpSource(), LewdSource { +class NHentai(context: Context) : HttpSource(), LewdSource { + override val metaClass = NHentaiSearchMetadata::class + override fun fetchPopularManga(page: Int): Observable { //TODO There is currently no way to get the most popular mangas //TODO Instead, we delegate this to the latest updates thing to avoid confusing users with an empty screen return fetchLatestUpdates(page) } - override fun popularMangaRequest(page: Int): Request { - TODO("Currently unavailable!") - } + override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException() - override fun popularMangaParse(response: Response): MangasPage { - TODO("Currently unavailable!") - } + override fun popularMangaParse(response: Response) = throw UnsupportedOperationException() //Support direct URL importing override fun fetchSearchManga(page: Int, query: String, filters: FilterList) = - urlImportFetchSearchManga(query, { - super.fetchSearchManga(page, query, filters) - }) + urlImportFetchSearchManga(query) { + searchMangaRequestObservable(page, query, filters).flatMap { + client.newCall(it).asObservableSuccess() + } .map { response -> + searchMangaParse(response) + } + } + + private fun searchMangaRequestObservable(page: Int, query: String, filters: FilterList): Observable { + val uri = if(query.isNotBlank()) { + Uri.parse("$baseUrl/search/").buildUpon().apply { + appendQueryParameter("q", query) + } + } else { + Uri.parse(baseUrl).buildUpon() + } + val sortFilter = filters.filterIsInstance().firstOrNull()?.state + ?: defaultSortFilterSelection() + + if(sortFilter.index == 1) { + if(query.isBlank()) error("You must specify a search query if you wish to sort by popularity!") + uri.appendQueryParameter("sort", "popular") + } + + if(sortFilter.ascending) { + return client.newCall(nhGet(uri.toString())) + .asObservableSuccess() + .map { + val doc = it.asJsoup() + + val lastPage = doc.selectFirst(".last") + ?.attr("href") + ?.substringAfterLast('=') + ?.toIntOrNull() ?: 1 + + val thisPage = lastPage - (page - 1) + + uri.appendQueryParameter(REVERSE_PARAM, (thisPage > 1).toString()) + uri.appendQueryParameter("page", thisPage.toString()) + + nhGet(uri.toString(), page) + } + } - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - //Currently we have no filters - //TODO Filter builder - val uri = Uri.parse("$baseUrl/api/galleries/search").buildUpon() - uri.appendQueryParameter("query", query) uri.appendQueryParameter("page", page.toString()) - return nhGet(uri.toString(), page) + + return Observable.just(nhGet(uri.toString(), page)) } + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) + = throw UnsupportedOperationException() + override fun searchMangaParse(response: Response) = parseResultPage(response) override fun latestUpdatesRequest(page: Int): Request { - val uri = Uri.parse("$baseUrl/api/galleries/all").buildUpon() + val uri = Uri.parse(baseUrl).buildUpon() uri.appendQueryParameter("page", page.toString()) return nhGet(uri.toString(), page) } @@ -70,124 +107,122 @@ class NHentai(context: Context) : HttpSource(), LewdSource { - return client.newCall(urlToDetailsRequest(manga.url)) + return client.newCall(mangaDetailsRequest(manga)) .asObservableSuccess() - .map { response -> - mangaDetailsParse(response).apply { initialized = true } + .flatMap { + parseToManga(manga, it).andThen(Observable.just(manga.apply { + initialized = true + })) } } override fun mangaDetailsRequest(manga: SManga) - = nhGet(manga.url) + = nhGet(baseUrl + manga.url) fun urlToDetailsRequest(url: String) = nhGet(baseUrl + "/api/gallery/" + url.split("/").last { it.isNotBlank() }) fun parseResultPage(response: Response): MangasPage { - val res = jsonParser.parse(response.body()!!.string()).asJsonObject + val doc = response.asJsoup() - val error = res.get("error") - if(error == null) { - val results = res.getAsJsonArray("result")?.map { - val obj = it.asJsonObject - parseToManga(NHentaiMetadata.Query(obj["id"].long), obj) + // TODO Parse lang + tags + + val mangas = doc.select(".gallery > a").map { + SManga.create().apply { + url = it.attr("href") + + title = it.selectFirst(".caption").text() + + // last() is a hack to ignore the lazy-loader placeholder image on the front page + thumbnail_url = it.select("img").last().attr("src") + // In some pages, the thumbnail url does not include the protocol + if(!thumbnail_url!!.startsWith("https:")) thumbnail_url = "https:$thumbnail_url" } - val numPages = res.get("num_pages")?.int - if(results != null && numPages != null) - return MangasPage(results, numPages > response.request().tag() as Int) + } + + val hasNextPage = if(!response.request().url().queryParameterNames().contains(REVERSE_PARAM)) { + doc.selectFirst(".next") != null } else { - Timber.w("An error occurred while performing the search: $error") + response.request().url().queryParameter(REVERSE_PARAM)!!.toBoolean() } - return MangasPage(emptyList(), false) + + return MangasPage(mangas, hasNextPage) } - override val metaParser: NHentaiMetadata.(JsonObject) -> Unit = { obj -> - nhId = obj["id"].asLong + override fun parseIntoMetadata(metadata: NHentaiSearchMetadata, input: Response) { + val json = GALLERY_JSON_REGEX.find(input.body()!!.string())!!.groupValues[1] + val obj = jsonParser.parse(json).asJsonObject - uploadDate = obj["upload_date"].nullLong + with(metadata) { + nhId = obj["id"].asLong - favoritesCount = obj["num_favorites"].nullLong + uploadDate = obj["upload_date"].nullLong - mediaId = obj["media_id"].nullString + favoritesCount = obj["num_favorites"].nullLong - obj["title"].nullObj?.let { it -> - japaneseTitle = it["japanese"].nullString - shortTitle = it["pretty"].nullString - englishTitle = it["english"].nullString - } + mediaId = obj["media_id"].nullString - obj["images"].nullObj?.let { - coverImageType = it["cover"]?.get("t").nullString - it["pages"].nullArray?.mapNotNull { - it?.asJsonObject?.get("t").nullString - }?.map { - PageImageType(it) - }?.let { - pageImageTypes.clear() - pageImageTypes.addAll(it) + obj["title"].nullObj?.let { title -> + japaneseTitle = title["japanese"].nullString + shortTitle = title["pretty"].nullString + englishTitle = title["english"].nullString } - thumbnailImageType = it["thumbnail"]?.get("t").nullString - } - scanlator = obj["scanlator"].nullString - - obj["tags"]?.asJsonArray?.map { - val asObj = it.asJsonObject - Pair(asObj["type"].nullString, asObj["name"].nullString) - }?.apply { - tags.clear() - }?.forEach { - if(it.first != null && it.second != null) - tags.add(Tag(it.first!!, it.second!!, false)) - } - } - - fun lazyLoadMetadata(url: String) = - defRealm { realm -> - val meta = NHentaiMetadata.UrlQuery(url).query(realm).findFirst() - if(meta == null) { - client.newCall(urlToDetailsRequest(url)) - .asObservableSuccess() - .map { - realmTrans { realm -> - realm.copyFromRealm(realm.createUUIDObj(queryAll().clazz.java).apply { - metaParser(this, - jsonParser.parse(it.body()!!.string()).asJsonObject) - }) - } - } - .first() - } else { - Observable.just(realm.copyFromRealm(meta)) + obj["images"].nullObj?.let { + coverImageType = it["cover"]?.get("t").nullString + it["pages"].nullArray?.mapNotNull { + it?.asJsonObject?.get("t").nullString + }?.let { + pageImageTypes = it } + thumbnailImageType = it["thumbnail"]?.get("t").nullString } + scanlator = obj["scanlator"].nullString + + obj["tags"]?.asJsonArray?.map { + val asObj = it.asJsonObject + Pair(asObj["type"].nullString, asObj["name"].nullString) + }?.apply { + tags.clear() + }?.forEach { + if (it.first != null && it.second != null) + tags.add(RaisedTag(it.first!!, it.second!!, TAG_TYPE_DEFAULT)) + } + } + } + + fun getOrLoadMetadata(mangaId: Long?, nhId: Long) = getOrLoadMetadata(mangaId) { + client.newCall(nhGet(baseUrl + NHentaiSearchMetadata.nhIdToPath(nhId))) + .asObservableSuccess() + .toSingle() + } + override fun fetchChapterList(manga: SManga) - = lazyLoadMetadata(manga.url).map { - listOf(SChapter.create().apply { - url = manga.url - name = "Chapter" - date_upload = ((it.uploadDate ?: 0) * 1000) - chapter_number = 1f - }) - }!! + = Observable.just(listOf(SChapter.create().apply { + url = manga.url + name = "Chapter" + chapter_number = 1f + })) override fun fetchPageList(chapter: SChapter) - = lazyLoadMetadata(chapter.url).map { metadata -> + = getOrLoadMetadata(chapter.mangaId, NHentaiSearchMetadata.nhUrlToId(chapter.url)).map { metadata -> if(metadata.mediaId == null) emptyList() else metadata.pageImageTypes.mapIndexed { index, s -> - val imageUrl = imageUrlFromType(metadata.mediaId!!, index + 1, s.type!!) + val imageUrl = imageUrlFromType(metadata.mediaId!!, index + 1, s) Page(index, imageUrl!!, imageUrl) } - }!! + }.toObservable() override fun fetchImageUrl(page: Page) = Observable.just(page.imageUrl!!)!! @@ -207,6 +242,14 @@ class NHentai(context: Context) : HttpSource(), LewdSource { + LewdSource { + /** + * The class of the metadata used by this source + */ + override val metaClass = PervEdenSearchMetadata::class override val supportsLatest = true override val name = "Perv Eden" @@ -48,9 +56,9 @@ class PervEden(override val id: Long, val pvLang: PervEdenLang) : ParsedHttpSour //Support direct URL importing override fun fetchSearchManga(page: Int, query: String, filters: FilterList) = - urlImportFetchSearchManga(query, { + urlImportFetchSearchManga(query) { super.fetchSearchManga(page, query, filters) - }) + } override fun searchMangaSelector() = "#mangaList > tbody > tr" @@ -79,7 +87,7 @@ class PervEden(override val id: Long, val pvLang: PervEdenLang) : ParsedHttpSour val titleElement = header.child(0) manga.url = titleElement.attr("href") manga.title = titleElement.text().trim() - manga.thumbnail_url = "http:" + titleElement.getElementsByClass("mangaImage").first().attr("tmpsrc") + manga.thumbnail_url = "https:" + header.parent().selectFirst(".mangaImage img").attr("tmpsrc") return manga } @@ -107,67 +115,90 @@ class PervEden(override val id: Long, val pvLang: PervEdenLang) : ParsedHttpSour throw NotImplementedError("Unused method called!") } - override val metaParser: PervEdenGalleryMetadata.(Document) -> Unit = { document -> - url = Uri.parse(document.location()).path + /** + * Returns an observable with the updated details for a manga. Normally it's not needed to + * override this method. + * + * @param manga the manga to be updated. + */ + override fun fetchMangaDetails(manga: SManga): Observable { + return client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .flatMap { + parseToManga(manga, it.asJsoup()).andThen(Observable.just(manga.apply { + initialized = true + })) + } + } - pvId = PervEdenGalleryMetadata.pvIdFromUrl(url!!) + /** + * Parse the supplied input into the supplied metadata object + */ + override fun parseIntoMetadata(metadata: PervEdenSearchMetadata, input: Document) { + with(metadata) { + url = Uri.parse(input.location()).path - lang = this@PervEden.lang + pvId = PervEdenGalleryMetadata.pvIdFromUrl(url!!) - title = document.getElementsByClass("manga-title").first()?.text() + lang = this@PervEden.lang - thumbnailUrl = "http:" + document.getElementsByClass("mangaImage2").first()?.child(0)?.attr("src") + title = input.getElementsByClass("manga-title").first()?.text() - val rightBoxElement = document.select(".rightBox:not(.info)").first() + thumbnailUrl = "http:" + input.getElementsByClass("mangaImage2").first()?.child(0)?.attr("src") - altTitles.clear() - tags.clear() - var inStatus: String? = null - rightBoxElement.childNodes().forEach { - if(it is Element && it.tagName().toLowerCase() == "h4") { - inStatus = it.text().trim() - } else { - when(inStatus) { - "Alternative name(s)" -> { - if(it is TextNode) { - val text = it.text().trim() - if(!text.isBlank()) - altTitles.add(PervEdenTitle(this, text)) + val rightBoxElement = input.select(".rightBox:not(.info)").first() + + val newAltTitles = mutableListOf() + tags.clear() + var inStatus: String? = null + rightBoxElement.childNodes().forEach { + if(it is Element && it.tagName().toLowerCase() == "h4") { + inStatus = it.text().trim() + } else { + when(inStatus) { + "Alternative name(s)" -> { + if(it is TextNode) { + val text = it.text().trim() + if(!text.isBlank()) + newAltTitles += text + } } - } - "Artist" -> { - if(it is Element && it.tagName() == "a") { - artist = it.text() - tags.add(Tag("artist", it.text().toLowerCase(), false)) + "Artist" -> { + if(it is Element && it.tagName() == "a") { + artist = it.text() + tags += RaisedTag("artist", it.text().toLowerCase(), TAG_TYPE_VIRTUAL) + } } - } - "Genres" -> { - if(it is Element && it.tagName() == "a") - tags.add(Tag(EMULATED_TAG_NAMESPACE, it.text().toLowerCase(), false)) - } - "Type" -> { - if(it is TextNode) { - val text = it.text().trim() - if(!text.isBlank()) - type = text + "Genres" -> { + if(it is Element && it.tagName() == "a") + tags += RaisedTag(null, it.text().toLowerCase(), TAG_TYPE_DEFAULT) } - } - "Status" -> { - if(it is TextNode) { - val text = it.text().trim() - if(!text.isBlank()) - status = text + "Type" -> { + if(it is TextNode) { + val text = it.text().trim() + if(!text.isBlank()) + type = text + } + } + "Status" -> { + if(it is TextNode) { + val text = it.text().trim() + if(!text.isBlank()) + status = text + } } } } } - } - rating = document.getElementById("rating-score")?.attr("value")?.toFloat() + altTitles = newAltTitles + + rating = input.getElementById("rating-score")?.attr("value")?.toFloat() + } } override fun mangaDetailsParse(document: Document): SManga - = parseToManga(queryFromUrl(document.location()), document) + = throw UnsupportedOperationException() override fun latestUpdatesRequest(page: Int): Request { val num = when (lang) { @@ -206,9 +237,6 @@ class PervEden(override val id: Long, val pvLang: PervEdenLang) : ParsedHttpSour override fun imageUrlParse(document: Document) = "http:" + document.getElementById("mainImg").attr("src")!! - override fun queryAll() = PervEdenGalleryMetadata.EmptyQuery() - override fun queryFromUrl(url: String) = PervEdenGalleryMetadata.UrlQuery(url, PervEdenLang.source(id)) - override fun getFilterList() = FilterList ( AuthorFilter(), ArtistFilter(), diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt index 295eb4dad..e873e3164 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt @@ -1,204 +1,91 @@ package eu.kanade.tachiyomi.source.online.english -import android.net.Uri -import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.source.model.* +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.LewdSource -import eu.kanade.tachiyomi.source.online.ParsedHttpSource import eu.kanade.tachiyomi.util.asJsoup -import exh.HENTAI_CAFE_SOURCE_ID -import exh.metadata.EMULATED_TAG_NAMESPACE -import exh.metadata.models.HentaiCafeMetadata -import exh.metadata.models.HentaiCafeMetadata.Companion.BASE_URL -import exh.metadata.models.Tag +import exh.metadata.metadata.HentaiCafeSearchMetadata +import exh.metadata.metadata.HentaiCafeSearchMetadata.Companion.TAG_TYPE_DEFAULT +import exh.metadata.metadata.base.RaisedSearchMetadata.Companion.TAG_TYPE_VIRTUAL +import exh.metadata.metadata.base.RaisedTag +import exh.source.DelegatedHttpSource import exh.util.urlImportFetchSearchManga -import okhttp3.Request +import okhttp3.HttpUrl import org.jsoup.nodes.Document -import org.jsoup.nodes.Element import rx.Observable -class HentaiCafe : ParsedHttpSource(), LewdSource { - override val id = HENTAI_CAFE_SOURCE_ID - +class HentaiCafe(delegate: HttpSource) : DelegatedHttpSource(delegate), + LewdSource { + /** + * An ISO 639-1 compliant language code (two letters in lower case). + */ override val lang = "en" + /** + * The class of the metadata used by this source + */ + override val metaClass = HentaiCafeSearchMetadata::class - override val supportsLatest = true - - override fun queryAll() = HentaiCafeMetadata.EmptyQuery() - override fun queryFromUrl(url: String) = HentaiCafeMetadata.UrlQuery(url) - - override val name = "Hentai Cafe" - override val baseUrl = "https://hentai.cafe" - - // Defer popular manga -> latest updates - override fun popularMangaSelector() = throw UnsupportedOperationException("Unused method called!") - override fun popularMangaFromElement(element: Element) = throw UnsupportedOperationException("Unused method called!") - override fun popularMangaNextPageSelector() = throw UnsupportedOperationException("Unused method called!") - override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException("Unused method called!") - override fun fetchPopularManga(page: Int) = fetchLatestUpdates(page) - //Support direct URL importing override fun fetchSearchManga(page: Int, query: String, filters: FilterList) = - urlImportFetchSearchManga(query, { + urlImportFetchSearchManga(query) { super.fetchSearchManga(page, query, filters) - }) - override fun searchMangaSelector() = "article.post:not(#post-0)" - override fun searchMangaFromElement(element: Element): SManga { - val thumb = element.select(".entry-thumb > img") - val title = element.select(".entry-title > a") - - return SManga.create().apply { - setUrlWithoutDomain(title.attr("href")) - this.title = title.text() - - thumbnail_url = thumb.attr("src") - } - } - override fun searchMangaNextPageSelector() = ".x-pagination > ul > li:last-child > a.prev-next" - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = if(query.isNotBlank()) { - //Filter by query - "$baseUrl/page/$page/?s=${Uri.encode(query)}" - } else if(filters.filterIsInstance().any { it.state }) { - //Filter by book - "$baseUrl/category/book/page/$page/" - } else { - //Filter by tag - val tagFilter = filters.filterIsInstance().first() - - if(tagFilter.state == 0) throw IllegalArgumentException("No filters active, no query active! What to filter?") - - val tag = tagFilter.values[tagFilter.state] - "$baseUrl/tag/${tag.id}/page/$page/" - } - - return GET(url) - } - - override fun latestUpdatesSelector() = searchMangaSelector() - override fun latestUpdatesFromElement(element: Element) = searchMangaFromElement(element) - override fun latestUpdatesNextPageSelector() = searchMangaNextPageSelector() - override fun latestUpdatesRequest(page: Int) = GET("$BASE_URL/page/$page/") - - override fun mangaDetailsParse(document: Document): SManga { - return parseToManga(queryFromUrl(document.location()), document) - } - - override fun chapterListSelector() = throw UnsupportedOperationException("Unused method called!") - override fun chapterFromElement(element: Element) = throw UnsupportedOperationException("Unused method called!") - - override fun fetchChapterList(manga: SManga): Observable> { - return lazyLoadMeta(queryFromUrl(manga.url), - client.newCall(mangaDetailsRequest(manga)).asObservableSuccess().map { it.asJsoup() } - ).map { - listOf(SChapter.create().apply { - url = "/manga/read/${it.readerId}/en/0/1/" - - name = "Chapter" - - chapter_number = 1f - }) - } - } - - override fun pageListParse(document: Document): List { - val pageItems = document.select(".dropdown > li > a") - - return pageItems.mapIndexed { index, element -> - Page(index, element.attr("href")) - } - } - - override fun imageUrlParse(document: Document) - = document.select("#page img").attr("src") - - override val metaParser: HentaiCafeMetadata.(Document) -> Unit = { - val content = it.getElementsByClass("content") - val eTitle = content.select("h3") - - url = Uri.decode(it.location()) - title = eTitle.text() - - thumbnailUrl = content.select("img").attr("src") - - tags.clear() - val eDetails = content.select("p > a[rel=tag]") - eDetails.forEach { - val href = it.attr("href") - val parsed = Uri.parse(href) - val firstPath = parsed.pathSegments.first() - - when(firstPath) { - "tag" -> tags.add(Tag(EMULATED_TAG_NAMESPACE, it.text(), false)) - "artist" -> { - artist = it.text() - tags.add(Tag("artist", it.text(), false)) - } } + + override fun fetchMangaDetails(manga: SManga): Observable { + return client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .flatMap { + parseToManga(manga, it.asJsoup()).andThen(Observable.just(manga.apply { + initialized = true + })) + } + } + + /** + * Parse the supplied input into the supplied metadata object + */ + override fun parseIntoMetadata(metadata: HentaiCafeSearchMetadata, input: Document) { + with(metadata) { + url = input.location() + title = input.select(".entry-title").text() + val contentElement = input.select(".entry-content").first() + thumbnailUrl = contentElement.child(0).child(0).attr("src") + + fun filterableTagsOfType(type: String) = contentElement.select("a") + .filter { "$baseUrl/$type/" in it.attr("href") } + .map { it.text() } + + tags.clear() + tags += filterableTagsOfType("tag").map { + RaisedTag(null, it, TAG_TYPE_DEFAULT) + } + + val artists = filterableTagsOfType("artist") + + artist = artists.joinToString() + tags += artists.map { + RaisedTag("artist", it, TAG_TYPE_VIRTUAL) + } + + readerId = HttpUrl.parse(input.select("[title=Read]").attr("href"))!!.pathSegments()[2] } - - readerId = Uri.parse(content.select("a[title=Read]").attr("href")).pathSegments[2] } - override fun getFilterList() = FilterList( - TagFilter(), - ShowBooksOnlyFilter() - ) - - class ShowBooksOnlyFilter : Filter.CheckBox("Show books only") - - class TagFilter : Filter.Select("Filter by tag", listOf( - "???" to "None", - - "ahegao" to "Ahegao", - "anal" to "Anal", - "big-ass" to "Big ass", - "big-breast" to "Big Breast", - "bondage" to "Bondage", - "cheating" to "Cheating", - "chubby" to "Chubby", - "condom" to "Condom", - "cosplay" to "Cosplay", - "cunnilingus" to "Cunnilingus", - "dark-skin" to "Dark skin", - "defloration" to "Defloration", - "exhibitionism" to "Exhibitionism", - "fellatio" to "Fellatio", - "femdom" to "Femdom", - "flat-chest" to "Flat chest", - "full-color" to "Full color", - "glasses" to "Glasses", - "group" to "Group", - "hairy" to "Hairy", - "handjob" to "Handjob", - "harem" to "Harem", - "housewife" to "Housewife", - "incest" to "Incest", - "large-breast" to "Large Breast", - "lingerie" to "Lingerie", - "loli" to "Loli", - "masturbation" to "Masturbation", - "nakadashi" to "Nakadashi", - "netorare" to "Netorare", - "office-lady" to "Office Lady", - "osananajimi" to "Osananajimi", - "paizuri" to "Paizuri", - "pettanko" to "Pettanko", - "rape" to "Rape", - "schoolgirl" to "Schoolgirl", - "sex-toys" to "Sex Toys", - "shota" to "Shota", - "stocking" to "Stocking", - "swimsuit" to "Swimsuit", - "teacher" to "Teacher", - "tsundere" to "Tsundere", - "uncensored" to "uncensored", - "x-ray" to "X-ray" - ).map { HCTag(it.first, it.second) }.toTypedArray() - ) - - class HCTag(val id: String, val displayName: String) { - override fun toString() = displayName - } + override fun fetchChapterList(manga: SManga) = getOrLoadMetadata(manga.id) { + client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .map { it.asJsoup() } + .toSingle() + }.map { + listOf( + SChapter.create().apply { + setUrlWithoutDomain("/manga/read/${it.readerId}/en/0/1/") + name = "Chapter" + chapter_number = 0.0f + } + ) + }.toObservable() } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/Tsumino.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/english/Tsumino.kt index b4436780a..b767154f1 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/Tsumino.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/english/Tsumino.kt @@ -18,10 +18,11 @@ import eu.kanade.tachiyomi.util.toast import exh.TSUMINO_SOURCE_ID import exh.ui.captcha.CaptchaCompletionVerifier import exh.ui.captcha.SolveCaptchaActivity -import exh.metadata.EMULATED_TAG_NAMESPACE -import exh.metadata.models.Tag -import exh.metadata.models.TsuminoMetadata -import exh.metadata.models.TsuminoMetadata.Companion.BASE_URL +import exh.metadata.metadata.TsuminoSearchMetadata +import exh.metadata.metadata.TsuminoSearchMetadata.Companion.BASE_URL +import exh.metadata.metadata.TsuminoSearchMetadata.Companion.TAG_TYPE_DEFAULT +import exh.metadata.metadata.base.RaisedSearchMetadata.Companion.TAG_TYPE_VIRTUAL +import exh.metadata.metadata.base.RaisedTag import exh.util.urlImportFetchSearchManga import okhttp3.* import org.jsoup.nodes.Document @@ -32,7 +33,9 @@ import uy.kohesive.injekt.injectLazy import java.text.SimpleDateFormat import java.util.* -class Tsumino(private val context: Context): ParsedHttpSource(), LewdSource, CaptchaCompletionVerifier { +class Tsumino(private val context: Context): ParsedHttpSource(), LewdSource, CaptchaCompletionVerifier { + override val metaClass = TsuminoSearchMetadata::class + private val preferences: PreferencesHelper by injectLazy() override val id = TSUMINO_SOURCE_ID @@ -41,77 +44,76 @@ class Tsumino(private val context: Context): ParsedHttpSource(), LewdSource Unit = { - url = it.location() - tags.clear() - - it.getElementById("Title")?.text()?.let { - title = it.trim() - } - - it.getElementById("Artist")?.children()?.first()?.text()?.trim()?.let { - tags.add(Tag("artist", it, false)) - artist = it - } - - it.getElementById("Uploader")?.children()?.first()?.text()?.trim()?.let { - tags.add(Tag("uploader", it, false)) - uploader = it - } - - it.getElementById("Uploaded")?.text()?.let { - uploadDate = TM_DATE_FORMAT.parse(it.trim()).time - } - - it.getElementById("Pages")?.text()?.let { - length = it.trim().toIntOrNull() - } - - it.getElementById("Rating")?.text()?.let { - ratingString = it.trim() - } - - it.getElementById("Category")?.children()?.first()?.text()?.let { - category = it.trim() - tags.add(Tag("genre", it, false)) - } - - it.getElementById("Collection")?.children()?.first()?.text()?.let { - collection = it.trim() - } - - it.getElementById("Group")?.children()?.first()?.text()?.let { - group = it.trim() - tags.add(Tag("group", it, false)) - } - - parody.clear() - it.getElementById("Parody")?.children()?.forEach { - val entry = it.text().trim() - parody.add(entry) - tags.add(Tag("parody", entry, false)) - } - - character.clear() - it.getElementById("Character")?.children()?.forEach { - val entry = it.text().trim() - character.add(entry) - tags.add(Tag("character", entry, false)) - } - - it.getElementById("Tag")?.children()?.let { - tags.addAll(it.map { - Tag(EMULATED_TAG_NAMESPACE, it.text().trim(), false) - }) + + override fun parseIntoMetadata(metadata: TsuminoSearchMetadata, input: Document) { + with(metadata) { + tmId = TsuminoSearchMetadata.tmIdFromUrl(input.location()).toInt() + tags.clear() + + input.getElementById("Title")?.text()?.let { + title = it.trim() + } + + input.getElementById("Artist")?.children()?.first()?.text()?.trim()?.let { + tags.add(RaisedTag("artist", it, TAG_TYPE_VIRTUAL)) + artist = it + } + + input.getElementById("Uploader")?.children()?.first()?.text()?.trim()?.let { + uploader = it + } + + input.getElementById("Uploaded")?.text()?.let { + uploadDate = TM_DATE_FORMAT.parse(it.trim()).time + } + + input.getElementById("Pages")?.text()?.let { + length = it.trim().toIntOrNull() + } + + input.getElementById("Rating")?.text()?.let { + ratingString = it.trim() + } + + input.getElementById("Category")?.children()?.first()?.text()?.let { + category = it.trim() + tags.add(RaisedTag("genre", it, TAG_TYPE_VIRTUAL)) + } + + input.getElementById("Collection")?.children()?.first()?.text()?.let { + collection = it.trim() + } + + input.getElementById("Group")?.children()?.first()?.text()?.let { + group = it.trim() + tags.add(RaisedTag("group", it, TAG_TYPE_VIRTUAL)) + } + + val newParody = mutableListOf() + input.getElementById("Parody")?.children()?.forEach { + val entry = it.text().trim() + newParody.add(entry) + tags.add(RaisedTag("parody", entry, TAG_TYPE_VIRTUAL)) + } + parody = newParody + + val newCharacter = mutableListOf() + input.getElementById("Character")?.children()?.forEach { + val entry = it.text().trim() + newCharacter.add(entry) + tags.add(RaisedTag("character", entry, TAG_TYPE_VIRTUAL)) + } + character = newCharacter + + input.getElementById("Tag")?.children()?.let { + tags.addAll(it.map { + RaisedTag(null, it.text().trim(), TAG_TYPE_DEFAULT) + }) + } } } - + fun genericMangaParse(response: Response): MangasPage { val json = jsonParser.parse(response.body()!!.string()!!).asJsonObject val hasNextPage = json["PageNumber"].int < json["PageCount"].int @@ -121,8 +123,8 @@ class Tsumino(private val context: Context): ParsedHttpSource(), LewdSource { + return client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .flatMap { + parseToManga(manga, it.asJsoup()).andThen(Observable.just(manga.apply { + initialized = true + })) + } + } + override fun mangaDetailsParse(document: Document) - = parseToManga(queryFromUrl(document.location()), document) - + = throw UnsupportedOperationException("Unused method called!") + override fun chapterListSelector() = throw UnsupportedOperationException("Unused method called!") override fun chapterFromElement(element: Element) = throw UnsupportedOperationException("Unused method called!") - override fun fetchChapterList(manga: SManga) = lazyLoadMeta(queryFromUrl(manga.url), - client.newCall(mangaDetailsRequest(manga)).asObservableSuccess().map { it.asJsoup() } - ).map { + override fun fetchChapterList(manga: SManga) = getOrLoadMetadata(manga.id) { + client.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .map { it.asJsoup() } + .toSingle() + }.map { trickTsumino(it.tmId) listOf( @@ -250,9 +266,9 @@ class Tsumino(private val context: Context): ParsedHttpSource(), LewdSource + presenter.loadSearches()?.let { + navView.setSavedSearches(it) + } ?: run { + MaterialDialog.Builder(navView.context) + .title("Failed to load saved searches!") + .content("An error occurred while loading your saved searches.") + .cancelable(true) + .canceledOnTouchOutside(true) + .show() + } + navView.onSaveClicked = { + MaterialDialog.Builder(navView.context) + .title("Save current search query?") + .input("My search name", "") { _, searchName -> + val oldSavedSearches = presenter.loadSearches() ?: emptyList() + if(searchName.isNotBlank() + && oldSavedSearches.size < CatalogueNavigationView.MAX_SAVED_SEARCHES) { + val newSearches = oldSavedSearches + EXHSavedSearch( + searchName.toString().trim(), + presenter.query, + presenter.sourceFilters.toList() + ) + presenter.saveSearches(newSearches) + navView.setSavedSearches(newSearches) + } + } + .positiveText("Save") + .negativeText("Cancel") + .cancelable(true) + .canceledOnTouchOutside(true) + .show() + } + + navView.onSavedSearchClicked = cb@{ indexToSearch -> + val savedSearches = presenter.loadSearches() + + if(savedSearches == null) { + MaterialDialog.Builder(navView.context) + .title("Failed to load saved searches!") + .content("An error occurred while loading your saved searches.") + .cancelable(true) + .canceledOnTouchOutside(true) + .show() + return@cb + } + + val search = savedSearches[indexToSearch] + + presenter.sourceFilters = FilterList(search.filterList) + navView.setFilters(presenter.filterItems) + val allDefault = presenter.sourceFilters == presenter.source.getFilterList() + + showProgressBar() + adapter?.clear() + drawer.closeDrawer(Gravity.END) + presenter.restartPager(search.query, if (allDefault) FilterList() else presenter.sourceFilters) + activity?.invalidateOptionsMenu() + } + + navView.onSavedSearchDeleteClicked = cb@{ indexToDelete -> + val savedSearches = presenter.loadSearches() + + if(savedSearches == null) { + MaterialDialog.Builder(navView.context) + .title("Failed to delete saved search!") + .content("An error occurred while deleting the search.") + .cancelable(true) + .canceledOnTouchOutside(true) + .show() + return@cb + } + + val search = savedSearches[indexToDelete] + + MaterialDialog.Builder(navView.context) + .title("Delete saved search query?") + .content("Are you sure you wish to delete your saved search query: '${search.name}'?") + .positiveText("Cancel") + .negativeText("Confirm") + .onNegative { _, _ -> + val newSearches = savedSearches.filterIndexed { index, _ -> + index != indexToDelete + } + presenter.saveSearches(newSearches) + navView.setSavedSearches(newSearches) + } + .cancelable(true) + .canceledOnTouchOutside(true) + .show() + } + // EXH <-- + navView.onSearchClicked = { val allDefault = presenter.sourceFilters == presenter.source.getFilterList() showProgressBar() diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/BrowseCataloguePresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/BrowseCataloguePresenter.kt index 61d57d749..f8821bea7 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/BrowseCataloguePresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/BrowseCataloguePresenter.kt @@ -1,6 +1,9 @@ package eu.kanade.tachiyomi.ui.catalogue.browse import android.os.Bundle +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import eu.davidea.flexibleadapter.items.IFlexible import eu.davidea.flexibleadapter.items.ISectionable import eu.kanade.tachiyomi.data.cache.CoverCache @@ -9,6 +12,7 @@ import eu.kanade.tachiyomi.data.database.models.Category import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.database.models.MangaCategory import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.SourceManager import eu.kanade.tachiyomi.source.model.Filter @@ -374,4 +378,23 @@ open class BrowseCataloguePresenter( } } + // EXH --> + private val mapper = jacksonObjectMapper().enableDefaultTyping() + fun saveSearches(searches: List) { + val serialized = mapper.writeValueAsString(searches.toTypedArray()) + prefs.eh_savedSearches().set(serialized) + } + + fun loadSearches(): List? { + val loaded = prefs.eh_savedSearches().getOrDefault() + return try { + if (!loaded.isEmpty()) mapper.readValue>(loaded).toList() + else emptyList() + } catch(t: JsonProcessingException) { + // Load failed + Timber.e(t, "Failed to load saved searches!") + null + } + } + // EXH <-- } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/CatalogueNavigationView.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/CatalogueNavigationView.kt index 1f06fc407..7ef11f190 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/CatalogueNavigationView.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/CatalogueNavigationView.kt @@ -2,14 +2,20 @@ package eu.kanade.tachiyomi.ui.catalogue.browse import android.content.Context import android.util.AttributeSet +import android.view.Gravity import android.view.ViewGroup +import android.view.ViewGroup.LayoutParams.MATCH_PARENT +import android.view.ViewGroup.LayoutParams.WRAP_CONTENT +import android.widget.LinearLayout +import android.widget.TextView import eu.davidea.flexibleadapter.FlexibleAdapter import eu.davidea.flexibleadapter.items.IFlexible -import eu.kanade.tachiyomi.R +import eu.kanade.tachiyomi.util.dpToPx import eu.kanade.tachiyomi.util.inflate import eu.kanade.tachiyomi.widget.SimpleNavigationView import kotlinx.android.synthetic.main.catalogue_drawer_content.view.* - +import android.util.TypedValue +import android.view.View class CatalogueNavigationView @JvmOverloads constructor(context: Context, attrs: AttributeSet? = null) : SimpleNavigationView(context, attrs) { @@ -22,13 +28,26 @@ class CatalogueNavigationView @JvmOverloads constructor(context: Context, attrs: var onResetClicked = {} + // EXH --> + var onSaveClicked = {} + // EXH <-- + + // EXH --> + var onSavedSearchClicked: (Int) -> Unit = {} + // EXH <-- + + // EXH --> + var onSavedSearchDeleteClicked: (Int) -> Unit = {} + // EXH <-- + init { recycler.adapter = adapter recycler.setHasFixedSize(true) - val view = inflate(R.layout.catalogue_drawer_content) + val view = inflate(eu.kanade.tachiyomi.R.layout.catalogue_drawer_content) ((view as ViewGroup).getChildAt(1) as ViewGroup).addView(recycler) addView(view) - title.text = context?.getString(R.string.source_search_options) + title.text = context?.getString(eu.kanade.tachiyomi.R.string.source_search_options) + save_search_btn.setOnClickListener { onSaveClicked() } search_btn.setOnClickListener { onSearchClicked() } reset_btn.setOnClickListener { onResetClicked() } } @@ -37,4 +56,33 @@ class CatalogueNavigationView @JvmOverloads constructor(context: Context, attrs: adapter.updateDataSet(items) } + // EXH --> + fun setSavedSearches(searches: List) { + saved_searches.removeAllViews() + + val outValue = TypedValue() + context.theme.resolveAttribute(android.R.attr.selectableItemBackground, outValue, true) + + save_search_btn.visibility = if(searches.size < 5) View.VISIBLE else View.GONE + + searches.forEachIndexed { index, search -> + val restoreBtn = TextView(context) + restoreBtn.text = search.name + val params = LinearLayout.LayoutParams(MATCH_PARENT, WRAP_CONTENT) + params.gravity = Gravity.CENTER + restoreBtn.layoutParams = params + restoreBtn.gravity = Gravity.CENTER + restoreBtn.setBackgroundResource(outValue.resourceId) + restoreBtn.setPadding(8.dpToPx, 8.dpToPx, 8.dpToPx, 8.dpToPx) + restoreBtn.setOnClickListener { onSavedSearchClicked(index) } + restoreBtn.setOnLongClickListener { onSavedSearchDeleteClicked(index); true } + saved_searches.addView(restoreBtn) + } + } + + companion object { + const val MAX_SAVED_SEARCHES = 5 + } + // EXH <-- + } \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/EXHSavedSearch.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/EXHSavedSearch.kt new file mode 100644 index 000000000..2ba1923e3 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/catalogue/browse/EXHSavedSearch.kt @@ -0,0 +1,7 @@ +package eu.kanade.tachiyomi.ui.catalogue.browse + +import eu.kanade.tachiyomi.source.model.Filter + +data class EXHSavedSearch(val name: String, + val query: String, + val filterList: List>) \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryAdapter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryAdapter.kt index fff19e16c..bbe42191d 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryAdapter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryAdapter.kt @@ -1,17 +1,14 @@ package eu.kanade.tachiyomi.ui.library +import com.pushtorefresh.storio.sqlite.queries.RawQuery import eu.davidea.flexibleadapter.FlexibleAdapter +import eu.kanade.tachiyomi.data.database.DatabaseHelper import eu.kanade.tachiyomi.data.database.models.Manga -import exh.* -import exh.metadata.metadataClass -import exh.metadata.models.SearchableGalleryMetadata -import exh.metadata.syncMangaIds +import exh.isLewdSource +import exh.metadata.sql.tables.SearchMetadataTable import exh.search.SearchEngine -import exh.util.defRealm -import io.realm.RealmResults import timber.log.Timber -import java.util.concurrent.ConcurrentHashMap -import kotlin.concurrent.thread +import uy.kohesive.injekt.injectLazy /** * Adapter storing a list of manga in a certain category. @@ -21,6 +18,7 @@ import kotlin.concurrent.thread class LibraryCategoryAdapter(val view: LibraryCategoryView) : FlexibleAdapter(null, view, true) { // --> EH + private val db: DatabaseHelper by injectLazy() private val searchEngine = SearchEngine() // <-- EH @@ -38,15 +36,6 @@ class LibraryCategoryAdapter(val view: LibraryCategoryView) : // A copy of manga always unfiltered. mangas = list.toList() - // Sync manga IDs in background (EH) - thread { - //Wait 1s to reduce UI stutter during animations - Thread.sleep(2000) - defRealm { - it.syncMangaIds(mangas) - } - } - performFilter() } @@ -61,104 +50,43 @@ class LibraryCategoryAdapter(val view: LibraryCategoryView) : fun performFilter() { if(searchText.isNotBlank()) { - if(cacheText != searchText) { - globalSearchCache.clear() - cacheText = searchText - } - + // EXH --> try { - val thisCache = globalSearchCache.getOrPut(view.category.name) { - SearchCache(mangas.size) + val startTime = System.currentTimeMillis() + + val parsedQuery = searchEngine.parseQuery(searchText) + val sqlQuery = searchEngine.queryToSql(parsedQuery) + val queryResult = db.lowLevel().rawQuery(RawQuery.builder() + .query(sqlQuery.first) + .args(*sqlQuery.second.toTypedArray()) + .build()) + + val convertedResult = ArrayList(queryResult.count) + val mangaIdCol = queryResult.getColumnIndex(SearchMetadataTable.COL_MANGA_ID) + queryResult.moveToFirst() + while(queryResult.count > 0 && !queryResult.isAfterLast) { + convertedResult += queryResult.getLong(mangaIdCol) + queryResult.moveToNext() } - if(thisCache.ready) { - //Skip everything if cache matches our query exactly - updateDataSet(mangas.filter { - thisCache.cache[it.manga.id] ?: false - }) - } else { - thisCache.cache.clear() - - val parsedQuery = searchEngine.parseQuery(searchText) - var totalFilteredSize = 0 - - val metadata = view.controller.meta!!.map { - val meta: RealmResults = if (it.value.isNotEmpty()) - searchEngine.filterResults(it.value.where(), - parsedQuery, - it.value.first()!!.titleFields) - .sort(SearchableGalleryMetadata::mangaId.name) - .findAll().apply { - totalFilteredSize += size - } - else - it.value - Pair(it.key, meta) - }.toMap() - - val out = ArrayList(mangas.size) - - var lewdMatches = 0 - - for(manga in mangas) { - // --> EH - try { - if (isLewdSource(manga.manga.source)) { - //Stop matching lewd manga if we have matched them all already! - if (lewdMatches >= totalFilteredSize) - continue - - val metaClass = manga.manga.metadataClass - val unfilteredMeta = view.controller.meta!![metaClass] - val filteredMeta = metadata[metaClass] - - val hasMeta = manga.hasMetadata ?: (unfilteredMeta - ?.where() - ?.equalTo(SearchableGalleryMetadata::mangaId.name, manga.manga.id) - ?.count() ?: 0 > 0) - - if (hasMeta) { - if (filteredMeta!!.where() - .equalTo(SearchableGalleryMetadata::mangaId.name, manga.manga.id) - .count() > 0) { - //Metadata match! - lewdMatches++ - thisCache.cache[manga.manga.id!!] = true - out += manga - continue - } - } - } - } catch (e: Exception) { - Timber.w(e, "Could not filter manga! %s", manga.manga) - } - - //Fallback to regular filter - val filterRes = manga.filter(searchText) - thisCache.cache[manga.manga.id!!] = filterRes - if(filterRes) out += manga - // <-- EH + val out = mangas.filter { + if(isLewdSource(it.manga.source)) { + convertedResult.binarySearch(it.manga.id) >= 0 + } else { + it.filter(searchText) } - thisCache.ready = true - updateDataSet(out) } + + Timber.d("===> Took %s milliseconds to filter manga!", System.currentTimeMillis() - startTime) + + updateDataSet(out) } catch(e: Exception) { Timber.w(e, "Could not filter mangas!") updateDataSet(mangas) } + // EXH <-- } else { - globalSearchCache.clear() updateDataSet(mangas) } } - - class SearchCache(size: Int) { - var ready = false - var cache = HashMap(size) - } - - companion object { - var cacheText: String? = null - val globalSearchCache = ConcurrentHashMap() - } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt index 5ef28138b..789b39ea6 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt @@ -114,7 +114,7 @@ class LibraryCategoryView @JvmOverloads constructor(context: Context, attrs: Att subscriptions += controller.searchRelay .doOnNext { adapter.searchText = it } .skip(1) - .debounce(350, TimeUnit.MILLISECONDS) + .debounce(500, TimeUnit.MILLISECONDS) .observeOn(AndroidSchedulers.mainThread()) .subscribe { adapter.performFilter() } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt index 21ba220ad..931fdf06d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt @@ -38,10 +38,6 @@ import eu.kanade.tachiyomi.util.inflate import eu.kanade.tachiyomi.util.toast import exh.favorites.FavoritesIntroDialog import exh.favorites.FavoritesSyncStatus -import exh.metadata.loadAllMetadata -import exh.metadata.models.SearchableGalleryMetadata -import io.realm.Realm -import io.realm.RealmResults import kotlinx.android.synthetic.main.library_controller.* import kotlinx.android.synthetic.main.main_activity.* import rx.Subscription @@ -51,7 +47,6 @@ import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import java.io.IOException import java.util.concurrent.TimeUnit -import kotlin.reflect.KClass class LibraryController( @@ -130,10 +125,6 @@ class LibraryController( private var searchViewSubscription: Subscription? = null // --> EH - //Cached realm - var realm: Realm? = null - //Cached metadata - var meta: Map, RealmResults>? = null //Sync dialog private var favSyncDialog: MaterialDialog? = null //Old sync status @@ -159,16 +150,6 @@ class LibraryController( return inflater.inflate(R.layout.library_controller, container, false) } - // --> EH - override fun onCreateView(inflater: LayoutInflater, container: ViewGroup, savedViewState: Bundle?): View { - //Load realm - realm = Realm.getDefaultInstance()?.apply { - meta = loadAllMetadata() - } - return super.onCreateView(inflater, container, savedViewState) - } - // <-- EH - override fun onViewCreated(view: View) { super.onViewCreated(view) @@ -205,12 +186,6 @@ class LibraryController( tabsVisibilitySubscription?.unsubscribe() tabsVisibilitySubscription = null super.onDestroyView(view) - - // --> EH - //Clean up realm - realm?.close() - meta = null - // <-- EH } override fun createSecondaryDrawer(drawer: DrawerLayout): ViewGroup { diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt index de4f1f007..b72805e45 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt @@ -15,7 +15,6 @@ import android.support.v7.graphics.drawable.DrawerArrowDrawable import android.support.v7.widget.Toolbar import android.view.ViewGroup import com.bluelinelabs.conductor.* -import eu.kanade.tachiyomi.Migrations import eu.kanade.tachiyomi.R import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.getOrDefault @@ -29,27 +28,20 @@ import eu.kanade.tachiyomi.ui.manga.MangaController import eu.kanade.tachiyomi.ui.recent_updates.RecentChaptersController import eu.kanade.tachiyomi.ui.recently_read.RecentlyReadController import eu.kanade.tachiyomi.ui.setting.SettingsMainController -import exh.metadata.loadAllMetadata import exh.uconfig.WarnConfigureDialogController import exh.ui.batchadd.BatchAddController import exh.ui.lock.LockChangeHandler import exh.ui.lock.LockController import exh.ui.lock.lockEnabled import exh.ui.lock.notifyLockSecurity -import exh.ui.migration.MetadataFetchDialog -import exh.util.defRealm import kotlinx.android.synthetic.main.main_activity.* import uy.kohesive.injekt.injectLazy import android.text.TextUtils import android.view.View -import eu.kanade.tachiyomi.source.SourceManager -import eu.kanade.tachiyomi.source.online.all.Hitomi import eu.kanade.tachiyomi.util.vibrate -import exh.HITOMI_SOURCE_ID -import rx.schedulers.Schedulers +import exh.EXHMigrations +import exh.ui.migration.MetadataFetchDialog import timber.log.Timber -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get class MainActivity : BaseActivity() { @@ -176,35 +168,32 @@ class MainActivity : BaseActivity() { notifyLockSecurity(this) } } - - // Early hitomi.la refresh - if(preferences.eh_hl_earlyRefresh().getOrDefault()) { - (Injekt.get().get(HITOMI_SOURCE_ID) as Hitomi) - .ensureCacheLoaded(false) - .subscribeOn(Schedulers.computation()) - .subscribe() - } // <-- EH syncActivityViewWithController(router.backstack.lastOrNull()?.controller()) if (savedInstanceState == null) { // Show changelog if needed - if (Migrations.upgrade(preferences)) { + // TODO +// if (Migrations.upgrade(preferences)) { +// ChangelogDialogController().showDialog(router) +// } + + // EXH --> + // Perform EXH specific migrations + if(EXHMigrations.upgrade(preferences)) { ChangelogDialogController().showDialog(router) } - // Migrate metadata if empty (EH) - if(!defRealm { - it.loadAllMetadata().any { - it.value.isNotEmpty() - } - }) MetadataFetchDialog().askMigration(this, false) + if(!preferences.migrateLibraryAsked().getOrDefault()) { + MetadataFetchDialog().askMigration(this, false) + } // Upload settings if(preferences.enableExhentai().getOrDefault() && preferences.eh_showSettingsUploadWarning().getOrDefault()) WarnConfigureDialogController.uploadSettings(router) + // EXH <-- } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt index bdcd597bc..1f1560d3f 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/info/MangaInfoController.kt @@ -53,6 +53,7 @@ import exh.ui.webview.WebViewActivity import jp.wasabeef.glide.transformations.CropSquareTransformation import jp.wasabeef.glide.transformations.MaskTransformation import kotlinx.android.synthetic.main.manga_info_controller.* +import timber.log.Timber import uy.kohesive.injekt.injectLazy import java.text.DateFormat import java.text.DecimalFormat @@ -139,7 +140,7 @@ class MangaInfoController : NucleusController(), var text = tag if(isEHentaiBasedSource()) { val parsed = parseTag(text) - text = wrapTag(parsed.first, parsed.second) + text = wrapTag(parsed.first, parsed.second.substringBefore('|').trim()) } performGlobalSearch(text) } @@ -386,6 +387,9 @@ class MangaInfoController : NucleusController(), fun onFetchMangaError(error: Throwable) { setRefreshing(false) activity?.toast(error.message) + // EXH --> + Timber.e(error, "Failed to fetch manga details!") + // EXH <-- } /** diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderActivity.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderActivity.kt index c3b8aefd8..9aa2ab706 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderActivity.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderActivity.kt @@ -14,23 +14,29 @@ import android.view.* import android.view.animation.Animation import android.view.animation.AnimationUtils import android.widget.SeekBar +import com.afollestad.materialdialogs.MaterialDialog import com.davemorrissey.labs.subscaleview.SubsamplingScaleImageView +import com.jakewharton.rxbinding.view.clicks +import com.jakewharton.rxbinding.widget.checkedChanges +import com.jakewharton.rxbinding.widget.textChanges import eu.kanade.tachiyomi.R import eu.kanade.tachiyomi.data.database.models.Chapter import eu.kanade.tachiyomi.data.database.models.Manga import eu.kanade.tachiyomi.data.preference.PreferencesHelper import eu.kanade.tachiyomi.data.preference.getOrDefault +import eu.kanade.tachiyomi.source.SourceManager +import eu.kanade.tachiyomi.source.model.Page +import eu.kanade.tachiyomi.source.online.all.EHentai import eu.kanade.tachiyomi.ui.base.activity.BaseRxActivity import eu.kanade.tachiyomi.ui.reader.ReaderPresenter.SetAsCoverResult.AddToLibraryFirst import eu.kanade.tachiyomi.ui.reader.ReaderPresenter.SetAsCoverResult.Error import eu.kanade.tachiyomi.ui.reader.ReaderPresenter.SetAsCoverResult.Success +import eu.kanade.tachiyomi.ui.reader.loader.HttpPageLoader import eu.kanade.tachiyomi.ui.reader.model.ReaderChapter import eu.kanade.tachiyomi.ui.reader.model.ReaderPage import eu.kanade.tachiyomi.ui.reader.model.ViewerChapters import eu.kanade.tachiyomi.ui.reader.viewer.BaseViewer -import eu.kanade.tachiyomi.ui.reader.viewer.pager.L2RPagerViewer -import eu.kanade.tachiyomi.ui.reader.viewer.pager.R2LPagerViewer -import eu.kanade.tachiyomi.ui.reader.viewer.pager.VerticalPagerViewer +import eu.kanade.tachiyomi.ui.reader.viewer.pager.* import eu.kanade.tachiyomi.ui.reader.viewer.webtoon.WebtoonViewer import eu.kanade.tachiyomi.util.* import eu.kanade.tachiyomi.widget.SimpleAnimationListener @@ -46,6 +52,7 @@ import timber.log.Timber import uy.kohesive.injekt.injectLazy import java.io.File import java.util.concurrent.TimeUnit +import kotlin.math.roundToLong /** * Activity containing the reader of Tachiyomi. This activity is mostly a container of the @@ -76,6 +83,15 @@ class ReaderActivity : BaseRxActivity() { var menuVisible = false private set + // --> EH + private var ehUtilsVisible = false + + private val exhSubscriptions = CompositeSubscription() + + private var autoscrollSubscription: Subscription? = null + private val sourceManager: SourceManager by injectLazy() + // <-- EH + /** * System UI helper to hide status & navigation bar on all different API levels. */ @@ -132,12 +148,49 @@ class ReaderActivity : BaseRxActivity() { if (savedState != null) { menuVisible = savedState.getBoolean(::menuVisible.name) + // --> EH + ehUtilsVisible = savedState.getBoolean(::ehUtilsVisible.name) + // <-- EH } config = ReaderConfig() initializeMenu() } + // --> EH + private fun setEhUtilsVisibility(visible: Boolean) { + if(visible) { + eh_utils.visible() + expand_eh_button.setImageResource(R.drawable.ic_keyboard_arrow_up_white_32dp) + } else { + eh_utils.gone() + expand_eh_button.setImageResource(R.drawable.ic_keyboard_arrow_down_white_32dp) + } + } + // <-- EH + + // --> EH + private fun setupAutoscroll(interval: Float) { + exhSubscriptions.remove(autoscrollSubscription) + autoscrollSubscription = null + + if(interval == -1f) return + + val intervalMs = (interval * 1000).roundToLong() + val sub = Observable.interval(intervalMs, intervalMs, TimeUnit.MILLISECONDS) + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { + viewer.let { v -> + if(v is PagerViewer) v.moveToNext() + else if(v is WebtoonViewer) v.scrollDown() + } + } + + autoscrollSubscription = sub + exhSubscriptions += sub + } + // <-- EH + /** * Called when the activity is destroyed. Cleans up the viewer, configuration and any view. */ @@ -157,6 +210,9 @@ class ReaderActivity : BaseRxActivity() { */ override fun onSaveInstanceState(outState: Bundle) { outState.putBoolean(::menuVisible.name, menuVisible) + // EXH --> + outState.putBoolean(::ehUtilsVisible.name, ehUtilsVisible) + // EXH <-- if (!isChangingConfigurations) { presenter.onSaveInstanceStateNonConfigurationChange() } @@ -257,10 +313,151 @@ class ReaderActivity : BaseRxActivity() { } } + // --> EH + exhSubscriptions += expand_eh_button.clicks().subscribe { + ehUtilsVisible = !ehUtilsVisible + setEhUtilsVisibility(ehUtilsVisible) + } + + eh_autoscroll_freq.setText(preferences.eh_utilAutoscrollInterval().getOrDefault().let { + if(it == -1f) + "" + else it.toString() + }) + + exhSubscriptions += eh_autoscroll.checkedChanges() + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { + setupAutoscroll(if(it) + preferences.eh_utilAutoscrollInterval().getOrDefault() + else -1f) + } + + exhSubscriptions += eh_autoscroll_freq.textChanges() + .observeOn(AndroidSchedulers.mainThread()) + .subscribe { + val parsed = it?.toString()?.toFloatOrNull() + + if (parsed == null || parsed <= 0 || parsed > 9999) { + eh_autoscroll_freq.error = "Invalid frequency" + preferences.eh_utilAutoscrollInterval().set(-1f) + eh_autoscroll.isEnabled = false + setupAutoscroll(-1f) + } else { + eh_autoscroll_freq.error = null + preferences.eh_utilAutoscrollInterval().set(parsed) + eh_autoscroll.isEnabled = true + setupAutoscroll(if(eh_autoscroll.isChecked) parsed else -1f) + } + } + + exhSubscriptions += eh_autoscroll_help.clicks().subscribe { + MaterialDialog.Builder(this) + .title("Autoscroll help") + .content("Automatically scroll to the next page in the specified interval. Interval is specified in seconds.") + .positiveText("Ok") + .show() + } + + exhSubscriptions += eh_retry_all.clicks().subscribe { + var retried = 0 + + presenter.viewerChaptersRelay.value + .currChapter + .pages + ?.forEachIndexed { index, page -> + var shouldQueuePage = false + if(page.status == Page.ERROR) { + shouldQueuePage = true + } else if(page.status == Page.LOAD_PAGE + || page.status == Page.DOWNLOAD_IMAGE) { + // Do nothing + } + + if(shouldQueuePage) { + page.status = Page.QUEUE + } else { + return@forEachIndexed + } + + //If we are using EHentai/ExHentai, get a new image URL + presenter.manga?.let { m -> + val src = sourceManager.get(m.source) + if(src is EHentai) + page.imageUrl = null + } + + val loader = page.chapter.pageLoader + if(page.index == exh_currentPage()?.index && loader is HttpPageLoader) { + loader.boostPage(page) + } else { + loader?.retryPage(page) + } + + retried++ + } + + toast("Retrying $retried failed pages...") + } + + exhSubscriptions += eh_retry_all_help.clicks().subscribe { + MaterialDialog.Builder(this) + .title("Retry all help") + .content("Re-add all failed pages to the download queue.") + .positiveText("Ok") + .show() + } + + exhSubscriptions += eh_boost_page.clicks().subscribe { + viewer?.let { viewer -> + val curPage = exh_currentPage() ?: run { + toast("This page cannot be boosted (invalid page)!") + return@let + } + + if(curPage.status == Page.ERROR) { + toast("Page failed to load, press the retry button instead!") + } else if(curPage.status == Page.LOAD_PAGE || curPage.status == Page.DOWNLOAD_IMAGE) { + toast("This page is already downloading!") + } else if(curPage.status == Page.READY) { + toast("This page has already been downloaded!") + } else { + val loader = (presenter.viewerChaptersRelay.value.currChapter.pageLoader as? HttpPageLoader) + if(loader != null) { + loader.boostPage(curPage) + toast("Boosted current page!") + } else { + toast("This page cannot be boosted (invalid page loader)!") + } + } + } + } + + exhSubscriptions += eh_boost_page_help.clicks().subscribe { + MaterialDialog.Builder(this) + .title("Boost page help") + .content("Normally the downloader can only download a specific amount of pages at the same time. This means you can be waiting for a page to download but the downloader will not start downloading the page until it has a free download slot. Pressing 'Boost page' will force the downloader to begin downloading the current page, regardless of whether or not there is an available slot.") + .positiveText("Ok") + .show() + } + // <-- EH + // Set initial visibility setMenuVisibility(menuVisible) + + // --> EH + setEhUtilsVisibility(ehUtilsVisible) + // <-- EH } + // EXH --> + private fun exh_currentPage(): ReaderPage? { + val currentPage = (((viewer as? PagerViewer)?.currentPage + ?: (viewer as? WebtoonViewer)?.currentPage) as? ReaderPage)?.index + return currentPage?.let { presenter.viewerChaptersRelay.value.currChapter.pages?.getOrNull(it) } + } + // EXH <-- + /** * Sets the visibility of the menu according to [visible] and with an optional parameter to * [animate] the views. @@ -282,7 +479,9 @@ class ReaderActivity : BaseRxActivity() { } } }) - toolbar.startAnimation(toolbarAnimation) + // EXH --> + header.startAnimation(toolbarAnimation) + // EXH <-- val bottomAnimation = AnimationUtils.loadAnimation(this, R.anim.enter_from_bottom) reader_menu_bottom.startAnimation(bottomAnimation) @@ -297,7 +496,9 @@ class ReaderActivity : BaseRxActivity() { reader_menu.visibility = View.GONE } }) - toolbar.startAnimation(toolbarAnimation) + // EXH --> + header.startAnimation(toolbarAnimation) + // EXH <-- val bottomAnimation = AnimationUtils.loadAnimation(this, R.anim.exit_to_bottom) reader_menu_bottom.startAnimation(bottomAnimation) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt index ab8b56ee8..035413b0e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/ReaderPresenter.kt @@ -70,7 +70,7 @@ class ReaderPresenter( /** * Relay for currently active viewer chapters. */ - private val viewerChaptersRelay = BehaviorRelay.create() + /* [EXH] private */ val viewerChaptersRelay = BehaviorRelay.create() /** * Relay used when loading prev/next chapter needed to lock the UI (with a dialog). diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt index 223e9811e..5da9ec570 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/loader/HttpPageLoader.kt @@ -1,6 +1,8 @@ package eu.kanade.tachiyomi.ui.reader.loader import eu.kanade.tachiyomi.data.cache.ChapterCache +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.ui.reader.model.ReaderChapter @@ -15,6 +17,7 @@ import rx.subscriptions.CompositeSubscription import timber.log.Timber import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get +import uy.kohesive.injekt.injectLazy import java.util.concurrent.PriorityBlockingQueue import java.util.concurrent.atomic.AtomicInteger @@ -26,6 +29,9 @@ class HttpPageLoader( private val source: HttpSource, private val chapterCache: ChapterCache = Injekt.get() ) : PageLoader() { + // EXH --> + private val prefs: PreferencesHelper by injectLazy() + // EXH <-- /** * A queue used to manage requests one by one while allowing priorities. @@ -38,17 +44,23 @@ class HttpPageLoader( private val subscriptions = CompositeSubscription() init { - subscriptions += Observable.defer { Observable.just(queue.take().page) } - .filter { it.status == Page.QUEUE } - .concatMap { source.fetchImageFromCacheThenNet(it) } - .repeat() - .subscribeOn(Schedulers.io()) - .subscribe({ - }, { error -> - if (error !is InterruptedException) { - Timber.e(error) - } - }) + // EXH --> + repeat(prefs.eh_readerThreads().getOrDefault()) { + // EXH <-- + subscriptions += Observable.defer { Observable.just(queue.take().page) } + .filter { it.status == Page.QUEUE } + .concatMap { source.fetchImageFromCacheThenNet(it) } + .repeat() + .subscribeOn(Schedulers.io()) + .subscribe({ + }, { error -> + if (error !is InterruptedException) { + Timber.e(error) + } + }) + // EXH --> + } + // EXH <-- } /** @@ -142,6 +154,9 @@ class HttpPageLoader( if (page.status == Page.ERROR) { page.status = Page.QUEUE } + // EXH --> + if(prefs.eh_readerInstantRetry().getOrDefault()) boostPage(page) + else // EXH <-- queue.offer(PriorityPage(page, 2)) } @@ -223,4 +238,20 @@ class HttpPageLoader( .doOnNext { chapterCache.putImageToCache(page.imageUrl!!, it) } .map { page } } + + // EXH --> + fun boostPage(page: ReaderPage) { + if(page.status == Page.QUEUE) { + subscriptions += Observable.just(page) + .concatMap { source.fetchImageFromCacheThenNet(it) } + .subscribeOn(Schedulers.io()) + .subscribe({ + }, { error -> + if (error !is InterruptedException) { + Timber.e(error) + } + }) + } + } + // EXH <-- } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/pager/PagerViewer.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/pager/PagerViewer.kt index 8f37257a5..785a772a9 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/pager/PagerViewer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/pager/PagerViewer.kt @@ -39,7 +39,7 @@ abstract class PagerViewer(val activity: ReaderActivity) : BaseViewer { /** * Currently active item. It can be a chapter page or a chapter transition. */ - private var currentPage: Any? = null + /* [EXH] private */ var currentPage: Any? = null /** * Viewer chapters to set when the pager enters idle mode. Otherwise, if the view was settling diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/webtoon/WebtoonViewer.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/webtoon/WebtoonViewer.kt index 6adee83c2..07a504b13 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/webtoon/WebtoonViewer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/reader/viewer/webtoon/WebtoonViewer.kt @@ -48,7 +48,7 @@ class WebtoonViewer(val activity: ReaderActivity) : BaseViewer { /** * Currently active item. It can be a chapter page or a chapter transition. */ - private var currentPage: Any? = null + /* [EXH] private */ var currentPage: Any? = null /** * Configuration used by this viewer, like allow taps, or crop image borders. @@ -200,7 +200,7 @@ class WebtoonViewer(val activity: ReaderActivity) : BaseViewer { /** * Scrolls down by [scrollDistance]. */ - private fun scrollDown() { + /* [EXH] private */ fun scrollDown() { recycler.smoothScrollBy(0, scrollDistance) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt index c12ca8d7d..7dd7614a4 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsAdvancedController.kt @@ -3,8 +3,8 @@ package eu.kanade.tachiyomi.ui.setting import android.app.Dialog import android.os.Bundle import android.support.v7.preference.PreferenceScreen +import android.text.Html import android.view.View -import android.widget.Toast import com.afollestad.materialdialogs.MaterialDialog import com.bluelinelabs.conductor.RouterTransaction import com.bluelinelabs.conductor.changehandler.FadeChangeHandler @@ -15,12 +15,13 @@ import eu.kanade.tachiyomi.data.library.LibraryUpdateService import eu.kanade.tachiyomi.data.library.LibraryUpdateService.Target import eu.kanade.tachiyomi.data.preference.PreferenceKeys import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.source.SourceManager.Companion.DELEGATED_SOURCES import eu.kanade.tachiyomi.ui.base.controller.DialogController +import eu.kanade.tachiyomi.ui.base.controller.withFadeTransaction import eu.kanade.tachiyomi.ui.library.LibraryController import eu.kanade.tachiyomi.util.toast +import exh.debug.SettingsDebugController import exh.ui.migration.MetadataFetchDialog -import exh.util.realmTrans -import io.realm.Realm import rx.Observable import rx.android.schedulers.AndroidSchedulers import rx.schedulers.Schedulers @@ -74,6 +75,8 @@ class SettingsAdvancedController : SettingsController() { onClick { LibraryUpdateService.start(context, target = Target.TRACKING) } } + + // --> EXH preferenceCategory { title = "Gallery metadata" isPersistent = false @@ -98,14 +101,29 @@ class SettingsAdvancedController : SettingsController() { summary = "Clear all library metadata. Disables tag searching in the library" onClick { - realmTrans { - it.deleteAll() + db.inTransaction { + db.deleteAllSearchMetadata().executeAsBlocking() + db.deleteAllSearchTags().executeAsBlocking() + db.deleteAllSearchTitle().executeAsBlocking() } context.toast("Library metadata cleared!") } } } + switchPreference { + title = "Enable delegated sources" + key = PreferenceKeys.eh_delegateSources + defaultValue = true + summary = "Apply TachiyomiEH enhancements to the following sources if they are installed: ${DELEGATED_SOURCES.values.joinToString { it.sourceName }}" + } + + preference { + title = "Open debug menu" + summary = Html.fromHtml("DO NOT TOUCH THIS MENU UNLESS YOU KNOW WHAT YOU ARE DOING! IT CAN CORRUPT YOUR LIBRARY!") + onClick { router.pushController(SettingsDebugController().withFadeTransaction()) } + } + // <-- EXH } private fun clearChapterCache() { diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsGeneralController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsGeneralController.kt index 78ea8d20f..bff749666 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsGeneralController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsGeneralController.kt @@ -1,6 +1,7 @@ package eu.kanade.tachiyomi.ui.setting import android.app.Dialog +import android.os.Build import android.os.Bundle import android.os.Handler import android.support.v7.preference.PreferenceScreen @@ -180,7 +181,7 @@ class SettingsGeneralController : SettingsController() { } } - // --> EH + // --> EXH switchPreference { key = Keys.eh_askCategoryOnLongPress title = "Long-press favorite button to specify category" @@ -193,6 +194,14 @@ class SettingsGeneralController : SettingsController() { defaultValue = false } + switchPreference { + key = Keys.eh_autoSolveCaptchas + title = "Automatically solve captcha" + summary = "Use HIGHLY EXPERIMENTAL automatic ReCAPTCHA solver. Will be grayed out if unsupported by your device." + defaultValue = false + shouldDisableView = Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP + } + switchPreference { key = Keys.eh_incogWebview title = "Incognito 'Open in browser'" @@ -228,7 +237,7 @@ class SettingsGeneralController : SettingsController() { defaultValue = false } } - // <-- EH + // <-- EXH } class LibraryColumnsDialog : DialogController() { diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsHlController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsHlController.kt index 1cccc5e16..356bab47e 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsHlController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsHlController.kt @@ -4,7 +4,6 @@ import android.support.v7.preference.PreferenceScreen import android.widget.Toast import eu.kanade.tachiyomi.data.preference.PreferenceKeys import eu.kanade.tachiyomi.source.SourceManager -import eu.kanade.tachiyomi.source.online.all.Hitomi import eu.kanade.tachiyomi.util.toast import exh.HITOMI_SOURCE_ID import uy.kohesive.injekt.Injekt @@ -18,41 +17,6 @@ class SettingsHlController : SettingsController() { override fun setupPreferenceScreen(screen: PreferenceScreen) = with(screen) { title = "hitomi.la" - editTextPreference { - title = "Search database refresh frequency" - summary = "How often to get new entries for the search database in hours. Setting this frequency too high may cause high CPU usage and network usage." - key = PreferenceKeys.eh_hl_refreshFrequency - defaultValue = "24" - - onChange { - it as String - - if((it.toLongOrNull() ?: -1) <= 0) { - context.toast("Invalid frequency. Frequency must be a positive whole number.") - false - } else true - } - } - - switchPreference { - title = "Begin refreshing search database on app launch" - summary = "Normally the search database gets refreshed (if required) when you open the hitomi.la catalogue. If you enable this option, the database gets refreshed in the background as soon as you open the app. It will result in higher data usage but may increase hitomi.la search speeds." - key = PreferenceKeys.eh_hl_earlyRefresh - defaultValue = false - } - - preference { - title = "Force refresh search database now" - summary = "Delete the local copy of the hitomi.la search database and download the new database now. Hitomi.la search will not work in the ~10mins that it takes to refresh the search database" - isPersistent = false - - onClick { - context.toast(if((Injekt.get().get(HITOMI_SOURCE_ID) as Hitomi).forceEnsureCacheLoaded()) { - "Refreshing database. You will NOT be notified when it is complete!" - } else { - "Could not begin refresh process as there is already one ongoing!" - }, Toast.LENGTH_LONG) - } - } + // TODO Thumbnail quality chooser } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsReaderController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsReaderController.kt index ab9b32fc1..c28a1bc5d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsReaderController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsReaderController.kt @@ -86,6 +86,7 @@ class SettingsReaderController : SettingsController() { defaultValue = false } } + // EXH --> intListPreference { key = Keys.eh_readerThreads title = "Download threads" @@ -147,6 +148,7 @@ class SettingsReaderController : SettingsController() { title = "Preserve reading position on read manga" defaultValue = false } + // EXH <-- preferenceCategory { titleRes = R.string.pager_viewer diff --git a/app/src/main/java/exh/EHSourceHelpers.kt b/app/src/main/java/exh/EHSourceHelpers.kt index 70db105b0..c43a8a54d 100755 --- a/app/src/main/java/exh/EHSourceHelpers.kt +++ b/app/src/main/java/exh/EHSourceHelpers.kt @@ -1,5 +1,7 @@ package exh +import eu.kanade.tachiyomi.source.SourceManager + /** * Source helpers */ @@ -15,13 +17,17 @@ const val PERV_EDEN_IT_SOURCE_ID = LEWD_SOURCE_SERIES + 6 const val NHENTAI_SOURCE_ID = LEWD_SOURCE_SERIES + 7 +@Deprecated("Now a delegated source") const val HENTAI_CAFE_SOURCE_ID = LEWD_SOURCE_SERIES + 8 const val TSUMINO_SOURCE_ID = LEWD_SOURCE_SERIES + 9 const val HITOMI_SOURCE_ID = LEWD_SOURCE_SERIES + 10 -fun isLewdSource(source: Long) = source in 6900..6999 +// TODO hentai.cafe is a lewd source! +fun isLewdSource(source: Long) = source in 6900..6999 || SourceManager.DELEGATED_SOURCES.any { + it.value.sourceId == source +} fun isEhSource(source: Long) = source == EH_SOURCE_ID || source == EH_METADATA_SOURCE_ID diff --git a/app/src/main/java/exh/EXHMigrations.kt b/app/src/main/java/exh/EXHMigrations.kt new file mode 100644 index 000000000..9eed2cb34 --- /dev/null +++ b/app/src/main/java/exh/EXHMigrations.kt @@ -0,0 +1,85 @@ +package exh + +import com.pushtorefresh.storio.sqlite.queries.Query +import com.pushtorefresh.storio.sqlite.queries.RawQuery +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.database.resolvers.MangaUrlPutResolver +import eu.kanade.tachiyomi.data.database.tables.MangaTable +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import uy.kohesive.injekt.injectLazy +import java.net.URI +import java.net.URISyntaxException + +object EXHMigrations { + private val db: DatabaseHelper by injectLazy() + + private const val CURRENT_MIGRATION_VERSION = 1 + + /** + * Performs a migration when the application is updated. + * + * @param preferences Preferences of the application. + * @return true if a migration is performed, false otherwise. + */ + fun upgrade(preferences: PreferencesHelper): Boolean { + val context = preferences.context + val oldVersion = preferences.eh_lastVersionCode().getOrDefault() + if (oldVersion < CURRENT_MIGRATION_VERSION) { + preferences.eh_lastVersionCode().set(CURRENT_MIGRATION_VERSION) + + if(oldVersion < 1) { + db.inTransaction { + // Migrate HentaiCafe source IDs + db.lowLevel().executeSQL(RawQuery.builder() + .query(""" + UPDATE ${MangaTable.TABLE} + SET ${MangaTable.COL_SOURCE} = 260868874183818481 + WHERE ${MangaTable.COL_SOURCE} = $HENTAI_CAFE_SOURCE_ID + """.trimIndent()) + .affectsTables(MangaTable.TABLE) + .build()) + + // Migrate nhentai URLs + val nhentaiManga = db.db.get() + .listOfObjects(Manga::class.java) + .withQuery(Query.builder() + .table(MangaTable.TABLE) + .where("${MangaTable.COL_SOURCE} = $NHENTAI_SOURCE_ID") + .build()) + .prepare() + .executeAsBlocking() + + nhentaiManga.forEach { + it.url = getUrlWithoutDomain(it.url) + } + + db.db.put() + .objects(nhentaiManga) + // Extremely slow without the resolver :/ + .withPutResolver(MangaUrlPutResolver()) + .prepare() + .executeAsBlocking() + } + } + + return true + } + return false + } + + private fun getUrlWithoutDomain(orig: String): String { + return try { + val uri = URI(orig) + var out = uri.path + if (uri.query != null) + out += "?" + uri.query + if (uri.fragment != null) + out += "#" + uri.fragment + out + } catch (e: URISyntaxException) { + orig + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/GalleryAdder.kt b/app/src/main/java/exh/GalleryAdder.kt index b3ba0a7de..c6d766dc5 100755 --- a/app/src/main/java/exh/GalleryAdder.kt +++ b/app/src/main/java/exh/GalleryAdder.kt @@ -138,7 +138,7 @@ class GalleryAdder { val cleanedUrl = when(source) { EH_SOURCE_ID, EXH_SOURCE_ID -> ExGalleryMetadata.normalizeUrl(getUrlWithoutDomain(realUrl)) - NHENTAI_SOURCE_ID -> realUrl //nhentai uses URLs directly (oops, my bad when implementing this source) + NHENTAI_SOURCE_ID -> getUrlWithoutDomain(realUrl) PERV_EDEN_EN_SOURCE_ID, PERV_EDEN_IT_SOURCE_ID -> getUrlWithoutDomain(realUrl) HENTAI_CAFE_SOURCE_ID -> getUrlWithoutDomain(realUrl) diff --git a/app/src/main/java/exh/debug/DebugFunctions.kt b/app/src/main/java/exh/debug/DebugFunctions.kt new file mode 100644 index 000000000..b336d8bef --- /dev/null +++ b/app/src/main/java/exh/debug/DebugFunctions.kt @@ -0,0 +1,38 @@ +package exh.debug + +import com.pushtorefresh.storio.sqlite.queries.RawQuery +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import eu.kanade.tachiyomi.data.database.tables.MangaTable +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import uy.kohesive.injekt.injectLazy + +object DebugFunctions { + val db: DatabaseHelper by injectLazy() + val prefs: PreferencesHelper by injectLazy() + + fun addAllMangaInDatabaseToLibrary() { + db.inTransaction { + db.lowLevel().executeSQL(RawQuery.builder() + .query(""" + UPDATE ${MangaTable.TABLE} + SET ${MangaTable.COL_FAVORITE} = 1 + """.trimIndent()) + .affectsTables(MangaTable.TABLE) + .build()) + } + } + + fun countMangaInDatabaseInLibrary() = db.getMangas().executeAsBlocking().count { it.favorite } + + fun countMangaInDatabaseNotInLibrary() = db.getMangas().executeAsBlocking().count { !it.favorite } + + fun countMangaInDatabase() = db.getMangas().executeAsBlocking().size + + fun countMetadataInDatabase() = db.getSearchMetadata().executeAsBlocking().size + + fun countMangaInLibraryWithMissingMetadata() = db.getMangas().executeAsBlocking().count { + it.favorite && db.getSearchMetadataForManga(it.id!!).executeAsBlocking() == null + } + + fun clearSavedSearches() = prefs.eh_savedSearches().set("") +} \ No newline at end of file diff --git a/app/src/main/java/exh/debug/SettingsDebugController.kt b/app/src/main/java/exh/debug/SettingsDebugController.kt new file mode 100644 index 000000000..5356859ac --- /dev/null +++ b/app/src/main/java/exh/debug/SettingsDebugController.kt @@ -0,0 +1,33 @@ +package exh.debug + +import android.support.v7.preference.PreferenceScreen +import android.util.Log +import com.afollestad.materialdialogs.MaterialDialog +import eu.kanade.tachiyomi.ui.setting.SettingsController +import eu.kanade.tachiyomi.ui.setting.onClick +import eu.kanade.tachiyomi.ui.setting.preference +import kotlin.reflect.full.declaredFunctions + +class SettingsDebugController : SettingsController() { + override fun setupPreferenceScreen(screen: PreferenceScreen) = with(screen) { + title = "DEBUG MENU" + + DebugFunctions::class.declaredFunctions.forEach { + preference { + title = it.name.replace(Regex("(.)(\\p{Upper})"), "$1 $2").toLowerCase().capitalize() + isPersistent = false + + onClick { + try { + val result = it.call(DebugFunctions) + MaterialDialog.Builder(context) + .content("Function returned result:\n\n$result") + } catch(t: Throwable) { + MaterialDialog.Builder(context) + .content("Function threw exception:\n\n${Log.getStackTraceString(t)}") + }.show() + } + } + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt index b615129af..c099dd72c 100644 --- a/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt +++ b/app/src/main/java/exh/favorites/FavoritesSyncHelper.kt @@ -87,12 +87,12 @@ class FavoritesSyncHelper(val context: Context) { ignore { wakeLock?.release() } wakeLock = ignore { context.powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, - "ExhFavoritesSyncWakelock") + "teh:ExhFavoritesSyncWakelock") } ignore { wifiLock?.release() } wifiLock = ignore { context.wifiManager.createWifiLock(WifiManager.WIFI_MODE_FULL, - "ExhFavoritesSyncWifi") + "teh:ExhFavoritesSyncWifi") } storage.getRealm().use { realm -> diff --git a/app/src/main/java/exh/hitomi/HitomiNozomi.kt b/app/src/main/java/exh/hitomi/HitomiNozomi.kt new file mode 100644 index 000000000..1dc068972 --- /dev/null +++ b/app/src/main/java/exh/hitomi/HitomiNozomi.kt @@ -0,0 +1,244 @@ +package exh.hitomi + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservable +import eu.kanade.tachiyomi.network.asObservableSuccess +import exh.metadata.metadata.HitomiSearchMetadata.Companion.LTN_BASE_URL +import okhttp3.Headers +import okhttp3.OkHttpClient +import okhttp3.Request +import org.vepta.vdm.ByteCursor +import rx.Observable +import rx.Single +import java.security.MessageDigest + +private typealias HashedTerm = ByteArray + +private data class DataPair(val offset: Long, val length: Int) +private data class Node(val keys: List, + val datas: List, + val subnodeAddresses: List) + +/** + * Kotlin port of the hitomi.la search algorithm + * @author NerdNumber9 + */ +class HitomiNozomi(private val client: OkHttpClient, + private val tagIndexVersion: Long, + private val galleriesIndexVersion: Long) { + fun getGalleryIdsForQuery(query: String): Single> { + val replacedQuery = query.replace('_', ' ') + + if(':' in replacedQuery) { + val sides = replacedQuery.split(':') + val namespace = sides[0] + var tag = sides[1] + + var area: String? = namespace + var language = "all" + if(namespace == "female" || namespace == "male") { + area = "tag" + tag = replacedQuery + } else if(namespace == "language") { + area = null + language = tag + tag = "index" + } + + return getGalleryIdsFromNozomi(area, tag, language) + } + + val key = hashTerm(query) + val field = "galleries" + + return getNodeAtAddress(field, 0).flatMap { node -> + if(node == null) { + Single.just(null) + } else { + BSearch(field, key, node).flatMap { data -> + if (data == null) { + Single.just(null) + } else { + getGalleryIdsFromData(data) + } + } + } + } + } + + private fun getGalleryIdsFromData(data: DataPair?): Single> { + if(data == null) + return Single.just(emptyList()) + + val url = "$LTN_BASE_URL/$GALLERIES_INDEX_DIR/galleries.$galleriesIndexVersion.data" + val (offset, length) = data + if(length > 100000000 || length <= 0) + return Single.just(emptyList()) + + return client.newCall(rangedGet(url, offset, offset + length - 1)) + .asObservable() + .map { + it.body()?.bytes() ?: ByteArray(0) + } + .onErrorReturn { ByteArray(0) } + .map { inbuf -> + if(inbuf.isEmpty()) + return@map emptyList() + + val view = ByteCursor(inbuf) + val numberOfGalleryIds = view.nextInt() + + val expectedLength = numberOfGalleryIds * 4 + 4 + + if(numberOfGalleryIds > 10000000 + || numberOfGalleryIds <= 0 + || inbuf.size != expectedLength) { + return@map emptyList() + } + + (1 .. numberOfGalleryIds).map { + view.nextInt() + } + }.toSingle() + } + + private fun BSearch(field: String, key: ByteArray, node: Node?): Single { + fun compareByteArrays(dv1: ByteArray, dv2: ByteArray): Int { + val top = Math.min(dv1.size, dv2.size) + for(i in 0 until top) { + val dv1i = dv1[i].toInt() and 0xFF + val dv2i = dv2[i].toInt() and 0xFF + if(dv1i < dv2i) + return -1 + else if(dv1i > dv2i) + return 1 + } + return 0 + } + + fun locateKey(key: ByteArray, node: Node): Pair { + var cmpResult = -1 + var lastI = 0 + for(nodeKey in node.keys) { + cmpResult = compareByteArrays(key, nodeKey) + if(cmpResult <= 0) break + lastI++ + } + return (cmpResult == 0) to lastI + } + + fun isLeaf(node: Node): Boolean { + return !node.subnodeAddresses.any { + it != 0L + } + } + + if(node == null || node.keys.isEmpty()) { + return Single.just(null) + } + + val (there, where) = locateKey(key, node) + if(there) { + return Single.just(node.datas[where]) + } else if(isLeaf(node)) { + return Single.just(null) + } + + return getNodeAtAddress(field, node.subnodeAddresses[where]).flatMap { newNode -> + BSearch(field, key, newNode) + } + } + + private fun decodeNode(data: ByteArray): Node { + val view = ByteCursor(data) + + val numberOfKeys = view.nextInt() + + val keys = (1 .. numberOfKeys).map { + val keySize = view.nextInt() + view.next(keySize) + } + + val numberOfDatas = view.nextInt() + val datas = (1 .. numberOfDatas).map { + val offset = view.nextLong() + val length = view.nextInt() + DataPair(offset, length) + } + + val numberOfSubnodeAddresses = B + 1 + val subnodeAddresses = (1 .. numberOfSubnodeAddresses).map { + view.nextLong() + } + + return Node(keys, datas, subnodeAddresses) + } + + private fun getNodeAtAddress(field: String, address: Long): Single { + var url = "$LTN_BASE_URL/$INDEX_DIR/$field.$tagIndexVersion.index" + if(field == "galleries") { + url = "$LTN_BASE_URL/$GALLERIES_INDEX_DIR/galleries.$galleriesIndexVersion.index" + } + + return client.newCall(rangedGet(url, address, address + MAX_NODE_SIZE - 1)) + .asObservableSuccess() + .map { + it.body()?.bytes() ?: ByteArray(0) + } + .onErrorReturn { ByteArray(0) } + .map { nodedata -> + if(nodedata.isNotEmpty()) { + decodeNode(nodedata) + } else null + }.toSingle() + } + + fun getGalleryIdsFromNozomi(area: String?, tag: String, language: String): Single> { + var nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$tag-$language$NOZOMI_EXTENSION" + if(area != null) { + nozomiAddress = "$LTN_BASE_URL/$COMPRESSED_NOZOMI_PREFIX/$area/$tag-$language$NOZOMI_EXTENSION" + } + + return client.newCall(Request.Builder() + .url(nozomiAddress) + .build()) + .asObservableSuccess() + .map { resp -> + val body = resp.body()!!.bytes() + val cursor = ByteCursor(body) + (1 .. body.size / 4).map { + cursor.nextInt() + } + }.toSingle() + } + + private fun hashTerm(query: String): HashedTerm { + val md = MessageDigest.getInstance("SHA-256") + md.update(query.toByteArray(HASH_CHARSET)) + return md.digest().copyOf(4) + } + + companion object { + private const val INDEX_DIR = "tagindex" + private const val GALLERIES_INDEX_DIR = "galleriesindex" + private const val COMPRESSED_NOZOMI_PREFIX = "n" + private const val NOZOMI_EXTENSION = ".nozomi" + private const val MAX_NODE_SIZE = 464 + private const val B = 16 + + private val HASH_CHARSET = Charsets.UTF_8 + + fun rangedGet(url: String, rangeBegin: Long, rangeEnd: Long?): Request { + return GET(url, Headers.Builder() + .add("Range", "bytes=$rangeBegin-${rangeEnd ?: ""}") + .build()) + } + + + fun getIndexVersion(httpClient: OkHttpClient, name: String): Observable { + return httpClient.newCall(GET("$LTN_BASE_URL/$name/version?_=${System.currentTimeMillis()}")) + .asObservableSuccess() + .map { it.body()!!.string().toLong() } + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/MetadataHelper.kt b/app/src/main/java/exh/metadata/MetadataHelper.kt index 11a25893d..fe8a09262 100755 --- a/app/src/main/java/exh/metadata/MetadataHelper.kt +++ b/app/src/main/java/exh/metadata/MetadataHelper.kt @@ -14,21 +14,21 @@ import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get import kotlin.reflect.KClass -fun Realm.loadAllMetadata(): Map, RealmResults> = - Injekt.get().getOnlineSources().filterIsInstance>().map { - it.queryAll() - }.associate { - it.clazz to it.query(this@loadAllMetadata).sort(SearchableGalleryMetadata::mangaId.name).findAll() - }.toMap() +//fun Realm.loadAllMetadata(): Map, RealmResults> = +// Injekt.get().getOnlineSources().filterIsInstance>().map { +// it.queryAll() +// }.associate { +// it.clazz to it.query(this@loadAllMetadata).sort(SearchableGalleryMetadata::mangaId.name).findAll() +// }.toMap() -fun Realm.queryMetadataFromManga(manga: Manga, - meta: RealmQuery? = null): - RealmQuery = - Injekt.get().get(manga.source)?.let { - (it as LewdSource<*, *>).queryFromUrl(manga.url) as GalleryQuery - }?.query(this, meta) ?: throw IllegalArgumentException("Unknown source type!") +//fun Realm.queryMetadataFromManga(manga: Manga, +// meta: RealmQuery? = null): +// RealmQuery = +// Injekt.get().get(manga.source)?.let { +// (it as LewdSource<*, *>).queryFromUrl(manga.url) as GalleryQuery +// }?.query(this, meta) ?: throw IllegalArgumentException("Unknown source type!") -fun Realm.syncMangaIds(mangas: List) { +/*fun Realm.syncMangaIds(mangas: List) { Timber.d("--> EH: Begin syncing ${mangas.size} manga IDs...") executeTransaction { mangas.forEach { manga -> @@ -46,7 +46,7 @@ fun Realm.syncMangaIds(mangas: List) { } } Timber.d("--> EH: Finish syncing ${mangas.size} manga IDs!") -} +}*/ -val Manga.metadataClass - get() = (Injekt.get().get(source) as? LewdSource<*, *>)?.queryAll()?.clazz +//val Manga.metadataClass +// get() = (Injekt.get().get(source) as? LewdSource<*, *>)?.queryAll()?.clazz diff --git a/app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt new file mode 100644 index 000000000..71f09570b --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/EHentaiSearchMetadata.kt @@ -0,0 +1,135 @@ +package exh.metadata.metadata + +import android.net.Uri +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import eu.kanade.tachiyomi.source.model.SManga +import exh.metadata.* +import exh.metadata.metadata.base.RaisedSearchMetadata +import exh.plusAssign +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import java.util.* + +class EHentaiSearchMetadata : RaisedSearchMetadata() { + var gId: String? + get() = indexedExtra + set(value) { indexedExtra = value } + + var gToken: String? = null + var exh: Boolean? = null + var thumbnailUrl: String? = null + + var title by titleDelegate(TITLE_TYPE_TITLE) + var altTitle by titleDelegate(TITLE_TYPE_ALT_TITLE) + + var genre: String? = null + + var datePosted: Long? = null + var parent: String? = null + var visible: String? = null //Not a boolean + var language: String? = null + var translated: Boolean? = null + var size: Long? = null + var length: Int? = null + var favorites: Int? = null + var ratingCount: Int? = null + var averageRating: Double? = null + + override fun copyTo(manga: SManga) { + gId?.let { gId -> + gToken?.let { gToken -> + manga.url = idAndTokenToUrl(gId, gToken) + } + } + thumbnailUrl?.let { manga.thumbnail_url = it } + + //No title bug? + val titleObj = if(Injekt.get().useJapaneseTitle().getOrDefault()) + altTitle ?: title + else + title + titleObj?.let { manga.title = it } + + //Set artist (if we can find one) + tags.filter { it.namespace == EH_ARTIST_NAMESPACE }.let { + if(it.isNotEmpty()) manga.artist = it.joinToString(transform = { it.name }) + } + + //Copy tags -> genres + manga.genre = tagsToGenreString() + + //Try to automatically identify if it is ongoing, we try not to be too lenient here to avoid making mistakes + //We default to completed + manga.status = SManga.COMPLETED + title?.let { t -> + ONGOING_SUFFIX.find { + t.endsWith(it, ignoreCase = true) + }?.let { + manga.status = SManga.ONGOING + } + } + + //Build a nice looking description out of what we know + val titleDesc = StringBuilder() + title?.let { titleDesc += "Title: $it\n" } + altTitle?.let { titleDesc += "Alternate Title: $it\n" } + + val detailsDesc = StringBuilder() + genre?.let { detailsDesc += "Genre: $it\n" } + uploader?.let { detailsDesc += "Uploader: $it\n" } + datePosted?.let { detailsDesc += "Posted: ${EX_DATE_FORMAT.format(Date(it))}\n" } + visible?.let { detailsDesc += "Visible: $it\n" } + language?.let { + detailsDesc += "Language: $it" + if(translated == true) detailsDesc += " TR" + detailsDesc += "\n" + } + size?.let { detailsDesc += "File size: ${humanReadableByteCount(it, true)}\n" } + length?.let { detailsDesc += "Length: $it pages\n" } + favorites?.let { detailsDesc += "Favorited: $it times\n" } + averageRating?.let { + detailsDesc += "Rating: $it" + ratingCount?.let { detailsDesc += " ($it)" } + detailsDesc += "\n" + } + + val tagsDesc = tagsToDescription() + + manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString()) + .filter(String::isNotBlank) + .joinToString(separator = "\n") + } + + companion object { + private const val TITLE_TYPE_TITLE = 0 + private const val TITLE_TYPE_ALT_TITLE = 1 + + const val TAG_TYPE_NORMAL = 0 + const val TAG_TYPE_LIGHT = 1 + + const val EH_GENRE_NAMESPACE = "genre" + private const val EH_ARTIST_NAMESPACE = "artist" + + private fun splitGalleryUrl(url: String) + = url.let { + //Only parse URL if is full URL + val pathSegments = if(it.startsWith("http")) + Uri.parse(it).pathSegments + else + it.split('/') + pathSegments.filterNot(String::isNullOrBlank) + } + + fun galleryId(url: String) = splitGalleryUrl(url)[1] + + fun galleryToken(url: String) = + splitGalleryUrl(url)[2] + + fun normalizeUrl(url: String) + = idAndTokenToUrl(galleryId(url), galleryToken(url)) + + fun idAndTokenToUrl(id: String, token: String) + = "/g/$id/$token/?nw=always" + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/metadata/HentaiCafeSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/HentaiCafeSearchMetadata.kt new file mode 100644 index 000000000..b3d847651 --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/HentaiCafeSearchMetadata.kt @@ -0,0 +1,55 @@ +package exh.metadata.metadata + +import eu.kanade.tachiyomi.source.model.SManga +import exh.metadata.metadata.base.RaisedSearchMetadata + +class HentaiCafeSearchMetadata : RaisedSearchMetadata() { + var hcId: String? = null + var readerId: String? = null + + var url get() = hcId?.let { "$BASE_URL/$it" } + set(a) { + a?.let { + hcId = hcIdFromUrl(a) + } + } + + var thumbnailUrl: String? = null + + var title by titleDelegate(TITLE_TYPE_MAIN) + + var artist: String? = null + + override fun copyTo(manga: SManga) { + thumbnailUrl?.let { manga.thumbnail_url = it } + + manga.title = title!! + manga.artist = artist + manga.author = artist + + //Not available + manga.status = SManga.UNKNOWN + + val detailsDesc = "Title: $title\n" + + "Artist: $artist\n" + + val tagsDesc = tagsToDescription() + + manga.genre = tagsToGenreString() + + manga.description = listOf(detailsDesc, tagsDesc.toString()) + .filter(String::isNotBlank) + .joinToString(separator = "\n") + } + + companion object { + private const val TITLE_TYPE_MAIN = 0 + + const val TAG_TYPE_DEFAULT = 0 + + val BASE_URL = "https://hentai.cafe" + + fun hcIdFromUrl(url: String) + = url.split("/").last { it.isNotBlank() } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/metadata/HitomiSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/HitomiSearchMetadata.kt new file mode 100644 index 000000000..ddc5c8ad9 --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/HitomiSearchMetadata.kt @@ -0,0 +1,102 @@ +package exh.metadata.metadata + +import eu.kanade.tachiyomi.source.model.SManga +import exh.metadata.EX_DATE_FORMAT +import exh.metadata.metadata.base.RaisedSearchMetadata +import exh.plusAssign +import java.util.* + +class HitomiSearchMetadata: RaisedSearchMetadata() { + var url get() = hlId?.let { urlFromHlId(it) } + set(a) { + a?.let { + hlId = hlIdFromUrl(a) + } + } + + var hlId: String? = null + + var title by titleDelegate(TITLE_TYPE_MAIN) + + var thumbnailUrl: String? = null + + var artists: List = emptyList() + + var group: String? = null + + var type: String? = null + + var language: String? = null + + var series: List = emptyList() + + var characters: List = emptyList() + + var uploadDate: Long? = null + + override fun copyTo(manga: SManga) { + thumbnailUrl?.let { manga.thumbnail_url = it } + + val titleDesc = StringBuilder() + + title?.let { + manga.title = it + titleDesc += "Title: $it\n" + } + + val detailsDesc = StringBuilder() + + manga.artist = artists.joinToString() + + detailsDesc += "Artist(s): ${manga.artist}\n" + + group?.let { + detailsDesc += "Group: $it\n" + } + + type?.let { + detailsDesc += "Type: ${it.capitalize()}\n" + } + + (language ?: "unknown").let { + detailsDesc += "Language: ${it.capitalize()}\n" + } + + if(series.isNotEmpty()) + detailsDesc += "Series: ${series.joinToString()}\n" + + if(characters.isNotEmpty()) + detailsDesc += "Characters: ${characters.joinToString()}\n" + + uploadDate?.let { + detailsDesc += "Upload date: ${EX_DATE_FORMAT.format(Date(it))}\n" + } + + manga.status = SManga.UNKNOWN + + //Copy tags -> genres + manga.genre = tagsToGenreString() + + val tagsDesc = tagsToDescription() + + manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString()) + .filter(String::isNotBlank) + .joinToString(separator = "\n") + } + + companion object { + private const val TITLE_TYPE_MAIN = 0 + + const val TAG_TYPE_DEFAULT = 0 + + val LTN_BASE_URL = "https://ltn.hitomi.la" + val BASE_URL = "https://hitomi.la" + val IMG_BASE_URL = "https://aa.hitomi.la/galleries" + + fun hlIdFromUrl(url: String) + = url.split('/').last().substringBeforeLast('.') + + fun urlFromHlId(id: String) + = "$BASE_URL/galleries/$id.html" + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/metadata/NHentaiSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/NHentaiSearchMetadata.kt new file mode 100644 index 000000000..43f40fec7 --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/NHentaiSearchMetadata.kt @@ -0,0 +1,120 @@ +package exh.metadata.metadata + +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import eu.kanade.tachiyomi.source.model.SManga +import exh.metadata.* +import exh.metadata.metadata.base.RaisedSearchMetadata +import exh.plusAssign +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import java.util.* + +class NHentaiSearchMetadata : RaisedSearchMetadata() { + var url get() = nhId?.let { BASE_URL + nhIdToPath(it) } + set(a) { + a?.let { + nhId = nhUrlToId(a) + } + } + + var nhId: Long? = null + + var uploadDate: Long? = null + + var favoritesCount: Long? = null + + var mediaId: String? = null + + var japaneseTitle by titleDelegate(TITLE_TYPE_JAPANESE) + var englishTitle by titleDelegate(TITLE_TYPE_ENGLISH) + var shortTitle by titleDelegate(TITLE_TYPE_SHORT) + + var coverImageType: String? = null + var pageImageTypes: List = emptyList() + var thumbnailImageType: String? = null + + var scanlator: String? = null + + override fun copyTo(manga: SManga) { + nhId?.let { manga.url = nhIdToPath(it) } + + if(mediaId != null) { + val hqThumbs = Injekt.get().eh_nh_useHighQualityThumbs().getOrDefault() + typeToExtension(if(hqThumbs) coverImageType else thumbnailImageType)?.let { + manga.thumbnail_url = "https://t.nhentai.net/galleries/$mediaId/${if(hqThumbs) + "cover" + else "thumb"}.$it" + } + } + + manga.title = englishTitle ?: japaneseTitle ?: shortTitle!! + + //Set artist (if we can find one) + tags.filter { it.namespace == NHENTAI_ARTIST_NAMESPACE }.let { + if(it.isNotEmpty()) manga.artist = it.joinToString(transform = { it.name }) + } + + var category: String? = null + tags.filter { it.namespace == NHENTAI_CATEGORIES_NAMESPACE }.let { + if(it.isNotEmpty()) category = it.joinToString(transform = { it.name }) + } + + //Copy tags -> genres + manga.genre = tagsToGenreString() + + //Try to automatically identify if it is ongoing, we try not to be too lenient here to avoid making mistakes + //We default to completed + manga.status = SManga.COMPLETED + englishTitle?.let { t -> + ONGOING_SUFFIX.find { + t.endsWith(it, ignoreCase = true) + }?.let { + manga.status = SManga.ONGOING + } + } + + val titleDesc = StringBuilder() + englishTitle?.let { titleDesc += "English Title: $it\n" } + japaneseTitle?.let { titleDesc += "Japanese Title: $it\n" } + shortTitle?.let { titleDesc += "Short Title: $it\n" } + + val detailsDesc = StringBuilder() + category?.let { detailsDesc += "Category: $it\n" } + uploadDate?.let { detailsDesc += "Upload Date: ${EX_DATE_FORMAT.format(Date(it * 1000))}\n" } + pageImageTypes.size.let { detailsDesc += "Length: $it pages\n" } + favoritesCount?.let { detailsDesc += "Favorited: $it times\n" } + scanlator?.nullIfBlank()?.let { detailsDesc += "Scanlator: $it\n" } + + val tagsDesc = tagsToDescription() + + manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString()) + .filter(String::isNotBlank) + .joinToString(separator = "\n") + } + + companion object { + private const val TITLE_TYPE_JAPANESE = 0 + private const val TITLE_TYPE_ENGLISH = 1 + private const val TITLE_TYPE_SHORT = 2 + + const val TAG_TYPE_DEFAULT = 0 + + val BASE_URL = "https://nhentai.net" + + private const val NHENTAI_ARTIST_NAMESPACE = "artist" + private const val NHENTAI_CATEGORIES_NAMESPACE = "category" + + fun typeToExtension(t: String?) = + when(t) { + "p" -> "png" + "j" -> "jpg" + else -> null + } + + fun nhUrlToId(url: String) + = url.split("/").last { it.isNotBlank() }.toLong() + + fun nhIdToPath(id: Long) = "/g/$id/" + } +} diff --git a/app/src/main/java/exh/metadata/metadata/PervEdenSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/PervEdenSearchMetadata.kt new file mode 100644 index 000000000..726c48b60 --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/PervEdenSearchMetadata.kt @@ -0,0 +1,108 @@ +package exh.metadata.metadata + +import android.net.Uri +import eu.kanade.tachiyomi.source.model.SManga +import exh.PERV_EDEN_EN_SOURCE_ID +import exh.PERV_EDEN_IT_SOURCE_ID +import exh.metadata.metadata.base.RaisedSearchMetadata +import exh.metadata.metadata.base.RaisedTitle +import exh.plusAssign + +class PervEdenSearchMetadata : RaisedSearchMetadata() { + var pvId: String? = null + + var url: String? = null + var thumbnailUrl: String? = null + + var title by titleDelegate(TITLE_TYPE_MAIN) + var altTitles + get() = titles.filter { it.type == TITLE_TYPE_ALT }.map { it.title } + set(value) { + titles.removeAll { it.type == TITLE_TYPE_ALT } + titles += value.map { RaisedTitle(it, TITLE_TYPE_ALT) } + } + + var artist: String? = null + + var type: String? = null + + var rating: Float? = null + + var status: String? = null + + var lang: String? = null + + override fun copyTo(manga: SManga) { + url?.let { manga.url = it } + thumbnailUrl?.let { manga.thumbnail_url = it } + + val titleDesc = StringBuilder() + title?.let { + manga.title = it + titleDesc += "Title: $it\n" + } + if(altTitles.isNotEmpty()) + titleDesc += "Alternate Titles: \n" + altTitles.map { + "▪ $it" + }.joinToString(separator = "\n", postfix = "\n") + + val detailsDesc = StringBuilder() + artist?.let { + manga.artist = it + detailsDesc += "Artist: $it\n" + } + + type?.let { + detailsDesc += "Type: $it\n" + } + + status?.let { + manga.status = when(it) { + "Ongoing" -> SManga.ONGOING + "Completed", "Suspended" -> SManga.COMPLETED + else -> SManga.UNKNOWN + } + detailsDesc += "Status: $it\n" + } + + rating?.let { + detailsDesc += "Rating: %.2\n".format(it) + } + + //Copy tags -> genres + manga.genre = tagsToGenreString() + + val tagsDesc = tagsToDescription() + + manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString()) + .filter(String::isNotBlank) + .joinToString(separator = "\n") + } + + + companion object { + private const val TITLE_TYPE_MAIN = 0 + private const val TITLE_TYPE_ALT = 1 + + const val TAG_TYPE_DEFAULT = 0 + + private fun splitGalleryUrl(url: String) + = url.let { + Uri.parse(it).pathSegments.filterNot(String::isNullOrBlank) + } + + fun pvIdFromUrl(url: String) = splitGalleryUrl(url).last() + } +} + +enum class PervEdenLang(val id: Long) { + //DO NOT RENAME THESE TO CAPITAL LETTERS! The enum names are used to build URLs + en(PERV_EDEN_EN_SOURCE_ID), + it(PERV_EDEN_IT_SOURCE_ID); + + companion object { + fun source(id: Long) + = PervEdenLang.values().find { it.id == id } + ?: throw IllegalArgumentException("Unknown source ID: $id!") + } +} diff --git a/app/src/main/java/exh/metadata/metadata/TsuminoSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/TsuminoSearchMetadata.kt new file mode 100644 index 000000000..ac5671312 --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/TsuminoSearchMetadata.kt @@ -0,0 +1,88 @@ +package exh.metadata.metadata + +import android.net.Uri +import eu.kanade.tachiyomi.source.model.SManga +import exh.metadata.EX_DATE_FORMAT +import exh.metadata.buildTagsDescription +import exh.metadata.joinEmulatedTagsToGenreString +import exh.metadata.metadata.base.RaisedSearchMetadata +import exh.plusAssign +import java.util.* + +class TsuminoSearchMetadata : RaisedSearchMetadata() { + var tmId: Int? = null + + var title by titleDelegate(TITLE_TYPE_MAIN) + + var artist: String? = null + + var uploadDate: Long? = null + + var length: Int? = null + + var ratingString: String? = null + + var category: String? = null + + var collection: String? = null + + var group: String? = null + + var parody: List = emptyList() + + var character: List = emptyList() + + override fun copyTo(manga: SManga) { + title?.let { manga.title = it } + manga.thumbnail_url = BASE_URL + thumbUrlFromId(tmId.toString()) + + artist?.let { manga.artist = it } + + manga.status = SManga.UNKNOWN + + val titleDesc = "Title: $title\n" + + val detailsDesc = StringBuilder() + uploader?.let { detailsDesc += "Uploader: $it\n" } + uploadDate?.let { detailsDesc += "Uploaded: ${EX_DATE_FORMAT.format(Date(it))}\n" } + length?.let { detailsDesc += "Length: $it pages\n" } + ratingString?.let { detailsDesc += "Rating: $it\n" } + category?.let { + detailsDesc += "Category: $it\n" + } + collection?.let { detailsDesc += "Collection: $it\n" } + group?.let { detailsDesc += "Group: $it\n" } + val parodiesString = parody.joinToString() + if(parodiesString.isNotEmpty()) { + detailsDesc += "Parody: $parodiesString\n" + } + val charactersString = character.joinToString() + if(charactersString.isNotEmpty()) { + detailsDesc += "Character: $charactersString\n" + } + + //Copy tags -> genres + manga.genre = tagsToGenreString() + + val tagsDesc = tagsToDescription() + + manga.description = listOf(titleDesc, detailsDesc.toString(), tagsDesc.toString()) + .filter(String::isNotBlank) + .joinToString(separator = "\n") + } + + companion object { + private const val TITLE_TYPE_MAIN = 0 + + const val TAG_TYPE_DEFAULT = 0 + + val BASE_URL = "https://www.tsumino.com" + + fun tmIdFromUrl(url: String) + = Uri.parse(url).pathSegments[2] + + fun mangaUrlFromId(id: String) = "/Book/Info/$id" + + fun thumbUrlFromId(id: String) = "/Image/Thumb/$id" + } +} diff --git a/app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt b/app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt new file mode 100644 index 000000000..ef732de37 --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/base/FlatMetadata.kt @@ -0,0 +1,91 @@ +package exh.metadata.metadata.base + +import com.pushtorefresh.storio.operations.PreparedOperation +import eu.kanade.tachiyomi.data.database.DatabaseHelper +import exh.metadata.sql.models.SearchMetadata +import exh.metadata.sql.models.SearchTag +import exh.metadata.sql.models.SearchTitle +import rx.Completable +import rx.Single +import kotlin.reflect.KClass + +data class FlatMetadata( + val metadata: SearchMetadata, + val tags: List, + val titles: List +) { + inline fun raise(): T = raise(T::class) + + fun raise(clazz: KClass) + = RaisedSearchMetadata.raiseFlattenGson + .fromJson(metadata.extra, clazz.java).apply { + fillBaseFields(this@FlatMetadata) + } +} + +fun DatabaseHelper.getFlatMetadataForManga(mangaId: Long): PreparedOperation { + fun getSingle() = getSearchMetadataForManga(mangaId).asRxSingle().flatMap { meta -> + if(meta == null) Single.just(null) + else Single.zip( + getSearchTagsForManga(mangaId).asRxSingle(), + getSearchTitlesForManga(mangaId).asRxSingle() + ) { tags, titles -> + FlatMetadata(meta, tags, titles) + } + } + + return object : PreparedOperation { + /** + * Creates [rx.Observable] that emits result of Operation. + * + * + * Observable may be "Hot" or "Cold", please read documentation of the concrete implementation. + * + * @return observable result of operation with only one [rx.Observer.onNext] call. + */ + override fun createObservable() = getSingle().toObservable() + + /** + * Executes operation synchronously in current thread. + * + * + * Notice: Blocking I/O operation should not be executed on the Main Thread, + * it can cause ANR (Activity Not Responding dialog), block the UI and drop animations frames. + * So please, execute blocking I/O operation only from background thread. + * See [WorkerThread]. + * + * @return nullable result of operation. + */ + override fun executeAsBlocking() = getSingle().toBlocking().value() + + /** + * Creates [rx.Observable] that emits result of Operation. + * + * + * Observable may be "Hot" (usually "Warm") or "Cold", please read documentation of the concrete implementation. + * + * @return observable result of operation with only one [rx.Observer.onNext] call. + */ + override fun asRxObservable() = getSingle().toObservable() + + /** + * Creates [rx.Single] that emits result of Operation lazily when somebody subscribes to it. + * + * + * + * @return single result of operation. + */ + override fun asRxSingle() = getSingle() + + } +} + +fun DatabaseHelper.insertFlatMetadata(flatMetadata: FlatMetadata) = Completable.fromCallable { + require(flatMetadata.metadata.mangaId != -1L) + + inTransaction { + insertSearchMetadata(flatMetadata.metadata).executeAsBlocking() + setSearchTagsForManga(flatMetadata.metadata.mangaId, flatMetadata.tags) + setSearchTitlesForManga(flatMetadata.metadata.mangaId, flatMetadata.titles) + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/metadata/base/RaisedSearchMetadata.kt b/app/src/main/java/exh/metadata/metadata/base/RaisedSearchMetadata.kt new file mode 100644 index 000000000..928b4027d --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/base/RaisedSearchMetadata.kt @@ -0,0 +1,137 @@ +package exh.metadata.metadata.base + +import com.google.gson.GsonBuilder +import eu.kanade.tachiyomi.source.model.SManga +import exh.metadata.forEach +import exh.metadata.sql.models.SearchMetadata +import exh.metadata.sql.models.SearchTag +import exh.metadata.sql.models.SearchTitle +import exh.plusAssign +import kotlin.properties.ReadWriteProperty +import kotlin.reflect.KProperty + +abstract class RaisedSearchMetadata { + @Transient + var mangaId: Long = -1 + + @Transient + var uploader: String? = null + + @Transient + protected open var indexedExtra: String? = null + + @Transient + val tags = mutableListOf() + + @Transient + val titles = mutableListOf() + + fun getTitleOfType(type: Int): String? = titles.find { it.type == type }?.title + + fun replaceTitleOfType(type: Int, newTitle: String?) { + titles.removeAll { it.type == type } + if(newTitle != null) titles += RaisedTitle(newTitle, type) + } + + abstract fun copyTo(manga: SManga) + + fun tagsToGenreString() + = tags.filter { it.type != TAG_TYPE_VIRTUAL } + .joinToString { (if(it.namespace != null) "${it.namespace}: " else "") + it.name } + + fun tagsToDescription() + = StringBuilder("Tags:\n").apply { + //BiConsumer only available in Java 8, don't bother calling forEach directly on 'tags' + val groupedTags = tags.filter { it.type != TAG_TYPE_VIRTUAL }.groupBy { + it.namespace + }.entries + + groupedTags.forEach { namespace, tags -> + if (tags.isNotEmpty()) { + val joinedTags = tags.joinToString(separator = " ", transform = { "<${it.name}>" }) + if(namespace != null) { + this += "▪ " + this += namespace + this += ": " + } + this += joinedTags + this += "\n" + } + } + } + + fun flatten(): FlatMetadata { + require(mangaId != -1L) + + val extra = raiseFlattenGson.toJson(this) + return FlatMetadata( + SearchMetadata( + mangaId, + uploader, + extra, + indexedExtra, + 0 + ), + tags.map { + SearchTag( + null, + mangaId, + it.namespace, + it.name, + it.type + ) + }, + titles.map { + SearchTitle( + null, + mangaId, + it.title, + it.type + ) + } + ) + } + + fun fillBaseFields(metadata: FlatMetadata) { + mangaId = metadata.metadata.mangaId + uploader = metadata.metadata.uploader + indexedExtra = metadata.metadata.indexedExtra + + this.tags.clear() + this.tags += metadata.tags.map { + RaisedTag(it.namespace, it.name, it.type) + } + + this.titles.clear() + this.titles += metadata.titles.map { + RaisedTitle(it.title, it.type) + } + } + + companion object { + // Virtual tags allow searching of otherwise unindexed fields + const val TAG_TYPE_VIRTUAL = -2 + + val raiseFlattenGson = GsonBuilder().create() + + fun titleDelegate(type: Int) = object : ReadWriteProperty { + /** + * Returns the value of the property for the given object. + * @param thisRef the object for which the value is requested. + * @param property the metadata for the property. + * @return the property value. + */ + override fun getValue(thisRef: RaisedSearchMetadata, property: KProperty<*>) + = thisRef.getTitleOfType(type) + + /** + * Sets the value of the property for the given object. + * @param thisRef the object for which the value is requested. + * @param property the metadata for the property. + * @param value the value to set. + */ + override fun setValue(thisRef: RaisedSearchMetadata, property: KProperty<*>, value: String?) + = thisRef.replaceTitleOfType(type, value) + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/metadata/base/RaisedTag.kt b/app/src/main/java/exh/metadata/metadata/base/RaisedTag.kt new file mode 100644 index 000000000..5e38d51c9 --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/base/RaisedTag.kt @@ -0,0 +1,5 @@ +package exh.metadata.metadata.base + +data class RaisedTag(val namespace: String?, + val name: String, + val type: Int) \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/metadata/base/RaisedTitle.kt b/app/src/main/java/exh/metadata/metadata/base/RaisedTitle.kt new file mode 100644 index 000000000..3f95c135d --- /dev/null +++ b/app/src/main/java/exh/metadata/metadata/base/RaisedTitle.kt @@ -0,0 +1,6 @@ +package exh.metadata.metadata.base + +data class RaisedTitle( + val title: String, + val type: Int = 0 +) \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt b/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt index 08b7f4940..cd217732f 100755 --- a/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt +++ b/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt @@ -29,7 +29,7 @@ open class NHentaiMetadata : RealmObject(), SearchableGalleryMetadata { var url get() = nhId?.let { "$BASE_URL/g/$it" } set(a) { a?.let { - nhId = nhIdFromUrl(a) + nhId = nhUrlToId(a) } } @@ -71,7 +71,7 @@ open class NHentaiMetadata : RealmObject(), SearchableGalleryMetadata { val url: String ) : GalleryQuery(NHentaiMetadata::class) { override fun transform() = Query( - nhIdFromUrl(url) + nhUrlToId(url) ) } @@ -154,7 +154,7 @@ open class NHentaiMetadata : RealmObject(), SearchableGalleryMetadata { else -> null } - fun nhIdFromUrl(url: String) + fun nhUrlToId(url: String) = url.split("/").last { it.isNotBlank() }.toLong() val TITLE_FIELDS = listOf( diff --git a/app/src/main/java/exh/metadata/models/PervEdenGalleryMetadata.kt b/app/src/main/java/exh/metadata/models/PervEdenGalleryMetadata.kt index d8736462a..de0269fed 100755 --- a/app/src/main/java/exh/metadata/models/PervEdenGalleryMetadata.kt +++ b/app/src/main/java/exh/metadata/models/PervEdenGalleryMetadata.kt @@ -104,29 +104,6 @@ open class PervEdenGalleryMetadata : RealmObject(), SearchableGalleryMetadata { .joinToString(separator = "\n") } - class EmptyQuery : GalleryQuery(PervEdenGalleryMetadata::class) - - class UrlQuery( - val url: String, - val lang: PervEdenLang - ) : GalleryQuery(PervEdenGalleryMetadata::class) { - override fun transform() = Query( - pvIdFromUrl(url), - lang - ) - } - - class Query(val pvId: String, - val lang: PervEdenLang - ) : GalleryQuery(PervEdenGalleryMetadata::class) { - override fun map() = mapOf( - PervEdenGalleryMetadata::pvId to Query::pvId - ) - - override fun override(meta: RealmQuery) - = meta.equalTo(PervEdenGalleryMetadata::lang.name, lang.name) - } - companion object { private fun splitGalleryUrl(url: String) = url.let { @@ -165,15 +142,3 @@ open class PervEdenTitle(var metadata: PervEdenGalleryMetadata? = null, override fun toString() = "PervEdenTitle(metadata=$metadata, title=$title)" } - -enum class PervEdenLang(val id: Long) { - //DO NOT RENAME THESE TO CAPITAL LETTERS! The enum names are used to build URLs - en(PERV_EDEN_EN_SOURCE_ID), - it(PERV_EDEN_IT_SOURCE_ID); - - companion object { - fun source(id: Long) - = PervEdenLang.values().find { it.id == id } - ?: throw IllegalArgumentException("Unknown source ID: $id!") - } -} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/sql/mappers/SearchMetadataTypeMapping.kt b/app/src/main/java/exh/metadata/sql/mappers/SearchMetadataTypeMapping.kt new file mode 100755 index 000000000..e5e7a2178 --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/mappers/SearchMetadataTypeMapping.kt @@ -0,0 +1,65 @@ +package exh.metadata.sql.mappers + +import android.content.ContentValues +import android.database.Cursor +import com.pushtorefresh.storio.sqlite.SQLiteTypeMapping +import com.pushtorefresh.storio.sqlite.operations.delete.DefaultDeleteResolver +import com.pushtorefresh.storio.sqlite.operations.get.DefaultGetResolver +import com.pushtorefresh.storio.sqlite.operations.put.DefaultPutResolver +import com.pushtorefresh.storio.sqlite.queries.DeleteQuery +import com.pushtorefresh.storio.sqlite.queries.InsertQuery +import com.pushtorefresh.storio.sqlite.queries.UpdateQuery +import exh.metadata.sql.models.SearchMetadata +import exh.metadata.sql.tables.SearchMetadataTable.COL_EXTRA +import exh.metadata.sql.tables.SearchMetadataTable.COL_EXTRA_VERSION +import exh.metadata.sql.tables.SearchMetadataTable.COL_INDEXED_EXTRA +import exh.metadata.sql.tables.SearchMetadataTable.COL_MANGA_ID +import exh.metadata.sql.tables.SearchMetadataTable.COL_UPLOADER +import exh.metadata.sql.tables.SearchMetadataTable.TABLE + +class SearchMetadataTypeMapping : SQLiteTypeMapping( + SearchMetadataPutResolver(), + SearchMetadataGetResolver(), + SearchMetadataDeleteResolver() +) + +class SearchMetadataPutResolver : DefaultPutResolver() { + + override fun mapToInsertQuery(obj: SearchMetadata) = InsertQuery.builder() + .table(TABLE) + .build() + + override fun mapToUpdateQuery(obj: SearchMetadata) = UpdateQuery.builder() + .table(TABLE) + .where("$COL_MANGA_ID = ?") + .whereArgs(obj.mangaId) + .build() + + override fun mapToContentValues(obj: SearchMetadata) = ContentValues(5).apply { + put(COL_MANGA_ID, obj.mangaId) + put(COL_UPLOADER, obj.uploader) + put(COL_EXTRA, obj.extra) + put(COL_INDEXED_EXTRA, obj.indexedExtra) + put(COL_EXTRA_VERSION, obj.extraVersion) + } +} + +class SearchMetadataGetResolver : DefaultGetResolver() { + + override fun mapFromCursor(cursor: Cursor): SearchMetadata = SearchMetadata( + mangaId = cursor.getLong(cursor.getColumnIndex(COL_MANGA_ID)), + uploader = cursor.getString(cursor.getColumnIndex(COL_UPLOADER)), + extra = cursor.getString(cursor.getColumnIndex(COL_EXTRA)), + indexedExtra = cursor.getString(cursor.getColumnIndex(COL_INDEXED_EXTRA)), + extraVersion = cursor.getInt(cursor.getColumnIndex(COL_EXTRA_VERSION)) + ) +} + +class SearchMetadataDeleteResolver : DefaultDeleteResolver() { + + override fun mapToDeleteQuery(obj: SearchMetadata) = DeleteQuery.builder() + .table(TABLE) + .where("$COL_MANGA_ID = ?") + .whereArgs(obj.mangaId) + .build() +} diff --git a/app/src/main/java/exh/metadata/sql/mappers/SearchTagTypeMapping.kt b/app/src/main/java/exh/metadata/sql/mappers/SearchTagTypeMapping.kt new file mode 100755 index 000000000..7eaf97e30 --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/mappers/SearchTagTypeMapping.kt @@ -0,0 +1,65 @@ +package exh.metadata.sql.mappers + +import android.content.ContentValues +import android.database.Cursor +import com.pushtorefresh.storio.sqlite.SQLiteTypeMapping +import com.pushtorefresh.storio.sqlite.operations.delete.DefaultDeleteResolver +import com.pushtorefresh.storio.sqlite.operations.get.DefaultGetResolver +import com.pushtorefresh.storio.sqlite.operations.put.DefaultPutResolver +import com.pushtorefresh.storio.sqlite.queries.DeleteQuery +import com.pushtorefresh.storio.sqlite.queries.InsertQuery +import com.pushtorefresh.storio.sqlite.queries.UpdateQuery +import exh.metadata.sql.models.SearchTag +import exh.metadata.sql.tables.SearchTagTable.COL_ID +import exh.metadata.sql.tables.SearchTagTable.COL_MANGA_ID +import exh.metadata.sql.tables.SearchTagTable.COL_NAME +import exh.metadata.sql.tables.SearchTagTable.COL_NAMESPACE +import exh.metadata.sql.tables.SearchTagTable.COL_TYPE +import exh.metadata.sql.tables.SearchTagTable.TABLE + +class SearchTagTypeMapping : SQLiteTypeMapping( + SearchTagPutResolver(), + SearchTagGetResolver(), + SearchTagDeleteResolver() +) + +class SearchTagPutResolver : DefaultPutResolver() { + + override fun mapToInsertQuery(obj: SearchTag) = InsertQuery.builder() + .table(TABLE) + .build() + + override fun mapToUpdateQuery(obj: SearchTag) = UpdateQuery.builder() + .table(TABLE) + .where("$COL_ID = ?") + .whereArgs(obj.id) + .build() + + override fun mapToContentValues(obj: SearchTag) = ContentValues(5).apply { + put(COL_ID, obj.id) + put(COL_MANGA_ID, obj.mangaId) + put(COL_NAMESPACE, obj.namespace) + put(COL_NAME, obj.name) + put(COL_TYPE, obj.type) + } +} + +class SearchTagGetResolver : DefaultGetResolver() { + + override fun mapFromCursor(cursor: Cursor): SearchTag = SearchTag( + id = cursor.getLong(cursor.getColumnIndex(COL_ID)), + mangaId = cursor.getLong(cursor.getColumnIndex(COL_MANGA_ID)), + namespace = cursor.getString(cursor.getColumnIndex(COL_NAMESPACE)), + name = cursor.getString(cursor.getColumnIndex(COL_NAME)), + type = cursor.getInt(cursor.getColumnIndex(COL_TYPE)) + ) +} + +class SearchTagDeleteResolver : DefaultDeleteResolver() { + + override fun mapToDeleteQuery(obj: SearchTag) = DeleteQuery.builder() + .table(TABLE) + .where("$COL_ID = ?") + .whereArgs(obj.id) + .build() +} diff --git a/app/src/main/java/exh/metadata/sql/mappers/SearchTitleTypeMapping.kt b/app/src/main/java/exh/metadata/sql/mappers/SearchTitleTypeMapping.kt new file mode 100755 index 000000000..3065d273c --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/mappers/SearchTitleTypeMapping.kt @@ -0,0 +1,62 @@ +package exh.metadata.sql.mappers + +import android.content.ContentValues +import android.database.Cursor +import com.pushtorefresh.storio.sqlite.SQLiteTypeMapping +import com.pushtorefresh.storio.sqlite.operations.delete.DefaultDeleteResolver +import com.pushtorefresh.storio.sqlite.operations.get.DefaultGetResolver +import com.pushtorefresh.storio.sqlite.operations.put.DefaultPutResolver +import com.pushtorefresh.storio.sqlite.queries.DeleteQuery +import com.pushtorefresh.storio.sqlite.queries.InsertQuery +import com.pushtorefresh.storio.sqlite.queries.UpdateQuery +import exh.metadata.sql.models.SearchTitle +import exh.metadata.sql.tables.SearchTitleTable.COL_ID +import exh.metadata.sql.tables.SearchTitleTable.COL_MANGA_ID +import exh.metadata.sql.tables.SearchTitleTable.COL_TITLE +import exh.metadata.sql.tables.SearchTitleTable.COL_TYPE +import exh.metadata.sql.tables.SearchTitleTable.TABLE + +class SearchTitleTypeMapping : SQLiteTypeMapping( + SearchTitlePutResolver(), + SearchTitleGetResolver(), + SearchTitleDeleteResolver() +) + +class SearchTitlePutResolver : DefaultPutResolver() { + + override fun mapToInsertQuery(obj: SearchTitle) = InsertQuery.builder() + .table(TABLE) + .build() + + override fun mapToUpdateQuery(obj: SearchTitle) = UpdateQuery.builder() + .table(TABLE) + .where("$COL_ID = ?") + .whereArgs(obj.id) + .build() + + override fun mapToContentValues(obj: SearchTitle) = ContentValues(4).apply { + put(COL_ID, obj.id) + put(COL_MANGA_ID, obj.mangaId) + put(COL_TITLE, obj.title) + put(COL_TYPE, obj.type) + } +} + +class SearchTitleGetResolver : DefaultGetResolver() { + + override fun mapFromCursor(cursor: Cursor): SearchTitle = SearchTitle( + id = cursor.getLong(cursor.getColumnIndex(COL_ID)), + mangaId = cursor.getLong(cursor.getColumnIndex(COL_MANGA_ID)), + title = cursor.getString(cursor.getColumnIndex(COL_TITLE)), + type = cursor.getInt(cursor.getColumnIndex(COL_TYPE)) + ) +} + +class SearchTitleDeleteResolver : DefaultDeleteResolver() { + + override fun mapToDeleteQuery(obj: SearchTitle) = DeleteQuery.builder() + .table(TABLE) + .where("$COL_ID = ?") + .whereArgs(obj.id) + .build() +} diff --git a/app/src/main/java/exh/metadata/sql/models/SearchMetadata.kt b/app/src/main/java/exh/metadata/sql/models/SearchMetadata.kt new file mode 100644 index 000000000..f0a1043cc --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/models/SearchMetadata.kt @@ -0,0 +1,21 @@ +package exh.metadata.sql.models + +data class SearchMetadata( + // Manga ID this gallery is linked to + val mangaId: Long, + + // Gallery uploader + val uploader: String?, + + // Extra data attached to this metadata, in JSON format + val extra: String, + + // Indexed extra data attached to this metadata + val indexedExtra: String?, + + // The version of this metadata's extra. Used to track changes to the 'extra' field's schema + val extraVersion: Int +) { + // Transient information attached to this piece of metadata, useful for caching + var transientCache: Map? = null +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/sql/models/SearchTag.kt b/app/src/main/java/exh/metadata/sql/models/SearchTag.kt new file mode 100644 index 000000000..1665fc9e4 --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/models/SearchTag.kt @@ -0,0 +1,18 @@ +package exh.metadata.sql.models + +data class SearchTag( + // Tag identifier, unique + val id: Long?, + + // Metadata this tag is attached to + val mangaId: Long, + + // Tag namespace + val namespace: String?, + + // Tag name + val name: String, + + // Tag type + val type: Int +) \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/sql/models/SearchTitle.kt b/app/src/main/java/exh/metadata/sql/models/SearchTitle.kt new file mode 100644 index 000000000..521cccb6d --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/models/SearchTitle.kt @@ -0,0 +1,15 @@ +package exh.metadata.sql.models + +data class SearchTitle( + // Title identifier, unique + val id: Long?, + + // Metadata this title is attached to + val mangaId: Long, + + // Title + val title: String, + + // Title type, useful for distinguishing between main/alt titles + val type: Int +) diff --git a/app/src/main/java/exh/metadata/sql/queries/SearchMetadataQueries.kt b/app/src/main/java/exh/metadata/sql/queries/SearchMetadataQueries.kt new file mode 100755 index 000000000..a9905d6d7 --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/queries/SearchMetadataQueries.kt @@ -0,0 +1,36 @@ +package exh.metadata.sql.queries + +import com.pushtorefresh.storio.sqlite.queries.DeleteQuery +import com.pushtorefresh.storio.sqlite.queries.Query +import eu.kanade.tachiyomi.data.database.DbProvider +import eu.kanade.tachiyomi.data.database.models.Manga +import exh.metadata.sql.models.SearchMetadata +import exh.metadata.sql.tables.SearchMetadataTable + +interface SearchMetadataQueries : DbProvider { + + fun getSearchMetadataForManga(mangaId: Long) = db.get() + .`object`(SearchMetadata::class.java) + .withQuery(Query.builder() + .table(SearchMetadataTable.TABLE) + .where("${SearchMetadataTable.COL_MANGA_ID} = ?") + .whereArgs(mangaId) + .build()) + .prepare() + + fun getSearchMetadata() = db.get() + .listOfObjects(SearchMetadata::class.java) + .withQuery(Query.builder() + .table(SearchMetadataTable.TABLE) + .build()) + .prepare() + + fun insertSearchMetadata(metadata: SearchMetadata) = db.put().`object`(metadata).prepare() + + fun deleteSearchMetadata(metadata: SearchMetadata) = db.delete().`object`(metadata).prepare() + + fun deleteAllSearchMetadata() = db.delete().byQuery(DeleteQuery.builder() + .table(SearchMetadataTable.TABLE) + .build()) + .prepare() +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/sql/queries/SearchTagQueries.kt b/app/src/main/java/exh/metadata/sql/queries/SearchTagQueries.kt new file mode 100755 index 000000000..ebd98528b --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/queries/SearchTagQueries.kt @@ -0,0 +1,45 @@ +package exh.metadata.sql.queries + +import com.pushtorefresh.storio.sqlite.queries.DeleteQuery +import com.pushtorefresh.storio.sqlite.queries.Query +import eu.kanade.tachiyomi.data.database.DbProvider +import eu.kanade.tachiyomi.data.database.inTransaction +import exh.metadata.sql.models.SearchTag +import exh.metadata.sql.tables.SearchTagTable + +interface SearchTagQueries : DbProvider { + fun getSearchTagsForManga(mangaId: Long) = db.get() + .listOfObjects(SearchTag::class.java) + .withQuery(Query.builder() + .table(SearchTagTable.TABLE) + .where("${SearchTagTable.COL_MANGA_ID} = ?") + .whereArgs(mangaId) + .build()) + .prepare() + + fun deleteSearchTagsForManga(mangaId: Long) = db.delete() + .byQuery(DeleteQuery.builder() + .table(SearchTagTable.TABLE) + .where("${SearchTagTable.COL_MANGA_ID} = ?") + .whereArgs(mangaId) + .build()) + .prepare() + + fun insertSearchTag(searchTag: SearchTag) = db.put().`object`(searchTag).prepare() + + fun insertSearchTags(searchTags: List) = db.put().objects(searchTags).prepare() + + fun deleteSearchTag(searchTag: SearchTag) = db.delete().`object`(searchTag).prepare() + + fun deleteAllSearchTags() = db.delete().byQuery(DeleteQuery.builder() + .table(SearchTagTable.TABLE) + .build()) + .prepare() + + fun setSearchTagsForManga(mangaId: Long, tags: List) { + db.inTransaction { + deleteSearchTagsForManga(mangaId).executeAsBlocking() + insertSearchTags(tags).executeAsBlocking() + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/sql/queries/SearchTitleQueries.kt b/app/src/main/java/exh/metadata/sql/queries/SearchTitleQueries.kt new file mode 100755 index 000000000..dc1c2ae25 --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/queries/SearchTitleQueries.kt @@ -0,0 +1,47 @@ +package exh.metadata.sql.queries + +import com.pushtorefresh.storio.sqlite.queries.DeleteQuery +import com.pushtorefresh.storio.sqlite.queries.Query +import eu.kanade.tachiyomi.data.database.DbProvider +import eu.kanade.tachiyomi.data.database.inTransaction +import eu.kanade.tachiyomi.data.database.models.Manga +import exh.metadata.sql.models.SearchMetadata +import exh.metadata.sql.models.SearchTitle +import exh.metadata.sql.tables.SearchTitleTable + +interface SearchTitleQueries : DbProvider { + fun getSearchTitlesForManga(mangaId: Long) = db.get() + .listOfObjects(SearchTitle::class.java) + .withQuery(Query.builder() + .table(SearchTitleTable.TABLE) + .where("${SearchTitleTable.COL_MANGA_ID} = ?") + .whereArgs(mangaId) + .build()) + .prepare() + + fun deleteSearchTitlesForManga(mangaId: Long) = db.delete() + .byQuery(DeleteQuery.builder() + .table(SearchTitleTable.TABLE) + .where("${SearchTitleTable.COL_MANGA_ID} = ?") + .whereArgs(mangaId) + .build()) + .prepare() + + fun insertSearchTitle(searchTitle: SearchTitle) = db.put().`object`(searchTitle).prepare() + + fun insertSearchTitles(searchTitles: List) = db.put().objects(searchTitles).prepare() + + fun deleteSearchTitle(searchTitle: SearchTitle) = db.delete().`object`(searchTitle).prepare() + + fun deleteAllSearchTitle() = db.delete().byQuery(DeleteQuery.builder() + .table(SearchTitleTable.TABLE) + .build()) + .prepare() + + fun setSearchTitlesForManga(mangaId: Long, titles: List) { + db.inTransaction { + deleteSearchTitlesForManga(mangaId).executeAsBlocking() + insertSearchTitles(titles).executeAsBlocking() + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/metadata/sql/tables/SearchMetadataTable.kt b/app/src/main/java/exh/metadata/sql/tables/SearchMetadataTable.kt new file mode 100755 index 000000000..efbaecd1a --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/tables/SearchMetadataTable.kt @@ -0,0 +1,35 @@ +package exh.metadata.sql.tables + +import eu.kanade.tachiyomi.data.database.tables.MangaTable + +object SearchMetadataTable { + const val TABLE = "search_metadata" + + const val COL_MANGA_ID = "manga_id" + + const val COL_UPLOADER = "uploader" + + const val COL_EXTRA = "extra" + + const val COL_INDEXED_EXTRA = "indexed_extra" + + const val COL_EXTRA_VERSION = "extra_version" + + // Insane foreign, primary key to avoid touch manga table + val createTableQuery: String + get() = """CREATE TABLE $TABLE( + $COL_MANGA_ID INTEGER NOT NULL PRIMARY KEY, + $COL_UPLOADER TEXT, + $COL_EXTRA TEXT NOT NULL, + $COL_INDEXED_EXTRA TEXT, + $COL_EXTRA_VERSION INT NOT NULL, + FOREIGN KEY($COL_MANGA_ID) REFERENCES ${MangaTable.TABLE} (${MangaTable.COL_ID}) + ON DELETE CASCADE + )""" + + val createUploaderIndexQuery: String + get() = "CREATE INDEX ${TABLE}_${COL_UPLOADER}_index ON $TABLE($COL_UPLOADER)" + + val createIndexedExtraIndexQuery: String + get() = "CREATE INDEX ${TABLE}_${COL_INDEXED_EXTRA}_index ON $TABLE($COL_INDEXED_EXTRA)" +} diff --git a/app/src/main/java/exh/metadata/sql/tables/SearchTagTable.kt b/app/src/main/java/exh/metadata/sql/tables/SearchTagTable.kt new file mode 100755 index 000000000..81be76583 --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/tables/SearchTagTable.kt @@ -0,0 +1,34 @@ +package exh.metadata.sql.tables + +import eu.kanade.tachiyomi.data.database.tables.MangaTable + +object SearchTagTable { + const val TABLE = "search_tags" + + const val COL_ID = "_id" + + const val COL_MANGA_ID = "manga_id" + + const val COL_NAMESPACE = "namespace" + + const val COL_NAME = "name" + + const val COL_TYPE = "type" + + val createTableQuery: String + get() = """CREATE TABLE $TABLE( + $COL_ID INTEGER NOT NULL PRIMARY KEY, + $COL_MANGA_ID INTEGER NOT NULL, + $COL_NAMESPACE TEXT, + $COL_NAME TEXT NOT NULL, + $COL_TYPE INT NOT NULL, + FOREIGN KEY($COL_MANGA_ID) REFERENCES ${MangaTable.TABLE} (${MangaTable.COL_ID}) + ON DELETE CASCADE + )""" + + val createMangaIdIndexQuery: String + get() = "CREATE INDEX ${TABLE}_${COL_MANGA_ID}_index ON $TABLE($COL_MANGA_ID)" + + val createNamespaceNameIndexQuery: String + get() = "CREATE INDEX ${TABLE}_${COL_NAMESPACE}_${COL_NAME}_index ON $TABLE($COL_NAMESPACE, $COL_NAME)" +} diff --git a/app/src/main/java/exh/metadata/sql/tables/SearchTitleTable.kt b/app/src/main/java/exh/metadata/sql/tables/SearchTitleTable.kt new file mode 100755 index 000000000..d72cb9b8c --- /dev/null +++ b/app/src/main/java/exh/metadata/sql/tables/SearchTitleTable.kt @@ -0,0 +1,31 @@ +package exh.metadata.sql.tables + +import eu.kanade.tachiyomi.data.database.tables.MangaTable + +object SearchTitleTable { + const val TABLE = "search_titles" + + const val COL_ID = "_id" + + const val COL_MANGA_ID = "manga_id" + + const val COL_TITLE = "title" + + const val COL_TYPE = "type" + + val createTableQuery: String + get() = """CREATE TABLE $TABLE( + $COL_ID INTEGER NOT NULL PRIMARY KEY, + $COL_MANGA_ID INTEGER NOT NULL, + $COL_TITLE TEXT NOT NULL, + $COL_TYPE INT NOT NULL, + FOREIGN KEY($COL_MANGA_ID) REFERENCES ${MangaTable.TABLE} (${MangaTable.COL_ID}) + ON DELETE CASCADE + )""" + + val createMangaIdIndexQuery: String + get() = "CREATE INDEX ${TABLE}_${COL_MANGA_ID}_index ON $TABLE($COL_MANGA_ID)" + + val createTitleIndexQuery: String + get() = "CREATE INDEX ${TABLE}_${COL_TITLE}_index ON $TABLE($COL_TITLE)" +} diff --git a/app/src/main/java/exh/search/MultiWildcard.kt b/app/src/main/java/exh/search/MultiWildcard.kt index b5cecbe57..d9a6d7dc8 100755 --- a/app/src/main/java/exh/search/MultiWildcard.kt +++ b/app/src/main/java/exh/search/MultiWildcard.kt @@ -1,3 +1,3 @@ package exh.search -class MultiWildcard : TextComponent() +class MultiWildcard(rawText: String) : TextComponent(rawText) diff --git a/app/src/main/java/exh/search/SearchEngine.kt b/app/src/main/java/exh/search/SearchEngine.kt index 227ed9bf3..07c18afc3 100755 --- a/app/src/main/java/exh/search/SearchEngine.kt +++ b/app/src/main/java/exh/search/SearchEngine.kt @@ -1,95 +1,122 @@ package exh.search -import exh.metadata.models.SearchableGalleryMetadata -import exh.metadata.models.Tag -import io.realm.Case -import io.realm.RealmQuery +import eu.kanade.tachiyomi.data.database.tables.MangaTable +import exh.metadata.sql.tables.SearchMetadataTable +import exh.metadata.sql.tables.SearchTagTable +import exh.metadata.sql.tables.SearchTitleTable class SearchEngine { private val queryCache = mutableMapOf>() - fun filterResults(rQuery: RealmQuery, - query: List, - titleFields: List): - RealmQuery { - var queryEmpty = true - - fun matchTagList(namespace: String?, - component: Text?, - excluded: Boolean) { - when { - excluded -> rQuery.not() - queryEmpty -> queryEmpty = false - else -> rQuery.or() - } - - rQuery.beginGroup() - //Match namespace if specified - namespace?.let { - rQuery.equalTo("${SearchableGalleryMetadata::tags.name}.${Tag::namespace.name}", - it, - Case.INSENSITIVE) - } - //Match tag name if specified - component?.let { - rQuery.beginGroup() - val q = if (!it.exact) + fun textToSubQueries(namespace: String?, + component: Text?): Pair>? { + val maybeLenientComponent = component?.let { + if (!it.exact) it.asLenientTagQueries() else listOf(it.asQuery()) - q.forEachIndexed { index, s -> - if(index > 0) - rQuery.or() - - rQuery.like("${SearchableGalleryMetadata::tags.name}.${Tag::name.name}", s, Case.INSENSITIVE) - } - rQuery.endGroup() - } - rQuery.endGroup() } + val componentTagQuery = maybeLenientComponent?.let { + val params = mutableListOf() + it.map { q -> + params += q + "${SearchTagTable.TABLE}.${SearchTagTable.COL_NAME} LIKE ?" + }.joinToString(separator = " OR ", prefix = "(", postfix = ")") to params + } + return if(namespace != null) { + var query = """ + (SELECT ${SearchTagTable.COL_MANGA_ID} AS $COL_MANGA_ID FROM ${SearchTagTable.TABLE} + WHERE ${SearchTagTable.COL_NAMESPACE} IS NOT NULL + AND ${SearchTagTable.COL_NAMESPACE} LIKE ? + """.trimIndent() + val params = mutableListOf(escapeLike(namespace)) + if(componentTagQuery != null) { + query += "\n AND ${componentTagQuery.first}" + params += componentTagQuery.second + } - for(component in query) { - if(component is Text) { - if(component.excluded) - rQuery.not() + "$query)" to params + } else if(component != null) { + // Match title + tags + val tagQuery = """ + SELECT ${SearchTagTable.COL_MANGA_ID} AS $COL_MANGA_ID FROM ${SearchTagTable.TABLE} + WHERE ${componentTagQuery!!.first} + """.trimIndent() to componentTagQuery.second - rQuery.beginGroup() + val titleQuery = """ + SELECT ${SearchTitleTable.COL_MANGA_ID} AS $COL_MANGA_ID FROM ${SearchTitleTable.TABLE} + WHERE ${SearchTitleTable.COL_TITLE} LIKE ? + """.trimIndent() to listOf(component.asLenientTitleQuery()) - //Match title - titleFields.forEachIndexed { index, s -> - queryEmpty = false - if(index > 0) - rQuery.or() + "(${tagQuery.first} UNION ${titleQuery.first})".trimIndent() to + (tagQuery.second + titleQuery.second) + } else null + } - rQuery.like(s, component.asLenientTitleQuery(), Case.INSENSITIVE) - } + fun queryToSql(q: List): Pair> { + val wheres = mutableListOf() + val whereParams = mutableListOf() - //Match tags - matchTagList(null, component, false) //We already deal with exclusions here - rQuery.endGroup() + val include = mutableListOf>>() + val exclude = mutableListOf>>() + + for(component in q) { + val query = if(component is Text) { + textToSubQueries(null, component) } else if(component is Namespace) { if(component.namespace == "uploader") { - queryEmpty = false - //Match uploader - rQuery.equalTo(SearchableGalleryMetadata::uploader.name, - component.tag!!.rawTextOnly(), - Case.INSENSITIVE) + wheres += "meta.${SearchMetadataTable.COL_UPLOADER} LIKE ?" + whereParams += component.tag!!.rawTextEscapedForLike() + null } else { if(component.tag!!.components.size > 0) { //Match namespace + tags - matchTagList(component.namespace, component.tag!!, component.tag!!.excluded) + textToSubQueries(component.namespace, component.tag) } else { //Perform namespace search - matchTagList(component.namespace, null, component.excluded) + textToSubQueries(component.namespace, null) } } + } else error("Unknown query component!") + + if(query != null) { + (if(component.excluded) exclude else include) += query } } - return rQuery + + val completeParams = mutableListOf() + var baseQuery = """ + SELECT ${SearchMetadataTable.COL_MANGA_ID} + FROM ${SearchMetadataTable.TABLE} meta + """.trimIndent() + + include.forEachIndexed { index, pair -> + baseQuery += "\n" + (""" + INNER JOIN ${pair.first} i$index + ON i$index.$COL_MANGA_ID = meta.${SearchMetadataTable.COL_MANGA_ID} + """.trimIndent()) + completeParams += pair.second + } + + + exclude.forEach { + wheres += """ + (meta.${SearchMetadataTable.COL_MANGA_ID} NOT IN ${it.first}) + """.trimIndent() + whereParams += it.second + } + if(wheres.isNotEmpty()) { + completeParams += whereParams + baseQuery += "\nWHERE\n" + baseQuery += wheres.joinToString("\nAND\n") + } + baseQuery += "\nORDER BY ${SearchMetadataTable.COL_MANGA_ID}" + + return baseQuery to completeParams } - fun parseQuery(query: String) = queryCache.getOrPut(query, { + fun parseQuery(query: String) = queryCache.getOrPut(query) { val res = mutableListOf() var inQuotes = false @@ -130,10 +157,10 @@ class SearchEngine { inQuotes = !inQuotes } else if(char == '?' || char == '_') { flushText() - queuedText.add(SingleWildcard()) + queuedText.add(SingleWildcard(char.toString())) } else if(char == '*' || char == '%') { flushText() - queuedText.add(MultiWildcard()) + queuedText.add(MultiWildcard(char.toString())) } else if(char == '-') { nextIsExcluded = true } else if(char == '$') { @@ -163,5 +190,16 @@ class SearchEngine { flushAll() res - }) + } + + companion object { + private const val COL_MANGA_ID = "cmid" + + fun escapeLike(string: String): String { + return string.replace("\\", "\\\\") + .replace("_", "\\_") + .replace("%", "\\%") + + } + } } diff --git a/app/src/main/java/exh/search/SingleWildcard.kt b/app/src/main/java/exh/search/SingleWildcard.kt index 503d751e1..72ed60726 100755 --- a/app/src/main/java/exh/search/SingleWildcard.kt +++ b/app/src/main/java/exh/search/SingleWildcard.kt @@ -1,3 +1,3 @@ package exh.search -class SingleWildcard : TextComponent() +class SingleWildcard(rawText: String) : TextComponent(rawText) diff --git a/app/src/main/java/exh/search/StringTextComponent.kt b/app/src/main/java/exh/search/StringTextComponent.kt index 736f8c225..d82576cad 100755 --- a/app/src/main/java/exh/search/StringTextComponent.kt +++ b/app/src/main/java/exh/search/StringTextComponent.kt @@ -1,3 +1,3 @@ package exh.search -class StringTextComponent(val value: String) : TextComponent() +class StringTextComponent(val value: String) : TextComponent(value) diff --git a/app/src/main/java/exh/search/Text.kt b/app/src/main/java/exh/search/Text.kt index 98a51fd72..21638befd 100755 --- a/app/src/main/java/exh/search/Text.kt +++ b/app/src/main/java/exh/search/Text.kt @@ -1,6 +1,7 @@ package exh.search import exh.plusAssign +import exh.search.SearchEngine.Companion.escapeLike class Text: QueryComponent() { val components = mutableListOf() @@ -19,7 +20,7 @@ class Text: QueryComponent() { fun asLenientTitleQuery(): String { if(lenientTitleQuery == null) { - lenientTitleQuery = StringBuilder("*").append(rBaseBuilder()).append("*").toString() + lenientTitleQuery = StringBuilder("%").append(rBaseBuilder()).append("%").toString() } return lenientTitleQuery!! } @@ -28,7 +29,7 @@ class Text: QueryComponent() { if(lenientTagQueries == null) { lenientTagQueries = listOf( //Match beginning of tag - rBaseBuilder().append("*").toString(), + rBaseBuilder().append("%").toString(), //Tag word matcher (that matches multiple words) //Can't make it match a single word in Realm :( StringBuilder(" ").append(rBaseBuilder()).append(" ").toString(), @@ -43,9 +44,9 @@ class Text: QueryComponent() { val builder = StringBuilder() for(component in components) { when(component) { - is StringTextComponent -> builder += component.value - is SingleWildcard -> builder += "?" - is MultiWildcard -> builder += "*" + is StringTextComponent -> builder += escapeLike(component.value) + is SingleWildcard -> builder += "_" + is MultiWildcard -> builder += "%" } } return builder @@ -55,10 +56,9 @@ class Text: QueryComponent() { rawText!! else { rawText = components - .filter { it is StringTextComponent } - .joinToString(separator = "", transform = { - (it as StringTextComponent).value - }) + .joinToString(separator = "", transform = { it.rawText }) rawText!! } + + fun rawTextEscapedForLike() = escapeLike(rawTextOnly()) } diff --git a/app/src/main/java/exh/search/TextComponent.kt b/app/src/main/java/exh/search/TextComponent.kt index 9b50051f5..b04adffec 100755 --- a/app/src/main/java/exh/search/TextComponent.kt +++ b/app/src/main/java/exh/search/TextComponent.kt @@ -1,3 +1,3 @@ package exh.search -open class TextComponent +open class TextComponent(val rawText: String) diff --git a/app/src/main/java/exh/source/DelegatedHttpSource.kt b/app/src/main/java/exh/source/DelegatedHttpSource.kt new file mode 100644 index 000000000..f306f4c5e --- /dev/null +++ b/app/src/main/java/exh/source/DelegatedHttpSource.kt @@ -0,0 +1,238 @@ +package exh.source + +import eu.kanade.tachiyomi.source.model.* +import eu.kanade.tachiyomi.source.online.HttpSource +import okhttp3.Request +import okhttp3.Response +import rx.Observable +import java.lang.RuntimeException + +abstract class DelegatedHttpSource(val delegate: HttpSource): HttpSource() { + /** + * Returns the request for the popular manga given the page. + * + * @param page the page number to retrieve. + */ + override fun popularMangaRequest(page: Int) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun popularMangaParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Returns the request for the search manga given the page. + * + * @param page the page number to retrieve. + * @param query the search query. + * @param filters the list of filters to apply. + */ + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun searchMangaParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Returns the request for latest manga given the page. + * + * @param page the page number to retrieve. + */ + override fun latestUpdatesRequest(page: Int) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun latestUpdatesParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns the details of a manga. + * + * @param response the response from the site. + */ + override fun mangaDetailsParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a list of chapters. + * + * @param response the response from the site. + */ + override fun chapterListParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a list of pages. + * + * @param response the response from the site. + */ + override fun pageListParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns the absolute url to the source image. + * + * @param response the response from the site. + */ + override fun imageUrlParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Base url of the website without the trailing slash, like: http://mysite.com + */ + override val baseUrl = delegate.baseUrl + + /** + * Whether the source has support for latest updates. + */ + override val supportsLatest = delegate.supportsLatest + /** + * Name of the source. + */ + final override val name = delegate.name + + // ===> OPTIONAL FIELDS + + /** + * Id of the source. By default it uses a generated id using the first 16 characters (64 bits) + * of the MD5 of the string: sourcename/language/versionId + * Note the generated id sets the sign bit to 0. + */ + override val id = delegate.id + /** + * Default network client for doing requests. + */ + override val client = delegate.client + + /** + * Visible name of the source. + */ + override fun toString() = delegate.toString() + + /** + * Returns an observable containing a page with a list of manga. Normally it's not needed to + * override this method. + * + * @param page the page number to retrieve. + */ + override fun fetchPopularManga(page: Int): Observable { + ensureDelegateCompatible() + return delegate.fetchPopularManga(page) + } + + /** + * Returns an observable containing a page with a list of manga. Normally it's not needed to + * override this method. + * + * @param page the page number to retrieve. + * @param query the search query. + * @param filters the list of filters to apply. + */ + override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { + ensureDelegateCompatible() + return delegate.fetchSearchManga(page, query, filters) + } + + /** + * Returns an observable containing a page with a list of latest manga updates. + * + * @param page the page number to retrieve. + */ + override fun fetchLatestUpdates(page: Int): Observable { + ensureDelegateCompatible() + return delegate.fetchLatestUpdates(page) + } + + /** + * Returns an observable with the updated details for a manga. Normally it's not needed to + * override this method. + * + * @param manga the manga to be updated. + */ + override fun fetchMangaDetails(manga: SManga): Observable { + ensureDelegateCompatible() + return delegate.fetchMangaDetails(manga) + } + + /** + * Returns the request for the details of a manga. Override only if it's needed to change the + * url, send different headers or request method like POST. + * + * @param manga the manga to be updated. + */ + override fun mangaDetailsRequest(manga: SManga): Request { + ensureDelegateCompatible() + return delegate.mangaDetailsRequest(manga) + } + + /** + * Returns an observable with the updated chapter list for a manga. Normally it's not needed to + * override this method. If a manga is licensed an empty chapter list observable is returned + * + * @param manga the manga to look for chapters. + */ + override fun fetchChapterList(manga: SManga): Observable> { + ensureDelegateCompatible() + return delegate.fetchChapterList(manga) + } + + /** + * Returns an observable with the page list for a chapter. + * + * @param chapter the chapter whose page list has to be fetched. + */ + override fun fetchPageList(chapter: SChapter): Observable> { + ensureDelegateCompatible() + return delegate.fetchPageList(chapter) + } + + /** + * Returns an observable with the page containing the source url of the image. If there's any + * error, it will return null instead of throwing an exception. + * + * @param page the page whose source image has to be fetched. + */ + override fun fetchImageUrl(page: Page): Observable { + ensureDelegateCompatible() + return delegate.fetchImageUrl(page) + } + + /** + * Called before inserting a new chapter into database. Use it if you need to override chapter + * fields, like the title or the chapter number. Do not change anything to [manga]. + * + * @param chapter the chapter to be added. + * @param manga the manga of the chapter. + */ + override fun prepareNewChapter(chapter: SChapter, manga: SManga) { + ensureDelegateCompatible() + return delegate.prepareNewChapter(chapter, manga) + } + + /** + * Returns the list of filters for the source. + */ + override fun getFilterList() = delegate.getFilterList() + + private fun ensureDelegateCompatible() { + if(versionId != delegate.versionId + || lang != delegate.lang) { + throw IncompatibleDelegateException("Delegate source is not compatible (versionId: $versionId <=> ${delegate.versionId}, lang: $lang <=> ${delegate.lang})!") + } + } + + class IncompatibleDelegateException(message: String) : RuntimeException(message) +} \ No newline at end of file diff --git a/app/src/main/java/exh/source/EnhancedHttpSource.kt b/app/src/main/java/exh/source/EnhancedHttpSource.kt new file mode 100644 index 000000000..7738ab801 --- /dev/null +++ b/app/src/main/java/exh/source/EnhancedHttpSource.kt @@ -0,0 +1,220 @@ +package exh.source + +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import eu.kanade.tachiyomi.source.model.* +import eu.kanade.tachiyomi.source.online.HttpSource +import okhttp3.Response +import uy.kohesive.injekt.injectLazy + +class EnhancedHttpSource(val originalSource: HttpSource, + val enchancedSource: HttpSource): HttpSource() { + private val prefs: PreferencesHelper by injectLazy() + + /** + * Returns the request for the popular manga given the page. + * + * @param page the page number to retrieve. + */ + override fun popularMangaRequest(page: Int) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun popularMangaParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Returns the request for the search manga given the page. + * + * @param page the page number to retrieve. + * @param query the search query. + * @param filters the list of filters to apply. + */ + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun searchMangaParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Returns the request for latest manga given the page. + * + * @param page the page number to retrieve. + */ + override fun latestUpdatesRequest(page: Int) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a [MangasPage] object. + * + * @param response the response from the site. + */ + override fun latestUpdatesParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns the details of a manga. + * + * @param response the response from the site. + */ + override fun mangaDetailsParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a list of chapters. + * + * @param response the response from the site. + */ + override fun chapterListParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns a list of pages. + * + * @param response the response from the site. + */ + override fun pageListParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Parses the response from the site and returns the absolute url to the source image. + * + * @param response the response from the site. + */ + override fun imageUrlParse(response: Response) + = throw UnsupportedOperationException("Should never be called!") + + /** + * Base url of the website without the trailing slash, like: http://mysite.com + */ + override val baseUrl = source().baseUrl + + /** + * Whether the source has support for latest updates. + */ + override val supportsLatest = source().supportsLatest + /** + * Name of the source. + */ + override val name = source().name + + /** + * An ISO 639-1 compliant language code (two letters in lower case). + */ + override val lang = source().lang + + // ===> OPTIONAL FIELDS + + /** + * Id of the source. By default it uses a generated id using the first 16 characters (64 bits) + * of the MD5 of the string: sourcename/language/versionId + * Note the generated id sets the sign bit to 0. + */ + override val id = source().id + /** + * Default network client for doing requests. + */ + override val client = source().client + + /** + * Visible name of the source. + */ + override fun toString() = source().toString() + + /** + * Returns an observable containing a page with a list of manga. Normally it's not needed to + * override this method. + * + * @param page the page number to retrieve. + */ + override fun fetchPopularManga(page: Int) = source().fetchPopularManga(page) + + /** + * Returns an observable containing a page with a list of manga. Normally it's not needed to + * override this method. + * + * @param page the page number to retrieve. + * @param query the search query. + * @param filters the list of filters to apply. + */ + override fun fetchSearchManga(page: Int, query: String, filters: FilterList) + = source().fetchSearchManga(page, query, filters) + + /** + * Returns an observable containing a page with a list of latest manga updates. + * + * @param page the page number to retrieve. + */ + override fun fetchLatestUpdates(page: Int) = source().fetchLatestUpdates(page) + + /** + * Returns an observable with the updated details for a manga. Normally it's not needed to + * override this method. + * + * @param manga the manga to be updated. + */ + override fun fetchMangaDetails(manga: SManga) = source().fetchMangaDetails(manga) + + /** + * Returns the request for the details of a manga. Override only if it's needed to change the + * url, send different headers or request method like POST. + * + * @param manga the manga to be updated. + */ + override fun mangaDetailsRequest(manga: SManga) = source().mangaDetailsRequest(manga) + + /** + * Returns an observable with the updated chapter list for a manga. Normally it's not needed to + * override this method. If a manga is licensed an empty chapter list observable is returned + * + * @param manga the manga to look for chapters. + */ + override fun fetchChapterList(manga: SManga) = source().fetchChapterList(manga) + + /** + * Returns an observable with the page list for a chapter. + * + * @param chapter the chapter whose page list has to be fetched. + */ + override fun fetchPageList(chapter: SChapter) = source().fetchPageList(chapter) + + /** + * Returns an observable with the page containing the source url of the image. If there's any + * error, it will return null instead of throwing an exception. + * + * @param page the page whose source image has to be fetched. + */ + override fun fetchImageUrl(page: Page) = source().fetchImageUrl(page) + + /** + * Called before inserting a new chapter into database. Use it if you need to override chapter + * fields, like the title or the chapter number. Do not change anything to [manga]. + * + * @param chapter the chapter to be added. + * @param manga the manga of the chapter. + */ + override fun prepareNewChapter(chapter: SChapter, manga: SManga) + = source().prepareNewChapter(chapter, manga) + + /** + * Returns the list of filters for the source. + */ + override fun getFilterList() = source().getFilterList() + + private fun source(): HttpSource { + return if(prefs.eh_delegateSources().getOrDefault()) { + enchancedSource + } else { + originalSource + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/uconfig/EhUConfigBuilder.kt b/app/src/main/java/exh/uconfig/EhUConfigBuilder.kt index 850da220e..3f1f671a8 100644 --- a/app/src/main/java/exh/uconfig/EhUConfigBuilder.kt +++ b/app/src/main/java/exh/uconfig/EhUConfigBuilder.kt @@ -92,10 +92,10 @@ object Entry { override val key = "tl" } - //Locked to list mode as that's what the parser and toplists use + //Locked to extended mode as that's what the parser and toplists use class DisplayMode: ConfigItem { override val key = "dm" - override val value = "0" + override val value = "2" } enum class SearchResultsCount(override val value: String): ConfigItem { diff --git a/app/src/main/java/exh/ui/captcha/AutoSolvingWebViewClient.kt b/app/src/main/java/exh/ui/captcha/AutoSolvingWebViewClient.kt new file mode 100644 index 000000000..c492df15a --- /dev/null +++ b/app/src/main/java/exh/ui/captcha/AutoSolvingWebViewClient.kt @@ -0,0 +1,36 @@ +package exh.ui.captcha + +import android.os.Build +import android.support.annotation.RequiresApi +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebView +import eu.kanade.tachiyomi.util.asJsoup +import exh.ui.captcha.SolveCaptchaActivity.Companion.CROSS_WINDOW_SCRIPT_INNER +import org.jsoup.nodes.DataNode +import org.jsoup.nodes.Element +import java.nio.charset.Charset + +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +class AutoSolvingWebViewClient(activity: SolveCaptchaActivity, + source: CaptchaCompletionVerifier, + injectScript: String?) + : BasicWebViewClient(activity, source, injectScript) { + + override fun shouldInterceptRequest(view: WebView, request: WebResourceRequest): WebResourceResponse? { + // Inject our custom script into the recaptcha iframes + val lastPathSegment = request.url.pathSegments.lastOrNull() + if(lastPathSegment == "anchor" || lastPathSegment == "bframe") { + val oReq = request.toOkHttpRequest() + val response = activity.httpClient.newCall(oReq).execute() + val doc = response.asJsoup() + doc.body().appendChild(Element("script").appendChild(DataNode(CROSS_WINDOW_SCRIPT_INNER))) + return WebResourceResponse( + "text/html", + "UTF-8", + doc.toString().byteInputStream(Charset.forName("UTF-8")).buffered() + ) + } + return super.shouldInterceptRequest(view, request) + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/captcha/BasicWebViewClient.kt b/app/src/main/java/exh/ui/captcha/BasicWebViewClient.kt new file mode 100644 index 000000000..c791ecfd0 --- /dev/null +++ b/app/src/main/java/exh/ui/captcha/BasicWebViewClient.kt @@ -0,0 +1,18 @@ +package exh.ui.captcha + +import android.webkit.WebView +import android.webkit.WebViewClient + +open class BasicWebViewClient(protected val activity: SolveCaptchaActivity, + protected val source: CaptchaCompletionVerifier, + private val injectScript: String?) : WebViewClient() { + override fun onPageFinished(view: WebView, url: String) { + super.onPageFinished(view, url) + + if(source.verifyNoCaptcha(url)) { + activity.finish() + } else { + if(injectScript != null) view.loadUrl("javascript:(function() {$injectScript})();") + } + } +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/captcha/SolveCaptchaActivity.kt b/app/src/main/java/exh/ui/captcha/SolveCaptchaActivity.kt index b4ee7116d..be782b87a 100644 --- a/app/src/main/java/exh/ui/captcha/SolveCaptchaActivity.kt +++ b/app/src/main/java/exh/ui/captcha/SolveCaptchaActivity.kt @@ -4,25 +4,48 @@ import android.content.Context import android.content.Intent import android.os.Build import android.os.Bundle +import android.support.annotation.RequiresApi import android.support.v7.app.AppCompatActivity -import android.webkit.CookieManager -import android.webkit.CookieSyncManager -import android.webkit.WebView -import android.webkit.WebViewClient -import eu.kanade.tachiyomi.R +import android.webkit.* +import com.github.salomonbrys.kotson.get +import com.github.salomonbrys.kotson.string +import com.google.gson.JsonParser +import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceManager import kotlinx.android.synthetic.main.eh_activity_captcha.* +import okhttp3.* +import rx.Single +import rx.schedulers.Schedulers +import timber.log.Timber import uy.kohesive.injekt.injectLazy import java.net.URL +import java.util.* +import android.view.MotionEvent +import android.os.SystemClock +import com.afollestad.materialdialogs.MaterialDialog +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import exh.util.melt +import rx.Observable class SolveCaptchaActivity : AppCompatActivity() { private val sourceManager: SourceManager by injectLazy() + private val preferencesHelper: PreferencesHelper by injectLazy() + + val httpClient = OkHttpClient() + private val jsonParser = JsonParser() + + private var currentLoopId: String? = null + private var validateCurrentLoopId: String? = null + private var strictValidationStartTime: Long? = null + + lateinit var credentialsObservable: Observable override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - setContentView(R.layout.eh_activity_captcha) + setContentView(eu.kanade.tachiyomi.R.layout.eh_activity_captcha) val sourceId = intent.getLongExtra(SOURCE_ID_EXTRA, -1) val source = if(sourceId != -1L) @@ -59,18 +82,56 @@ class SolveCaptchaActivity : AppCompatActivity() { webview.settings.javaScriptEnabled = true webview.settings.domStorageEnabled = true - webview.webViewClient = object : WebViewClient() { - override fun onPageFinished(view: WebView, url: String) { - super.onPageFinished(view, url) + var loadedInners = 0 - if(source.verify(url)) { - finish() - } else { - view.loadUrl("javascript:(function() {$script})();") + webview.webChromeClient = object : WebChromeClient() { + override fun onJsAlert(view: WebView?, url: String?, message: String, result: JsResult): Boolean { + if(message.startsWith("exh-")) { + loadedInners++ + // Wait for both inner scripts to be loaded + if(loadedInners >= 2) { + // Attempt to autosolve captcha + if(preferencesHelper.eh_autoSolveCaptchas().getOrDefault() + && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + webview.post { + // 10 seconds to auto-solve captcha + strictValidationStartTime = System.currentTimeMillis() + 1000 * 10 + beginSolveLoop() + beginValidateCaptchaLoop() + webview.evaluateJavascript(SOLVE_UI_SCRIPT_HIDE) { + webview.evaluateJavascript(SOLVE_UI_SCRIPT_SHOW, null) + } + } + } + } + result.confirm() + return true } + return false } } + webview.webViewClient = if (preferencesHelper.eh_autoSolveCaptchas().getOrDefault() + && Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + // Fetch auto-solve credentials early for speed + credentialsObservable = httpClient.newCall(Request.Builder() + // Rob demo credentials + .url("https://speech-to-text-demo.ng.bluemix.net/api/v1/credentials") + .build()) + .asObservableSuccess() + .subscribeOn(Schedulers.io()) + .map { + val json = jsonParser.parse(it.body()!!.string()) + it.close() + json["token"].string + }.melt() + + webview.addJavascriptInterface(this@SolveCaptchaActivity, "exh") + AutoSolvingWebViewClient(this, source, script) + } else { + BasicWebViewClient(this, source, script) + } + webview.loadUrl(url) } @@ -91,22 +152,458 @@ class SolveCaptchaActivity : AppCompatActivity() { return true } + @RequiresApi(Build.VERSION_CODES.KITKAT) + fun captchaSolveFail() { + currentLoopId = null + validateCurrentLoopId = null + Timber.e(IllegalStateException("Captcha solve failure!")) + runOnUiThread { + webview.evaluateJavascript(SOLVE_UI_SCRIPT_HIDE, null) + MaterialDialog.Builder(this) + .title("Captcha solve failure") + .content("Failed to auto-solve the captcha!") + .cancelable(true) + .canceledOnTouchOutside(true) + .positiveText("Ok") + .show() + } + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + @JavascriptInterface + fun callback(result: String?, loopId: String, stage: Int) { + if(loopId != currentLoopId) return + + when(stage) { + STAGE_CHECKBOX -> { + if(result!!.toBoolean()) { + webview.postDelayed({ + getAudioButtonLocation(loopId) + }, 250) + } else { + webview.postDelayed({ + doStageCheckbox(loopId) + }, 250) + } + } + STAGE_GET_AUDIO_BTN_LOCATION -> { + if(result != null) { + val splitResult = result.split(" ").map { it.toFloat() } + val origX = splitResult[0] + val origY = splitResult[1] + val iw = splitResult[2] + val ih = splitResult[3] + val x = webview.x + origX / iw * webview.width + val y = webview.y + origY / ih * webview.height + Timber.d("Found audio button coords: %f %f", x, y) + simulateClick(x + 50, y + 50) + webview.post { + doStageDownloadAudio(loopId) + } + } else { + webview.postDelayed({ + getAudioButtonLocation(loopId) + }, 250) + } + } + STAGE_DOWNLOAD_AUDIO -> { + if(result != null) { + Timber.d("Got audio URL: $result") + performRecognize(result) + .observeOn(Schedulers.io()) + .subscribe ({ + Timber.d("Got audio transcript: $it") + webview.post { + typeResult(loopId, it!! + .replace(TRANSCRIPT_CLEANER_REGEX, "") + .replace(SPACE_DEDUPE_REGEX, " ") + .trim()) + } + }, { + captchaSolveFail() + }) + } else { + webview.postDelayed({ + doStageDownloadAudio(loopId) + }, 250) + } + } + STAGE_TYPE_RESULT -> { + if(result!!.toBoolean()) { + // Fail if captcha still not solved after 1.5s + strictValidationStartTime = System.currentTimeMillis() + 1500 + } else { + captchaSolveFail() + } + } + } + } + + fun performRecognize(url: String): Single { + return credentialsObservable.flatMap { token -> + httpClient.newCall(Request.Builder() + .url(url) + .build()).asObservableSuccess().map { + token to it + } + }.flatMap { (token, response) -> + val audioFile = response.body()!!.bytes() + + httpClient.newCall(Request.Builder() + .url(HttpUrl.parse("https://stream.watsonplatform.net/speech-to-text/api/v1/recognize")!! + .newBuilder() + .addQueryParameter("watson-token", token) + .build()) + .post(MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart("jsonDescription", RECOGNIZE_JSON) + .addFormDataPart("audio.mp3", + "audio.mp3", + RequestBody.create(MediaType.parse("audio/mp3"), audioFile)) + .build()) + .build()).asObservableSuccess() + }.map { response -> + jsonParser.parse(response.body()!!.string())["results"][0]["alternatives"][0]["transcript"].string.trim() + }.toSingle() + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun doStageCheckbox(loopId: String) { + if(loopId != currentLoopId) return + + webview.evaluateJavascript(""" + (function() { + $CROSS_WINDOW_SCRIPT_OUTER + + let exh_cframe = document.querySelector('iframe[role=presentation][name|=a]'); + + if(exh_cframe != null) { + cwmExec(exh_cframe, ` + let exh_cb = document.getElementsByClassName('recaptcha-checkbox-checkmark')[0]; + if(exh_cb != null) { + exh_cb.click(); + return "true"; + } else { + return "false"; + } + `, function(result) { + exh.callback(result, '$loopId', $STAGE_CHECKBOX); + }); + } else { + exh.callback("false", '$loopId', $STAGE_CHECKBOX); + } + })(); + """.trimIndent().replace("\n", ""), null) + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun getAudioButtonLocation(loopId: String) { + webview.evaluateJavascript(""" + (function() { + $CROSS_WINDOW_SCRIPT_OUTER + + let exh_bframe = document.querySelector("iframe[title='recaptcha challenge'][name|=c]"); + + if(exh_bframe != null) { + let bfb = exh_bframe.getBoundingClientRect(); + let iw = window.innerWidth; + let ih = window.innerHeight; + if(bfb.left < 0 || bfb.top < 0) { + exh.callback(null, '$loopId', $STAGE_GET_AUDIO_BTN_LOCATION); + } else { + cwmExec(exh_bframe, ` let exh_ab = document.getElementById("recaptcha-audio-button"); + if(exh_ab != null) { + let bounds = exh_ab.getBoundingClientRect(); + return (${'$'}{bfb.left} + bounds.left) + " " + (${'$'}{bfb.top} + bounds.top) + " " + ${'$'}{iw} + " " + ${'$'}{ih}; + } else { + return null; + } + `, function(result) { + exh.callback(result, '$loopId', $STAGE_GET_AUDIO_BTN_LOCATION); + }); + } + } else { + exh.callback(null, '$loopId', $STAGE_GET_AUDIO_BTN_LOCATION); + } + })(); + """.trimIndent().replace("\n", ""), null) + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun doStageDownloadAudio(loopId: String) { + webview.evaluateJavascript(""" + (function() { + $CROSS_WINDOW_SCRIPT_OUTER + + let exh_bframe = document.querySelector("iframe[title='recaptcha challenge'][name|=c]"); + + if(exh_bframe != null) { + cwmExec(exh_bframe, ` + let exh_as = document.getElementById("audio-source"); + if(exh_as != null) { + return exh_as.src; + } else { + return null; + } + `, function(result) { + exh.callback(result, '$loopId', $STAGE_DOWNLOAD_AUDIO); + }); + } else { + exh.callback(null, '$loopId', $STAGE_DOWNLOAD_AUDIO); + } + })(); + """.trimIndent().replace("\n", ""), null) + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun typeResult(loopId: String, result: String) { + webview.evaluateJavascript(""" + (function() { + $CROSS_WINDOW_SCRIPT_OUTER + + let exh_bframe = document.querySelector("iframe[title='recaptcha challenge'][name|=c]"); + + if(exh_bframe != null) { + cwmExec(exh_bframe, ` + let exh_as = document.getElementById("audio-response"); + let exh_vb = document.getElementById("recaptcha-verify-button"); + if(exh_as != null && exh_vb != null) { + exh_as.value = "$result"; + exh_vb.click(); + return "true"; + } else { + return "false"; + } + `, function(result) { + exh.callback(result, '$loopId', $STAGE_TYPE_RESULT); + }); + } else { + exh.callback("false", '$loopId', $STAGE_TYPE_RESULT); + } + })(); + """.trimIndent().replace("\n", ""), null) + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun beginSolveLoop() { + val loopId = UUID.randomUUID().toString() + currentLoopId = loopId + doStageCheckbox(loopId) + } + + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + @JavascriptInterface + fun validateCaptchaCallback(result: Boolean, loopId: String) { + if(loopId != validateCurrentLoopId) return + + if(result) { + Timber.d("Captcha solved!") + webview.post { + webview.evaluateJavascript(SOLVE_UI_SCRIPT_HIDE, null) + } + val asbtn = intent.getStringExtra(ASBTN_EXTRA) + if(asbtn != null) { + webview.post { + webview.evaluateJavascript("(function() {document.querySelector('$asbtn').click();})();", null) + } + } + } else { + val savedStrictValidationStartTime = strictValidationStartTime + if(savedStrictValidationStartTime != null + && System.currentTimeMillis() > savedStrictValidationStartTime) { + captchaSolveFail() + } else { + webview.postDelayed({ + runValidateCaptcha(loopId) + }, 250) + } + } + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun runValidateCaptcha(loopId: String) { + if(loopId != validateCurrentLoopId) return + + webview.evaluateJavascript(""" + (function() { + $CROSS_WINDOW_SCRIPT_OUTER + + let exh_cframe = document.querySelector('iframe[role=presentation][name|=a]'); + + if(exh_cframe != null) { + cwmExec(exh_cframe, ` + let exh_cb = document.querySelector(".recaptcha-checkbox[aria-checked=true]"); + if(exh_cb != null) { + return true; + } else { + return false; + } + `, function(result) { + exh.validateCaptchaCallback(result, '$loopId'); + }); + } else { + exh.validateCaptchaCallback(false, '$loopId'); + } + })(); + """.trimIndent().replace("\n", ""), null) + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun beginValidateCaptchaLoop() { + val loopId = UUID.randomUUID().toString() + validateCurrentLoopId = loopId + runValidateCaptcha(loopId) + } + + private fun simulateClick(x: Float, y: Float) { + val downTime = SystemClock.uptimeMillis() + val eventTime = SystemClock.uptimeMillis() + val properties = arrayOfNulls(1) + val pp1 = MotionEvent.PointerProperties().apply { + id = 0 + toolType = MotionEvent.TOOL_TYPE_FINGER + } + properties[0] = pp1 + val pointerCoords = arrayOfNulls(1) + val pc1 = MotionEvent.PointerCoords().apply { + this.x = x + this.y = y + pressure = 1f + size = 1f + } + pointerCoords[0] = pc1 + var motionEvent = MotionEvent.obtain(downTime, eventTime, MotionEvent.ACTION_DOWN, 1, properties, pointerCoords, 0, 0, 1f, 1f, 0, 0, 0, 0) + dispatchTouchEvent(motionEvent) + motionEvent.recycle() + motionEvent = MotionEvent.obtain(downTime, eventTime, MotionEvent.ACTION_UP, 1, properties, pointerCoords, 0, 0, 1f, 1f, 0, 0, 0, 0) + dispatchTouchEvent(motionEvent) + motionEvent.recycle() + } + companion object { const val SOURCE_ID_EXTRA = "source_id_extra" const val COOKIES_EXTRA = "cookies_extra" const val SCRIPT_EXTRA = "script_extra" const val URL_EXTRA = "url_extra" + const val ASBTN_EXTRA = "asbtn_extra" + + const val STAGE_CHECKBOX = 0 + const val STAGE_GET_AUDIO_BTN_LOCATION = 1 + const val STAGE_DOWNLOAD_AUDIO = 2 + const val STAGE_TYPE_RESULT = 3 + + val CROSS_WINDOW_SCRIPT_OUTER = """ + function cwmExec(element, code, cb) { + console.log(">>> [CWM-Outer] Running: " + code); + let runId = Math.random(); + if(cb != null) { + let listener; + listener = function(event) { + if(typeof event.data === "string" && event.data.startsWith("exh-")) { + let response = JSON.parse(event.data.substring(4)); + if(response.id === runId) { + cb(response.result); + window.removeEventListener('message', listener); + console.log(">>> [CWM-Outer] Finished: " + response.id + " ==> " + response.result); + } + } + }; + window.addEventListener('message', listener, false); + } + let runRequest = { id: runId, code: code }; + element.contentWindow.postMessage("exh-" + JSON.stringify(runRequest), "*"); + } + """.trimIndent().replace("\n", "") + + val CROSS_WINDOW_SCRIPT_INNER = """ + window.addEventListener('message', function(event) { + if(typeof event.data === "string" && event.data.startsWith("exh-")) { + let request = JSON.parse(event.data.substring(4)); + console.log(">>> [CWM-Inner] Incoming: " + request.id); + let result = eval("(function() {" + request.code + "})();"); + let response = { id: request.id, result: result }; + console.log(">>> [CWM-Inner] Outgoing: " + response.id + " ==> " + response.result); + event.source.postMessage("exh-" + JSON.stringify(response), event.origin); + } + }, false); + console.log(">>> [CWM-Inner] Loaded!"); + alert("exh-"); + """.trimIndent() + + val SOLVE_UI_SCRIPT_SHOW = """ + (function() { + let exh_overlay = document.createElement("div"); + exh_overlay.id = "exh_overlay"; + exh_overlay.style.zIndex = 2000000001; + exh_overlay.style.backgroundColor = "rgba(0, 0, 0, 0.8)"; + exh_overlay.style.position = "fixed"; + exh_overlay.style.top = 0; + exh_overlay.style.left = 0; + exh_overlay.style.width = "100%"; + exh_overlay.style.height = "100%"; + exh_overlay.style.pointerEvents = "none"; + document.body.appendChild(exh_overlay); + let exh_otext = document.createElement("div"); + exh_otext.id = "exh_otext"; + exh_otext.style.zIndex = 2000000002; + exh_otext.style.position = "fixed"; + exh_otext.style.top = "50%"; + exh_otext.style.left = 0; + exh_otext.style.transform = "translateY(-50%)"; + exh_otext.style.color = "white"; + exh_otext.style.fontSize = "25pt"; + exh_otext.style.pointerEvents = "none"; + exh_otext.style.width = "100%"; + exh_otext.style.textAlign = "center"; + exh_otext.textContent = "Solving captcha..." + document.body.appendChild(exh_otext); + })(); + """.trimIndent() + + val SOLVE_UI_SCRIPT_HIDE = """ + (function() { + let exh_overlay = document.getElementById("exh_overlay"); + let exh_otext = document.getElementById("exh_otext"); + if(exh_overlay != null) exh_overlay.remove(); + if(exh_otext != null) exh_otext.remove(); + })(); + """.trimIndent() + + val RECOGNIZE_JSON = """ + { + "part_content_type": "audio/mp3", + "keywords": [], + "profanity_filter": false, + "max_alternatives": 1, + "speaker_labels": false, + "firstReadyInSession": false, + "preserveAdaptation": false, + "timestamps": false, + "inactivity_timeout": 30, + "word_confidence": false, + "audioMetrics": false, + "latticeGeneration": true, + "customGrammarWords": [], + "action": "recognize" + } + """.trimIndent() + + val TRANSCRIPT_CLEANER_REGEX = Regex("[^0-9a-zA-Z_ -]") + val SPACE_DEDUPE_REGEX = Regex(" +") fun launch(context: Context, source: CaptchaCompletionVerifier, cookies: Map, script: String, - url: String) { + url: String, + autoSolveSubmitBtnSelector: String? = null) { val intent = Intent(context, SolveCaptchaActivity::class.java).apply { putExtra(SOURCE_ID_EXTRA, source.id) putExtra(COOKIES_EXTRA, HashMap(cookies)) putExtra(SCRIPT_EXTRA, script) putExtra(URL_EXTRA, url) + putExtra(ASBTN_EXTRA, autoSolveSubmitBtnSelector) } context.startActivity(intent) @@ -115,6 +612,6 @@ class SolveCaptchaActivity : AppCompatActivity() { } interface CaptchaCompletionVerifier : Source { - fun verify(url: String): Boolean + fun verifyNoCaptcha(url: String): Boolean } diff --git a/app/src/main/java/exh/ui/captcha/WebViewUtil.kt b/app/src/main/java/exh/ui/captcha/WebViewUtil.kt new file mode 100644 index 000000000..1d1447e06 --- /dev/null +++ b/app/src/main/java/exh/ui/captcha/WebViewUtil.kt @@ -0,0 +1,19 @@ +package exh.ui.captcha + +import android.os.Build +import android.support.annotation.RequiresApi +import android.webkit.WebResourceRequest +import okhttp3.Request + +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +fun WebResourceRequest.toOkHttpRequest(): Request { + val request = Request.Builder() + .url(url.toString()) + .method(method, null) + + requestHeaders.entries.forEach { (t, u) -> + request.addHeader(t, u) + } + + return request.build() +} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/migration/MetadataFetchDialog.kt b/app/src/main/java/exh/ui/migration/MetadataFetchDialog.kt index a977fee25..2e2ded458 100755 --- a/app/src/main/java/exh/ui/migration/MetadataFetchDialog.kt +++ b/app/src/main/java/exh/ui/migration/MetadataFetchDialog.kt @@ -11,8 +11,6 @@ import eu.kanade.tachiyomi.data.preference.getOrDefault import eu.kanade.tachiyomi.source.SourceManager import exh.isExSource import exh.isLewdSource -import exh.metadata.queryMetadataFromManga -import exh.util.defRealm import timber.log.Timber import uy.kohesive.injekt.injectLazy import kotlin.concurrent.thread @@ -29,54 +27,71 @@ class MetadataFetchDialog { //Too lazy to actually deal with orientation changes context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR + var running = true + val progressDialog = MaterialDialog.Builder(context) .title("Fetching library metadata") .content("Preparing library") .progress(false, 0, true) + .negativeText("Stop") + .onNegative { dialog, which -> + running = false + dialog.dismiss() + notifyMigrationStopped(context) + } .cancelable(false) .canceledOnTouchOutside(false) .show() thread { - defRealm { realm -> - db.deleteMangasNotInLibrary().executeAsBlocking() + val libraryMangas = db.getLibraryMangas().executeAsBlocking() + .filter { isLewdSource(it.source) } + .distinctBy { it.id } - val libraryMangas = db.getLibraryMangas() - .executeAsBlocking() - .filter { - isLewdSource(it.source) - && realm.queryMetadataFromManga(it).findFirst() == null + context.runOnUiThread { + progressDialog.maxProgress = libraryMangas.size + } + + val mangaWithMissingMetadata = libraryMangas + .filterIndexed { index, libraryManga -> + if(index % 100 == 0) { + context.runOnUiThread { + progressDialog.setContent("[Stage 1/2] Scanning for missing metadata...") + progressDialog.setProgress(index + 1) + } } - - context.runOnUiThread { - progressDialog.maxProgress = libraryMangas.size - } - - //Actual metadata fetch code - libraryMangas.forEachIndexed { i, manga -> - context.runOnUiThread { - progressDialog.setContent("Processing: ${manga.title}") - progressDialog.setProgress(i + 1) + db.getSearchMetadataForManga(libraryManga.id!!).executeAsBlocking() == null } - try { - val source = sourceManager.get(manga.source) - source?.let { - manga.copyFrom(it.fetchMangaDetails(manga).toBlocking().first()) - realm.queryMetadataFromManga(manga).findFirst()?.copyTo(manga) - } - } catch (t: Throwable) { - Timber.e(t, "Could not migrate manga!") - } - } + .toList() + context.runOnUiThread { + progressDialog.maxProgress = mangaWithMissingMetadata.size + } + + //Actual metadata fetch code + for((i, manga) in mangaWithMissingMetadata.withIndex()) { + if(!running) break context.runOnUiThread { - progressDialog.dismiss() - - //Enable orientation changes again - context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR - - displayMigrationComplete(context) + progressDialog.setContent("[Stage 2/2] Processing: ${manga.title}") + progressDialog.setProgress(i + 1) } + try { + val source = sourceManager.get(manga.source) + source?.let { + manga.copyFrom(it.fetchMangaDetails(manga).toBlocking().first()) + } + } catch (t: Throwable) { + Timber.e(t, "Could not migrate manga!") + } + } + + context.runOnUiThread { + progressDialog.dismiss() + + //Enable orientation changes again + context.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_NOSENSOR + + if(running) displayMigrationComplete(context) } } } @@ -85,7 +100,9 @@ class MetadataFetchDialog { var extra = "" db.getLibraryMangas().asRxSingle().subscribe { if(!explicit && it.none { isLewdSource(it.source) }) { - //Do not open dialog on startup if no manga + // Do not open dialog on startup if no manga + // Also do not check again + preferenceHelper.migrateLibraryAsked().set(true) } else { //Not logged in but have ExHentai galleries if (!preferenceHelper.enableExhentai().getOrDefault()) { @@ -97,13 +114,14 @@ class MetadataFetchDialog { MaterialDialog.Builder(activity) .title("Fetch library metadata") .content(Html.fromHtml("You need to fetch your library's metadata before tag searching in the library will function.

" + - "This process may take a long time depending on your library size and will also use up a significant amount of internet bandwidth.

" + + "This process may take a long time depending on your library size and will also use up a significant amount of internet bandwidth but can be stopped and started whenever you wish.

" + extra + "This process can be done later if required.")) .positiveText("Migrate") .negativeText("Later") .onPositive { _, _ -> show(activity) } - .onNegative({ _, _ -> adviseMigrationLater(activity) }) + .onNegative { _, _ -> adviseMigrationLater(activity) } + .onAny { _, _ -> preferenceHelper.migrateLibraryAsked().set(true) } .cancelable(false) .canceledOnTouchOutside(false) .show() @@ -124,6 +142,17 @@ class MetadataFetchDialog { .show() } + fun notifyMigrationStopped(activity: Activity) { + MaterialDialog.Builder(activity) + .title("Metadata fetch stopped") + .content("Library metadata fetch has been stopped.\n\n" + + "You can continue this operation later by going to: Settings > Advanced > Migrate library metadata") + .positiveText("Ok") + .cancelable(true) + .canceledOnTouchOutside(true) + .show() + } + fun displayMigrationComplete(activity: Activity) { MaterialDialog.Builder(activity) .title("Migration complete") diff --git a/app/src/main/java/exh/ui/migration/UrlMigrator.kt b/app/src/main/java/exh/ui/migration/UrlMigrator.kt deleted file mode 100755 index 3844928de..000000000 --- a/app/src/main/java/exh/ui/migration/UrlMigrator.kt +++ /dev/null @@ -1,81 +0,0 @@ -package exh.ui.migration - -import eu.kanade.tachiyomi.data.database.DatabaseHelper -import eu.kanade.tachiyomi.data.database.models.Manga -import eu.kanade.tachiyomi.data.preference.PreferencesHelper -import eu.kanade.tachiyomi.data.preference.getOrDefault -import exh.isExSource -import exh.isLewdSource -import exh.metadata.models.ExGalleryMetadata -import exh.util.realmTrans -import uy.kohesive.injekt.injectLazy - -class UrlMigrator { - private val db: DatabaseHelper by injectLazy() - - private val prefs: PreferencesHelper by injectLazy() - - fun perform() { - db.inTransaction { - val dbMangas = db.getMangas() - .executeAsBlocking() - - //Find all EX mangas - val qualifyingMangas = dbMangas.asSequence().filter { - isLewdSource(it.source) - } - - val possibleDups = mutableListOf() - val badMangas = mutableListOf() - - qualifyingMangas.forEach { - if(it.url.startsWith("g/")) //Missing slash at front so we are bad - badMangas.add(it) - else - possibleDups.add(it) - } - - //Sort possible dups so we can use binary search on it - possibleDups.sortBy { it.url } - - realmTrans { realm -> - badMangas.forEach { manga -> - //Build fixed URL - val urlWithSlash = "/" + manga.url - //Fix metadata if required - val metadata = ExGalleryMetadata.UrlQuery(manga.url, isExSource(manga.source)) - .query(realm) - .findFirst() - metadata?.url?.let { - if (it.startsWith("g/")) { //Check if metadata URL has no slash - metadata.url = urlWithSlash //Fix it - } - } - //If we have a dup (with the fixed url), use the dup instead - val possibleDup = possibleDups.binarySearchBy(urlWithSlash, selector = { it.url }) - if (possibleDup >= 0) { - //Make sure it is favorited if we are - if (manga.favorite) { - val dup = possibleDups[possibleDup] - dup.favorite = true - db.insertManga(dup).executeAsBlocking() //Update DB with changes - } - //Delete ourself (but the dup is still there) - db.deleteManga(manga).executeAsBlocking() - return@forEach - } - //No dup, correct URL and reinsert ourselves - manga.url = urlWithSlash - db.insertManga(manga).executeAsBlocking() - } - } - } - } - - fun tryMigration() { - if(!prefs.hasPerformedURLMigration().getOrDefault()) { - perform() - prefs.hasPerformedURLMigration().set(true) - } - } -} diff --git a/app/src/main/java/exh/ui/webview/NestedWebView.java b/app/src/main/java/exh/ui/webview/NestedWebView.java deleted file mode 100644 index d5d162b61..000000000 --- a/app/src/main/java/exh/ui/webview/NestedWebView.java +++ /dev/null @@ -1,126 +0,0 @@ -package exh.ui.webview; - -import android.content.Context; -import android.support.v4.view.MotionEventCompat; -import android.support.v4.view.NestedScrollingChild; -import android.support.v4.view.NestedScrollingChildHelper; -import android.support.v4.view.ViewCompat; -import android.util.AttributeSet; -import android.view.MotionEvent; -import android.webkit.WebView; - -public class NestedWebView extends WebView implements NestedScrollingChild { - private int mLastY; - private final int[] mScrollOffset = new int[2]; - private final int[] mScrollConsumed = new int[2]; - private int mNestedOffsetY; - private NestedScrollingChildHelper mChildHelper; - - public NestedWebView(Context context) { - this(context, null); - } - - public NestedWebView(Context context, AttributeSet attrs) { - this(context, attrs, android.R.attr.webViewStyle); - } - - public NestedWebView(Context context, AttributeSet attrs, int defStyleAttr) { - super(context, attrs, defStyleAttr); - mChildHelper = new NestedScrollingChildHelper(this); - setNestedScrollingEnabled(true); - } - - @Override - public boolean onTouchEvent(MotionEvent ev) { - boolean returnValue = false; - - MotionEvent event = MotionEvent.obtain(ev); - final int action = MotionEventCompat.getActionMasked(event); - if (action == MotionEvent.ACTION_DOWN) { - mNestedOffsetY = 0; - } - int eventY = (int) event.getY(); - event.offsetLocation(0, mNestedOffsetY); - switch (action) { - case MotionEvent.ACTION_MOVE: - int deltaY = mLastY - eventY; - // NestedPreScroll - if (dispatchNestedPreScroll(0, deltaY, mScrollConsumed, mScrollOffset)) { - deltaY -= mScrollConsumed[1]; - mLastY = eventY - mScrollOffset[1]; - event.offsetLocation(0, -mScrollOffset[1]); - mNestedOffsetY += mScrollOffset[1]; - } - returnValue = super.onTouchEvent(event); - - // NestedScroll - if (dispatchNestedScroll(0, mScrollOffset[1], 0, deltaY, mScrollOffset)) { - event.offsetLocation(0, mScrollOffset[1]); - mNestedOffsetY += mScrollOffset[1]; - mLastY -= mScrollOffset[1]; - } - break; - case MotionEvent.ACTION_DOWN: - returnValue = super.onTouchEvent(event); - mLastY = eventY; - // start NestedScroll - startNestedScroll(ViewCompat.SCROLL_AXIS_VERTICAL); - break; - case MotionEvent.ACTION_UP: - case MotionEvent.ACTION_CANCEL: - returnValue = super.onTouchEvent(event); - // end NestedScroll - stopNestedScroll(); - break; - } - return returnValue; - } - - // Nested Scroll implements - @Override - public void setNestedScrollingEnabled(boolean enabled) { - mChildHelper.setNestedScrollingEnabled(enabled); - } - - @Override - public boolean isNestedScrollingEnabled() { - return mChildHelper.isNestedScrollingEnabled(); - } - - @Override - public boolean startNestedScroll(int axes) { - return mChildHelper.startNestedScroll(axes); - } - - @Override - public void stopNestedScroll() { - mChildHelper.stopNestedScroll(); - } - - @Override - public boolean hasNestedScrollingParent() { - return mChildHelper.hasNestedScrollingParent(); - } - - @Override - public boolean dispatchNestedScroll(int dxConsumed, int dyConsumed, int dxUnconsumed, int dyUnconsumed, - int[] offsetInWindow) { - return mChildHelper.dispatchNestedScroll(dxConsumed, dyConsumed, dxUnconsumed, dyUnconsumed, offsetInWindow); - } - - @Override - public boolean dispatchNestedPreScroll(int dx, int dy, int[] consumed, int[] offsetInWindow) { - return mChildHelper.dispatchNestedPreScroll(dx, dy, consumed, offsetInWindow); - } - - @Override - public boolean dispatchNestedFling(float velocityX, float velocityY, boolean consumed) { - return mChildHelper.dispatchNestedFling(velocityX, velocityY, consumed); - } - - @Override - public boolean dispatchNestedPreFling(float velocityX, float velocityY) { - return mChildHelper.dispatchNestedPreFling(velocityX, velocityY); - } - -} \ No newline at end of file diff --git a/app/src/main/java/exh/ui/webview/WebViewActivity.kt b/app/src/main/java/exh/ui/webview/WebViewActivity.kt index aa24b667c..fab5c5990 100644 --- a/app/src/main/java/exh/ui/webview/WebViewActivity.kt +++ b/app/src/main/java/exh/ui/webview/WebViewActivity.kt @@ -57,6 +57,10 @@ class WebViewActivity : BaseActivity() { webview.settings.javaScriptEnabled = true webview.settings.domStorageEnabled = true webview.settings.databaseEnabled = true + webview.settings.useWideViewPort = true + webview.settings.loadWithOverviewMode = true + webview.settings.builtInZoomControls = true + webview.settings.displayZoomControls = false webview.webViewClient = object : WebViewClient() { override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) { super.onPageStarted(view, url, favicon) @@ -134,7 +138,6 @@ class WebViewActivity : BaseActivity() { override fun onPrepareOptionsMenu(menu: Menu?): Boolean { menu?.findItem(R.id.action_forward)?.isEnabled = webview.canGoForward() - menu?.findItem(R.id.action_desktop_site)?.isChecked = isDesktop return super.onPrepareOptionsMenu(menu) } @@ -156,26 +159,6 @@ class WebViewActivity : BaseActivity() { android.R.id.home -> finish() R.id.action_refresh -> webview.reload() R.id.action_forward -> webview.goForward() - R.id.action_desktop_site -> { - isDesktop = !item.isChecked - item.isChecked = isDesktop - - (if(isDesktop) { - mobileUserAgent?.replace("\\([^(]*(Mobile|Android)[^)]*\\)" - .toRegex(RegexOption.IGNORE_CASE), "") - ?.replace("Mobile", "", true) - ?.replace("Android", "", true) - } else { - mobileUserAgent - })?.let { - webview.settings.userAgentString = it - } - - webview.settings.useWideViewPort = isDesktop - webview.settings.loadWithOverviewMode = isDesktop - - webview.reload() - } R.id.action_open_in_browser -> startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(webview.url))) } diff --git a/app/src/main/java/exh/util/RxUtil.kt b/app/src/main/java/exh/util/RxUtil.kt new file mode 100644 index 000000000..3548f2f57 --- /dev/null +++ b/app/src/main/java/exh/util/RxUtil.kt @@ -0,0 +1,27 @@ +package exh.util + +import rx.Observable +import rx.Single +import rx.subjects.ReplaySubject + +/** + * Transform a cold single to a hot single + * + * Note: Behaves like a ReplaySubject + * All generated items are buffered in memory! + */ +fun Single.melt(): Single { + return toObservable().melt().toSingle() +} + +/** + * Transform a cold observable to a hot observable + * + * Note: Behaves like a ReplaySubject + * All generated items are buffered in memory! + */ +fun Observable.melt(): Observable { + val rs = ReplaySubject.create() + subscribe(rs) + return rs +} \ No newline at end of file diff --git a/app/src/main/java/org/vepta/vdm/ByteCursor.kt b/app/src/main/java/org/vepta/vdm/ByteCursor.kt new file mode 100644 index 000000000..bd6e5e17e --- /dev/null +++ b/app/src/main/java/org/vepta/vdm/ByteCursor.kt @@ -0,0 +1,89 @@ +package org.vepta.vdm + +import java.nio.ByteBuffer + +/** + * Simple cursor for use on byte arrays + * @author nulldev + */ +class ByteCursor(val content: ByteArray) { + var index = -1 + private set + private var mark = -1 + + fun mark() { + mark = index + } + + fun jumpToMark() { + index = mark + } + + fun jumpToIndex(index: Int) { + this.index = index + } + + fun next(): Byte { + return content[++index] + } + + fun next(count: Int): ByteArray { + val res = content.sliceArray(index + 1 .. index + count) + skip(count) + return res + } + + //Used to perform conversions + private fun byteBuffer(count: Int): ByteBuffer { + return ByteBuffer.wrap(next(count)) + } + + //Epic hack to get an unsigned short properly... + fun fakeNextShortInt(): Int = ByteBuffer + .wrap(arrayOf(0x00, 0x00, *next(2).toTypedArray()).toByteArray()) + .getInt(0) + + // fun nextShort(): Short = byteBuffer(2).getShort(0) + fun nextInt(): Int = byteBuffer(4).getInt(0) + fun nextLong(): Long = byteBuffer(8).getLong(0) + fun nextFloat(): Float = byteBuffer(4).getFloat(0) + fun nextDouble(): Double = byteBuffer(8).getDouble(0) + + fun skip(count: Int) { + index += count + } + + fun expect(vararg bytes: Byte) { + if(bytes.size > remaining()) + throw IllegalStateException("Unexpected end of content!") + + for(i in 0 .. bytes.lastIndex) { + val expected = bytes[i] + val actual = content[index + i + 1] + + if(expected != actual) + throw IllegalStateException("Unexpected content (expected: $expected, actual: $actual)!") + } + + index += bytes.size + } + + fun checkEqual(vararg bytes: Byte): Boolean { + if(bytes.size > remaining()) + return false + + for(i in 0 .. bytes.lastIndex) { + val expected = bytes[i] + val actual = content[index + i + 1] + + if(expected != actual) + return false + } + + return true + } + + fun atEnd() = index >= content.size - 1 + + fun remaining() = content.size - index - 1 +} \ No newline at end of file diff --git a/app/src/main/res/drawable/ic_save_black_24dp.xml b/app/src/main/res/drawable/ic_save_black_24dp.xml new file mode 100644 index 000000000..a561d632a --- /dev/null +++ b/app/src/main/res/drawable/ic_save_black_24dp.xml @@ -0,0 +1,9 @@ + + + diff --git a/app/src/main/res/layout/activity_webview.xml b/app/src/main/res/layout/activity_webview.xml index b320363c2..f59d4a656 100644 --- a/app/src/main/res/layout/activity_webview.xml +++ b/app/src/main/res/layout/activity_webview.xml @@ -1,5 +1,5 @@ - + android:theme="?attr/actionBarTheme" + app:layout_constraintEnd_toEndOf="parent" + app:layout_constraintStart_toStartOf="parent" + app:layout_constraintTop_toTopOf="parent"> - - - - - + android:layout_height="0dp" + android:isScrollContainer="false" + app:layout_constraintBottom_toBottomOf="parent" + app:layout_constraintEnd_toEndOf="parent" + app:layout_constraintStart_toStartOf="parent" + app:layout_constraintTop_toBottomOf="@+id/appbar" /> + diff --git a/app/src/main/res/layout/catalogue_drawer_content.xml b/app/src/main/res/layout/catalogue_drawer_content.xml index b2b621a9c..5cb504781 100755 --- a/app/src/main/res/layout/catalogue_drawer_content.xml +++ b/app/src/main/res/layout/catalogue_drawer_content.xml @@ -1,11 +1,13 @@ + + + +