From 87a2ac7887e794f50dcc5414a2deeef78d468aaf Mon Sep 17 00:00:00 2001 From: NerdNumber9 Date: Tue, 13 Mar 2018 15:21:31 -0400 Subject: [PATCH] Attempt to add hitomi.la source (still broken) and code cleanup --- .../data/preference/PreferenceKeys.kt | 4 + .../data/preference/PreferencesHelper.kt | 5 + .../kanade/tachiyomi/source/SourceManager.kt | 2 + .../tachiyomi/source/online/LewdSource.kt | 1 - .../tachiyomi/source/online/all/Hitomi.kt | 309 ++++++++++++++++++ .../source/online/english/HentaiCafe.kt | 1 + app/src/main/java/exh/EHSourceHelpers.kt | 22 +- .../exh/metadata/models/ExGalleryMetadata.kt | 6 +- .../exh/metadata/models/HentaiCafeMetadata.kt | 2 +- .../metadata/models/HitomiGalleryMetadata.kt | 153 +++++++++ .../exh/metadata/models/NHentaiMetadata.kt | 2 +- .../exh/metadata/models/TsuminoMetadata.kt | 4 +- 12 files changed, 493 insertions(+), 18 deletions(-) create mode 100644 app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt create mode 100644 app/src/main/java/exh/metadata/models/HitomiGalleryMetadata.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt index c9abc2c0c..ffa841f95 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt @@ -144,4 +144,8 @@ object PreferenceKeys { const val eh_ts_aspNetCookie = "eh_ts_aspNetCookie" const val eh_showSettingsUploadWarning = "eh_showSettingsUploadWarning1" + + const val eh_hl_refreshFrequency = "eh_nh_refresh_frequency" + + const val eh_hl_lastRefresh = "eh_nh_last_refresh" } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt index 5a4e61839..bc6821c1b 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt @@ -222,5 +222,10 @@ class PreferencesHelper(val context: Context) { fun eh_ts_aspNetCookie() = rxPrefs.getString(Keys.eh_ts_aspNetCookie, "") fun eh_showSettingsUploadWarning() = rxPrefs.getBoolean(Keys.eh_showSettingsUploadWarning, true) + + // Default is 24h, refresh daily + fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24") + + fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L) // <-- EH } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt b/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt index dd3d35681..5f3189610 100755 --- a/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt @@ -88,6 +88,8 @@ open class SourceManager(private val context: Context) { exSrcs += NHentai(context) exSrcs += HentaiCafe() exSrcs += Tsumino(context) + // Mysteriously broken +// exSrcs += Hitomi(context) return exSrcs } } diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt index 12d8e5041..5a2118842 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt @@ -3,7 +3,6 @@ package eu.kanade.tachiyomi.source.online import eu.kanade.tachiyomi.source.CatalogueSource import eu.kanade.tachiyomi.source.model.SManga import exh.metadata.models.GalleryQuery -import exh.metadata.models.PervEdenGalleryMetadata import exh.metadata.models.SearchableGalleryMetadata import exh.util.createUUIDObj import exh.util.defRealm diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt new file mode 100644 index 000000000..14c1cf0d4 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt @@ -0,0 +1,309 @@ +package eu.kanade.tachiyomi.source.online.all + +import android.content.Context +import com.github.salomonbrys.kotson.* +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.squareup.duktape.Duktape +import eu.kanade.tachiyomi.data.preference.PreferencesHelper +import eu.kanade.tachiyomi.data.preference.getOrDefault +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservableSuccess +import eu.kanade.tachiyomi.source.model.* +import eu.kanade.tachiyomi.source.online.HttpSource +import eu.kanade.tachiyomi.source.online.LewdSource +import eu.kanade.tachiyomi.util.asJsoup +import exh.HITOMI_SOURCE_ID +import exh.metadata.models.HitomiGalleryMetadata +import exh.metadata.models.HitomiGalleryMetadata.Companion.BASE_URL +import exh.metadata.models.HitomiGalleryMetadata.Companion.urlFromHlId +import exh.metadata.models.Tag +import okhttp3.Request +import okhttp3.Response +import org.jsoup.nodes.Document +import rx.Observable +import uy.kohesive.injekt.injectLazy +import java.io.File +import java.util.concurrent.locks.ReentrantLock + +class Hitomi(private val context: Context) + :HttpSource(), LewdSource { + override fun queryAll() = HitomiGalleryMetadata.EmptyQuery() + override fun queryFromUrl(url: String) = HitomiGalleryMetadata.UrlQuery(url) + + override val metaParser: HitomiGalleryMetadata.(HitomiGallery) -> Unit = { + hlId = it.id.toString() + title = it.name + thumbnailUrl = resolveImage("//g.hitomi.la/galleries/$hlId/001.jpg") + artist = it.artists.firstOrNull() + group = it.groups.firstOrNull() + type = it.type + languageSimple = it.language + series.clear() + series.addAll(it.parodies) + characters.clear() + characters.addAll(it.characters) + + tags.clear() + it.tags.mapTo(tags) { Tag(it.key, it.value) } + } + + override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Unused method called!") + + override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!") + + override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException("Unused method called!") + + override fun fetchMangaDetails(manga: SManga): Observable { + return loadGalleryMetadata(manga.url).map { + parseToManga(queryFromUrl(manga.url), it) + } + } + + override fun fetchChapterList(manga: SManga): Observable> { + return lazyLoadMeta(queryFromUrl(manga.url), + loadAllGalleryMetadata().map { + val mid = HitomiGalleryMetadata.hlIdFromUrl(manga.url) + it.find { it.id.toString() == mid } + } + ).map { + listOf(SChapter.create().apply { + url = "$BASE_URL/reader/${it.hlId}.html" + + name = "Chapter" + + chapter_number = 1f + }) + } + } + + override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Unused method called!") + + override fun pageListParse(response: Response): List { + val doc = response.asJsoup() + return doc.select(".img-url").mapIndexed { index, element -> + val resolved = resolveImage(element.text()) + Page(index, resolved, resolved) + } + } + + override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Unused method called!") + + override val name = "hitomi.la" + + override val baseUrl = BASE_URL + + override val lang = "all" + + override val id = HITOMI_SOURCE_ID + + override val supportsLatest = true + + private val prefs: PreferencesHelper by injectLazy() + + private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) { + JsonParser() + } + + private val cacheLock = ReentrantLock() + + private var metaCache: List? = null + + override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html") + + override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!") + override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException("Unused method called!") + + override fun latestUpdatesRequest(page: Int) = GET("$BASE_URL/index-all-2.html") + + private fun resolveMangaIds(doc: Document, data: List): List { + return doc.select(".gallery-content > div > a").mapNotNull { + val id = HitomiGalleryMetadata.hlIdFromUrl(it.attr("href")) + data.find { it.id.toString() == id } + } + } + + private fun fetchAndResolveRequest(request: Request): Observable { + return loadAllGalleryMetadata().flatMap { + client.newCall(request) + .asObservableSuccess() + .map { response -> + val doc = response.asJsoup() + val res = resolveMangaIds(doc, it) + val sManga = res.map { + parseToManga(queryFromUrl(urlFromHlId(it.id.toString())), it) + } + val hasNextPage = doc.select(".page-container > ul > li:last-child > a").isNotEmpty() + MangasPage(sManga, hasNextPage) + } + } + } + + override fun fetchPopularManga(page: Int) + = fetchAndResolveRequest(popularMangaRequest(page)) + override fun fetchLatestUpdates(page: Int) + = fetchAndResolveRequest(latestUpdatesRequest(page)) + + private fun galleryFile(index: Int) + = File(context.cacheDir.absoluteFile, "hitomi/galleries$index.json") + + private fun shouldRefreshGalleryFiles(): Boolean { + val timeDiff = System.currentTimeMillis() - prefs.eh_hl_lastRefresh().getOrDefault() + return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L + } + + private inline fun lockCache(block: () -> T): T { + cacheLock.lock() + try { + return block() + } finally { + cacheLock.unlock() + } + } + + private fun loadGalleryMetadata(url: String): Observable { + return loadAllGalleryMetadata().map { + val mid = HitomiGalleryMetadata.hlIdFromUrl(url) + it.find { it.id.toString() == mid } + } + } + + private fun loadAllGalleryMetadata(): Observable> { + val shouldRefresh = shouldRefreshGalleryFiles() + + metaCache?.let { + if(!shouldRefresh) { + return Observable.just(metaCache) + } + } + + var obs: Observable> = Observable.just(emptyList()) + + var refresh = false + + for (i in 0 until GALLERY_CHUNK_COUNT) { + val cacheFile = galleryFile(i) + val newObs = if(shouldRefresh || !cacheFile.exists()) { + val url = "https://ltn.hitomi.la/galleries$i.json" + + refresh = true + + client.newCall(GET(url)).asObservableSuccess().map { + it.body()!!.string().apply { + lockCache { + cacheFile.parentFile.mkdirs() + cacheFile.writeText(this) + } + } + } + } else { + // Load galleries from cache + Observable.fromCallable { + lockCache { + cacheFile.readText() + } + } + } + + obs = obs.flatMap { l -> + newObs.map { + l + it + } + } + } + + // Update refresh time if we refreshed + if(refresh) + prefs.eh_hl_lastRefresh().set(System.currentTimeMillis()) + + return obs.map { + val res = it.flatMap { + jsonParser.parse(it).array.map { + HitomiGallery.fromJson(it.obj) + } + } + + metaCache = res + res + } + } + + private fun resolveImage(url: String): String { + return Duktape.create().use { + it.evaluate(IMAGE_RESOLVER.replace(IMAGE_RESOLVER_URL_VAR, url)) as String + } + } + + companion object { + private val GALLERY_CHUNK_COUNT = 20 + private val IMAGE_RESOLVER_URL_VAR = "%IMAGE_URL%" + private val IMAGE_RESOLVER = """ + (function() { +var adapose = false; // Currently not sure what this does, it switches out frontend URL when we right click??? +var number_of_frontends = 2; +function subdomain_from_galleryid(g) { + if (adapose) { + return '0'; + } + return String.fromCharCode(97 + (g % number_of_frontends)); +} +function subdomain_from_url(url, base) { + var retval = 'a'; + if (base) { + retval = base; + } + + var r = /\/(\d+)\//; + var m = r.exec(url); + var g; + if (m) { + g = parseInt(m[1]); + } + if (g) { + retval = subdomain_from_galleryid(g) + retval; + } + + return retval; +} +function url_from_url(url, base) { + return url.replace(/\/\/..?\.hitomi\.la\//, '//'+subdomain_from_url(url, base)+'.hitomi.la/'); +} + +return url_from_url('$IMAGE_RESOLVER_URL_VAR'); +})(); + """.trimIndent() + } +} + +data class HitomiGallery(val artists: List, + val parodies: List, + val id: Int, + val name: String, + val groups: List, + val tags: Map, + val characters: List, + val type: String, + val language: String?) { + companion object { + fun fromJson(obj: JsonObject): HitomiGallery + = HitomiGallery( + obj.mapNullStringList("a"), + obj.mapNullStringList("p"), + obj["id"].int, + obj["n"].string, + obj.mapNullStringList("g"), + obj["t"]?.nullArray?.associate { + val str = it.string + if(str.contains(":")) + str.substringBefore(':') to str.substringAfter(':') + else + "tag" to str + } ?: emptyMap(), + obj.mapNullStringList("c"), + obj["type"].string, + obj["l"].nullString) + + private fun JsonObject.mapNullStringList(key: String) + = this[key]?.nullArray?.map { it.string } ?: emptyList() + } +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt b/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt index 1b3b64268..295eb4dad 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt @@ -31,6 +31,7 @@ class HentaiCafe : ParsedHttpSource(), LewdSource override val name = "Hentai Cafe" override val baseUrl = "https://hentai.cafe" + // Defer popular manga -> latest updates override fun popularMangaSelector() = throw UnsupportedOperationException("Unused method called!") override fun popularMangaFromElement(element: Element) = throw UnsupportedOperationException("Unused method called!") override fun popularMangaNextPageSelector() = throw UnsupportedOperationException("Unused method called!") diff --git a/app/src/main/java/exh/EHSourceHelpers.kt b/app/src/main/java/exh/EHSourceHelpers.kt index 28f752f81..70db105b0 100755 --- a/app/src/main/java/exh/EHSourceHelpers.kt +++ b/app/src/main/java/exh/EHSourceHelpers.kt @@ -4,20 +4,22 @@ package exh * Source helpers */ -val LEWD_SOURCE_SERIES = 6900L -val EH_SOURCE_ID = LEWD_SOURCE_SERIES + 1 -val EXH_SOURCE_ID = LEWD_SOURCE_SERIES + 2 -val EH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 3 -val EXH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 4 +const val LEWD_SOURCE_SERIES = 6900L +const val EH_SOURCE_ID = LEWD_SOURCE_SERIES + 1 +const val EXH_SOURCE_ID = LEWD_SOURCE_SERIES + 2 +const val EH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 3 +const val EXH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 4 -val PERV_EDEN_EN_SOURCE_ID = LEWD_SOURCE_SERIES + 5 -val PERV_EDEN_IT_SOURCE_ID = LEWD_SOURCE_SERIES + 6 +const val PERV_EDEN_EN_SOURCE_ID = LEWD_SOURCE_SERIES + 5 +const val PERV_EDEN_IT_SOURCE_ID = LEWD_SOURCE_SERIES + 6 -val NHENTAI_SOURCE_ID = LEWD_SOURCE_SERIES + 7 +const val NHENTAI_SOURCE_ID = LEWD_SOURCE_SERIES + 7 -val HENTAI_CAFE_SOURCE_ID = LEWD_SOURCE_SERIES + 8 +const val HENTAI_CAFE_SOURCE_ID = LEWD_SOURCE_SERIES + 8 -val TSUMINO_SOURCE_ID = LEWD_SOURCE_SERIES + 9 +const val TSUMINO_SOURCE_ID = LEWD_SOURCE_SERIES + 9 + +const val HITOMI_SOURCE_ID = LEWD_SOURCE_SERIES + 10 fun isLewdSource(source: Long) = source in 6900..6999 diff --git a/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt b/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt index 0ebe4aeb7..12983be96 100755 --- a/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt +++ b/app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt @@ -90,9 +90,9 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata { val exh: Boolean ) : GalleryQuery(ExGalleryMetadata::class) { override fun map() = mapOf( - ExGalleryMetadata::gId to Query::gId, - ExGalleryMetadata::gToken to Query::gToken, - ExGalleryMetadata::exh to Query::exh + ::gId to Query::gId, + ::gToken to Query::gToken, + ::exh to Query::exh ) } diff --git a/app/src/main/java/exh/metadata/models/HentaiCafeMetadata.kt b/app/src/main/java/exh/metadata/models/HentaiCafeMetadata.kt index ecb634e00..f07c0d363 100644 --- a/app/src/main/java/exh/metadata/models/HentaiCafeMetadata.kt +++ b/app/src/main/java/exh/metadata/models/HentaiCafeMetadata.kt @@ -40,7 +40,7 @@ open class HentaiCafeMetadata : RealmObject(), SearchableGalleryMetadata { @Ignore override val titleFields = listOf( - HentaiCafeMetadata::title.name + ::title.name ) @Index diff --git a/app/src/main/java/exh/metadata/models/HitomiGalleryMetadata.kt b/app/src/main/java/exh/metadata/models/HitomiGalleryMetadata.kt new file mode 100644 index 000000000..4cd3a4d7a --- /dev/null +++ b/app/src/main/java/exh/metadata/models/HitomiGalleryMetadata.kt @@ -0,0 +1,153 @@ +package exh.metadata.models + +import eu.kanade.tachiyomi.source.model.SManga +import exh.metadata.EX_DATE_FORMAT +import exh.metadata.buildTagsDescription +import exh.metadata.joinTagsToGenreString +import exh.plusAssign +import io.realm.RealmList +import io.realm.RealmObject +import io.realm.annotations.Ignore +import io.realm.annotations.Index +import io.realm.annotations.PrimaryKey +import io.realm.annotations.RealmClass +import java.util.* + +@RealmClass +open class HitomiGalleryMetadata : RealmObject(), SearchableGalleryMetadata { + @PrimaryKey + override var uuid: String = UUID.randomUUID().toString() + + @Index + var hlId: String? = null + + var thumbnailUrl: String? = null + + var artist: String? = null + + var group: String? = null + + var type: String? = null + + var language: String? = null + + var languageSimple: String? = null + + var series: RealmList = RealmList() + + var characters: RealmList = RealmList() + + var buyLink: String? = null + + var uploadDate: Long? = null + + override var tags: RealmList = RealmList() + + // Sites does not show uploader + override var uploader: String? + get() = "admin" + set(value) {} + + var url get() = hlId?.let { urlFromHlId(it) } + set(a) { + a?.let { + hlId = hlIdFromUrl(a) + } + } + + @Index + override var mangaId: Long? = null + + @Index + var title: String? = null + + override fun getTitles() = listOfNotNull(title) + + @Ignore + override val titleFields = listOf( + ::title.name + ) + + class EmptyQuery : GalleryQuery(HitomiGalleryMetadata::class) + + class UrlQuery( + val url: String + ) : GalleryQuery(HitomiGalleryMetadata::class) { + override fun transform() = Query( + hlIdFromUrl(url) + ) + } + + class Query(val hlId: String): GalleryQuery(HitomiGalleryMetadata::class) { + override fun map() = mapOf( + HitomiGalleryMetadata::hlId to Query::hlId + ) + } + + override fun copyTo(manga: SManga) { + thumbnailUrl?.let { manga.thumbnail_url = it } + + val titleDesc = StringBuilder() + + title?.let { + manga.title = it + titleDesc += "Title: $it\n" + } + + val detailsDesc = StringBuilder() + + artist?.let { + manga.artist = it + manga.author = it + + detailsDesc += "Artist: $it\n" + } + + group?.let { + detailsDesc += "Group: $it\n" + } + + type?.let { + detailsDesc += "Type: $it\n" + } + + (language ?: languageSimple ?: "none").let { + detailsDesc += "Language: $it\n" + } + + if(series.isNotEmpty()) + detailsDesc += "Series: ${series.joinToString()}\n" + + if(characters.isNotEmpty()) + detailsDesc += "Characters: ${characters.joinToString()}\n" + + uploadDate?.let { + detailsDesc += "Upload date: ${EX_DATE_FORMAT.format(Date(it))}\n" + } + + buyLink?.let { + detailsDesc += "Buy at: $it" + } + + manga.status = SManga.UNKNOWN + + //Copy tags -> genres + manga.genre = joinTagsToGenreString(this) + + val tagsDesc = buildTagsDescription(this) + + manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString()) + .filter(String::isNotBlank) + .joinToString(separator = "\n") + } + + companion object { + val BASE_URL = "https://hitomi.la" + + fun hlIdFromUrl(url: String) + = url.split('/').last().substringBeforeLast('.') + + fun urlFromHlId(id: String) + = "$BASE_URL/galleries/$id" + } +} diff --git a/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt b/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt index c8cb53e3a..08b7f4940 100755 --- a/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt +++ b/app/src/main/java/exh/metadata/models/NHentaiMetadata.kt @@ -79,7 +79,7 @@ open class NHentaiMetadata : RealmObject(), SearchableGalleryMetadata { val nhId: Long ) : GalleryQuery(NHentaiMetadata::class) { override fun map() = mapOf( - NHentaiMetadata::nhId to Query::nhId + ::nhId to Query::nhId ) } diff --git a/app/src/main/java/exh/metadata/models/TsuminoMetadata.kt b/app/src/main/java/exh/metadata/models/TsuminoMetadata.kt index 2210464f5..eb0369edf 100644 --- a/app/src/main/java/exh/metadata/models/TsuminoMetadata.kt +++ b/app/src/main/java/exh/metadata/models/TsuminoMetadata.kt @@ -57,7 +57,7 @@ open class TsuminoMetadata : RealmObject(), SearchableGalleryMetadata { @Ignore override val titleFields = listOf( - TsuminoMetadata::title.name + ::title.name ) @Index @@ -77,7 +77,7 @@ open class TsuminoMetadata : RealmObject(), SearchableGalleryMetadata { val tmId: String ) : GalleryQuery(TsuminoMetadata::class) { override fun map() = mapOf( - TsuminoMetadata::tmId to Query::tmId + ::tmId to Query::tmId ) }