Draff/TachiyomiSY-Plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Attempt to add hitomi.la source (still broken) and code cleanup

Browse Source
This commit is contained in:
NerdNumber9 2018-03-13 15:21:31 -04:00
parent 07ce90ab8c
commit 87a2ac7887
12 changed files with 493 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt
View File

@ -144,4 +144,8 @@ object PreferenceKeys {
const val eh_ts_aspNetCookie = "eh_ts_aspNetCookie"
const val eh_showSettingsUploadWarning = "eh_showSettingsUploadWarning1"
const val eh_hl_refreshFrequency = "eh_nh_refresh_frequency"
const val eh_hl_lastRefresh = "eh_nh_last_refresh"
}

5
app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt
View File

@ -222,5 +222,10 @@ class PreferencesHelper(val context: Context) {
fun eh_ts_aspNetCookie() = rxPrefs.getString(Keys.eh_ts_aspNetCookie, "")
fun eh_showSettingsUploadWarning() = rxPrefs.getBoolean(Keys.eh_showSettingsUploadWarning, true)
// Default is 24h, refresh daily
fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24")
fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L)
// <-- EH
}

2
app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt
View File

@ -88,6 +88,8 @@ open class SourceManager(private val context: Context) {
exSrcs += NHentai(context)
exSrcs += HentaiCafe()
exSrcs += Tsumino(context)
// Mysteriously broken
// exSrcs += Hitomi(context)
return exSrcs
}
}

1
app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt
View File

@ -3,7 +3,6 @@ package eu.kanade.tachiyomi.source.online
import eu.kanade.tachiyomi.source.CatalogueSource
import eu.kanade.tachiyomi.source.model.SManga
import exh.metadata.models.GalleryQuery
import exh.metadata.models.PervEdenGalleryMetadata
import exh.metadata.models.SearchableGalleryMetadata
import exh.util.createUUIDObj
import exh.util.defRealm

309
app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt Normal file
View File

@ -0,0 +1,309 @@
package eu.kanade.tachiyomi.source.online.all
import android.content.Context
import com.github.salomonbrys.kotson.*
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import com.squareup.duktape.Duktape
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.preference.getOrDefault
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.LewdSource
import eu.kanade.tachiyomi.util.asJsoup
import exh.HITOMI_SOURCE_ID
import exh.metadata.models.HitomiGalleryMetadata
import exh.metadata.models.HitomiGalleryMetadata.Companion.BASE_URL
import exh.metadata.models.HitomiGalleryMetadata.Companion.urlFromHlId
import exh.metadata.models.Tag
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.io.File
import java.util.concurrent.locks.ReentrantLock
class Hitomi(private val context: Context)
:HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiGallery> {
override fun queryAll() = HitomiGalleryMetadata.EmptyQuery()
override fun queryFromUrl(url: String) = HitomiGalleryMetadata.UrlQuery(url)
override val metaParser: HitomiGalleryMetadata.(HitomiGallery) -> Unit = {
hlId = it.id.toString()
title = it.name
thumbnailUrl = resolveImage("//g.hitomi.la/galleries/$hlId/001.jpg")
artist = it.artists.firstOrNull()
group = it.groups.firstOrNull()
type = it.type
languageSimple = it.language
series.clear()
series.addAll(it.parodies)
characters.clear()
characters.addAll(it.characters)
tags.clear()
it.tags.mapTo(tags) { Tag(it.key, it.value) }
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Unused method called!")
override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
return loadGalleryMetadata(manga.url).map {
parseToManga(queryFromUrl(manga.url), it)
}
}
override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
return lazyLoadMeta(queryFromUrl(manga.url),
loadAllGalleryMetadata().map {
val mid = HitomiGalleryMetadata.hlIdFromUrl(manga.url)
it.find { it.id.toString() == mid }
}
).map {
listOf(SChapter.create().apply {
url = "$BASE_URL/reader/${it.hlId}.html"
name = "Chapter"
chapter_number = 1f
})
}
}
override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
override fun pageListParse(response: Response): List<Page> {
val doc = response.asJsoup()
return doc.select(".img-url").mapIndexed { index, element ->
val resolved = resolveImage(element.text())
Page(index, resolved, resolved)
}
}
override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
override val name = "hitomi.la"
override val baseUrl = BASE_URL
override val lang = "all"
override val id = HITOMI_SOURCE_ID
override val supportsLatest = true
private val prefs: PreferencesHelper by injectLazy()
private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) {
JsonParser()
}
private val cacheLock = ReentrantLock()
private var metaCache: List<HitomiGallery>? = null
override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html")
override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
override fun latestUpdatesRequest(page: Int) = GET("$BASE_URL/index-all-2.html")
private fun resolveMangaIds(doc: Document, data: List<HitomiGallery>): List<HitomiGallery> {
return doc.select(".gallery-content > div > a").mapNotNull {
val id = HitomiGalleryMetadata.hlIdFromUrl(it.attr("href"))
data.find { it.id.toString() == id }
}
}
private fun fetchAndResolveRequest(request: Request): Observable<MangasPage> {
return loadAllGalleryMetadata().flatMap {
client.newCall(request)
.asObservableSuccess()
.map { response ->
val doc = response.asJsoup()
val res = resolveMangaIds(doc, it)
val sManga = res.map {
parseToManga(queryFromUrl(urlFromHlId(it.id.toString())), it)
}
val hasNextPage = doc.select(".page-container > ul > li:last-child > a").isNotEmpty()
MangasPage(sManga, hasNextPage)
}
}
}
override fun fetchPopularManga(page: Int)
= fetchAndResolveRequest(popularMangaRequest(page))
override fun fetchLatestUpdates(page: Int)
= fetchAndResolveRequest(latestUpdatesRequest(page))
private fun galleryFile(index: Int)
= File(context.cacheDir.absoluteFile, "hitomi/galleries$index.json")
private fun shouldRefreshGalleryFiles(): Boolean {
val timeDiff = System.currentTimeMillis() - prefs.eh_hl_lastRefresh().getOrDefault()
return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L
}
private inline fun <T> lockCache(block: () -> T): T {
cacheLock.lock()
try {
return block()
} finally {
cacheLock.unlock()
}
}
private fun loadGalleryMetadata(url: String): Observable<HitomiGallery> {
return loadAllGalleryMetadata().map {
val mid = HitomiGalleryMetadata.hlIdFromUrl(url)
it.find { it.id.toString() == mid }
}
}
private fun loadAllGalleryMetadata(): Observable<List<HitomiGallery>> {
val shouldRefresh = shouldRefreshGalleryFiles()
metaCache?.let {
if(!shouldRefresh) {
return Observable.just(metaCache)
}
}
var obs: Observable<List<String>> = Observable.just(emptyList())
var refresh = false
for (i in 0 until GALLERY_CHUNK_COUNT) {
val cacheFile = galleryFile(i)
val newObs = if(shouldRefresh || !cacheFile.exists()) {
val url = "https://ltn.hitomi.la/galleries$i.json"
refresh = true
client.newCall(GET(url)).asObservableSuccess().map {
it.body()!!.string().apply {
lockCache {
cacheFile.parentFile.mkdirs()
cacheFile.writeText(this)
}
}
}
} else {
// Load galleries from cache
Observable.fromCallable {
lockCache {
cacheFile.readText()
}
}
}
obs = obs.flatMap { l ->
newObs.map {
l + it
}
}
}
// Update refresh time if we refreshed
if(refresh)
prefs.eh_hl_lastRefresh().set(System.currentTimeMillis())
return obs.map {
val res = it.flatMap {
jsonParser.parse(it).array.map {
HitomiGallery.fromJson(it.obj)
}
}
metaCache = res
res
}
}
private fun resolveImage(url: String): String {
return Duktape.create().use {
it.evaluate(IMAGE_RESOLVER.replace(IMAGE_RESOLVER_URL_VAR, url)) as String
}
}
companion object {
private val GALLERY_CHUNK_COUNT = 20
private val IMAGE_RESOLVER_URL_VAR = "%IMAGE_URL%"
private val IMAGE_RESOLVER = """
(function() {
var adapose = false; // Currently not sure what this does, it switches out frontend URL when we right click???
var number_of_frontends = 2;
function subdomain_from_galleryid(g) {
if (adapose) {
return '0';
}
return String.fromCharCode(97 + (g % number_of_frontends));
}
function subdomain_from_url(url, base) {
var retval = 'a';
if (base) {
retval = base;
}
var r = /\/(\d+)\//;
var m = r.exec(url);
var g;
if (m) {
g = parseInt(m[1]);
}
if (g) {
retval = subdomain_from_galleryid(g) + retval;
}
return retval;
}
function url_from_url(url, base) {
return url.replace(/\/\/..?\.hitomi\.la\//, '//'+subdomain_from_url(url, base)+'.hitomi.la/');
}
return url_from_url('$IMAGE_RESOLVER_URL_VAR');
})();
""".trimIndent()
}
}
data class HitomiGallery(val artists: List<String>,
val parodies: List<String>,
val id: Int,
val name: String,
val groups: List<String>,
val tags: Map<String, String>,
val characters: List<String>,
val type: String,
val language: String?) {
companion object {
fun fromJson(obj: JsonObject): HitomiGallery
= HitomiGallery(
obj.mapNullStringList("a"),
obj.mapNullStringList("p"),
obj["id"].int,
obj["n"].string,
obj.mapNullStringList("g"),
obj["t"]?.nullArray?.associate {
val str = it.string
if(str.contains(":"))
str.substringBefore(':') to str.substringAfter(':')
else
"tag" to str
} ?: emptyMap(),
obj.mapNullStringList("c"),
obj["type"].string,
obj["l"].nullString)
private fun JsonObject.mapNullStringList(key: String)
= this[key]?.nullArray?.map { it.string } ?: emptyList()
}
}

1
app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt
View File

@ -31,6 +31,7 @@ class HentaiCafe : ParsedHttpSource(), LewdSource<HentaiCafeMetadata, Document>
override val name = "Hentai Cafe"
override val baseUrl = "https://hentai.cafe"
// Defer popular manga -> latest updates
override fun popularMangaSelector() = throw UnsupportedOperationException("Unused method called!")
override fun popularMangaFromElement(element: Element) = throw UnsupportedOperationException("Unused method called!")
override fun popularMangaNextPageSelector() = throw UnsupportedOperationException("Unused method called!")

22
app/src/main/java/exh/EHSourceHelpers.kt
View File

@ -4,20 +4,22 @@ package exh
* Source helpers
*/
val LEWD_SOURCE_SERIES = 6900L
val EH_SOURCE_ID = LEWD_SOURCE_SERIES + 1
val EXH_SOURCE_ID = LEWD_SOURCE_SERIES + 2
val EH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 3
val EXH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 4
const val LEWD_SOURCE_SERIES = 6900L
const val EH_SOURCE_ID = LEWD_SOURCE_SERIES + 1
const val EXH_SOURCE_ID = LEWD_SOURCE_SERIES + 2
const val EH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 3
const val EXH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 4
val PERV_EDEN_EN_SOURCE_ID = LEWD_SOURCE_SERIES + 5
val PERV_EDEN_IT_SOURCE_ID = LEWD_SOURCE_SERIES + 6
const val PERV_EDEN_EN_SOURCE_ID = LEWD_SOURCE_SERIES + 5
const val PERV_EDEN_IT_SOURCE_ID = LEWD_SOURCE_SERIES + 6
val NHENTAI_SOURCE_ID = LEWD_SOURCE_SERIES + 7
const val NHENTAI_SOURCE_ID = LEWD_SOURCE_SERIES + 7
val HENTAI_CAFE_SOURCE_ID = LEWD_SOURCE_SERIES + 8
const val HENTAI_CAFE_SOURCE_ID = LEWD_SOURCE_SERIES + 8
val TSUMINO_SOURCE_ID = LEWD_SOURCE_SERIES + 9
const val TSUMINO_SOURCE_ID = LEWD_SOURCE_SERIES + 9
const val HITOMI_SOURCE_ID = LEWD_SOURCE_SERIES + 10
fun isLewdSource(source: Long) = source in 6900..6999

6
app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt
View File

@ -90,9 +90,9 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
val exh: Boolean
) : GalleryQuery<ExGalleryMetadata>(ExGalleryMetadata::class) {
override fun map() = mapOf(
ExGalleryMetadata::gId to Query::gId,
ExGalleryMetadata::gToken to Query::gToken,
ExGalleryMetadata::exh to Query::exh
::gId to Query::gId,
::gToken to Query::gToken,
::exh to Query::exh
)
}

2
app/src/main/java/exh/metadata/models/HentaiCafeMetadata.kt
View File

@ -40,7 +40,7 @@ open class HentaiCafeMetadata : RealmObject(), SearchableGalleryMetadata {
@Ignore
override val titleFields = listOf(
HentaiCafeMetadata::title.name
::title.name
)
@Index

153
app/src/main/java/exh/metadata/models/HitomiGalleryMetadata.kt Normal file
View File

@ -0,0 +1,153 @@
package exh.metadata.models
import eu.kanade.tachiyomi.source.model.SManga
import exh.metadata.EX_DATE_FORMAT
import exh.metadata.buildTagsDescription
import exh.metadata.joinTagsToGenreString
import exh.plusAssign
import io.realm.RealmList
import io.realm.RealmObject
import io.realm.annotations.Ignore
import io.realm.annotations.Index
import io.realm.annotations.PrimaryKey
import io.realm.annotations.RealmClass
import java.util.*
@RealmClass
open class HitomiGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
@PrimaryKey
override var uuid: String = UUID.randomUUID().toString()
@Index
var hlId: String? = null
var thumbnailUrl: String? = null
var artist: String? = null
var group: String? = null
var type: String? = null
var language: String? = null
var languageSimple: String? = null
var series: RealmList<String> = RealmList()
var characters: RealmList<String> = RealmList()
var buyLink: String? = null
var uploadDate: Long? = null
override var tags: RealmList<Tag> = RealmList()
// Sites does not show uploader
override var uploader: String?
get() = "admin"
set(value) {}
var url get() = hlId?.let { urlFromHlId(it) }
set(a) {
a?.let {
hlId = hlIdFromUrl(a)
}
}
@Index
override var mangaId: Long? = null
@Index
var title: String? = null
override fun getTitles() = listOfNotNull(title)
@Ignore
override val titleFields = listOf(
::title.name
)
class EmptyQuery : GalleryQuery<HitomiGalleryMetadata>(HitomiGalleryMetadata::class)
class UrlQuery(
val url: String
) : GalleryQuery<HitomiGalleryMetadata>(HitomiGalleryMetadata::class) {
override fun transform() = Query(
hlIdFromUrl(url)
)
}
class Query(val hlId: String): GalleryQuery<HitomiGalleryMetadata>(HitomiGalleryMetadata::class) {
override fun map() = mapOf(
HitomiGalleryMetadata::hlId to Query::hlId
)
}
override fun copyTo(manga: SManga) {
thumbnailUrl?.let { manga.thumbnail_url = it }
val titleDesc = StringBuilder()
title?.let {
manga.title = it
titleDesc += "Title: $it\n"
}
val detailsDesc = StringBuilder()
artist?.let {
manga.artist = it
manga.author = it
detailsDesc += "Artist: $it\n"
}
group?.let {
detailsDesc += "Group: $it\n"
}
type?.let {
detailsDesc += "Type: $it\n"
}
(language ?: languageSimple ?: "none").let {
detailsDesc += "Language: $it\n"
}
if(series.isNotEmpty())
detailsDesc += "Series: ${series.joinToString()}\n"
if(characters.isNotEmpty())
detailsDesc += "Characters: ${characters.joinToString()}\n"
uploadDate?.let {
detailsDesc += "Upload date: ${EX_DATE_FORMAT.format(Date(it))}\n"
}
buyLink?.let {
detailsDesc += "Buy at: $it"
}
manga.status = SManga.UNKNOWN
//Copy tags -> genres
manga.genre = joinTagsToGenreString(this)
val tagsDesc = buildTagsDescription(this)
manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString())
.filter(String::isNotBlank)
.joinToString(separator = "\n")
}
companion object {
val BASE_URL = "https://hitomi.la"
fun hlIdFromUrl(url: String)
= url.split('/').last().substringBeforeLast('.')
fun urlFromHlId(id: String)
= "$BASE_URL/galleries/$id"
}
}

2
app/src/main/java/exh/metadata/models/NHentaiMetadata.kt
View File

@ -79,7 +79,7 @@ open class NHentaiMetadata : RealmObject(), SearchableGalleryMetadata {
val nhId: Long
) : GalleryQuery<NHentaiMetadata>(NHentaiMetadata::class) {
override fun map() = mapOf(
NHentaiMetadata::nhId to Query::nhId
::nhId to Query::nhId
)
}

4
app/src/main/java/exh/metadata/models/TsuminoMetadata.kt
View File

@ -57,7 +57,7 @@ open class TsuminoMetadata : RealmObject(), SearchableGalleryMetadata {
@Ignore
override val titleFields = listOf(
TsuminoMetadata::title.name
::title.name
)
@Index
@ -77,7 +77,7 @@ open class TsuminoMetadata : RealmObject(), SearchableGalleryMetadata {
val tmId: String
) : GalleryQuery<TsuminoMetadata>(TsuminoMetadata::class) {
override fun map() = mapOf(
TsuminoMetadata::tmId to Query::tmId
::tmId to Query::tmId
)
}