Attempt to add hitomi.la source (still broken) and code cleanup

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt

@@ -144,4 +144,8 @@ object PreferenceKeys {
     const val eh_ts_aspNetCookie = "eh_ts_aspNetCookie"
 
     const val eh_showSettingsUploadWarning = "eh_showSettingsUploadWarning1"
+
+    const val eh_hl_refreshFrequency = "eh_nh_refresh_frequency"
+
+    const val eh_hl_lastRefresh = "eh_nh_last_refresh"
 }

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt

@@ -222,5 +222,10 @@ class PreferencesHelper(val context: Context) {
     fun eh_ts_aspNetCookie() = rxPrefs.getString(Keys.eh_ts_aspNetCookie, "")
 
     fun eh_showSettingsUploadWarning() = rxPrefs.getBoolean(Keys.eh_showSettingsUploadWarning, true)
+
+    // Default is 24h, refresh daily
+    fun eh_hl_refreshFrequency() = rxPrefs.getString(Keys.eh_hl_refreshFrequency, "24")
+
+    fun eh_hl_lastRefresh() = rxPrefs.getLong(Keys.eh_hl_lastRefresh, 0L)
     // <-- EH
 }

app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt

@@ -88,6 +88,8 @@ open class SourceManager(private val context: Context) {
         exSrcs += NHentai(context)
         exSrcs += HentaiCafe()
         exSrcs += Tsumino(context)
+        // Mysteriously broken
+//        exSrcs += Hitomi(context)
         return exSrcs
     }
 }

app/src/main/java/eu/kanade/tachiyomi/source/online/LewdSource.kt

@@ -3,7 +3,6 @@ package eu.kanade.tachiyomi.source.online
 import eu.kanade.tachiyomi.source.CatalogueSource
 import eu.kanade.tachiyomi.source.model.SManga
 import exh.metadata.models.GalleryQuery
-import exh.metadata.models.PervEdenGalleryMetadata
 import exh.metadata.models.SearchableGalleryMetadata
 import exh.util.createUUIDObj
 import exh.util.defRealm

app/src/main/java/eu/kanade/tachiyomi/source/online/all/Hitomi.kt

@@ -0,0 +1,309 @@
+package eu.kanade.tachiyomi.source.online.all
+
+import android.content.Context
+import com.github.salomonbrys.kotson.*
+import com.google.gson.JsonObject
+import com.google.gson.JsonParser
+import com.squareup.duktape.Duktape
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import eu.kanade.tachiyomi.data.preference.getOrDefault
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.source.model.*
+import eu.kanade.tachiyomi.source.online.HttpSource
+import eu.kanade.tachiyomi.source.online.LewdSource
+import eu.kanade.tachiyomi.util.asJsoup
+import exh.HITOMI_SOURCE_ID
+import exh.metadata.models.HitomiGalleryMetadata
+import exh.metadata.models.HitomiGalleryMetadata.Companion.BASE_URL
+import exh.metadata.models.HitomiGalleryMetadata.Companion.urlFromHlId
+import exh.metadata.models.Tag
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.nodes.Document
+import rx.Observable
+import uy.kohesive.injekt.injectLazy
+import java.io.File
+import java.util.concurrent.locks.ReentrantLock
+
+class Hitomi(private val context: Context)
+    :HttpSource(), LewdSource<HitomiGalleryMetadata, HitomiGallery> {
+    override fun queryAll() = HitomiGalleryMetadata.EmptyQuery()
+    override fun queryFromUrl(url: String) = HitomiGalleryMetadata.UrlQuery(url)
+
+    override val metaParser: HitomiGalleryMetadata.(HitomiGallery) -> Unit = {
+        hlId = it.id.toString()
+        title = it.name
+        thumbnailUrl = resolveImage("//g.hitomi.la/galleries/$hlId/001.jpg")
+        artist = it.artists.firstOrNull()
+        group = it.groups.firstOrNull()
+        type = it.type
+        languageSimple = it.language
+        series.clear()
+        series.addAll(it.parodies)
+        characters.clear()
+        characters.addAll(it.characters)
+
+        tags.clear()
+        it.tags.mapTo(tags) { Tag(it.key, it.value) }
+    }
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException("Unused method called!")
+
+    override fun searchMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
+
+    override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
+
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
+        return loadGalleryMetadata(manga.url).map {
+            parseToManga(queryFromUrl(manga.url), it)
+        }
+    }
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+        return lazyLoadMeta(queryFromUrl(manga.url),
+            loadAllGalleryMetadata().map {
+                val mid = HitomiGalleryMetadata.hlIdFromUrl(manga.url)
+                it.find { it.id.toString() == mid }
+            }
+        ).map {
+            listOf(SChapter.create().apply {
+                url = "$BASE_URL/reader/${it.hlId}.html"
+
+                name = "Chapter"
+
+                chapter_number = 1f
+            })
+        }
+    }
+
+    override fun chapterListParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
+
+    override fun pageListParse(response: Response): List<Page> {
+        val doc = response.asJsoup()
+        return doc.select(".img-url").mapIndexed { index, element ->
+            val resolved = resolveImage(element.text())
+            Page(index, resolved, resolved)
+        }
+    }
+
+    override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
+
+    override val name = "hitomi.la"
+
+    override val baseUrl = BASE_URL
+
+    override val lang = "all"
+
+    override val id = HITOMI_SOURCE_ID
+
+    override val supportsLatest = true
+
+    private val prefs: PreferencesHelper by injectLazy()
+
+    private val jsonParser by lazy(LazyThreadSafetyMode.PUBLICATION) {
+        JsonParser()
+    }
+
+    private val cacheLock = ReentrantLock()
+
+    private var metaCache: List<HitomiGallery>? = null
+
+    override fun popularMangaRequest(page: Int) = GET("$BASE_URL/popular-all-$page.html")
+
+    override fun popularMangaParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
+    override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException("Unused method called!")
+
+    override fun latestUpdatesRequest(page: Int) = GET("$BASE_URL/index-all-2.html")
+
+    private fun resolveMangaIds(doc: Document, data: List<HitomiGallery>): List<HitomiGallery> {
+        return doc.select(".gallery-content > div > a").mapNotNull {
+            val id = HitomiGalleryMetadata.hlIdFromUrl(it.attr("href"))
+            data.find { it.id.toString() == id }
+        }
+    }
+
+    private fun fetchAndResolveRequest(request: Request): Observable<MangasPage> {
+        return loadAllGalleryMetadata().flatMap {
+            client.newCall(request)
+                    .asObservableSuccess()
+                    .map { response ->
+                        val doc = response.asJsoup()
+                        val res = resolveMangaIds(doc, it)
+                        val sManga = res.map {
+                            parseToManga(queryFromUrl(urlFromHlId(it.id.toString())), it)
+                        }
+                        val hasNextPage = doc.select(".page-container > ul > li:last-child > a").isNotEmpty()
+                        MangasPage(sManga, hasNextPage)
+                    }
+        }
+    }
+
+    override fun fetchPopularManga(page: Int)
+            = fetchAndResolveRequest(popularMangaRequest(page))
+    override fun fetchLatestUpdates(page: Int)
+            = fetchAndResolveRequest(latestUpdatesRequest(page))
+
+    private fun galleryFile(index: Int)
+            = File(context.cacheDir.absoluteFile, "hitomi/galleries$index.json")
+
+    private fun shouldRefreshGalleryFiles(): Boolean {
+        val timeDiff = System.currentTimeMillis() - prefs.eh_hl_lastRefresh().getOrDefault()
+        return timeDiff > prefs.eh_hl_refreshFrequency().getOrDefault().toLong() * 60L * 60L * 1000L
+    }
+
+    private inline fun <T> lockCache(block: () -> T): T {
+        cacheLock.lock()
+        try {
+            return block()
+        } finally {
+            cacheLock.unlock()
+        }
+    }
+
+    private fun loadGalleryMetadata(url: String): Observable<HitomiGallery> {
+        return loadAllGalleryMetadata().map {
+            val mid = HitomiGalleryMetadata.hlIdFromUrl(url)
+            it.find { it.id.toString() == mid }
+        }
+    }
+
+    private fun loadAllGalleryMetadata(): Observable<List<HitomiGallery>> {
+        val shouldRefresh = shouldRefreshGalleryFiles()
+
+        metaCache?.let {
+            if(!shouldRefresh) {
+                return Observable.just(metaCache)
+            }
+        }
+
+        var obs: Observable<List<String>> = Observable.just(emptyList())
+
+        var refresh = false
+
+        for (i in 0 until GALLERY_CHUNK_COUNT) {
+            val cacheFile = galleryFile(i)
+            val newObs = if(shouldRefresh || !cacheFile.exists()) {
+                val url = "https://ltn.hitomi.la/galleries$i.json"
+
+                refresh = true
+
+                client.newCall(GET(url)).asObservableSuccess().map {
+                    it.body()!!.string().apply {
+                        lockCache {
+                            cacheFile.parentFile.mkdirs()
+                            cacheFile.writeText(this)
+                        }
+                    }
+                }
+            } else {
+                // Load galleries from cache
+                Observable.fromCallable {
+                    lockCache {
+                        cacheFile.readText()
+                    }
+                }
+            }
+
+            obs = obs.flatMap { l ->
+                newObs.map {
+                    l + it
+                }
+            }
+        }
+
+        // Update refresh time if we refreshed
+        if(refresh)
+            prefs.eh_hl_lastRefresh().set(System.currentTimeMillis())
+
+        return obs.map {
+            val res = it.flatMap {
+                jsonParser.parse(it).array.map {
+                    HitomiGallery.fromJson(it.obj)
+                }
+            }
+
+            metaCache = res
+            res
+        }
+    }
+
+    private fun resolveImage(url: String): String {
+        return Duktape.create().use {
+            it.evaluate(IMAGE_RESOLVER.replace(IMAGE_RESOLVER_URL_VAR, url)) as String
+        }
+    }
+
+    companion object {
+        private val GALLERY_CHUNK_COUNT = 20
+        private val IMAGE_RESOLVER_URL_VAR = "%IMAGE_URL%"
+        private val IMAGE_RESOLVER = """
+            (function() {
+var adapose = false; // Currently not sure what this does, it switches out frontend URL when we right click???
+var number_of_frontends = 2;
+function subdomain_from_galleryid(g) {
+        if (adapose) {
+                return '0';
+        }
+        return String.fromCharCode(97 + (g % number_of_frontends));
+}
+function subdomain_from_url(url, base) {
+        var retval = 'a';
+        if (base) {
+                retval = base;
+        }
+
+        var r = /\/(\d+)\//;
+        var m = r.exec(url);
+        var g;
+        if (m) {
+                g = parseInt(m[1]);
+        }
+        if (g) {
+                retval = subdomain_from_galleryid(g) + retval;
+        }
+
+        return retval;
+}
+function url_from_url(url, base) {
+        return url.replace(/\/\/..?\.hitomi\.la\//, '//'+subdomain_from_url(url, base)+'.hitomi.la/');
+}
+
+return url_from_url('$IMAGE_RESOLVER_URL_VAR');
+})();
+            """.trimIndent()
+    }
+}
+
+data class HitomiGallery(val artists: List<String>,
+                         val parodies: List<String>,
+                         val id: Int,
+                         val name: String,
+                         val groups: List<String>,
+                         val tags: Map<String, String>,
+                         val characters: List<String>,
+                         val type: String,
+                         val language: String?) {
+    companion object {
+        fun fromJson(obj: JsonObject): HitomiGallery
+                = HitomiGallery(
+                obj.mapNullStringList("a"),
+                obj.mapNullStringList("p"),
+                obj["id"].int,
+                obj["n"].string,
+                obj.mapNullStringList("g"),
+                obj["t"]?.nullArray?.associate {
+                    val str = it.string
+                    if(str.contains(":"))
+                        str.substringBefore(':') to str.substringAfter(':')
+                    else
+                        "tag" to str
+                } ?: emptyMap(),
+                obj.mapNullStringList("c"),
+                obj["type"].string,
+                obj["l"].nullString)
+
+        private fun JsonObject.mapNullStringList(key: String)
+                = this[key]?.nullArray?.map { it.string } ?: emptyList()
+    }
+}

app/src/main/java/eu/kanade/tachiyomi/source/online/english/HentaiCafe.kt

@@ -31,6 +31,7 @@ class HentaiCafe : ParsedHttpSource(), LewdSource<HentaiCafeMetadata, Document>
     override val name = "Hentai Cafe"
     override val baseUrl = "https://hentai.cafe"
 
+    // Defer popular manga -> latest updates
     override fun popularMangaSelector() = throw UnsupportedOperationException("Unused method called!")
     override fun popularMangaFromElement(element: Element) = throw UnsupportedOperationException("Unused method called!")
     override fun popularMangaNextPageSelector() = throw UnsupportedOperationException("Unused method called!")

app/src/main/java/exh/EHSourceHelpers.kt

@@ -4,20 +4,22 @@ package exh
  * Source helpers
  */
 
-val LEWD_SOURCE_SERIES = 6900L
+const val LEWD_SOURCE_SERIES = 6900L
-val EH_SOURCE_ID = LEWD_SOURCE_SERIES + 1
+const val EH_SOURCE_ID = LEWD_SOURCE_SERIES + 1
-val EXH_SOURCE_ID = LEWD_SOURCE_SERIES + 2
+const val EXH_SOURCE_ID = LEWD_SOURCE_SERIES + 2
-val EH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 3
+const val EH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 3
-val EXH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 4
+const val EXH_METADATA_SOURCE_ID = LEWD_SOURCE_SERIES + 4
 
-val PERV_EDEN_EN_SOURCE_ID = LEWD_SOURCE_SERIES + 5
+const val PERV_EDEN_EN_SOURCE_ID = LEWD_SOURCE_SERIES + 5
-val PERV_EDEN_IT_SOURCE_ID = LEWD_SOURCE_SERIES + 6
+const val PERV_EDEN_IT_SOURCE_ID = LEWD_SOURCE_SERIES + 6
 
-val NHENTAI_SOURCE_ID = LEWD_SOURCE_SERIES + 7
+const val NHENTAI_SOURCE_ID = LEWD_SOURCE_SERIES + 7
 
-val HENTAI_CAFE_SOURCE_ID = LEWD_SOURCE_SERIES + 8
+const val HENTAI_CAFE_SOURCE_ID = LEWD_SOURCE_SERIES + 8
 
-val TSUMINO_SOURCE_ID = LEWD_SOURCE_SERIES + 9
+const val TSUMINO_SOURCE_ID = LEWD_SOURCE_SERIES + 9
+
+const val HITOMI_SOURCE_ID = LEWD_SOURCE_SERIES + 10
 
 fun isLewdSource(source: Long) = source in 6900..6999
 

app/src/main/java/exh/metadata/models/ExGalleryMetadata.kt

@@ -90,9 +90,9 @@ open class ExGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
                 val exh: Boolean
     ) : GalleryQuery<ExGalleryMetadata>(ExGalleryMetadata::class) {
         override fun map() = mapOf(
-                ExGalleryMetadata::gId to Query::gId,
+                ::gId to Query::gId,
-                ExGalleryMetadata::gToken to Query::gToken,
+                ::gToken to Query::gToken,
-                ExGalleryMetadata::exh to Query::exh
+                ::exh to Query::exh
         )
     }
 

app/src/main/java/exh/metadata/models/HentaiCafeMetadata.kt

@@ -40,7 +40,7 @@ open class HentaiCafeMetadata : RealmObject(), SearchableGalleryMetadata {
 
     @Ignore
     override val titleFields = listOf(
-            HentaiCafeMetadata::title.name
+            ::title.name
     )
 
     @Index

app/src/main/java/exh/metadata/models/HitomiGalleryMetadata.kt

@@ -0,0 +1,153 @@
+package exh.metadata.models
+
+import eu.kanade.tachiyomi.source.model.SManga
+import exh.metadata.EX_DATE_FORMAT
+import exh.metadata.buildTagsDescription
+import exh.metadata.joinTagsToGenreString
+import exh.plusAssign
+import io.realm.RealmList
+import io.realm.RealmObject
+import io.realm.annotations.Ignore
+import io.realm.annotations.Index
+import io.realm.annotations.PrimaryKey
+import io.realm.annotations.RealmClass
+import java.util.*
+
+@RealmClass
+open class HitomiGalleryMetadata : RealmObject(), SearchableGalleryMetadata {
+    @PrimaryKey
+    override var uuid: String = UUID.randomUUID().toString()
+
+    @Index
+    var hlId: String? = null
+
+    var thumbnailUrl: String? = null
+
+    var artist: String? = null
+
+    var group: String? = null
+
+    var type: String? = null
+
+    var language: String? = null
+
+    var languageSimple: String? = null
+
+    var series: RealmList<String> = RealmList()
+    
+    var characters: RealmList<String> = RealmList()
+
+    var buyLink: String? = null
+
+    var uploadDate: Long? = null
+
+    override var tags: RealmList<Tag> = RealmList()
+
+    // Sites does not show uploader
+    override var uploader: String?
+        get() = "admin"
+        set(value) {}
+
+    var url get() = hlId?.let { urlFromHlId(it) }
+        set(a) {
+            a?.let {
+                hlId = hlIdFromUrl(a)
+            }
+        }
+
+    @Index
+    override var mangaId: Long? = null
+
+    @Index
+    var title: String? = null
+
+    override fun getTitles() = listOfNotNull(title)
+
+    @Ignore
+    override val titleFields = listOf(
+            ::title.name
+    )
+
+    class EmptyQuery : GalleryQuery<HitomiGalleryMetadata>(HitomiGalleryMetadata::class)
+
+    class UrlQuery(
+            val url: String
+    ) : GalleryQuery<HitomiGalleryMetadata>(HitomiGalleryMetadata::class) {
+        override fun transform() = Query(
+                hlIdFromUrl(url)
+        )
+    }
+
+    class Query(val hlId: String): GalleryQuery<HitomiGalleryMetadata>(HitomiGalleryMetadata::class) {
+        override fun map() = mapOf(
+                HitomiGalleryMetadata::hlId to Query::hlId
+        )
+    }
+
+    override fun copyTo(manga: SManga) {
+        thumbnailUrl?.let { manga.thumbnail_url = it }
+
+        val titleDesc = StringBuilder()
+
+        title?.let {
+            manga.title = it
+            titleDesc += "Title: $it\n"
+        }
+
+        val detailsDesc = StringBuilder()
+
+        artist?.let {
+            manga.artist = it
+            manga.author = it
+
+            detailsDesc += "Artist: $it\n"
+        }
+
+        group?.let {
+            detailsDesc += "Group: $it\n"
+        }
+
+        type?.let {
+            detailsDesc += "Type: $it\n"
+        }
+
+        (language ?: languageSimple ?: "none").let {
+            detailsDesc += "Language: $it\n"
+        }
+
+        if(series.isNotEmpty())
+            detailsDesc += "Series: ${series.joinToString()}\n"
+
+        if(characters.isNotEmpty())
+            detailsDesc += "Characters: ${characters.joinToString()}\n"
+
+        uploadDate?.let {
+            detailsDesc += "Upload date: ${EX_DATE_FORMAT.format(Date(it))}\n"
+        }
+
+        buyLink?.let {
+            detailsDesc += "Buy at: $it"
+        }
+
+        manga.status = SManga.UNKNOWN
+
+        //Copy tags -> genres
+        manga.genre = joinTagsToGenreString(this)
+
+        val tagsDesc = buildTagsDescription(this)
+
+        manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString())
+                .filter(String::isNotBlank)
+                .joinToString(separator = "\n")
+    }
+
+    companion object {
+        val BASE_URL = "https://hitomi.la"
+
+        fun hlIdFromUrl(url: String)
+                = url.split('/').last().substringBeforeLast('.')
+
+        fun urlFromHlId(id: String)
+                = "$BASE_URL/galleries/$id"
+    }
+}

app/src/main/java/exh/metadata/models/NHentaiMetadata.kt

@@ -79,7 +79,7 @@ open class NHentaiMetadata : RealmObject(), SearchableGalleryMetadata {
             val nhId: Long
     ) : GalleryQuery<NHentaiMetadata>(NHentaiMetadata::class) {
         override fun map() = mapOf(
-                NHentaiMetadata::nhId to Query::nhId
+                ::nhId to Query::nhId
         )
     }
 

app/src/main/java/exh/metadata/models/TsuminoMetadata.kt

@@ -57,7 +57,7 @@ open class TsuminoMetadata : RealmObject(), SearchableGalleryMetadata {
     
     @Ignore
     override val titleFields = listOf(
-            TsuminoMetadata::title.name
+            ::title.name
     )
     
     @Index
@@ -77,7 +77,7 @@ open class TsuminoMetadata : RealmObject(), SearchableGalleryMetadata {
             val tmId: String
     ) : GalleryQuery<TsuminoMetadata>(TsuminoMetadata::class) {
         override fun map() = mapOf(
-                TsuminoMetadata::tmId to Query::tmId
+                ::tmId to Query::tmId
         )
     }