Add Pepper&Carrot (#13327)

* Add Pepper&Carrot

* better language support, caching, launcher icons

* update launcher icons

* add artwork entries

* show language key in title

* no search prompt

* add ProtoBuf note comment

* use constant for source name

* move lang pref to filters, improve title and chapter number parsing

* disable artwork chapter number parsing

src/all/peppercarrot/AndroidManifest.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest package="eu.kanade.tachiyomi.extension" />

src/all/peppercarrot/build.gradle

@@ -0,0 +1,12 @@
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
+
+ext {
+    extName = 'Pepper&Carrot'
+    pkgNameSuffix = 'all.peppercarrot'
+    extClass = '.PepperCarrot'
+    extVersionCode = 1
+}
+
+apply from: "$rootDir/common.gradle"

src/all/peppercarrot/src/eu/kanade/tachiyomi/extension/all/peppercarrot/Dto.kt

@@ -0,0 +1,35 @@
+package eu.kanade.tachiyomi.extension.all.peppercarrot
+
+import kotlinx.serialization.Serializable
+
+typealias LangsDto = Map<String, LangDto>
+
+@Serializable
+class LangDto(
+    val translators: List<String>,
+    val local_name: String,
+    val iso_code: String,
+)
+
+// ProtoBuf: should not change field type and order
+@Serializable
+class LangData(
+    val key: String,
+    val name: String,
+    val progress: String,
+    val translators: String,
+    val title: String?,
+)
+
+class Lang(
+    val key: String,
+    val name: String,
+    val code: String,
+    val translators: String,
+    val translatedCount: Int,
+)
+
+@Serializable
+class EpisodeDto(
+    val translated_languages: List<String>,
+)

src/all/peppercarrot/src/eu/kanade/tachiyomi/extension/all/peppercarrot/Filters.kt

@@ -0,0 +1,28 @@
+package eu.kanade.tachiyomi.extension.all.peppercarrot
+
+import android.content.SharedPreferences
+import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.source.model.FilterList
+
+fun getFilters(preferences: SharedPreferences): FilterList {
+    val langData = preferences.langData
+    val list: List<Filter<*>> = if (langData.isEmpty()) {
+        listOf(Filter.Header("Tap 'Reset' to load languages"))
+    } else buildList(langData.size + 1) {
+        add(Filter.Header("Languages"))
+        val lang = preferences.lang.toHashSet()
+        langData.mapTo(this) {
+            LangFilter(it.key, "${it.name} (${it.progress})", it.key in lang)
+        }
+    }
+    return FilterList(list)
+}
+
+fun SharedPreferences.saveFrom(filters: FilterList) {
+    val langFilters = filters.filterIsInstance<LangFilter>().ifEmpty { return }
+    val selected = langFilters.filter { it.state }.mapTo(LinkedHashSet()) { it.key }
+    val result = lang.filterTo(LinkedHashSet()) { it in selected }.apply { addAll(selected) }
+    edit().setLang(result).apply()
+}
+
+class LangFilter(val key: String, name: String, state: Boolean) : Filter.CheckBox(name, state)

src/all/peppercarrot/src/eu/kanade/tachiyomi/extension/all/peppercarrot/PepperCarrot.kt

@@ -0,0 +1,228 @@
+package eu.kanade.tachiyomi.extension.all.peppercarrot
+
+import android.app.Application
+import androidx.preference.PreferenceScreen
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.ConfigurableSource
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.HttpSource
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.CacheControl
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.nodes.TextNode
+import org.jsoup.select.Evaluator
+import rx.Observable
+import rx.Single
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import java.text.SimpleDateFormat
+import java.util.Locale
+import java.util.concurrent.TimeUnit
+
+class PepperCarrot : HttpSource(), ConfigurableSource {
+    override val name = TITLE
+    override val lang = "all"
+    override val supportsLatest = false
+
+    override val baseUrl = BASE_URL
+
+    private val preferences by lazy {
+        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)!!
+    }
+
+    override fun fetchPopularManga(page: Int): Observable<MangasPage> = Single.create<MangasPage> {
+        updateLangData(client, headers, preferences)
+        val lang = preferences.lang.ifEmpty {
+            throw Exception("Please select language in the filter")
+        }
+        val langMap = preferences.langData.associateBy { langData -> langData.key }
+        val mangas = lang.map { key -> langMap[key]!!.toSManga() }
+        val result = MangasPage(mangas + getArtworkList(), false)
+        it.onSuccess(result)
+    }.toObservable()
+
+    private fun getArtworkList(): List<SManga> = arrayOf(
+        "artworks", "wallpapers", "sketchbook", "misc",
+        "book-publishing", "comissions", "eshop", "framasoft", "press", "references", "wiki"
+    ).map(::getArtworkEntry)
+
+    override fun getFilterList() = getFilters(preferences)
+
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        if (filters.isNotEmpty()) preferences.saveFrom(filters)
+        return fetchPopularManga(page)
+    }
+
+    override fun popularMangaRequest(page: Int) = throw UnsupportedOperationException()
+    override fun popularMangaParse(response: Response) = throw UnsupportedOperationException()
+    override fun latestUpdatesRequest(page: Int) = throw UnsupportedOperationException()
+    override fun latestUpdatesParse(response: Response) = throw UnsupportedOperationException()
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw UnsupportedOperationException()
+    override fun searchMangaParse(response: Response) = throw UnsupportedOperationException()
+
+    override fun fetchMangaDetails(manga: SManga): Observable<SManga> = Single.create<SManga> {
+        updateLangData(client, headers, preferences)
+        val key = manga.url
+        val result = if (key.startsWith('#')) {
+            getArtworkEntry(key.substring(1))
+        } else {
+            preferences.langData.find { lang -> lang.key == key }!!.toSManga()
+        }
+        it.onSuccess(result)
+    }.toObservable()
+
+    override fun mangaDetailsRequest(manga: SManga): Request {
+        val key = manga.url
+        val url = if (key.startsWith('#')) { // artwork
+            "$BASE_URL/en/files/${key.substring(1)}.html"
+        } else {
+            "$BASE_URL/$key/webcomics/index.html"
+        }
+        return GET(url, headers)
+    }
+
+    override fun mangaDetailsParse(response: Response) = throw UnsupportedOperationException()
+
+    private fun LangData.toSManga() = SManga.create().apply {
+        url = key
+        title = this@toSManga.title ?: if (key == "en") TITLE else "$TITLE (${key.uppercase()})"
+        author = AUTHOR
+        description = this@toSManga.run {
+            "Language: $name\nTranslators: $translators"
+        }
+        status = SManga.ONGOING
+        thumbnail_url = "$BASE_URL/0_sources/0ther/artworks/low-res/2016-02-24_vertical-cover_remake_by-David-Revoy.jpg"
+        initialized = true
+    }
+
+    private fun getArtworkEntry(key: String) = SManga.create().apply {
+        url = "#$key"
+        title = when (key) {
+            "comissions" -> "Commissions"
+            "eshop" -> "Shop"
+            else -> key.replaceFirstChar { it.uppercase() }
+        }
+        author = AUTHOR
+        status = SManga.ONGOING
+        thumbnail_url = "$BASE_URL/0_sources/0ther/press/low-res/2015-10-12_logo_by-David-Revoy.jpg"
+        initialized = true
+    }
+
+    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> = Single.create<List<SChapter>> {
+        updateLangData(client, headers, preferences)
+        val response = client.newCall(chapterListRequest(manga)).execute()
+        it.onSuccess(chapterListParse(response))
+    }.toObservable()
+
+    override fun chapterListRequest(manga: SManga): Request {
+        val key = manga.url
+        val url = if (key.startsWith('#')) { // artwork
+            "$BASE_URL/0_sources/0ther/${key.substring(1)}/low-res/"
+        } else {
+            "$BASE_URL/$key/webcomics/index.html"
+        }
+        val lastUpdated = preferences.lastUpdated
+        if (lastUpdated == 0L) return GET(url, headers)
+
+        val seconds = System.currentTimeMillis() / 1000 - lastUpdated
+        val cache = CacheControl.Builder().maxStale(seconds.toInt(), TimeUnit.SECONDS).build()
+        return GET(url, headers, cache)
+    }
+
+    override fun chapterListParse(response: Response): List<SChapter> {
+        if (response.request.url.pathSegments[0] == "0_sources") return parseArtwork(response)
+
+        val translatedChapters = response.asJsoup()
+            .select(Evaluator.Tag("figure"))
+            .let { (it.size downTo 1) zip it }
+            .filter { it.second.hasClass("translated") }
+
+        return translatedChapters.map { (number, it) ->
+            SChapter.create().apply {
+                url = it.selectFirst(Evaluator.Tag("a")).attr("href").removePrefix(BASE_URL)
+                name = it.selectFirst(Evaluator.Tag("img")).attr("title").run {
+                    val index = lastIndexOf('（')
+                    when {
+                        index >= 0 -> substring(0, index).trimEnd()
+                        else -> substringBeforeLast('(').trimEnd()
+                    }
+                }
+                date_upload = it.selectFirst(Evaluator.Tag("figcaption")).ownText().let {
+                    val date = dateRegex.find(it)!!.value
+                    dateFormat.parse(date)!!.time
+                }
+                chapter_number = number.toFloat()
+            }
+        }
+    }
+
+    private fun parseArtwork(response: Response): List<SChapter> {
+        val baseDir = response.request.url.toString().removePrefix(BASE_URL)
+        return response.asJsoup().select(Evaluator.Tag("a")).asReversed().mapNotNull {
+            val filename = it.attr("href")!!
+            if (!filename.endsWith(".jpg")) return@mapNotNull null
+
+            val file = filename.removeSuffix(".jpg").removeSuffix("_by-David-Revoy")
+            val fileStripped: String
+            val date: Long
+            if (file.length >= 10 && dateRegex.matches(file.substring(0, 10))) {
+                fileStripped = file.substring(10)
+                date = dateFormat.parse(file.substring(0, 10))!!.time
+            } else {
+                fileStripped = file
+                val lastModified = it.nextSibling() as? TextNode
+                date = if (lastModified == null) 0 else dateFormat.parse(lastModified.text())!!.time
+            }
+            val fileNormalized = fileStripped
+                .replace('_', ' ')
+                .replace('-', ' ')
+                .trim()
+                .replaceFirstChar { char -> char.uppercase() }
+
+            SChapter.create().apply {
+                url = baseDir + filename
+                name = fileNormalized
+                date_upload = date
+                chapter_number = -2f
+            }
+        }
+    }
+
+    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
+        val url = chapter.url
+        return if (url.endsWith(".jpg")) {
+            Observable.just(listOf(Page(0, imageUrl = BASE_URL + url)))
+        } else {
+            super.fetchPageList(chapter)
+        }
+    }
+
+    override fun pageListParse(response: Response): List<Page> {
+        val document = response.asJsoup()
+        val urls = document.select(Evaluator.Class("comicpage")).map { it.attr("src")!! }
+        val thumbnail = urls[0].replace("P00.jpg", ".jpg")
+        return (listOf(thumbnail) + urls).mapIndexed { index, imageUrl ->
+            Page(index, imageUrl = imageUrl)
+        }
+    }
+
+    override fun imageUrlParse(response: Response) = throw UnsupportedOperationException()
+
+    override fun imageRequest(page: Page): Request {
+        val url = page.imageUrl!!
+        val newUrl = if (preferences.isHiRes) url.replace("/low-res/", "/hi-res/") else url
+        return GET(newUrl, headers)
+    }
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        getPreferences(screen.context).forEach(screen::addPreference)
+    }
+
+    private val dateRegex by lazy { Regex("""\d{4}-\d{2}-\d{2}""") }
+    private val dateFormat by lazy { SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) }
+}

src/all/peppercarrot/src/eu/kanade/tachiyomi/extension/all/peppercarrot/Preferences.kt

@@ -0,0 +1,145 @@
+package eu.kanade.tachiyomi.extension.all.peppercarrot
+
+import android.content.Context
+import android.content.SharedPreferences
+import android.util.Base64
+import androidx.preference.SwitchPreferenceCompat
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.decodeFromByteArray
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToByteArray
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.protobuf.ProtoBuf
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+import okhttp3.Response
+import org.jsoup.parser.Parser.unescapeEntities
+import org.jsoup.select.Evaluator
+import uy.kohesive.injekt.injectLazy
+import java.util.Locale
+
+fun getPreferences(context: Context) = arrayOf(
+    SwitchPreferenceCompat(context).apply {
+        key = HI_RES_PREF
+        title = "High resolution images"
+        summary = "Changes will not be applied to images that are already cached or downloaded " +
+            "until you clear the chapter cache or delete the chapter download."
+        setDefaultValue(false)
+    },
+)
+
+val SharedPreferences.isHiRes get() = getBoolean(HI_RES_PREF, false)
+val SharedPreferences.lastUpdated get() = getLong(LAST_UPDATED_PREF, 0)
+
+val SharedPreferences.lang: List<String>
+    get() {
+        val lang = getString(LANG_PREF, "")!!
+        if (lang.isEmpty()) return emptyList()
+        return lang.split(", ")
+    }
+
+fun SharedPreferences.Editor.setLang(value: Iterable<String>): SharedPreferences.Editor =
+    putString(LANG_PREF, value.joinToString())
+
+val SharedPreferences.langData: List<LangData>
+    get() {
+        val data = getString(LANG_DATA_PREF, "")!!
+        if (data.isEmpty()) return emptyList()
+        return ProtoBuf.decodeFromBase64(data)
+    }
+
+@Synchronized
+fun updateLangData(client: OkHttpClient, headers: Headers, preferences: SharedPreferences) {
+    val lastUpdated = client.newCall(GET("$BASE_URL/0_sources/last_updated.txt", headers))
+        .execute().body!!.string().substringBefore('\n').toLong()
+    if (lastUpdated <= preferences.lastUpdated) return
+
+    val editor = preferences.edit().putLong(LAST_UPDATED_PREF, lastUpdated)
+
+    val episodes = client.newCall(GET("$BASE_URL/0_sources/episodes.json", headers))
+        .execute().parseAs<List<EpisodeDto>>()
+    val total = episodes.size
+    val translatedCount = episodes.flatMap { it.translated_languages }
+        .groupingBy { it }.eachCount()
+
+    val titles = fetchTitles(client, headers)
+
+    val langs = client.newCall(GET("$BASE_URL/0_sources/langs.json", headers))
+        .execute().parseAs<LangsDto>().entries.map { (key, dto) ->
+            Lang(
+                key = key,
+                name = dto.local_name,
+                code = dto.iso_code.ifEmpty { key },
+                translators = dto.translators.joinToString(),
+                translatedCount = translatedCount[key] ?: 0
+            )
+        }
+        .filter { it.translatedCount > 0 }
+        .groupBy { it.code }.values
+        .flatMap { it.sortedByDescending { lang -> lang.translatedCount } }
+        .also { if (preferences.lang.isEmpty()) editor.chooseLang(it) }
+        .map {
+            val progress = "${it.translatedCount}/$total translated"
+            LangData(it.key, it.name, progress, it.translators, titles[it.key])
+        }
+
+    editor.putString(LANG_DATA_PREF, ProtoBuf.encodeToBase64(langs)).apply()
+}
+
+private fun SharedPreferences.Editor.chooseLang(langs: List<Lang>) {
+    val language = Locale.getDefault().language
+    val result = langs.filter { it.code == language }.mapTo(ArrayList()) { it.key }
+    if (result.isEmpty()) return
+    if (language != "en") result.add("en")
+    setLang(result)
+}
+
+private fun fetchTitles(client: OkHttpClient, headers: Headers): Map<String, String> {
+    val url = "https://framagit.org/search?project_id=76196&search=core/mod-header.php:4"
+    val document = client.newCall(GET(url, headers)).execute().asJsoup()
+    val result = hashMapOf<String, String>()
+    for (file in document.selectFirst(Evaluator.Class("search-results")).children()) {
+        val filename = file.selectFirst(Evaluator.Tag("strong")).ownText()
+        if (!filename.endsWith(".po") || !filename.startsWith("po/")) continue
+        val lang = filename.substring(3, filename.length - 3)
+
+        val lines = file.select(Evaluator.Class("line"))
+        for (i in lines.indices) {
+            if (lines[i].ownText() == "msgid \"Pepper&amp;Carrot\"" && i + 1 < lines.size) {
+                val title = lines[i + 1].ownText().removePrefix("msgstr \"").removeSuffix("\"")
+                val unescaped = unescapeEntities(title, false).trim()
+                if (unescaped.isNotEmpty() && unescaped != TITLE) result[lang] = unescaped
+                break
+            }
+        }
+    }
+
+    for (sameTitleList in result.entries.groupBy { it.value }.values) {
+        if (sameTitleList.size == 1) continue
+        for (entry in sameTitleList) {
+            entry.setValue("${entry.value} (${entry.key.uppercase()})")
+        }
+    }
+
+    return result
+}
+
+private inline fun <reified T> Response.parseAs(): T = json.decodeFromString(body!!.string())
+
+private inline fun <reified T> ProtoBuf.decodeFromBase64(base64: String): T =
+    decodeFromByteArray(Base64.decode(base64, Base64.NO_WRAP))
+
+private inline fun <reified T> ProtoBuf.encodeToBase64(value: T): String =
+    Base64.encodeToString(encodeToByteArray(value), Base64.NO_WRAP)
+
+private val json: Json by injectLazy()
+
+const val BASE_URL = "https://www.peppercarrot.com"
+const val TITLE = "Pepper&Carrot"
+const val AUTHOR = "David Revoy"
+
+private const val LANG_PREF = "LANG"
+private const val LANG_DATA_PREF = "LANG_DATA"
+private const val LAST_UPDATED_PREF = "LAST_UPDATED"
+private const val HI_RES_PREF = "HI_RES"