Add WeebDex (#11113)

* Add WeebDex

* fixes and add scanlator info

* add placeholder icons

* Refactoring with reviewed changes

* minor refactoring to fix build

* Url and Chapter title fixes

fix: Add manga URL correctly.
feat: Improve chapter title format by including volume and chapter numbers, and a "Oneshot" fallback.
fix: An issue where oneshot chapters show Chapter null

* Remove extraneous json field

src/en/weebdex/build.gradle

@@ -0,0 +1,7 @@
+ext {
+    extName = 'WeebDex'
+    extClass = '.WeebDex'
+    extVersionCode = 1
+    isNsfw = true
+}
+apply from: "$rootDir/common.gradle"

src/en/weebdex/src/eu/kanade/tachiyomi/extension/en/weebdex/WeebDex.kt

@@ -0,0 +1,173 @@
+package eu.kanade.tachiyomi.extension.en.weebdex
+
+import eu.kanade.tachiyomi.extension.en.weebdex.dto.ChapterDto
+import eu.kanade.tachiyomi.extension.en.weebdex.dto.ChapterListDto
+import eu.kanade.tachiyomi.extension.en.weebdex.dto.MangaDto
+import eu.kanade.tachiyomi.extension.en.weebdex.dto.MangaListDto
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.interceptor.rateLimit
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.HttpSource
+import keiyoushi.utils.parseAs
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Request
+import okhttp3.Response
+
+class WeebDex : HttpSource() {
+    override val name = "WeebDex"
+    override val baseUrl = "https://weebdex.org"
+    override val lang = "en"
+    override val supportsLatest = true
+
+    override val client = network.cloudflareClient.newBuilder()
+        .rateLimit(WeebDexConstants.RATE_LIMIT)
+        .build()
+
+    override fun headersBuilder(): Headers.Builder = super.headersBuilder()
+        .add("Referer", "$baseUrl/")
+
+    // -------------------- Popular --------------------
+
+    override fun popularMangaRequest(page: Int): Request {
+        val url = WeebDexConstants.API_MANGA_URL.toHttpUrl().newBuilder()
+            .addQueryParameter("page", page.toString())
+            .addQueryParameter("sort", "views")
+            .addQueryParameter("order", "desc")
+            .addQueryParameter("hasChapters", "1")
+            .build()
+        return GET(url, headers)
+    }
+
+    override fun popularMangaParse(response: Response): MangasPage {
+        val mangaListDto = response.parseAs<MangaListDto>()
+        val mangas = mangaListDto.toSMangaList()
+        return MangasPage(mangas, mangaListDto.hasNextPage)
+    }
+
+    // -------------------- Latest --------------------
+    override fun latestUpdatesRequest(page: Int): Request {
+        val url = WeebDexConstants.API_MANGA_URL.toHttpUrl().newBuilder()
+            .addQueryParameter("page", page.toString())
+            .addQueryParameter("sort", "updatedAt")
+            .addQueryParameter("order", "desc")
+            .addQueryParameter("hasChapters", "1")
+            .build()
+        return GET(url, headers)
+    }
+
+    override fun latestUpdatesParse(response: Response): MangasPage = popularMangaParse(response)
+
+    // -------------------- Search --------------------
+    override fun getFilterList(): FilterList = buildFilterList()
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        val urlBuilder = WeebDexConstants.API_MANGA_URL.toHttpUrl().newBuilder()
+            .addQueryParameter("page", page.toString())
+
+        if (query.isNotBlank()) {
+            urlBuilder.addQueryParameter("title", query)
+        } else {
+            filters.forEach { filter ->
+                when (filter) {
+                    is TagList -> {
+                        filter.state.forEach { tag ->
+                            if (tag.state) {
+                                WeebDexConstants.tags[tag.name]?.let { tagId ->
+                                    urlBuilder.addQueryParameter("tag", tagId)
+                                }
+                            }
+                        }
+                    }
+                    is TagsExcludeFilter -> {
+                        filter.state.forEach { tag ->
+                            if (tag.state) {
+                                WeebDexConstants.tags[tag.name]?.let { tagId ->
+                                    urlBuilder.addQueryParameter("tagx", tagId)
+                                }
+                            }
+                        }
+                    }
+                    is TagModeFilter -> urlBuilder.addQueryParameter("tmod", filter.state.toString())
+                    is TagExcludeModeFilter -> urlBuilder.addQueryParameter("txmod", filter.state.toString())
+                    else -> { /* Do Nothing */ }
+                }
+            }
+        }
+
+        // Separated explicitly to be applied even when a search query is applied.
+        filters.forEach { filter ->
+            when (filter) {
+                is SortFilter -> urlBuilder.addQueryParameter("sort", filter.selected)
+                is OrderFilter -> urlBuilder.addQueryParameter("order", filter.selected)
+                is StatusFilter -> filter.selected?.let { urlBuilder.addQueryParameter("status", it) }
+                is DemographicFilter -> filter.selected?.let { urlBuilder.addQueryParameter("demographic", it) }
+                is ContentRatingFilter -> filter.selected?.let { urlBuilder.addQueryParameter("contentRating", it) }
+                is LangFilter -> filter.query?.let { urlBuilder.addQueryParameter("lang", it) }
+                is HasChaptersFilter -> if (filter.state) urlBuilder.addQueryParameter("hasChapters", "1")
+                is YearFromFilter -> filter.state.takeIf { it.isNotEmpty() }?.let { urlBuilder.addQueryParameter("yearFrom", it) }
+                is YearToFilter -> filter.state.takeIf { it.isNotEmpty() }?.let { urlBuilder.addQueryParameter("yearTo", it) }
+                else -> { /* Do Nothing */ }
+            }
+        }
+
+        return GET(urlBuilder.build(), headers)
+    }
+
+    override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response)
+
+    // -------------------- Manga details --------------------
+
+    override fun mangaDetailsRequest(manga: SManga): Request {
+        return GET("${WeebDexConstants.API_URL}${manga.url}", headers)
+    }
+
+    override fun mangaDetailsParse(response: Response): SManga {
+        val manga = response.parseAs<MangaDto>()
+        return manga.toSManga()
+    }
+
+    // -------------------- Chapters --------------------
+
+    override fun chapterListRequest(manga: SManga): Request {
+        // chapter list is paginated; get all pages
+        return GET("${WeebDexConstants.API_URL}${manga.url}/chapters?order=desc", headers)
+    }
+
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val chapters = mutableListOf<SChapter>()
+
+        // Recursively parse pages
+        fun parsePage(chapterListDto: ChapterListDto) {
+            chapters.addAll(chapterListDto.toSChapterList())
+            if (chapterListDto.hasNextPage) {
+                val nextUrl = response.request.url.newBuilder()
+                    .setQueryParameter("page", (chapterListDto.page + 1).toString())
+                    .build()
+                val nextResponse = client.newCall(GET(nextUrl, headers)).execute()
+                val nextChapterListDto = nextResponse.parseAs<ChapterListDto>()
+                parsePage(nextChapterListDto)
+            }
+        }
+
+        parsePage(response.parseAs<ChapterListDto>())
+        return chapters
+    }
+
+    // -------------------- Pages --------------------
+
+    override fun pageListRequest(chapter: SChapter): Request {
+        return GET("${WeebDexConstants.API_URL}${chapter.url}", headers)
+    }
+
+    override fun pageListParse(response: Response): List<Page> {
+        val chapter = response.parseAs<ChapterDto>()
+        return chapter.toPageList()
+    }
+
+    override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException("Not used")
+}

src/en/weebdex/src/eu/kanade/tachiyomi/extension/en/weebdex/WeebDexConstants.kt

@@ -0,0 +1,109 @@
+package eu.kanade.tachiyomi.extension.en.weebdex
+
+object WeebDexConstants {
+
+    // API Base URLs
+    const val API_URL = "https://api.weebdex.org"
+    const val CDN_URL = "https://srv.notdelta.xyz"
+
+    // API Endpoints
+    const val API_MANGA_URL = "$API_URL/manga"
+
+    // CDN Endpoints
+    const val CDN_COVER_URL = "$CDN_URL/covers"
+    const val CDN_DATA_URL = "$CDN_URL/data"
+
+    // Rate Limit (API is 5 req/s, using conservative value)
+    const val RATE_LIMIT = 3
+
+    // Tags Map
+    val tags = mapOf(
+        // Formats
+        "Oneshot" to "99q3m1plnt",
+        "Web Comic" to "1utcekkc70",
+        "Doujinshi" to "fnvjk3jg1b",
+        "Adaptation" to "pbst9p8bd4",
+        "Full Color" to "6amsrv3w16",
+        "4-Koma" to "jnqtucy8q3",
+
+        // Genres
+        "Action" to "g0eao31zjw",
+        "Adventure" to "pjl8oxd1ld",
+        "Boys' Love" to "1cnfhxwshb",
+        "Comedy" to "onj03z2gnf",
+        "Crime" to "bwec51tbms",
+        "Drama" to "00xq9oqthh",
+        "Fantasy" to "3lhj8r7s6n",
+        "Girls' Love" to "i9w6sjikyd",
+        "Historical" to "mmf28hr2co",
+        "Horror" to "rclreo8b25",
+        "Magical Girls" to "hy189x450f",
+        "Mystery" to "hv0hsu8kje",
+        "Romance" to "o0rm4pweru",
+        "Slice of Life" to "13x7xvq10k",
+        "Sports" to "zsvyg4whkp",
+        "Tragedy" to "85hmqw16y9",
+
+        // Themes
+        "Cooking" to "9wm2j2zl1e",
+        "Crossdressing" to "arjr4qdpgc",
+        "Delinquents" to "h5ioz14hix",
+        "Genderswap" to "25k4gcfnfp",
+        "Magic" to "evt7r78scn",
+        "Monster Girls" to "ddjrvi8vsu",
+        "School Life" to "hobsiukk71",
+        "Shota" to "lu0sbwbs3r",
+        "Supernatural" to "c4rnaci8q6",
+        "Traditional Games" to "aqfqkul8rg",
+        "Vampires" to "djs29flsq6",
+        "Video Games" to "axstzcu7pc",
+        "Office Workers" to "6uytt2873o",
+        "Martial Arts" to "577a4hd52b",
+        "Zombies" to "szg24cwbrm",
+        "Survival" to "mt4vdanhfc",
+        "Police" to "acai4usl79",
+        "Mafia" to "qjuief8bi1",
+
+        // Content Tags
+        "Gore" to "hceia50cf9",
+        "Sexual Violence" to "xh9k4t31ll",
+    )
+
+    // Demographics
+    val demographics = listOf(
+        "Any" to null,
+        "Shounen" to "shounen",
+        "Shoujo" to "shoujo",
+        "Josei" to "josei",
+        "Seinen" to "seinen",
+    )
+
+    // Publication Status
+    val statusList = listOf(
+        "Any" to null,
+        "Ongoing" to "ongoing",
+        "Completed" to "completed",
+        "Hiatus" to "hiatus",
+        "Cancelled" to "cancelled",
+    )
+
+    // Languages
+    val langList = listOf(
+        "Any" to null,
+        "English" to "en",
+        "Japanese" to "ja",
+    )
+
+    // Sort Options
+    val sortList = listOf(
+        "Views" to "views",
+        "Updated At" to "updatedAt",
+        "Created At" to "createdAt",
+        "Chapter Update" to "lastUploadedChapterAt",
+        "Title" to "title",
+        "Year" to "year",
+        "Rating" to "rating",
+        "Follows" to "follows",
+        "Chapters" to "chapters",
+    )
+}

src/en/weebdex/src/eu/kanade/tachiyomi/extension/en/weebdex/WeebDexFilters.kt

@@ -0,0 +1,113 @@
+package eu.kanade.tachiyomi.extension.en.weebdex
+
+import eu.kanade.tachiyomi.source.model.Filter
+import eu.kanade.tachiyomi.source.model.FilterList
+
+internal fun buildFilterList() = FilterList(
+    Filter.Header("NOTE: Not all filters work on all sources."),
+    Filter.Separator(),
+    SortFilter(),
+    OrderFilter(),
+    StatusFilter(),
+    LangFilter(),
+    DemographicFilter(),
+    HasChaptersFilter(),
+    Filter.Header("Tags (ignored if searching by text)"),
+    TagModeFilter(),
+    TagList(WeebDexConstants.tags.keys.toTypedArray()),
+    Filter.Header("Tags to exclude (ignored if searching by text)"),
+    TagExcludeModeFilter(),
+    TagsExcludeFilter(WeebDexConstants.tags.keys.toTypedArray()),
+    ContentRatingFilter(),
+    YearFromFilter(),
+    YearToFilter(),
+)
+
+internal class SortFilter : Filter.Select<String>(
+    "Sort by",
+    WeebDexConstants.sortList.map { it.first }.toTypedArray(),
+    0,
+) {
+    val selected: String
+        get() = WeebDexConstants.sortList[state].second
+}
+
+internal class OrderFilter : Filter.Select<String>(
+    "Order",
+    arrayOf("Descending", "Ascending"),
+    0,
+) {
+    val selected: String
+        get() = if (state == 0) "desc" else "asc"
+}
+
+internal class StatusFilter : Filter.Select<String>(
+    "Status",
+    WeebDexConstants.statusList.map { it.first }.toTypedArray(),
+    0,
+) {
+    val selected: String?
+        get() = WeebDexConstants.statusList[state].second
+}
+
+class TagCheckBox(name: String) : Filter.CheckBox(name, false)
+class TagList(tags: Array<String>) : Filter.Group<TagCheckBox>("Tags", tags.map { TagCheckBox(it) })
+class TagsExcludeFilter(tags: Array<String>) : Filter.Group<TagCheckBox>(
+    "Tags to Exclude",
+    tags.map { TagCheckBox(it) },
+)
+
+class TagModeFilter : Filter.Select<String>(
+    "Tag mode",
+    arrayOf("AND", "OR"), // what user sees
+    0,
+) {
+    val selected: String
+        get() = if (state == 0) "0" else "1" // backend wants 0=AND, 1=OR
+}
+
+class TagExcludeModeFilter : Filter.Select<String>(
+    "Exclude tag mode",
+    arrayOf("OR", "AND"), // what user sees
+    0,
+) {
+    val selected: String
+        get() = if (state == 0) "0" else "1" // backend wants 0=OR, 1=AND
+}
+
+internal class DemographicFilter : Filter.Select<String>(
+    "Demographic",
+    WeebDexConstants.demographics.map { it.first }.toTypedArray(),
+    0,
+) {
+    val selected: String?
+        get() = WeebDexConstants.demographics[state].second
+}
+
+internal class ContentRatingFilter : Filter.Select<String>(
+    "Content Rating",
+    arrayOf("Any", "Safe", "Suggestive", "Erotica", "Pornographic"),
+    0,
+) {
+    private val apiValues = arrayOf(null, "safe", "suggestive", "erotica", "pornographic")
+
+    val selected: String?
+        get() = apiValues[state]
+}
+
+internal class LangFilter : Filter.Select<String>(
+    "Original Language",
+    WeebDexConstants.langList.map { it.first }.toTypedArray(),
+    0,
+) {
+    val query: String?
+        get() = WeebDexConstants.langList[state].second
+}
+
+internal class HasChaptersFilter : Filter.CheckBox("Has Chapters", true) {
+    val selected: String?
+        get() = if (state) "1" else null
+}
+
+internal class YearFromFilter : Filter.Text("Year (from)")
+internal class YearToFilter : Filter.Text("Year (to)")

src/en/weebdex/src/eu/kanade/tachiyomi/extension/en/weebdex/WeebDexHelper.kt

@@ -0,0 +1,31 @@
+import eu.kanade.tachiyomi.extension.en.weebdex.WeebDexConstants
+import eu.kanade.tachiyomi.source.model.SManga
+import keiyoushi.utils.tryParse
+import java.text.SimpleDateFormat
+import java.util.Locale
+import java.util.TimeZone
+
+class WeebDexHelper {
+    private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH).apply {
+        timeZone = TimeZone.getTimeZone("UTC")
+    }
+    fun parseStatus(status: String?): Int {
+        return when (status?.lowercase(Locale.ROOT)) {
+            "ongoing" -> SManga.ONGOING
+            "completed" -> SManga.COMPLETED
+            "hiatus" -> SManga.ON_HIATUS
+            "cancelled" -> SManga.CANCELLED
+            else -> SManga.UNKNOWN
+        }
+    }
+
+    fun buildCoverUrl(mangaId: String, cover: eu.kanade.tachiyomi.extension.en.weebdex.dto.CoverDto?): String? {
+        if (cover == null) return null
+        val ext = cover.ext
+        return "${WeebDexConstants.CDN_COVER_URL}/$mangaId/${cover.id}$ext"
+    }
+
+    fun parseDate(dateStr: String): Long {
+        return dateFormat.tryParse(dateStr)
+    }
+}

src/en/weebdex/src/eu/kanade/tachiyomi/extension/en/weebdex/dto/ChapterDto.kt

@@ -0,0 +1,101 @@
+package eu.kanade.tachiyomi.extension.en.weebdex.dto
+
+import WeebDexHelper
+import eu.kanade.tachiyomi.extension.en.weebdex.WeebDexConstants
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import kotlinx.serialization.Contextual
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import org.jsoup.parser.Parser
+
+@Serializable
+class ChapterListDto(
+    private val data: List<ChapterDto> = emptyList(),
+    val page: Int = 1,
+    val limit: Int = 0,
+    val total: Int = 0,
+) {
+    val hasNextPage: Boolean
+        get() = page * limit < total
+    fun toSChapterList(): List<SChapter> {
+        return data.map { it.toSChapter() }
+    }
+}
+
+@Serializable
+class ChapterDto(
+    private val id: String,
+    private val title: String? = null,
+    private val chapter: String? = null,
+    private val volume: String? = null,
+    @SerialName("published_at") private val publishedAt: String = "",
+    private val data: List<PageData>? = null,
+    @SerialName("data_optimized") private val dataOptimized: List<PageData>? = null,
+    private val relationships: ChapterRelationshipsDto? = null,
+) {
+    @Contextual
+    private val helper = WeebDexHelper()
+
+    fun toSChapter(): SChapter {
+        val chapterName = mutableListOf<String>()
+        // Build chapter name
+        volume?.let {
+            if (it.isNotEmpty()) {
+                chapterName.add("Vol.$it")
+            }
+        }
+
+        chapter?.let {
+            if (it.isNotEmpty()) {
+                chapterName.add("Ch.$it")
+            }
+        }
+
+        title?.let {
+            if (it.isNotEmpty()) {
+                if (chapterName.isNotEmpty()) {
+                    chapterName.add("-")
+                }
+                chapterName.add(it)
+            }
+        }
+
+        // if volume, chapter and title is empty its a oneshot
+        if (chapterName.isEmpty()) {
+            chapterName.add("Oneshot")
+        }
+
+        return SChapter.create().apply {
+            url = "/chapter/$id"
+            name = Parser.unescapeEntities(chapterName.joinToString(" "), false)
+            chapter_number = chapter?.toFloat() ?: -2F
+            date_upload = helper.parseDate(publishedAt)
+            scanlator = relationships?.groups?.joinToString(", ") { it.name }
+        }
+    }
+    fun toPageList(): List<Page> {
+        val pagesArray = dataOptimized ?: data ?: emptyList()
+        val pages = mutableListOf<Page>()
+
+        pagesArray.forEachIndexed { index, pageData ->
+            // pages in spec have 'name' field and images served from /data/{id}/{filename}
+            val filename = pageData.name
+            val chapterId = id
+            val imageUrl = filename?.takeIf { it.isNotBlank() && chapterId.isNotBlank() }
+                ?.let { "${WeebDexConstants.CDN_DATA_URL}/$chapterId/$it" }
+            pages.add(Page(index, imageUrl = imageUrl))
+        }
+        return pages
+    }
+}
+
+@Serializable
+class ChapterRelationshipsDto(
+    val groups: List<NamedEntity> = emptyList(),
+)
+
+@Serializable
+class PageData(
+    val name: String? = null,
+)

src/en/weebdex/src/eu/kanade/tachiyomi/extension/en/weebdex/dto/MangaDto.kt

@@ -0,0 +1,65 @@
+package eu.kanade.tachiyomi.extension.en.weebdex.dto
+
+import WeebDexHelper
+import eu.kanade.tachiyomi.source.model.SManga
+import kotlinx.serialization.Contextual
+import kotlinx.serialization.Serializable
+
+@Serializable
+class MangaListDto(
+    private val data: List<MangaDto> = emptyList(),
+    val page: Int = 1,
+    val limit: Int = 0,
+    val total: Int = 0,
+) {
+    val hasNextPage: Boolean
+        get() = page * limit < total
+    fun toSMangaList(): List<SManga> {
+        return data.map { it.toSManga() }
+    }
+}
+
+@Serializable
+class MangaDto(
+    private val id: String,
+    private val title: String,
+    private val description: String = "",
+    private val status: String? = null,
+    val relationships: RelationshipsDto? = null,
+) {
+    @Contextual
+    private val helper = WeebDexHelper()
+    fun toSManga(): SManga {
+        return SManga.create().apply {
+            title = this@MangaDto.title
+            description = this@MangaDto.description
+            status = helper.parseStatus(this@MangaDto.status)
+            thumbnail_url = helper.buildCoverUrl(id, relationships?.cover)
+            url = "/manga/$id"
+            relationships?.let { rel ->
+                author = rel.authors.joinToString(", ") { it.name }
+                artist = rel.artists.joinToString(", ") { it.name }
+                genre = rel.tags.joinToString(", ") { it.name }
+            }
+        }
+    }
+}
+
+@Serializable
+class RelationshipsDto(
+    val authors: List<NamedEntity> = emptyList(),
+    val artists: List<NamedEntity> = emptyList(),
+    val tags: List<NamedEntity> = emptyList(),
+    val cover: CoverDto? = null,
+)
+
+@Serializable
+class NamedEntity(
+    val name: String,
+)
+
+@Serializable
+class CoverDto(
+    val id: String,
+    val ext: String = ".jpg",
+)