Webnovel.com: Use API (#14168)

* Webnovel.com: Use API

* Show migration message instead of bumping "versionId"
This commit is contained in:
AntsyLich 2022-11-14 01:43:40 +06:00 committed by GitHub
parent 945aca879b
commit f8845b1fb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 418 additions and 158 deletions

View File

@ -1,11 +1,12 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
apply plugin: 'kotlinx-serialization'
ext {
extName = 'Webnovel.com'
pkgNameSuffix = 'en.webnovel'
extClass = '.Webnovel'
extVersionCode = 4
extVersionCode = 5
}
apply from: "$rootDir/common.gradle"

View File

@ -1,220 +1,337 @@
package eu.kanade.tachiyomi.extension.en.webnovel
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import okhttp3.Headers
import eu.kanade.tachiyomi.source.online.HttpSource
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.text.SimpleDateFormat
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.io.IOException
import java.util.Calendar
import java.util.Locale
import java.util.Date
class Webnovel : ParsedHttpSource() {
class Webnovel : HttpSource() {
override val name = "Webnovel.com"
override val baseUrl = "https://www.webnovel.com"
private val baseApiUrl = "$baseUrl$BASE_API_ENDPOINT"
private val baseCoverURl = baseUrl.replace("www", "img")
private val baseCdnUrl = baseUrl.replace("www", "comic-image")
override val lang = "en"
override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient
.newBuilder()
.addNetworkInterceptor(::csrfTokenInterceptor)
.addNetworkInterceptor(::expiredImageUrlInterceptor)
.build()
private val dateFormat: SimpleDateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US)
private val json: Json by injectLazy()
override fun headersBuilder() = Headers.Builder().apply {
add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0 ")
add("Referer", baseUrl)
}
// Popular
override fun popularMangaRequest(page: Int): Request = searchMangaRequest(
page = page,
query = "",
filters = FilterList(
SortByFilter(default = 1)
)
)
// popular
override fun popularMangaRequest(page: Int) = GET("$baseUrl/category/0_comic_page$page", headers)
override fun popularMangaParse(response: Response): MangasPage = searchMangaParse(response)
override fun popularMangaSelector() = "a.g_thumb, div.j_bookList .g_book_item a:has(img)"
// Latest
override fun latestUpdatesRequest(page: Int): Request = searchMangaRequest(
page = page,
query = "",
filters = FilterList(
SortByFilter(default = 5)
)
)
override fun popularMangaFromElement(element: Element): SManga {
val manga = SManga.create()
manga.url = element.attr("abs:href").substringAfter(baseUrl)
manga.title = element.attr("title")
manga.thumbnail_url = element.select("img").attr("abs:data-original")
return manga
}
override fun latestUpdatesParse(response: Response): MangasPage = searchMangaParse(response)
override fun popularMangaNextPageSelector() = "[rel=next]"
// latest
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/category/0_comic_page$page?orderBy=5", headers)
override fun latestUpdatesSelector() = popularMangaSelector()
override fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element)
override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
// search
// Search
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val filters = if (filters.isEmpty()) getFilterList() else filters
val genre = filters.findInstance<GenreList>()?.toUriPart()
val order = filters.findInstance<OrderByFilter>()?.toUriPart()
val status = filters.findInstance<StatusFilter>()?.toUriPart()
if (query.isNotBlank()) {
val url = "$baseApiUrl$QUERY_SEARCH_PATH?type=manga&pageIndex=$page".toHttpUrl()
.newBuilder()
.addQueryParameter("keywords", query)
.toString()
return when {
query!!.isNotEmpty() -> GET("$baseUrl/search?keywords=$query&type=2&pageIndex=$page", headers)
else -> GET("$baseUrl/category/$genre" + "_comic_page1?&orderBy=$order&bookStatus=$status")
return GET(url, headers)
}
val sort = filters.firstInstanceOrNull<SortByFilter>()?.selectedValue.orEmpty()
val contentStatus = filters.firstInstanceOrNull<ContentStatusFilter>()?.selectedValue.orEmpty()
val genre = filters.firstInstanceOrNull<GenreFilter>()?.selectedValue.orEmpty()
val url = "$baseApiUrl$FILTER_SEARCH_PATH?categoryType=2&pageIndex=$page" +
"&categoryId=$genre&bookStatus=$contentStatus&orderBy=$sort"
return GET(url, headers)
}
override fun searchMangaSelector() = popularMangaSelector()
override fun searchMangaParse(response: Response): MangasPage {
val browseResponseDto = if (response.request.url.toString().contains(QUERY_SEARCH_PATH)) {
response.checkAndParseAs<QuerySearchResponseDto>().browseResponse
} else {
// Due to the previous line this automatically parses as "BrowseResponseDto"
response.checkAndParseAs()
}
override fun searchMangaFromElement(element: Element): SManga {
val manga = SManga.create()
manga.url = element.attr("abs:href").substringAfter(baseUrl)
manga.title = element.attr("title")
manga.thumbnail_url = element.select("img").attr("abs:src")
return manga
val manga = browseResponseDto.items.map {
SManga.create().apply {
title = it.name
url = it.id
thumbnail_url = getCoverUrl(it.id)
}
}
return MangasPage(manga, browseResponseDto.isLast == 0)
}
override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
// Manga details
// TODO: Cleanup this block when ext-lib 1.4 is released
override fun mangaDetailsRequest(manga: SManga): Request {
return GET("$baseUrl/comic/${manga.getId}", headers)
}
// manga details
override fun mangaDetailsParse(document: Document) = SManga.create().apply {
thumbnail_url = document.select("i.g_thumb img:first-child").attr("abs:src")
title = document.select("h1").text()
description = document.select(".j_synopsis p").text()
override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
return client.newCall(internalMangaDetailsRequest(manga))
.asObservableSuccess()
.map { response ->
mangaDetailsParse(response)
}
}
private fun internalMangaDetailsRequest(manga: SManga): Request {
return GET("$baseApiUrl/comic/getComicDetailPage?comicId=${manga.getId}", headers)
}
override fun mangaDetailsParse(response: Response): SManga {
val comic = response.checkAndParseAs<ComicDetailInfoResponseDto>().info
return SManga.create().apply {
title = comic.name
url = comic.id
thumbnail_url = getCoverUrl(comic.id)
author = comic.authorName
description = comic.description
genre = comic.categoryName
status = when (comic.actionStatus) {
1 -> SManga.ONGOING
2 -> SManga.COMPLETED
else -> SManga.UNKNOWN
}
}
}
// chapters
override fun chapterListRequest(manga: SManga) = GET(baseUrl + manga.url + "/catalog", headers)
override fun chapterListSelector() = ".volume-item li a"
override fun chapterFromElement(element: Element) = SChapter.create().apply {
setUrlWithoutDomain(element.attr("href"))
name = if (element.select("svg").hasAttr("class")) { "\uD83D\uDD12 " } else { "" } +
element.attr("title")
date_upload = parseChapterDate(element.select(".oh small").text())
override fun chapterListRequest(manga: SManga): Request {
return GET("$baseApiUrl/comic/getChapterList?comicId=${manga.getId}", headers)
}
override fun chapterListParse(response: Response): List<SChapter> {
return super.chapterListParse(response).reversed()
val chapterList = response.checkAndParseAs<ComicChapterListDto>()
val comic = chapterList.comicInfo
val chapters = chapterList.comicChapters.reversed().asSequence()
val updateTimes = chapters.map { it.publishTime.toDate() }
val filteredChapters = chapters
// You can pay to get some chapter earlier than others. This privilege is divided into some tiers
// We check if user has same tier unlocked as chapter's.
.filter { it.userLevel == it.chapterLevel }
// When new privileged chapter is released oldest privileged chapter becomes normal one (in most cases)
// but since those normal chapter retain the original upload time we improvise. (This isn't optimal but meh)
return filteredChapters.zip(updateTimes) { chapter, updateTime ->
val namePrefix = when {
chapter.isPremium && !chapter.isAccessibleByUser -> "\uD83D\uDD12 "
else -> ""
}
SChapter.create().apply {
name = namePrefix + chapter.name
url = "${comic.id}:${chapter.id}"
date_upload = updateTime
}
}.toList()
}
fun parseChapterDate(date: String): Long {
return if (date.contains("ago")) {
val value = date.split(' ')[0].toInt()
when {
"min" in date -> Calendar.getInstance().apply {
add(Calendar.MINUTE, value * -1)
}.timeInMillis
"hour" in date -> Calendar.getInstance().apply {
add(Calendar.HOUR_OF_DAY, value * -1)
}.timeInMillis
"day" in date -> Calendar.getInstance().apply {
add(Calendar.DATE, value * -1)
}.timeInMillis
"week" in date -> Calendar.getInstance().apply {
add(Calendar.DATE, value * 7 * -1)
}.timeInMillis
"month" in date -> Calendar.getInstance().apply {
add(Calendar.MONTH, value * -1)
}.timeInMillis
"year" in date -> Calendar.getInstance().apply {
add(Calendar.YEAR, value * -1)
}.timeInMillis
else -> {
0L
}
}
} else {
try {
dateFormat.parse(date)?.time ?: 0
} catch (_: Exception) {
0L
}
private val ComicChapterDto.isPremium get() = isVip != 0 || price != 0
// This can mean the chapter is free or user has paid to unlock it (check with [isPremium] for this case)
private val ComicChapterDto.isAccessibleByUser get() = isAuth == 1
private fun String.toDate(): Long {
if (contains("now", ignoreCase = true)) return Date().time
val number = DIGIT_REGEX.find(this)?.value?.toIntOrNull() ?: return 0
val cal = Calendar.getInstance()
return when {
contains("yr") -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
contains("mth") -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
contains("d") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
contains("h") -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
contains("min") -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
else -> 0
}
}
// pages
override fun pageListParse(document: Document): List<Page> {
return document.select("#comicPageContainer img").mapIndexed { i, element ->
Page(i, "", element.attr("data-original"))
// Pages
// TODO: Cleanup this block when ext-lib 1.4 is released
override fun pageListRequest(chapter: SChapter): Request {
val (comicId, chapterId) = chapter.getMangaAndChapterId
return GET("$baseUrl/comic/$comicId/$chapterId", headers)
}
override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
return client.newCall(internalPageListRequest(chapter))
.asObservableSuccess()
.map { response ->
pageListParse(response)
}
}
private fun internalPageListRequest(chapter: SChapter): Request {
val (comicId, chapterId) = chapter.getMangaAndChapterId
return pageListRequest(comicId, chapterId)
}
private fun pageListRequest(comicId: String, chapterId: String): Request {
return GET("$baseApiUrl/comic/getContent?comicId=$comicId&chapterId=$chapterId")
}
// LinkedHashMap with a capacity of 25. When exceeding the capacity the oldest entry is removed.
private val chapterPageCache = object : LinkedHashMap<Long, List<ChapterPageDto>>() {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Long, List<ChapterPageDto>>?): Boolean {
return size > 25
}
}
override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not Used")
override fun pageListParse(response: Response): List<Page> {
val chapterContent = response.checkAndParseAs<ChapterContentResponseDto>().chapterContent
chapterPageCache[chapterContent.chapterId] = chapterContent.chapterPage
return chapterContent.chapterPage.mapIndexed { i, page -> Page(i, imageUrl = page.url) }
}
override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException("Not Used")
// filter
override fun getFilterList() = FilterList(
Filter.Header("NOTE: Ignored if using text search!"),
Filter.Separator(),
StatusFilter(),
OrderByFilter(),
GenreList()
ContentStatusFilter(),
SortByFilter(),
GenreFilter()
)
private class StatusFilter : UriPartFilter(
"Status",
arrayOf(
Pair("0", "All"),
Pair("1", "Ongoing"),
Pair("2", "Completed")
)
)
private val SManga.getId: String
get() {
if (url.toLongOrNull() == null) throw Exception(MIGRATE_MESSAGE)
return url
}
private class OrderByFilter : UriPartFilter(
"Order By",
arrayOf(
Pair("1", "Default"),
Pair("1", "Popular"),
Pair("2", "Recommendation"),
Pair("3", "Collection"),
Pair("4", "Rates"),
Pair("5", "Updated")
)
)
private class GenreList : UriPartFilter(
"Select Genre",
arrayOf(
Pair("0", "All"),
Pair("60002", "Action"),
Pair("60014", "Adventure"),
Pair("60011", "Comedy"),
Pair("60009", "Cooking"),
Pair("60027", "Diabolical"),
Pair("60024", "Drama"),
Pair("60006", "Eastern"),
Pair("60022", "Fantasy"),
Pair("60017", "Harem"),
Pair("60018", "History"),
Pair("60015", "Horror"),
Pair("60013", "Inspiring"),
Pair("60029", "LGBT+"),
Pair("60016", "Magic"),
Pair("60008", "Mystery"),
Pair("60003", "Romance"),
Pair("60007", "School"),
Pair("60004", "Sci-fi"),
Pair("60019", "Slice of Life"),
Pair("60023", "Sports"),
Pair("60012", "Transmigration"),
Pair("60005", "Urban"),
Pair("60010", "Wuxia")
)
)
private val SChapter.getMangaAndChapterId: Pair<String, String>
get() {
val (comicId, chapterId) = url.split(":")
if (listOf(comicId, chapterId).any { it.toLongOrNull() == null }) throw Exception(MIGRATE_MESSAGE)
return comicId to chapterId
}
private open class UriPartFilter(displayName: String, val vals: Array<Pair<String, String>>) :
Filter.Select<String>(displayName, vals.map { it.second }.toTypedArray()) {
fun toUriPart() = vals[state].first
private fun getCoverUrl(comicId: String): String {
return "$baseCoverURl/bookcover/$comicId/0/600.jpg"
}
private inline fun <reified T> Iterable<*>.findInstance() = find { it is T } as? T
private fun csrfTokenInterceptor(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
val originalRequestUrl = originalRequest.url
if (!originalRequestUrl.toString().contains(BASE_API_ENDPOINT)) return chain.proceed(originalRequest)
val csrfToken = originalRequest.header("cookie")
?.takeIf { csrfTokenName in it }
?.substringAfter("$csrfTokenName=")
?.substringBefore(";")
?: throw IOException("'$csrfTokenName' cookie not found.\nOpen in webview to set it.")
val newUrl = originalRequestUrl.newBuilder()
.addQueryParameter(csrfTokenName, csrfToken)
.build()
val newRequest = originalRequest.newBuilder().url(newUrl).build()
return chain.proceed(newRequest)
}
private fun expiredImageUrlInterceptor(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
val originalRequestUrl = originalRequest.url
// If original request is not a page url or the url is still valid we just continue with og request
if (!originalRequestUrl.toString().contains(baseCdnUrl) || isPageUrlStillValid(originalRequestUrl))
return chain.proceed(originalRequest)
val (_, comicId, chapterId, pageFileName) = originalRequest.url.pathSegments
// Page url is not valid anymore so we check if cache has updated one
val pageId = pageFileName.substringBefore("!")
val cachedPageUrl = chapterPageCache[chapterId.toLong()]?.firstOrNull { it.id == pageId }?.url
if (cachedPageUrl != null && isPageUrlStillValid(cachedPageUrl.toHttpUrl())) return chain.proceed(originalRequest)
// Time to get it from api
val pageListResponse = chain.proceed(pageListRequest(comicId, chapterId))
val chapterContent = pageListResponse.checkAndParseAs<ChapterContentResponseDto>().chapterContent
pageListResponse.close()
chapterPageCache[chapterContent.chapterId] = chapterContent.chapterPage
val newPageUrl = chapterContent.chapterPage.firstOrNull { it.id == pageId }?.url?.toHttpUrl()
?: throw IOException("Couldn't regenerate expired image url")
val newRequest = originalRequest.newBuilder().url(newPageUrl).build()
return chain.proceed(newRequest)
}
private fun isPageUrlStillValid(imageUrl: HttpUrl): Boolean {
val urlGenerationTime = imageUrl.queryParameter("t")?.toLongOrNull()
?: throw IOException("Parameter 't' missing from page url or isn't a long")
// Urls are valid for 10 minutes after generation. We check for 9min and 30s just to be safe
return (Date().time / 1000) - urlGenerationTime <= 570
}
private inline fun <reified T> Response.checkAndParseAs(): T = use {
val parsed = json.decodeFromString<ResponseDto<T>>(it.body?.string().orEmpty())
if (parsed.code != 0) error("Error ${parsed.code}: ${parsed.msg}")
requireNotNull(parsed.data) { "Response data is null" }
}
private inline fun <reified T> List<*>.firstInstanceOrNull() = firstOrNull { it is T } as? T
companion object {
private const val BASE_API_ENDPOINT = "/go/pcm"
private const val QUERY_SEARCH_PATH = "/search/result"
private const val FILTER_SEARCH_PATH = "/category/categoryAjax"
private const val MIGRATE_MESSAGE = "Migrate this entry from \"Webnovel.com\" to \"Webnovel.com\" to update url"
private val DIGIT_REGEX = "(\\d+)".toRegex()
private const val csrfTokenName = "_csrfToken"
}
}

View File

@ -0,0 +1,79 @@
package eu.kanade.tachiyomi.extension.en.webnovel
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonNames
@Serializable
data class ResponseDto<T>(
val code: Int,
val data: T?,
val msg: String,
)
@Serializable
data class QuerySearchResponseDto(
@SerialName("comicInfo") val browseResponse: BrowseResponseDto,
)
@Serializable
data class BrowseResponseDto(
val isLast: Int,
@JsonNames("comicItems") val items: List<ComicInfoDto>,
)
@Serializable
data class ComicInfoDto(
@JsonNames("bookId", "comicId") val id: String,
@JsonNames("bookName", "comicName") val name: String,
)
@Serializable
data class ComicDetailInfoResponseDto(
@SerialName("comicInfo") val info: ComicDetailInfoDto
)
@Serializable
data class ComicDetailInfoDto(
@SerialName("comicId") val id: String,
@SerialName("comicName") val name: String,
val actionStatus: Int,
val authorName: String,
val categoryName: String,
val description: String
)
@Serializable
data class ComicChapterListDto(
val comicInfo: ComicInfoDto,
val comicChapters: List<ComicChapterDto>
)
@Serializable
data class ComicChapterDto(
@SerialName("chapterId") val id: String,
@SerialName("chapterName") val name: String,
val publishTime: String,
val price: Int,
val isVip: Int,
val isAuth: Int,
val chapterLevel: Int,
val userLevel: Int,
)
@Serializable
data class ChapterContentResponseDto(
@SerialName("chapterInfo") val chapterContent: ChapterContentDto
)
@Serializable
data class ChapterContentDto(
val chapterId: Long,
val chapterPage: List<ChapterPageDto>
)
@Serializable
data class ChapterPageDto(
@SerialName("pageId") val id: String,
val url: String
)

View File

@ -0,0 +1,63 @@
package eu.kanade.tachiyomi.extension.en.webnovel
import eu.kanade.tachiyomi.source.model.Filter
data class FilterOption(val displayName: String, val value: String)
open class EnhancedSelect(name: String, private val _values: List<FilterOption>, state: Int = 0) :
Filter.Select<String>(name, _values.map { it.displayName }.toTypedArray(), state) {
val selectedValue: String?
get() = _values.getOrNull(state)?.value
}
class SortByFilter(default: Int = 1) : EnhancedSelect(
"Sort By",
listOf(
FilterOption("Popular", "1"),
FilterOption("Recommended", "2"),
FilterOption("Most collections", "3"),
FilterOption("Rating", "4"),
FilterOption("Time updated", "5"),
),
default - 1
)
class ContentStatusFilter : EnhancedSelect(
"Content status",
listOf(
FilterOption("All", "0"),
FilterOption("Ongoing", "1"),
FilterOption("Completed", "2"),
)
)
class GenreFilter : EnhancedSelect(
"Genre",
listOf(
FilterOption("All", "0"),
FilterOption("Action", "60002"),
FilterOption("Adventure", "60014"),
FilterOption("Comedy", "60011"),
FilterOption("Cooking", "60009"),
FilterOption("Diabolical", "60027"),
FilterOption("Drama", "60024"),
FilterOption("Eastern", "60006"),
FilterOption("Fantasy", "60022"),
FilterOption("Harem", "60017"),
FilterOption("History", "60018"),
FilterOption("Horror", "60015"),
FilterOption("Inspiring", "60013"),
FilterOption("LGBT+", "60029"),
FilterOption("Magic", "60016"),
FilterOption("Mystery", "60008"),
FilterOption("Romance", "60003"),
FilterOption("School", "60007"),
FilterOption("Sci-fi", "60004"),
FilterOption("Slice of Life", "60019"),
FilterOption("Sports", "60023"),
FilterOption("Transmigration", "60012"),
FilterOption("Urban", "60005"),
FilterOption("Wuxia", "60010"),
)
)