Webnovel.com: Use API (#14168)

* Webnovel.com: Use API

* Show migration message instead of bumping "versionId"
This commit is contained in:
AntsyLich 2022-11-14 01:43:40 +06:00 committed by GitHub
parent 945aca879b
commit f8845b1fb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 418 additions and 158 deletions

View File

@ -1,11 +1,12 @@
apply plugin: 'com.android.application' apply plugin: 'com.android.application'
apply plugin: 'kotlin-android' apply plugin: 'kotlin-android'
apply plugin: 'kotlinx-serialization'
ext { ext {
extName = 'Webnovel.com' extName = 'Webnovel.com'
pkgNameSuffix = 'en.webnovel' pkgNameSuffix = 'en.webnovel'
extClass = '.Webnovel' extClass = '.Webnovel'
extVersionCode = 4 extVersionCode = 5
} }
apply from: "$rootDir/common.gradle" apply from: "$rootDir/common.gradle"

View File

@ -1,220 +1,337 @@
package eu.kanade.tachiyomi.extension.en.webnovel package eu.kanade.tachiyomi.extension.en.webnovel
import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.ParsedHttpSource import eu.kanade.tachiyomi.source.online.HttpSource
import okhttp3.Headers import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Interceptor
import okhttp3.OkHttpClient import okhttp3.OkHttpClient
import okhttp3.Request import okhttp3.Request
import okhttp3.Response import okhttp3.Response
import org.jsoup.nodes.Document import rx.Observable
import org.jsoup.nodes.Element import uy.kohesive.injekt.injectLazy
import java.text.SimpleDateFormat import java.io.IOException
import java.util.Calendar import java.util.Calendar
import java.util.Locale import java.util.Date
class Webnovel : ParsedHttpSource() { class Webnovel : HttpSource() {
override val name = "Webnovel.com" override val name = "Webnovel.com"
override val baseUrl = "https://www.webnovel.com" override val baseUrl = "https://www.webnovel.com"
private val baseApiUrl = "$baseUrl$BASE_API_ENDPOINT"
private val baseCoverURl = baseUrl.replace("www", "img")
private val baseCdnUrl = baseUrl.replace("www", "comic-image")
override val lang = "en" override val lang = "en"
override val supportsLatest = true override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient override val client: OkHttpClient = network.cloudflareClient
.newBuilder()
.addNetworkInterceptor(::csrfTokenInterceptor)
.addNetworkInterceptor(::expiredImageUrlInterceptor)
.build()
private val dateFormat: SimpleDateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) private val json: Json by injectLazy()
override fun headersBuilder() = Headers.Builder().apply { // Popular
add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0 ") override fun popularMangaRequest(page: Int): Request = searchMangaRequest(
add("Referer", baseUrl) page = page,
} query = "",
filters = FilterList(
SortByFilter(default = 1)
)
)
// popular override fun popularMangaParse(response: Response): MangasPage = searchMangaParse(response)
override fun popularMangaRequest(page: Int) = GET("$baseUrl/category/0_comic_page$page", headers)
override fun popularMangaSelector() = "a.g_thumb, div.j_bookList .g_book_item a:has(img)" // Latest
override fun latestUpdatesRequest(page: Int): Request = searchMangaRequest(
page = page,
query = "",
filters = FilterList(
SortByFilter(default = 5)
)
)
override fun popularMangaFromElement(element: Element): SManga { override fun latestUpdatesParse(response: Response): MangasPage = searchMangaParse(response)
val manga = SManga.create()
manga.url = element.attr("abs:href").substringAfter(baseUrl)
manga.title = element.attr("title")
manga.thumbnail_url = element.select("img").attr("abs:data-original")
return manga
}
override fun popularMangaNextPageSelector() = "[rel=next]" // Search
// latest
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/category/0_comic_page$page?orderBy=5", headers)
override fun latestUpdatesSelector() = popularMangaSelector()
override fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element)
override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
// search
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val filters = if (filters.isEmpty()) getFilterList() else filters if (query.isNotBlank()) {
val genre = filters.findInstance<GenreList>()?.toUriPart() val url = "$baseApiUrl$QUERY_SEARCH_PATH?type=manga&pageIndex=$page".toHttpUrl()
val order = filters.findInstance<OrderByFilter>()?.toUriPart() .newBuilder()
val status = filters.findInstance<StatusFilter>()?.toUriPart() .addQueryParameter("keywords", query)
.toString()
return when { return GET(url, headers)
query!!.isNotEmpty() -> GET("$baseUrl/search?keywords=$query&type=2&pageIndex=$page", headers) }
else -> GET("$baseUrl/category/$genre" + "_comic_page1?&orderBy=$order&bookStatus=$status") val sort = filters.firstInstanceOrNull<SortByFilter>()?.selectedValue.orEmpty()
val contentStatus = filters.firstInstanceOrNull<ContentStatusFilter>()?.selectedValue.orEmpty()
val genre = filters.firstInstanceOrNull<GenreFilter>()?.selectedValue.orEmpty()
val url = "$baseApiUrl$FILTER_SEARCH_PATH?categoryType=2&pageIndex=$page" +
"&categoryId=$genre&bookStatus=$contentStatus&orderBy=$sort"
return GET(url, headers)
}
override fun searchMangaParse(response: Response): MangasPage {
val browseResponseDto = if (response.request.url.toString().contains(QUERY_SEARCH_PATH)) {
response.checkAndParseAs<QuerySearchResponseDto>().browseResponse
} else {
// Due to the previous line this automatically parses as "BrowseResponseDto"
response.checkAndParseAs()
}
val manga = browseResponseDto.items.map {
SManga.create().apply {
title = it.name
url = it.id
thumbnail_url = getCoverUrl(it.id)
} }
} }
override fun searchMangaSelector() = popularMangaSelector() return MangasPage(manga, browseResponseDto.isLast == 0)
override fun searchMangaFromElement(element: Element): SManga {
val manga = SManga.create()
manga.url = element.attr("abs:href").substringAfter(baseUrl)
manga.title = element.attr("title")
manga.thumbnail_url = element.select("img").attr("abs:src")
return manga
} }
override fun searchMangaNextPageSelector() = popularMangaNextPageSelector() // Manga details
// TODO: Cleanup this block when ext-lib 1.4 is released
override fun mangaDetailsRequest(manga: SManga): Request {
return GET("$baseUrl/comic/${manga.getId}", headers)
}
// manga details override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
override fun mangaDetailsParse(document: Document) = SManga.create().apply { return client.newCall(internalMangaDetailsRequest(manga))
thumbnail_url = document.select("i.g_thumb img:first-child").attr("abs:src") .asObservableSuccess()
title = document.select("h1").text() .map { response ->
description = document.select(".j_synopsis p").text() mangaDetailsParse(response)
}
}
private fun internalMangaDetailsRequest(manga: SManga): Request {
return GET("$baseApiUrl/comic/getComicDetailPage?comicId=${manga.getId}", headers)
}
override fun mangaDetailsParse(response: Response): SManga {
val comic = response.checkAndParseAs<ComicDetailInfoResponseDto>().info
return SManga.create().apply {
title = comic.name
url = comic.id
thumbnail_url = getCoverUrl(comic.id)
author = comic.authorName
description = comic.description
genre = comic.categoryName
status = when (comic.actionStatus) {
1 -> SManga.ONGOING
2 -> SManga.COMPLETED
else -> SManga.UNKNOWN
}
}
} }
// chapters // chapters
override fun chapterListRequest(manga: SManga) = GET(baseUrl + manga.url + "/catalog", headers) override fun chapterListRequest(manga: SManga): Request {
return GET("$baseApiUrl/comic/getChapterList?comicId=${manga.getId}", headers)
override fun chapterListSelector() = ".volume-item li a"
override fun chapterFromElement(element: Element) = SChapter.create().apply {
setUrlWithoutDomain(element.attr("href"))
name = if (element.select("svg").hasAttr("class")) { "\uD83D\uDD12 " } else { "" } +
element.attr("title")
date_upload = parseChapterDate(element.select(".oh small").text())
} }
override fun chapterListParse(response: Response): List<SChapter> { override fun chapterListParse(response: Response): List<SChapter> {
return super.chapterListParse(response).reversed() val chapterList = response.checkAndParseAs<ComicChapterListDto>()
val comic = chapterList.comicInfo
val chapters = chapterList.comicChapters.reversed().asSequence()
val updateTimes = chapters.map { it.publishTime.toDate() }
val filteredChapters = chapters
// You can pay to get some chapter earlier than others. This privilege is divided into some tiers
// We check if user has same tier unlocked as chapter's.
.filter { it.userLevel == it.chapterLevel }
// When new privileged chapter is released oldest privileged chapter becomes normal one (in most cases)
// but since those normal chapter retain the original upload time we improvise. (This isn't optimal but meh)
return filteredChapters.zip(updateTimes) { chapter, updateTime ->
val namePrefix = when {
chapter.isPremium && !chapter.isAccessibleByUser -> "\uD83D\uDD12 "
else -> ""
}
SChapter.create().apply {
name = namePrefix + chapter.name
url = "${comic.id}:${chapter.id}"
date_upload = updateTime
}
}.toList()
} }
fun parseChapterDate(date: String): Long { private val ComicChapterDto.isPremium get() = isVip != 0 || price != 0
return if (date.contains("ago")) { // This can mean the chapter is free or user has paid to unlock it (check with [isPremium] for this case)
val value = date.split(' ')[0].toInt() private val ComicChapterDto.isAccessibleByUser get() = isAuth == 1
when {
"min" in date -> Calendar.getInstance().apply { private fun String.toDate(): Long {
add(Calendar.MINUTE, value * -1) if (contains("now", ignoreCase = true)) return Date().time
}.timeInMillis
"hour" in date -> Calendar.getInstance().apply { val number = DIGIT_REGEX.find(this)?.value?.toIntOrNull() ?: return 0
add(Calendar.HOUR_OF_DAY, value * -1) val cal = Calendar.getInstance()
}.timeInMillis
"day" in date -> Calendar.getInstance().apply { return when {
add(Calendar.DATE, value * -1) contains("yr") -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis
}.timeInMillis contains("mth") -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis
"week" in date -> Calendar.getInstance().apply { contains("d") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis
add(Calendar.DATE, value * 7 * -1) contains("h") -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis
}.timeInMillis contains("min") -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis
"month" in date -> Calendar.getInstance().apply { else -> 0
add(Calendar.MONTH, value * -1)
}.timeInMillis
"year" in date -> Calendar.getInstance().apply {
add(Calendar.YEAR, value * -1)
}.timeInMillis
else -> {
0L
}
}
} else {
try {
dateFormat.parse(date)?.time ?: 0
} catch (_: Exception) {
0L
}
} }
} }
// pages // Pages
override fun pageListParse(document: Document): List<Page> { // TODO: Cleanup this block when ext-lib 1.4 is released
return document.select("#comicPageContainer img").mapIndexed { i, element -> override fun pageListRequest(chapter: SChapter): Request {
Page(i, "", element.attr("data-original")) val (comicId, chapterId) = chapter.getMangaAndChapterId
return GET("$baseUrl/comic/$comicId/$chapterId", headers)
}
override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
return client.newCall(internalPageListRequest(chapter))
.asObservableSuccess()
.map { response ->
pageListParse(response)
} }
} }
override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not Used") private fun internalPageListRequest(chapter: SChapter): Request {
val (comicId, chapterId) = chapter.getMangaAndChapterId
return pageListRequest(comicId, chapterId)
}
private fun pageListRequest(comicId: String, chapterId: String): Request {
return GET("$baseApiUrl/comic/getContent?comicId=$comicId&chapterId=$chapterId")
}
// LinkedHashMap with a capacity of 25. When exceeding the capacity the oldest entry is removed.
private val chapterPageCache = object : LinkedHashMap<Long, List<ChapterPageDto>>() {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Long, List<ChapterPageDto>>?): Boolean {
return size > 25
}
}
override fun pageListParse(response: Response): List<Page> {
val chapterContent = response.checkAndParseAs<ChapterContentResponseDto>().chapterContent
chapterPageCache[chapterContent.chapterId] = chapterContent.chapterPage
return chapterContent.chapterPage.mapIndexed { i, page -> Page(i, imageUrl = page.url) }
}
override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException("Not Used")
// filter
override fun getFilterList() = FilterList( override fun getFilterList() = FilterList(
Filter.Header("NOTE: Ignored if using text search!"), Filter.Header("NOTE: Ignored if using text search!"),
Filter.Separator(), Filter.Separator(),
StatusFilter(), ContentStatusFilter(),
OrderByFilter(), SortByFilter(),
GenreList() GenreFilter()
) )
private class StatusFilter : UriPartFilter( private val SManga.getId: String
"Status", get() {
arrayOf( if (url.toLongOrNull() == null) throw Exception(MIGRATE_MESSAGE)
Pair("0", "All"), return url
Pair("1", "Ongoing"),
Pair("2", "Completed")
)
)
private class OrderByFilter : UriPartFilter(
"Order By",
arrayOf(
Pair("1", "Default"),
Pair("1", "Popular"),
Pair("2", "Recommendation"),
Pair("3", "Collection"),
Pair("4", "Rates"),
Pair("5", "Updated")
)
)
private class GenreList : UriPartFilter(
"Select Genre",
arrayOf(
Pair("0", "All"),
Pair("60002", "Action"),
Pair("60014", "Adventure"),
Pair("60011", "Comedy"),
Pair("60009", "Cooking"),
Pair("60027", "Diabolical"),
Pair("60024", "Drama"),
Pair("60006", "Eastern"),
Pair("60022", "Fantasy"),
Pair("60017", "Harem"),
Pair("60018", "History"),
Pair("60015", "Horror"),
Pair("60013", "Inspiring"),
Pair("60029", "LGBT+"),
Pair("60016", "Magic"),
Pair("60008", "Mystery"),
Pair("60003", "Romance"),
Pair("60007", "School"),
Pair("60004", "Sci-fi"),
Pair("60019", "Slice of Life"),
Pair("60023", "Sports"),
Pair("60012", "Transmigration"),
Pair("60005", "Urban"),
Pair("60010", "Wuxia")
)
)
private open class UriPartFilter(displayName: String, val vals: Array<Pair<String, String>>) :
Filter.Select<String>(displayName, vals.map { it.second }.toTypedArray()) {
fun toUriPart() = vals[state].first
} }
private inline fun <reified T> Iterable<*>.findInstance() = find { it is T } as? T private val SChapter.getMangaAndChapterId: Pair<String, String>
get() {
val (comicId, chapterId) = url.split(":")
if (listOf(comicId, chapterId).any { it.toLongOrNull() == null }) throw Exception(MIGRATE_MESSAGE)
return comicId to chapterId
}
private fun getCoverUrl(comicId: String): String {
return "$baseCoverURl/bookcover/$comicId/0/600.jpg"
}
private fun csrfTokenInterceptor(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
val originalRequestUrl = originalRequest.url
if (!originalRequestUrl.toString().contains(BASE_API_ENDPOINT)) return chain.proceed(originalRequest)
val csrfToken = originalRequest.header("cookie")
?.takeIf { csrfTokenName in it }
?.substringAfter("$csrfTokenName=")
?.substringBefore(";")
?: throw IOException("'$csrfTokenName' cookie not found.\nOpen in webview to set it.")
val newUrl = originalRequestUrl.newBuilder()
.addQueryParameter(csrfTokenName, csrfToken)
.build()
val newRequest = originalRequest.newBuilder().url(newUrl).build()
return chain.proceed(newRequest)
}
private fun expiredImageUrlInterceptor(chain: Interceptor.Chain): Response {
val originalRequest = chain.request()
val originalRequestUrl = originalRequest.url
// If original request is not a page url or the url is still valid we just continue with og request
if (!originalRequestUrl.toString().contains(baseCdnUrl) || isPageUrlStillValid(originalRequestUrl))
return chain.proceed(originalRequest)
val (_, comicId, chapterId, pageFileName) = originalRequest.url.pathSegments
// Page url is not valid anymore so we check if cache has updated one
val pageId = pageFileName.substringBefore("!")
val cachedPageUrl = chapterPageCache[chapterId.toLong()]?.firstOrNull { it.id == pageId }?.url
if (cachedPageUrl != null && isPageUrlStillValid(cachedPageUrl.toHttpUrl())) return chain.proceed(originalRequest)
// Time to get it from api
val pageListResponse = chain.proceed(pageListRequest(comicId, chapterId))
val chapterContent = pageListResponse.checkAndParseAs<ChapterContentResponseDto>().chapterContent
pageListResponse.close()
chapterPageCache[chapterContent.chapterId] = chapterContent.chapterPage
val newPageUrl = chapterContent.chapterPage.firstOrNull { it.id == pageId }?.url?.toHttpUrl()
?: throw IOException("Couldn't regenerate expired image url")
val newRequest = originalRequest.newBuilder().url(newPageUrl).build()
return chain.proceed(newRequest)
}
private fun isPageUrlStillValid(imageUrl: HttpUrl): Boolean {
val urlGenerationTime = imageUrl.queryParameter("t")?.toLongOrNull()
?: throw IOException("Parameter 't' missing from page url or isn't a long")
// Urls are valid for 10 minutes after generation. We check for 9min and 30s just to be safe
return (Date().time / 1000) - urlGenerationTime <= 570
}
private inline fun <reified T> Response.checkAndParseAs(): T = use {
val parsed = json.decodeFromString<ResponseDto<T>>(it.body?.string().orEmpty())
if (parsed.code != 0) error("Error ${parsed.code}: ${parsed.msg}")
requireNotNull(parsed.data) { "Response data is null" }
}
private inline fun <reified T> List<*>.firstInstanceOrNull() = firstOrNull { it is T } as? T
companion object {
private const val BASE_API_ENDPOINT = "/go/pcm"
private const val QUERY_SEARCH_PATH = "/search/result"
private const val FILTER_SEARCH_PATH = "/category/categoryAjax"
private const val MIGRATE_MESSAGE = "Migrate this entry from \"Webnovel.com\" to \"Webnovel.com\" to update url"
private val DIGIT_REGEX = "(\\d+)".toRegex()
private const val csrfTokenName = "_csrfToken"
}
} }

View File

@ -0,0 +1,79 @@
package eu.kanade.tachiyomi.extension.en.webnovel
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonNames
@Serializable
data class ResponseDto<T>(
val code: Int,
val data: T?,
val msg: String,
)
@Serializable
data class QuerySearchResponseDto(
@SerialName("comicInfo") val browseResponse: BrowseResponseDto,
)
@Serializable
data class BrowseResponseDto(
val isLast: Int,
@JsonNames("comicItems") val items: List<ComicInfoDto>,
)
@Serializable
data class ComicInfoDto(
@JsonNames("bookId", "comicId") val id: String,
@JsonNames("bookName", "comicName") val name: String,
)
@Serializable
data class ComicDetailInfoResponseDto(
@SerialName("comicInfo") val info: ComicDetailInfoDto
)
@Serializable
data class ComicDetailInfoDto(
@SerialName("comicId") val id: String,
@SerialName("comicName") val name: String,
val actionStatus: Int,
val authorName: String,
val categoryName: String,
val description: String
)
@Serializable
data class ComicChapterListDto(
val comicInfo: ComicInfoDto,
val comicChapters: List<ComicChapterDto>
)
@Serializable
data class ComicChapterDto(
@SerialName("chapterId") val id: String,
@SerialName("chapterName") val name: String,
val publishTime: String,
val price: Int,
val isVip: Int,
val isAuth: Int,
val chapterLevel: Int,
val userLevel: Int,
)
@Serializable
data class ChapterContentResponseDto(
@SerialName("chapterInfo") val chapterContent: ChapterContentDto
)
@Serializable
data class ChapterContentDto(
val chapterId: Long,
val chapterPage: List<ChapterPageDto>
)
@Serializable
data class ChapterPageDto(
@SerialName("pageId") val id: String,
val url: String
)

View File

@ -0,0 +1,63 @@
package eu.kanade.tachiyomi.extension.en.webnovel
import eu.kanade.tachiyomi.source.model.Filter
data class FilterOption(val displayName: String, val value: String)
open class EnhancedSelect(name: String, private val _values: List<FilterOption>, state: Int = 0) :
Filter.Select<String>(name, _values.map { it.displayName }.toTypedArray(), state) {
val selectedValue: String?
get() = _values.getOrNull(state)?.value
}
class SortByFilter(default: Int = 1) : EnhancedSelect(
"Sort By",
listOf(
FilterOption("Popular", "1"),
FilterOption("Recommended", "2"),
FilterOption("Most collections", "3"),
FilterOption("Rating", "4"),
FilterOption("Time updated", "5"),
),
default - 1
)
class ContentStatusFilter : EnhancedSelect(
"Content status",
listOf(
FilterOption("All", "0"),
FilterOption("Ongoing", "1"),
FilterOption("Completed", "2"),
)
)
class GenreFilter : EnhancedSelect(
"Genre",
listOf(
FilterOption("All", "0"),
FilterOption("Action", "60002"),
FilterOption("Adventure", "60014"),
FilterOption("Comedy", "60011"),
FilterOption("Cooking", "60009"),
FilterOption("Diabolical", "60027"),
FilterOption("Drama", "60024"),
FilterOption("Eastern", "60006"),
FilterOption("Fantasy", "60022"),
FilterOption("Harem", "60017"),
FilterOption("History", "60018"),
FilterOption("Horror", "60015"),
FilterOption("Inspiring", "60013"),
FilterOption("LGBT+", "60029"),
FilterOption("Magic", "60016"),
FilterOption("Mystery", "60008"),
FilterOption("Romance", "60003"),
FilterOption("School", "60007"),
FilterOption("Sci-fi", "60004"),
FilterOption("Slice of Life", "60019"),
FilterOption("Sports", "60023"),
FilterOption("Transmigration", "60012"),
FilterOption("Urban", "60005"),
FilterOption("Wuxia", "60010"),
)
)