From 121b012a3465d163a620f28ec0c075b0151373b2 Mon Sep 17 00:00:00 2001 From: Alessandro Jean Date: Sat, 29 Jan 2022 08:29:29 -0300 Subject: [PATCH] Add ability to read already paid chapters in Bilibili (#10593) * Add ability to read already paid chapters in Bilibili. * Add a README to the extension. --- src/all/bilibili/README.md | 46 ++++ src/all/bilibili/build.gradle | 2 +- .../extension/all/bilibili/Bilibili.kt | 106 ++++---- .../extension/all/bilibili/BilibiliComics.kt | 239 ++++++++++++++++++ .../extension/all/bilibili/BilibiliDto.kt | 39 ++- .../extension/all/bilibili/BilibiliFactory.kt | 3 - 6 files changed, 369 insertions(+), 66 deletions(-) create mode 100644 src/all/bilibili/README.md create mode 100644 src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliComics.kt diff --git a/src/all/bilibili/README.md b/src/all/bilibili/README.md new file mode 100644 index 000000000..b20de4a6f --- /dev/null +++ b/src/all/bilibili/README.md @@ -0,0 +1,46 @@ +# Bilibili + +Table of Content +- [FAQ](#FAQ) + - [Why are some chapters missing?](#why-are-some-chapters-missing) +- [Guides](#Guides) + - [Reading already paid chapters](#reading-already-paid-chapters) + +Don't find the question you are looking for? Go check out our general FAQs and Guides +over at [Extension FAQ] or [Getting Started]. + +[Extension FAQ]: https://tachiyomi.org/help/faq/#extensions +[Getting Started]: https://tachiyomi.org/help/guides/getting-started/#installation + +## FAQ + +### Why are some chapters missing? + +Bilibili now have series with paid chapters. These will be filtered out from +the chapter list by default if you didn't buy it before or if you're not signed in. +To sign in with your existing account, follow the guide available above. + +## Guides + +### Reading already paid chapters + +The **Bilibili Comics** sources allows the reading of paid chapters in your account. +Follow the following steps to be able to sign in and get access to them: + +1. Open the popular or latest section of the source. +2. Open the WebView by clicking the button with a globe icon. +3. Do the login with your existing account *(read the observations section)*. +4. Close the WebView and refresh the chapter list of the titles + you want to read the already paid chapters. + +#### Observations + +- Sign in with your Google account is not supported due to WebView restrictions + access that Google have. You need to have a simple account in order to be able + to login via WebView. +- You may sometime face the *"Failed to refresh the token"* error. To fix it, + you just need to open the WebView, await for the website to completely load. + After that, you can close the WebView and try again. +- The extension **will not** bypass any payment requirement. You still do need + to buy the chapters you want to read or wait until they become available and + added to your account. diff --git a/src/all/bilibili/build.gradle b/src/all/bilibili/build.gradle index e63be226d..e4aee3ce0 100644 --- a/src/all/bilibili/build.gradle +++ b/src/all/bilibili/build.gradle @@ -6,7 +6,7 @@ ext { extName = 'BILIBILI' pkgNameSuffix = 'all.bilibili' extClass = '.BilibiliFactory' - extVersionCode = 4 + extVersionCode = 5 } dependencies { diff --git a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/Bilibili.kt b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/Bilibili.kt index f5a9519df..33bf8e7bb 100644 --- a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/Bilibili.kt +++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/Bilibili.kt @@ -125,7 +125,9 @@ abstract class Bilibili( Injekt.get().getSharedPreferences("source_$id", 0x0000) } - private val json: Json by injectLazy() + protected val json: Json by injectLazy() + + protected open val signedIn: Boolean = false private val chapterImageQuality: String get() = preferences.getString("${IMAGE_QUALITY_PREF_KEY}_$lang", IMAGE_QUALITY_PREF_DEFAULT_VALUE)!! @@ -146,18 +148,12 @@ abstract class Bilibili( } val requestBody = requestPayload.toString().toRequestBody(JSON_MEDIA_TYPE) - val newHeaders = headersBuilder() - .add("Content-Length", requestBody.contentLength().toString()) - .add("Content-Type", requestBody.contentType().toString()) - .build() - - val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ClassPage".toHttpUrl().newBuilder() - .addQueryParameter("device", "pc") - .addQueryParameter("platform", "web") - .addLanguageParameters() + val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ClassPage".toHttpUrl() + .newBuilder() + .addCommonParameters() .toString() - return POST(apiUrl, newHeaders, requestBody) + return POST(apiUrl, headers, requestBody) } override fun popularMangaParse(response: Response): MangasPage { @@ -192,18 +188,12 @@ abstract class Bilibili( } val requestBody = requestPayload.toString().toRequestBody(JSON_MEDIA_TYPE) - val newHeaders = headersBuilder() - .add("Content-Length", requestBody.contentLength().toString()) - .add("Content-Type", requestBody.contentType().toString()) - .build() - - val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ClassPage".toHttpUrl().newBuilder() - .addQueryParameter("device", "pc") - .addQueryParameter("platform", "web") - .addLanguageParameters() + val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ClassPage".toHttpUrl() + .newBuilder() + .addCommonParameters() .toString() - return POST(apiUrl, newHeaders, requestBody) + return POST(apiUrl, headers, requestBody) } override fun latestUpdatesParse(response: Response): MangasPage { @@ -273,16 +263,12 @@ abstract class Bilibili( .toString() val newHeaders = headersBuilder() - .add("Content-Length", requestBody.contentLength().toString()) - .add("Content-Type", requestBody.contentType().toString()) .set("Referer", refererUrl) .build() - val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/".toHttpUrl().newBuilder() + val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/".toHttpUrl().newBuilder() .addPathSegment(if (query.isBlank()) "ClassPage" else "Search") - .addQueryParameter("device", "pc") - .addQueryParameter("platform", "web") - .addLanguageParameters() + .addCommonParameters() .toString() return POST(apiUrl, newHeaders, requestBody) @@ -343,15 +329,12 @@ abstract class Bilibili( val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE) val newHeaders = headersBuilder() - .add("Content-Length", requestBody.contentLength().toString()) - .add("Content-Type", requestBody.contentType().toString()) .set("Referer", baseUrl + mangaUrl) .build() - val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ComicDetail".toHttpUrl().newBuilder() - .addQueryParameter("device", "pc") - .addQueryParameter("platform", "web") - .addLanguageParameters() + val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ComicDetail".toHttpUrl() + .newBuilder() + .addCommonParameters() .toString() return POST(apiUrl, newHeaders, requestBody) @@ -369,7 +352,7 @@ abstract class Bilibili( thumbnail_url = comic.verticalCover + THUMBNAIL_RESOLUTION url = "/detail/mc" + comic.id - if (comic.episodeList.any { episode -> episode.payMode == 1 && episode.payGold > 0 }) { + if (comic.hasPaidChapters && !signedIn) { description += "\n\n$hasPaidChaptersWarning" } } @@ -380,46 +363,49 @@ abstract class Bilibili( override fun chapterListParse(response: Response): List { val result = response.parseAs() - if (result.code != 0) + if (result.code != 0) { return emptyList() + } return result.data!!.episodeList .filter { episode -> episode.payMode == 0 && episode.payGold == 0 } .map { ep -> chapterFromObject(ep, result.data.id) } } - private fun chapterFromObject(episode: BilibiliEpisodeDto, comicId: Int): SChapter = SChapter.create().apply { + protected fun chapterFromObject(episode: BilibiliEpisodeDto, comicId: Int): SChapter = SChapter.create().apply { name = episodePrefix + episode.shortTitle + (if (episode.title.isNotBlank()) " - " + episode.title else "") date_upload = episode.publicationTime.substringBefore("T").toDate() url = "/mc$comicId/${episode.id}" } - override fun pageListRequest(chapter: SChapter): Request { - val chapterId = chapter.url.substringAfterLast("/").toInt() + override fun pageListRequest(chapter: SChapter): Request = + imageIndexRequest(chapter.url, "") + + override fun pageListParse(response: Response): List = imageIndexParse(response) + + protected fun imageIndexRequest(chapterUrl: String, credential: String): Request { + val chapterId = chapterUrl.substringAfterLast("/").toInt() val jsonPayload = buildJsonObject { - put("credential", "") + put("credential", credential) put("ep_id", chapterId) } val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE) val newHeaders = headersBuilder() - .add("Content-Length", requestBody.contentLength().toString()) - .add("Content-Type", requestBody.contentType().toString()) - .set("Referer", baseUrl + chapter.url) + .set("Referer", baseUrl + chapterUrl) .build() - val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/GetImageIndex".toHttpUrl().newBuilder() - .addQueryParameter("device", "pc") - .addQueryParameter("platform", "web") - .addLanguageParameters() + val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/GetImageIndex".toHttpUrl() + .newBuilder() + .addCommonParameters() .toString() return POST(apiUrl, newHeaders, requestBody) } - override fun pageListParse(response: Response): List { + protected fun imageIndexParse(response: Response): List { val result = response.parseAs() if (result.code != 0) { @@ -444,18 +430,12 @@ abstract class Bilibili( } val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE) - val newHeaders = headersBuilder() - .add("Content-Length", requestBody.contentLength().toString()) - .add("Content-Type", requestBody.contentType().toString()) - .build() - - val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ImageToken".toHttpUrl().newBuilder() - .addQueryParameter("device", "pc") - .addQueryParameter("platform", "web") - .addLanguageParameters() + val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ImageToken".toHttpUrl() + .newBuilder() + .addCommonParameters() .toString() - return POST(apiUrl, newHeaders, requestBody) + return POST(apiUrl, headers, requestBody) } override fun imageUrlParse(response: Response): String = "" @@ -554,16 +534,19 @@ abstract class Bilibili( return response } - private fun HttpUrl.Builder.addLanguageParameters(): HttpUrl.Builder = let { + protected fun HttpUrl.Builder.addCommonParameters(): HttpUrl.Builder = let { if (name == "BILIBILI COMICS") { addQueryParameter("lang", apiLang) addQueryParameter("sys_lang", apiLang) } + addQueryParameter("device", "pc") + addQueryParameter("platform", "web") + return@let it } - private inline fun Response.parseAs(): BilibiliResultDto = use { + protected inline fun Response.parseAs(): BilibiliResultDto = use { json.decodeFromString(it.body?.string().orEmpty()) } @@ -576,11 +559,12 @@ abstract class Bilibili( private const val CDN_URL = "https://manga.hdslb.com" private const val COVER_CDN_URL = "https://i0.hdslb.com" - private const val BASE_API_ENDPOINT = "twirp/comic.v1.Comic" + const val BASE_API_COMIC_ENDPOINT = "twirp/comic.v1.Comic" + const val BASE_API_USER_ENDPOINT = "twirp/comic.v1.User" private const val ACCEPT_JSON = "application/json, text/plain, */*" - private val JSON_MEDIA_TYPE = "application/json;charset=UTF-8".toMediaType() + val JSON_MEDIA_TYPE = "application/json;charset=UTF-8".toMediaType() private const val POPULAR_PER_PAGE = 18 private const val SEARCH_PER_PAGE = 9 diff --git a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliComics.kt b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliComics.kt new file mode 100644 index 000000000..ac6136062 --- /dev/null +++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliComics.kt @@ -0,0 +1,239 @@ +package eu.kanade.tachiyomi.extension.all.bilibili + +import eu.kanade.tachiyomi.network.POST +import eu.kanade.tachiyomi.source.model.Page +import eu.kanade.tachiyomi.source.model.SChapter +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import okio.Buffer +import java.io.IOException +import java.net.URLDecoder + +abstract class BilibiliComics(lang: String) : Bilibili( + "BILIBILI COMICS", + "https://www.bilibilicomics.com", + lang +) { + override val client: OkHttpClient = super.client.newBuilder() + .addInterceptor(::signedInIntercept) + .build() + + override val signedIn: Boolean + get() = accessTokenCookie != null + + private var accessTokenCookie: BilibiliAccessTokenCookie? = null + + override fun chapterListParse(response: Response): List { + if (!signedIn) { + return super.chapterListParse(response) + } + + val result = response.parseAs() + + if (result.code != 0) { + return emptyList() + } + + val comic = result.data!! + + val userEpisodesRequest = userEpisodesRequest(comic.id) + val userEpisodesResponse = client.newCall(userEpisodesRequest).execute() + val unlockedEpisodes = userEpisodesParse(userEpisodesResponse) + + return comic.episodeList + .filter { episode -> + (episode.payMode == 0 && episode.payGold == 0) || + episode.id in unlockedEpisodes + } + .map { ep -> chapterFromObject(ep, comic.id) } + } + + private fun userEpisodesRequest(comicId: Int): Request { + val jsonPayload = buildJsonObject { put("comic_id", comicId) } + val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE) + + val newHeaders = headersBuilder() + .set("Referer", baseUrl) + .build() + + val apiUrl = "$GLOBAL_API_URL/$GLOBAL_BASE_API_COMIC_ENDPOINT/GetUserEpisodes".toHttpUrl() + .newBuilder() + .addCommonParameters() + .toString() + + return POST(apiUrl, newHeaders, requestBody) + } + + private fun userEpisodesParse(response: Response): List { + if (!response.isSuccessful) { + throw Exception("HTTP error ${response.code}") + } + + val result = response.parseAs() + + if (result.code != 0) { + return emptyList() + } + + return result.data!!.unlockedEpisodes.orEmpty() + .map(BilibiliUnlockedEpisode::id) + } + + override fun pageListRequest(chapter: SChapter): Request { + if (!signedIn) { + return super.pageListRequest(chapter) + } + + val chapterPaths = (baseUrl + chapter.url).toHttpUrl().pathSegments + val comicId = chapterPaths[0].removePrefix("mc").toInt() + val episodeId = chapterPaths[1].toInt() + + val jsonPayload = BilibiliGetCredential(comicId, episodeId, 1) + val requestBody = json.encodeToString(jsonPayload).toRequestBody(JSON_MEDIA_TYPE) + + val newHeaders = headersBuilder() + .set("Referer", baseUrl + chapter.url) + .build() + + val apiUrl = "$GLOBAL_API_URL/$GLOBAL_BASE_API_USER_ENDPOINT/GetCredential".toHttpUrl() + .newBuilder() + .addCommonParameters() + .toString() + + return POST(apiUrl, newHeaders, requestBody) + } + + override fun pageListParse(response: Response): List { + if (!signedIn) { + return super.pageListParse(response) + } + + if (!response.isSuccessful) { + throw Exception("HTTP error ${response.code}") + } + + val result = response.parseAs() + val credential = result.data?.credential ?: "" + + val requestPayload = response.request.bodyString + val credentialInfo = json.decodeFromString(requestPayload) + val chapterUrl = "/mc${credentialInfo.comicId}/${credentialInfo.episodeId}" + + val imageIndexRequest = imageIndexRequest(chapterUrl, credential) + val imageIndexResponse = client.newCall(imageIndexRequest).execute() + + return super.pageListParse(imageIndexResponse) + } + + private fun signedInIntercept(chain: Interceptor.Chain): Response { + var request = chain.request() + val requestUrl = request.url.toString() + + if (!requestUrl.startsWith(baseUrl) && !requestUrl.startsWith(GLOBAL_API_URL)) { + return chain.proceed(request) + } + + val authCookie = client.cookieJar.loadForRequest(request.url) + .firstOrNull { cookie -> cookie.name == ACCESS_TOKEN_COOKIE_NAME } + ?.let { cookie -> URLDecoder.decode(cookie.value, "UTF-8") } + ?.let { jsonString -> json.decodeFromString(jsonString) } + + if (accessTokenCookie == null) { + accessTokenCookie = authCookie + } else if (authCookie == null) { + accessTokenCookie = null + } + + if (!accessTokenCookie?.accessToken.isNullOrEmpty()) { + request = request.newBuilder() + .addHeader("Authorization", "Bearer ${accessTokenCookie!!.accessToken}") + .build() + } + + val response = chain.proceed(request) + + // Try to refresh the token if it expired. + if (response.code == 401 && !accessTokenCookie?.refreshToken.isNullOrEmpty()) { + response.close() + + val refreshTokenRequest = refreshTokenRequest( + accessTokenCookie!!.accessToken, + accessTokenCookie!!.refreshToken + ) + val refreshTokenResponse = chain.proceed(refreshTokenRequest) + + accessTokenCookie = refreshTokenParse(refreshTokenResponse) ?: accessTokenCookie + + request = request.newBuilder() + .header("Authorization", "Bearer ${accessTokenCookie!!.accessToken}") + .build() + return chain.proceed(request) + } + + return response + } + + private fun refreshTokenRequest(accessToken: String, refreshToken: String): Request { + val jsonPayload = buildJsonObject { put("refresh_token", refreshToken) } + val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE) + + val newHeaders = headersBuilder() + .add("Authorization", "Bearer $accessToken") + .set("Referer", baseUrl) + .build() + + val apiUrl = "$GLOBAL_API_URL/$GLOBAL_BASE_API_USER_ENDPOINT/RefreshToken".toHttpUrl() + .newBuilder() + .addCommonParameters() + .toString() + + return POST(apiUrl, newHeaders, requestBody) + } + + private fun refreshTokenParse(response: Response): BilibiliAccessTokenCookie? { + if (!response.isSuccessful) { + throw IOException(FAILED_TO_REFRESH_TOKEN) + } + + val result = response.parseAs() + + if (result.code != 0) { + return null + } + + val accessToken = result.data!! + + return BilibiliAccessTokenCookie( + accessToken.accessToken, + accessToken.refreshToken + ) + } + + private val Request.bodyString: String + get() { + val requestCopy = newBuilder().build() + val buffer = Buffer() + + return runCatching { buffer.apply { requestCopy.body!!.writeTo(this) }.readUtf8() } + .getOrNull() ?: "" + } + + companion object { + private const val ACCESS_TOKEN_COOKIE_NAME = "access_token" + + private const val GLOBAL_API_URL = "https://us-user.bilibilicomics.com" + private const val GLOBAL_BASE_API_USER_ENDPOINT = "twirp/global.v1.User" + private const val GLOBAL_BASE_API_COMIC_ENDPOINT = "twirp/comic.v1.User" + + private const val FAILED_TO_REFRESH_TOKEN = + "Failed to refresh the token. Open the WebView to fix this error." + } +} diff --git a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliDto.kt b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliDto.kt index 042f56b8c..c5e23533a 100644 --- a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliDto.kt +++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliDto.kt @@ -27,7 +27,10 @@ data class BilibiliComicDto( val styles: List = emptyList(), val title: String, @SerialName("vertical_cover") val verticalCover: String = "" -) +) { + val hasPaidChapters: Boolean + get() = episodeList.any { episode -> episode.payMode == 1 && episode.payGold > 0 } +} @Serializable data class BilibiliEpisodeDto( @@ -54,3 +57,37 @@ data class BilibiliPageDto( val token: String, val url: String ) + +@Serializable +data class BilibiliAccessTokenCookie( + val accessToken: String, + val refreshToken: String +) + +@Serializable +data class BilibiliAccessToken( + @SerialName("access_token") val accessToken: String, + @SerialName("refresh_token") val refreshToken: String +) + +@Serializable +data class BilibiliUserEpisodes( + @SerialName("unlocked_eps") val unlockedEpisodes: List? = emptyList() +) + +@Serializable +data class BilibiliUnlockedEpisode( + @SerialName("ep_id") val id: Int = 0 +) + +@Serializable +data class BilibiliGetCredential( + @SerialName("comic_id") val comicId: Int, + @SerialName("ep_id") val episodeId: Int, + val type: Int +) + +@Serializable +data class BilibiliCredential( + val credential: String +) diff --git a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliFactory.kt b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliFactory.kt index f25f89471..ca059bb2b 100644 --- a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliFactory.kt +++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliFactory.kt @@ -12,9 +12,6 @@ class BilibiliFactory : SourceFactory { ) } -abstract class BilibiliComics(lang: String) : - Bilibili("BILIBILI COMICS", "https://www.bilibilicomics.com", lang) - class BilibiliComicsEn : BilibiliComics("en") { override fun getAllGenres(): Array = arrayOf(