Add ability to read already paid chapters in Bilibili (#10593)

* Add ability to read already paid chapters in Bilibili. * Add a README to the extension.
2022-01-29 08:29:29 -03:00 · 2022-01-29 08:29:29 -03:00 · 121b012a34
parent 201463e1f5
commit 121b012a34
6 changed files with 369 additions and 66 deletions
--- a/src/all/bilibili/README.md
+++ b/src/all/bilibili/README.md
@ -0,0 +1,46 @@
+# Bilibili
+
+Table of Content
+- [FAQ](#FAQ)
+  - [Why are some chapters missing?](#why-are-some-chapters-missing)
+- [Guides](#Guides)
+  - [Reading already paid chapters](#reading-already-paid-chapters)
+    
+Don't find the question you are looking for? Go check out our general FAQs and Guides
+over at [Extension FAQ] or [Getting Started].
+
+[Extension FAQ]: https://tachiyomi.org/help/faq/#extensions
+[Getting Started]: https://tachiyomi.org/help/guides/getting-started/#installation
+
+## FAQ
+
+### Why are some chapters missing?
+
+Bilibili now have series with paid chapters. These will be filtered out from
+the chapter list by default if you didn't buy it before or if you're not signed in.
+To sign in with your existing account, follow the guide available above.
+
+## Guides
+
+### Reading already paid chapters
+
+The **Bilibili Comics** sources allows the reading of paid chapters in your account.
+Follow the following steps to be able to sign in and get access to them:
+
+1. Open the popular or latest section of the source.
+2. Open the WebView by clicking the button with a globe icon.
+3. Do the login with your existing account *(read the observations section)*.
+4. Close the WebView and refresh the chapter list of the titles
+   you want to read the already paid chapters.
+
+#### Observations
+
+- Sign in with your Google account is not supported due to WebView restrictions
+  access that Google have. You need to have a simple account in order to be able
+  to login via WebView.
+- You may sometime face the *"Failed to refresh the token"* error. To fix it,
+  you just need to open the WebView, await for the website to completely load.
+  After that, you can close the WebView and try again.
+- The extension **will not** bypass any payment requirement. You still do need
+  to buy the chapters you want to read or wait until they become available and
+  added to your account.
--- a/src/all/bilibili/build.gradle
+++ b/src/all/bilibili/build.gradle
@ -6,7 +6,7 @@ ext {
    extName = 'BILIBILI'
    pkgNameSuffix = 'all.bilibili'
    extClass = '.BilibiliFactory'
-    extVersionCode = 4
+    extVersionCode = 5
 }

 dependencies {
--- a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/Bilibili.kt
+++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/Bilibili.kt
@ -125,7 +125,9 @@ abstract class Bilibili(
        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
    }

-    private val json: Json by injectLazy()
+    protected val json: Json by injectLazy()
+
+    protected open val signedIn: Boolean = false

    private val chapterImageQuality: String
        get() = preferences.getString("${IMAGE_QUALITY_PREF_KEY}_$lang", IMAGE_QUALITY_PREF_DEFAULT_VALUE)!!
@ -146,18 +148,12 @@ abstract class Bilibili(
        }
        val requestBody = requestPayload.toString().toRequestBody(JSON_MEDIA_TYPE)

-        val newHeaders = headersBuilder()
-            .add("Content-Length", requestBody.contentLength().toString())
-            .add("Content-Type", requestBody.contentType().toString())
-            .build()
-
-        val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ClassPage".toHttpUrl().newBuilder()
-            .addQueryParameter("device", "pc")
-            .addQueryParameter("platform", "web")
-            .addLanguageParameters()
+        val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ClassPage".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
            .toString()

-        return POST(apiUrl, newHeaders, requestBody)
+        return POST(apiUrl, headers, requestBody)
    }

    override fun popularMangaParse(response: Response): MangasPage {
@ -192,18 +188,12 @@ abstract class Bilibili(
        }
        val requestBody = requestPayload.toString().toRequestBody(JSON_MEDIA_TYPE)

-        val newHeaders = headersBuilder()
-            .add("Content-Length", requestBody.contentLength().toString())
-            .add("Content-Type", requestBody.contentType().toString())
-            .build()
-
-        val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ClassPage".toHttpUrl().newBuilder()
-            .addQueryParameter("device", "pc")
-            .addQueryParameter("platform", "web")
-            .addLanguageParameters()
+        val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ClassPage".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
            .toString()

-        return POST(apiUrl, newHeaders, requestBody)
+        return POST(apiUrl, headers, requestBody)
    }

    override fun latestUpdatesParse(response: Response): MangasPage {
@ -273,16 +263,12 @@ abstract class Bilibili(
                .toString()

        val newHeaders = headersBuilder()
-            .add("Content-Length", requestBody.contentLength().toString())
-            .add("Content-Type", requestBody.contentType().toString())
            .set("Referer", refererUrl)
            .build()

-        val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/".toHttpUrl().newBuilder()
+        val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/".toHttpUrl().newBuilder()
            .addPathSegment(if (query.isBlank()) "ClassPage" else "Search")
-            .addQueryParameter("device", "pc")
-            .addQueryParameter("platform", "web")
-            .addLanguageParameters()
+            .addCommonParameters()
            .toString()

        return POST(apiUrl, newHeaders, requestBody)
@ -343,15 +329,12 @@ abstract class Bilibili(
        val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE)

        val newHeaders = headersBuilder()
-            .add("Content-Length", requestBody.contentLength().toString())
-            .add("Content-Type", requestBody.contentType().toString())
            .set("Referer", baseUrl + mangaUrl)
            .build()

-        val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ComicDetail".toHttpUrl().newBuilder()
-            .addQueryParameter("device", "pc")
-            .addQueryParameter("platform", "web")
-            .addLanguageParameters()
+        val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ComicDetail".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
            .toString()

        return POST(apiUrl, newHeaders, requestBody)
@ -369,7 +352,7 @@ abstract class Bilibili(
        thumbnail_url = comic.verticalCover + THUMBNAIL_RESOLUTION
        url = "/detail/mc" + comic.id

-        if (comic.episodeList.any { episode -> episode.payMode == 1 && episode.payGold > 0 }) {
+        if (comic.hasPaidChapters && !signedIn) {
            description += "\n\n$hasPaidChaptersWarning"
        }
    }
@ -380,46 +363,49 @@ abstract class Bilibili(
    override fun chapterListParse(response: Response): List<SChapter> {
        val result = response.parseAs<BilibiliComicDto>()

-        if (result.code != 0)
+        if (result.code != 0) {
            return emptyList()
+        }

        return result.data!!.episodeList
            .filter { episode -> episode.payMode == 0 && episode.payGold == 0 }
            .map { ep -> chapterFromObject(ep, result.data.id) }
    }

-    private fun chapterFromObject(episode: BilibiliEpisodeDto, comicId: Int): SChapter = SChapter.create().apply {
+    protected fun chapterFromObject(episode: BilibiliEpisodeDto, comicId: Int): SChapter = SChapter.create().apply {
        name = episodePrefix + episode.shortTitle +
            (if (episode.title.isNotBlank()) " - " + episode.title else "")
        date_upload = episode.publicationTime.substringBefore("T").toDate()
        url = "/mc$comicId/${episode.id}"
    }

-    override fun pageListRequest(chapter: SChapter): Request {
-        val chapterId = chapter.url.substringAfterLast("/").toInt()
+    override fun pageListRequest(chapter: SChapter): Request =
+        imageIndexRequest(chapter.url, "")
+
+    override fun pageListParse(response: Response): List<Page> = imageIndexParse(response)
+
+    protected fun imageIndexRequest(chapterUrl: String, credential: String): Request {
+        val chapterId = chapterUrl.substringAfterLast("/").toInt()

        val jsonPayload = buildJsonObject {
-            put("credential", "")
+            put("credential", credential)
            put("ep_id", chapterId)
        }
        val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE)

        val newHeaders = headersBuilder()
-            .add("Content-Length", requestBody.contentLength().toString())
-            .add("Content-Type", requestBody.contentType().toString())
-            .set("Referer", baseUrl + chapter.url)
+            .set("Referer", baseUrl + chapterUrl)
            .build()

-        val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/GetImageIndex".toHttpUrl().newBuilder()
-            .addQueryParameter("device", "pc")
-            .addQueryParameter("platform", "web")
-            .addLanguageParameters()
+        val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/GetImageIndex".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
            .toString()

        return POST(apiUrl, newHeaders, requestBody)
    }

-    override fun pageListParse(response: Response): List<Page> {
+    protected fun imageIndexParse(response: Response): List<Page> {
        val result = response.parseAs<BilibiliReader>()

        if (result.code != 0) {
@ -444,18 +430,12 @@ abstract class Bilibili(
        }
        val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE)

-        val newHeaders = headersBuilder()
-            .add("Content-Length", requestBody.contentLength().toString())
-            .add("Content-Type", requestBody.contentType().toString())
-            .build()
-
-        val apiUrl = "$baseUrl/$BASE_API_ENDPOINT/ImageToken".toHttpUrl().newBuilder()
-            .addQueryParameter("device", "pc")
-            .addQueryParameter("platform", "web")
-            .addLanguageParameters()
+        val apiUrl = "$baseUrl/$BASE_API_COMIC_ENDPOINT/ImageToken".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
            .toString()

-        return POST(apiUrl, newHeaders, requestBody)
+        return POST(apiUrl, headers, requestBody)
    }

    override fun imageUrlParse(response: Response): String = ""
@ -554,16 +534,19 @@ abstract class Bilibili(
        return response
    }

-    private fun HttpUrl.Builder.addLanguageParameters(): HttpUrl.Builder = let {
+    protected fun HttpUrl.Builder.addCommonParameters(): HttpUrl.Builder = let {
        if (name == "BILIBILI COMICS") {
            addQueryParameter("lang", apiLang)
            addQueryParameter("sys_lang", apiLang)
        }

+        addQueryParameter("device", "pc")
+        addQueryParameter("platform", "web")
+
        return@let it
    }

-    private inline fun <reified T> Response.parseAs(): BilibiliResultDto<T> = use {
+    protected inline fun <reified T> Response.parseAs(): BilibiliResultDto<T> = use {
        json.decodeFromString(it.body?.string().orEmpty())
    }

@ -576,11 +559,12 @@ abstract class Bilibili(
        private const val CDN_URL = "https://manga.hdslb.com"
        private const val COVER_CDN_URL = "https://i0.hdslb.com"

-        private const val BASE_API_ENDPOINT = "twirp/comic.v1.Comic"
+        const val BASE_API_COMIC_ENDPOINT = "twirp/comic.v1.Comic"
+        const val BASE_API_USER_ENDPOINT = "twirp/comic.v1.User"

        private const val ACCEPT_JSON = "application/json, text/plain, */*"

-        private val JSON_MEDIA_TYPE = "application/json;charset=UTF-8".toMediaType()
+        val JSON_MEDIA_TYPE = "application/json;charset=UTF-8".toMediaType()

        private const val POPULAR_PER_PAGE = 18
        private const val SEARCH_PER_PAGE = 9
--- a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliComics.kt
+++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliComics.kt
@ -0,0 +1,239 @@
+package eu.kanade.tachiyomi.extension.all.bilibili
+
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Interceptor
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
+import okhttp3.Response
+import okio.Buffer
+import java.io.IOException
+import java.net.URLDecoder
+
+abstract class BilibiliComics(lang: String) : Bilibili(
+    "BILIBILI COMICS",
+    "https://www.bilibilicomics.com",
+    lang
+) {
+    override val client: OkHttpClient = super.client.newBuilder()
+        .addInterceptor(::signedInIntercept)
+        .build()
+
+    override val signedIn: Boolean
+        get() = accessTokenCookie != null
+
+    private var accessTokenCookie: BilibiliAccessTokenCookie? = null
+
+    override fun chapterListParse(response: Response): List<SChapter> {
+        if (!signedIn) {
+            return super.chapterListParse(response)
+        }
+
+        val result = response.parseAs<BilibiliComicDto>()
+
+        if (result.code != 0) {
+            return emptyList()
+        }
+
+        val comic = result.data!!
+
+        val userEpisodesRequest = userEpisodesRequest(comic.id)
+        val userEpisodesResponse = client.newCall(userEpisodesRequest).execute()
+        val unlockedEpisodes = userEpisodesParse(userEpisodesResponse)
+
+        return comic.episodeList
+            .filter { episode ->
+                (episode.payMode == 0 && episode.payGold == 0) ||
+                    episode.id in unlockedEpisodes
+            }
+            .map { ep -> chapterFromObject(ep, comic.id) }
+    }
+
+    private fun userEpisodesRequest(comicId: Int): Request {
+        val jsonPayload = buildJsonObject { put("comic_id", comicId) }
+        val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE)
+
+        val newHeaders = headersBuilder()
+            .set("Referer", baseUrl)
+            .build()
+
+        val apiUrl = "$GLOBAL_API_URL/$GLOBAL_BASE_API_COMIC_ENDPOINT/GetUserEpisodes".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
+            .toString()
+
+        return POST(apiUrl, newHeaders, requestBody)
+    }
+
+    private fun userEpisodesParse(response: Response): List<Int> {
+        if (!response.isSuccessful) {
+            throw Exception("HTTP error ${response.code}")
+        }
+
+        val result = response.parseAs<BilibiliUserEpisodes>()
+
+        if (result.code != 0) {
+            return emptyList()
+        }
+
+        return result.data!!.unlockedEpisodes.orEmpty()
+            .map(BilibiliUnlockedEpisode::id)
+    }
+
+    override fun pageListRequest(chapter: SChapter): Request {
+        if (!signedIn) {
+            return super.pageListRequest(chapter)
+        }
+
+        val chapterPaths = (baseUrl + chapter.url).toHttpUrl().pathSegments
+        val comicId = chapterPaths[0].removePrefix("mc").toInt()
+        val episodeId = chapterPaths[1].toInt()
+
+        val jsonPayload = BilibiliGetCredential(comicId, episodeId, 1)
+        val requestBody = json.encodeToString(jsonPayload).toRequestBody(JSON_MEDIA_TYPE)
+
+        val newHeaders = headersBuilder()
+            .set("Referer", baseUrl + chapter.url)
+            .build()
+
+        val apiUrl = "$GLOBAL_API_URL/$GLOBAL_BASE_API_USER_ENDPOINT/GetCredential".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
+            .toString()
+
+        return POST(apiUrl, newHeaders, requestBody)
+    }
+
+    override fun pageListParse(response: Response): List<Page> {
+        if (!signedIn) {
+            return super.pageListParse(response)
+        }
+
+        if (!response.isSuccessful) {
+            throw Exception("HTTP error ${response.code}")
+        }
+
+        val result = response.parseAs<BilibiliCredential>()
+        val credential = result.data?.credential ?: ""
+
+        val requestPayload = response.request.bodyString
+        val credentialInfo = json.decodeFromString<BilibiliGetCredential>(requestPayload)
+        val chapterUrl = "/mc${credentialInfo.comicId}/${credentialInfo.episodeId}"
+
+        val imageIndexRequest = imageIndexRequest(chapterUrl, credential)
+        val imageIndexResponse = client.newCall(imageIndexRequest).execute()
+
+        return super.pageListParse(imageIndexResponse)
+    }
+
+    private fun signedInIntercept(chain: Interceptor.Chain): Response {
+        var request = chain.request()
+        val requestUrl = request.url.toString()
+
+        if (!requestUrl.startsWith(baseUrl) && !requestUrl.startsWith(GLOBAL_API_URL)) {
+            return chain.proceed(request)
+        }
+
+        val authCookie = client.cookieJar.loadForRequest(request.url)
+            .firstOrNull { cookie -> cookie.name == ACCESS_TOKEN_COOKIE_NAME }
+            ?.let { cookie -> URLDecoder.decode(cookie.value, "UTF-8") }
+            ?.let { jsonString -> json.decodeFromString<BilibiliAccessTokenCookie>(jsonString) }
+
+        if (accessTokenCookie == null) {
+            accessTokenCookie = authCookie
+        } else if (authCookie == null) {
+            accessTokenCookie = null
+        }
+
+        if (!accessTokenCookie?.accessToken.isNullOrEmpty()) {
+            request = request.newBuilder()
+                .addHeader("Authorization", "Bearer ${accessTokenCookie!!.accessToken}")
+                .build()
+        }
+
+        val response = chain.proceed(request)
+
+        // Try to refresh the token if it expired.
+        if (response.code == 401 && !accessTokenCookie?.refreshToken.isNullOrEmpty()) {
+            response.close()
+
+            val refreshTokenRequest = refreshTokenRequest(
+                accessTokenCookie!!.accessToken,
+                accessTokenCookie!!.refreshToken
+            )
+            val refreshTokenResponse = chain.proceed(refreshTokenRequest)
+
+            accessTokenCookie = refreshTokenParse(refreshTokenResponse) ?: accessTokenCookie
+
+            request = request.newBuilder()
+                .header("Authorization", "Bearer ${accessTokenCookie!!.accessToken}")
+                .build()
+            return chain.proceed(request)
+        }
+
+        return response
+    }
+
+    private fun refreshTokenRequest(accessToken: String, refreshToken: String): Request {
+        val jsonPayload = buildJsonObject { put("refresh_token", refreshToken) }
+        val requestBody = jsonPayload.toString().toRequestBody(JSON_MEDIA_TYPE)
+
+        val newHeaders = headersBuilder()
+            .add("Authorization", "Bearer $accessToken")
+            .set("Referer", baseUrl)
+            .build()
+
+        val apiUrl = "$GLOBAL_API_URL/$GLOBAL_BASE_API_USER_ENDPOINT/RefreshToken".toHttpUrl()
+            .newBuilder()
+            .addCommonParameters()
+            .toString()
+
+        return POST(apiUrl, newHeaders, requestBody)
+    }
+
+    private fun refreshTokenParse(response: Response): BilibiliAccessTokenCookie? {
+        if (!response.isSuccessful) {
+            throw IOException(FAILED_TO_REFRESH_TOKEN)
+        }
+
+        val result = response.parseAs<BilibiliAccessToken>()
+
+        if (result.code != 0) {
+            return null
+        }
+
+        val accessToken = result.data!!
+
+        return BilibiliAccessTokenCookie(
+            accessToken.accessToken,
+            accessToken.refreshToken
+        )
+    }
+
+    private val Request.bodyString: String
+        get() {
+            val requestCopy = newBuilder().build()
+            val buffer = Buffer()
+
+            return runCatching { buffer.apply { requestCopy.body!!.writeTo(this) }.readUtf8() }
+                .getOrNull() ?: ""
+        }
+
+    companion object {
+        private const val ACCESS_TOKEN_COOKIE_NAME = "access_token"
+
+        private const val GLOBAL_API_URL = "https://us-user.bilibilicomics.com"
+        private const val GLOBAL_BASE_API_USER_ENDPOINT = "twirp/global.v1.User"
+        private const val GLOBAL_BASE_API_COMIC_ENDPOINT = "twirp/comic.v1.User"
+
+        private const val FAILED_TO_REFRESH_TOKEN =
+            "Failed to refresh the token. Open the WebView to fix this error."
+    }
+}
--- a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliDto.kt
+++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliDto.kt
@ -27,7 +27,10 @@ data class BilibiliComicDto(
    val styles: List<String> = emptyList(),
    val title: String,
    @SerialName("vertical_cover") val verticalCover: String = ""
-)
+) {
+    val hasPaidChapters: Boolean
+        get() = episodeList.any { episode -> episode.payMode == 1 && episode.payGold > 0 }
+}

@Serializable
 data class BilibiliEpisodeDto(
@ -54,3 +57,37 @@ data class BilibiliPageDto(
    val token: String,
    val url: String
 )
+
+@Serializable
+data class BilibiliAccessTokenCookie(
+    val accessToken: String,
+    val refreshToken: String
+)
+
+@Serializable
+data class BilibiliAccessToken(
+    @SerialName("access_token") val accessToken: String,
+    @SerialName("refresh_token") val refreshToken: String
+)
+
+@Serializable
+data class BilibiliUserEpisodes(
+    @SerialName("unlocked_eps") val unlockedEpisodes: List<BilibiliUnlockedEpisode>? = emptyList()
+)
+
+@Serializable
+data class BilibiliUnlockedEpisode(
+    @SerialName("ep_id") val id: Int = 0
+)
+
+@Serializable
+data class BilibiliGetCredential(
+    @SerialName("comic_id") val comicId: Int,
+    @SerialName("ep_id") val episodeId: Int,
+    val type: Int
+)
+
+@Serializable
+data class BilibiliCredential(
+    val credential: String
+)
--- a/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliFactory.kt
+++ b/src/all/bilibili/src/eu/kanade/tachiyomi/extension/all/bilibili/BilibiliFactory.kt
@ -12,9 +12,6 @@ class BilibiliFactory : SourceFactory {
    )
 }

-abstract class BilibiliComics(lang: String) :
-    Bilibili("BILIBILI COMICS", "https://www.bilibilicomics.com", lang)
-
 class BilibiliComicsEn : BilibiliComics("en") {

    override fun getAllGenres(): Array<BilibiliTag> = arrayOf(