BookWalker Global: high-res thumbnails and details from api (#11654)

* thumbnails

* get details from api

* typo

* adjustments

* use select

src/en/bookwalker/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'BookWalker Global'
     extClass = '.BookWalker'
-    extVersionCode = 1
+    extVersionCode = 2
     isNsfw = true
 }
 

src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalker.kt

@@ -8,6 +8,7 @@ import androidx.preference.EditTextPreference
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
 import androidx.preference.SwitchPreferenceCompat
+import eu.kanade.tachiyomi.extension.en.bookwalker.dto.BookUpdateDto
 import eu.kanade.tachiyomi.extension.en.bookwalker.dto.HoldBookEntityDto
 import eu.kanade.tachiyomi.extension.en.bookwalker.dto.HoldBooksInfoDto
 import eu.kanade.tachiyomi.extension.en.bookwalker.dto.SeriesDto
@@ -31,6 +32,7 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.modules.SerializersModule
 import kotlinx.serialization.modules.plus
@@ -52,13 +54,18 @@ import java.util.regex.PatternSyntaxException
 class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences {
 
     override val name = "BookWalker Global"
+    private val domain = "bookwalker.jp"
 
-    override val baseUrl = "https://global.bookwalker.jp"
+    override val baseUrl = "https://global.$domain"
 
     override val lang = "en"
 
     override val supportsLatest = true
 
+    private val rimgUrl = "https://rimg.$domain"
+    private val cUrl = "https://c.$domain"
+    private val memberApiUrl = "https://member-app.$domain/api"
+
     override val client = network.client.newBuilder()
         .addInterceptor(BookWalkerImageRequestInterceptor(this))
         .build()
@@ -318,19 +325,28 @@ class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences
 
     override fun popularMangaParse(response: Response): MangasPage {
         if (showLibraryInPopular) {
-            val manga = response.parseAs<HoldBooksInfoDto>().holdBookList.entities
+            val manga = runBlocking(Dispatchers.IO) {
-                .map {
+                response.parseAs<HoldBooksInfoDto>().holdBookList.entities
-                    when (it) {
+                    .mapNotNull { entity ->
-                        is SeriesDto -> SManga.create().apply {
+                        when (entity) {
-                            url = "/series/${it.seriesId}/"
+                            is SeriesDto -> {
-                            title = it.seriesName.cleanTitle()
+                                SManga.create().apply {
-                            thumbnail_url = it.imageUrl
+                                    url = "/series/${entity.seriesId}/"
+                                    title = entity.seriesName.cleanTitle()
+                                    thumbnail_url = getHiResCoverFromLegacyUrl(entity.imageUrl)
+                                }
+                            }
+                            is SingleDto -> {
+                                val bookUpdate = fetchBookUpdate(entity.uuid)
+                                bookUpdate?.let { bookUpdate ->
+                                    SManga.create().apply {
+                                        url = "/de${entity.uuid}/"
+                                        title = bookUpdate.seriesName?.cleanTitle() ?: bookUpdate.productName.cleanTitle()
+                                        thumbnail_url = bookUpdate.coverImageUrl
+                                        author = bookUpdate.authors.joinToString { it.authorName }
+                                    }
+                                }
                             }
-                        is SingleDto -> SManga.create().apply {
-                            url = it.detailUrl.substring(baseUrl.length)
-                            title = it.title.cleanTitle()
-                            thumbnail_url = it.imageUrl
-                            author = it.authors.joinToString { a -> a.authorName }
                         }
                     }
             }
@@ -351,8 +367,9 @@ class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences
         return SManga.create().apply {
             url = titleElt.attr("href").substring(baseUrl.length)
             title = titleElt.attr("title").cleanTitle()
-            thumbnail_url = element.select(".a-tile-thumb-img > img")
+            thumbnail_url = getHiResCoverFromLegacyUrl(
-                .attr("data-srcset").getHighestQualitySrcset()
+                element.selectFirst(".a-tile-thumb-img > img")?.attr("data-srcset")?.getHighestQualitySrcset(),
+            )
         }
     }
 
@@ -417,43 +434,48 @@ class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences
                 .asJsoup()
                 .let { validateLogin(it) }
 
-            val mangaDetails = SManga.create().apply {
-                updateDetailsFromSeriesPage(seriesPage)
-            }
-
             // It generally doesn't matter which chapter we take the description from, but for
             // series that release in volumes we want the earliest one, which will _usually_ be the
             // last one on the page. With that said, it's not worth it to paginate in order to find
             // the earliest volume, and volume releases don't usually have 60+ volumes anyways.
-            val chapterUrl = seriesPage
+            val firstItem = seriesPage.selectFirst(".o-tile:not(:has(.a-ribbon-pre-order)) .a-tile-ttl a")
-                .select(".o-tile .a-tile-ttl a").last()
+            val uuid = firstItem!!.absUrl("href").substringAfter("/de").substringBefore("/")
-                ?.attr("href")
+            val bookUpdate = fetchBookUpdate(uuid)
-                ?: return@rxSingle mangaDetails
 
-            val chapterPage = client.newCall(GET(chapterUrl, callHeaders)).await().asJsoup()
+            SManga.create().apply {
-
+                title = bookUpdate!!.seriesName?.cleanTitle() ?: bookUpdate.productName.cleanTitle()
-            mangaDetails.apply {
+                author = bookUpdate.authors.joinToString { it.authorName }
-                description = getDescriptionFromChapterPage(chapterPage)
+                description = listOfNotNull(bookUpdate.productExplanationShort, bookUpdate.productExplanationDetails)
+                    .joinToString("\n\n")
+                    .trim()
+                thumbnail_url = bookUpdate.coverImageUrl
+                genre = getAvailableFilterNames(seriesPage, "side-genre").joinToString()
+                val statusIndicators = seriesPage.select("ul.side-others > li > a").map { it.ownText() }
+                status = parseStatus(statusIndicators)
             }
         }.toObservable()
     }
 
     private fun fetchSingleMangaDetails(manga: SManga): Observable<SManga> {
         return rxSingle {
-            val document = client.newCall(GET(baseUrl + manga.url, callHeaders))
+            val uuid = manga.url.substringAfter("/de").substringBefore("/")
-                .awaitSuccess()
+            val bookUpdate = fetchBookUpdate(uuid)
-                .asJsoup()
 
             SManga.create().apply {
-                title = getTitleFromChapterPage(document)?.cleanTitle().orEmpty()
+                title = bookUpdate!!.seriesName?.cleanTitle() ?: bookUpdate.productName.cleanTitle()
+                author = bookUpdate.authors.joinToString { it.authorName }
+                description = listOfNotNull(bookUpdate.productExplanationShort, bookUpdate.productExplanationDetails)
+                    .joinToString("\n\n")
+                    .trim()
+                thumbnail_url = bookUpdate.coverImageUrl
 
-                description = getDescriptionFromChapterPage(document)
                 // From the browse pages we can't distinguish between a true one-shot and a
                 // serial manga with only one chapter, but we can detect if there's a series
                 // reference in the chapter page. If there is, we should let the user know that
                 // they may need to take some action in the future to correct the error.
+                val document = client.newCall(GET(baseUrl + manga.url, callHeaders)).awaitSuccess().asJsoup()
                 if (document.selectFirst(".product-detail th:contains(Series Title)") != null) {
-                    description = (
+                    this.description = (
                         "WARNING: This entry is being treated as a one-shot but appears to " +
                             "have an associated series. If another chapter is released in " +
                             "the future, you will likely need to migrate this to itself." +
@@ -464,22 +486,8 @@ class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences
         }.toObservable()
     }
 
-    private fun SManga.updateDetailsFromSeriesPage(document: Document) = run {
+    private fun parseStatus(statusIndicators: List<String>): Int {
-        // Take the thumbnail from the first chapter that is not on pre-order.
+        return if (statusIndicators.any { it.startsWith("Completed") }) {
-        // Pre-order chapters often just have a gray rectangle with "NOW PRINTING" as their
-        // thumbnail, which doesn't look very pretty for the catalog.
-        thumbnail_url = document
-            .select(".o-tile:not(:has(.a-ribbon-pre-order)) .a-tile-thumb-img > img")
-            .attr("data-srcset")
-            .getHighestQualitySrcset()
-        title = document.selectFirst(".title-main-inner")!!.ownText().cleanTitle()
-        author = getAvailableFilterNames(document, "side-author").joinToString()
-        genre = getAvailableFilterNames(document, "side-genre").joinToString()
-
-        val statusIndicators = document.select("ul.side-others > li > a").map { it.ownText() }
-
-        status =
-            if (statusIndicators.any { it.startsWith("Completed") }) {
             if (statusIndicators.any { it.startsWith("Pre-Order") }) {
                 SManga.PUBLISHING_FINISHED
             } else {
@@ -494,14 +502,6 @@ class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences
         return document.selectFirst(".detail-book-title-box h1[itemprop='name']")?.ownText()
     }
 
-    private fun getDescriptionFromChapterPage(document: Document): String {
-        return buildString {
-            append(document.select(".synopsis-lead").text())
-            append("\n\n")
-            append(document.select(".synopsis-text").text())
-        }.trim()
-    }
-
     override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
         return rxSingle {
             if (!manga.url.startsWith("/series/")) {
@@ -750,6 +750,41 @@ class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences
         return srcsetPairs.maxByOrNull { it.first ?: 0 }?.second
     }
 
+    // This function works if the cover is from before mid-dec'23 (non-hexadecimal).
+    // If it's a newer cover, it will fall back to the low-res version.
+    private fun getHiResCoverFromLegacyUrl(url: String?): String? {
+        if (url.isNullOrEmpty()) return url
+        return try {
+            val extension = url.substringAfterLast(".")
+            val numericId = when {
+                url.startsWith(rimgUrl) -> {
+                    val id = url.substringAfter("$rimgUrl/").substringBefore('/')
+                    id.reversed().toLongOrNull()
+                }
+                // For legacy covers of "series" from the user's library.
+                url.contains("thumbnailImage_") -> {
+                    val id = url.substringAfter("thumbnailImage_").substringBefore(".$extension")
+                    id.toLongOrNull()
+                }
+                else -> null
+            }
+            numericId?.let { "$cUrl/coverImage_${it - 1}.$extension" } ?: url
+        } catch (e: Exception) {
+            url
+        }
+    }
+
+    // Fetch manga details form api.
+    private suspend fun fetchBookUpdate(uuid: String): BookUpdateDto? {
+        val apiUrl = "$memberApiUrl/books/updates".toHttpUrl().newBuilder()
+            .addQueryParameter("fileType", "EPUB")
+            .addQueryParameter(uuid, "0")
+            .build()
+
+        return client.newCall(GET(apiUrl, headers)).awaitSuccess()
+            .parseAs<List<BookUpdateDto>>().firstOrNull()
+    }
+
     private fun String.parseChapterNumber(): Pair<String, Float>? {
         for (pattern in CHAPTER_NUMBER_PATTERNS) {
             val match = pattern.find(this)

src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/DetailsDto.kt

@@ -0,0 +1,18 @@
+package eu.kanade.tachiyomi.extension.en.bookwalker.dto
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+class BookUpdateDto(
+    val productName: String,
+    val seriesName: String?,
+    val productExplanationShort: String?,
+    val productExplanationDetails: String,
+    val coverImageUrl: String,
+    val authors: List<AuthorUpdateDto>,
+)
+
+@Serializable
+class AuthorUpdateDto(
+    val authorName: String,
+)

src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/SingleDto.kt

@@ -6,13 +6,5 @@ import kotlinx.serialization.Serializable
 @Serializable
 @SerialName("normal")
 class SingleDto(
-    val detailUrl: String,
+    val uuid: String,
-    val title: String,
-    val imageUrl: String,
-    val authors: List<AuthorDto>,
 ) : HoldBookEntityDto()
-
-@Serializable
-class AuthorDto(
-    val authorName: String,
-)