MangaPlus: Update web API versions and clean up stuff (#1305)

* MangaPlus: Update API versions

* Add more stuff to the titleCache

src/all/mangaplus/src/eu/kanade/tachiyomi/extension/all/mangaplus/MangaPlus.kt

@@ -12,6 +12,7 @@ import androidx.preference.PreferenceScreen
 import androidx.preference.SwitchPreferenceCompat
 import eu.kanade.tachiyomi.lib.i18n.Intl
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.network.interceptor.rateLimitHost
 import eu.kanade.tachiyomi.source.ConfigurableSource
 import eu.kanade.tachiyomi.source.model.FilterList
@@ -50,9 +51,9 @@ class MangaPlus(
 
     override fun headersBuilder(): Headers.Builder = Headers.Builder()
         .add("Origin", baseUrl)
-        .add("Referer", baseUrl)
+        .add("Referer", "$baseUrl/")
         .add("User-Agent", USER_AGENT)
-        .add("Session-Token", UUID.randomUUID().toString())
+        .add("SESSION-TOKEN", UUID.randomUUID().toString())
 
     override val client: OkHttpClient = network.client.newBuilder()
         .addInterceptor(::imageIntercept)
@@ -81,17 +82,22 @@ class MangaPlus(
      * in Tachiyomi database is expired. It's also used during the chapter deeplink
      * handling to avoid an additional request if possible.
      */
-    private var titleCache: Map<Int, Title>? = null
+    private val titleCache = mutableMapOf<Int, Title>()
+    private lateinit var directory: List<Title>
 
-    override fun popularMangaRequest(page: Int): Request {
+    override fun fetchPopularManga(page: Int): Observable<MangasPage> {
-        val newHeaders = headersBuilder()
+        return if (page == 1) {
-            .set("Referer", "$baseUrl/manga_list/hot")
+            client.newCall(popularMangaRequest(page))
-            .set("X-Page", page.toString())
+                .asObservableSuccess()
-            .build()
+                .map { popularMangaParse(it) }
-
+        } else {
-        return GET("$API_URL/title_list/ranking?format=json", newHeaders)
+            Observable.just(parseDirectory(page))
+        }
     }
 
+    override fun popularMangaRequest(page: Int) =
+        GET("$API_URL/title_list/rankingV2?lang=$internalLang&type=hottest&clang=$internalLang&format=json", headers)
+
     override fun popularMangaParse(response: Response): MangasPage {
         val result = response.asMangaPlusResponse()
 
@@ -99,28 +105,36 @@ class MangaPlus(
             result.error!!.langPopup(langCode)?.body ?: intl["unknown_error"]
         }
 
-        val titleList = result.success.titleRankingView!!.titles
+        directory = result.success.titleRankingViewV2!!.rankedTitles
+            .flatMap(RankedTitle::titles)
             .filter { it.language == langCode }
+        titleCache.putAll(directory.associateBy(Title::titleId))
 
-        titleCache = titleList.associateBy(Title::titleId)
+        return parseDirectory(1)
+    }
 
-        val page = response.request.headers["X-Page"]!!.toInt()
+    private fun parseDirectory(page: Int): MangasPage {
-        val pageList = titleList
+        val pageList = directory
             .drop((page - 1) * LISTING_ITEMS_PER_PAGE)
             .take(LISTING_ITEMS_PER_PAGE)
-        val hasNextPage = (page + 1) * LISTING_ITEMS_PER_PAGE <= titleList.size
+        val hasNextPage = (page + 1) * LISTING_ITEMS_PER_PAGE <= directory.size
 
         return MangasPage(pageList.map(Title::toSManga), hasNextPage)
     }
 
-    override fun latestUpdatesRequest(page: Int): Request {
+    override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
-        val newHeaders = headersBuilder()
+        return if (page == 1) {
-            .set("Referer", "$baseUrl/updates")
+            client.newCall(latestUpdatesRequest(page))
-            .build()
+                .asObservableSuccess()
-
+                .map { latestUpdatesParse(it) }
-        return GET("$API_URL/web/web_homeV3?lang=$internalLang&format=json", newHeaders)
+        } else {
+            Observable.just(parseDirectory(page))
+        }
     }
 
+    override fun latestUpdatesRequest(page: Int) =
+        GET("$API_URL/web/web_homeV4?lang=$internalLang&clang=$internalLang&format=json", headers)
+
     override fun latestUpdatesParse(response: Response): MangasPage {
         val result = response.asMangaPlusResponse()
 
@@ -128,25 +142,42 @@ class MangaPlus(
             result.error!!.langPopup(langCode)?.body ?: intl["unknown_error"]
         }
 
-        // Fetch all titles to get newer thumbnail URLs in the interceptor.
+        directory = result.success.webHomeViewV4!!.groups
-        val popularResponse = client.newCall(popularMangaRequest(1)).execute()
-            .asMangaPlusResponse()
-
-        if (popularResponse.success != null) {
-            titleCache = popularResponse.success.titleRankingView!!.titles
-                .filter { it.language == langCode }
-                .associateBy(Title::titleId)
-        }
-
-        val mangas = result.success.webHomeViewV3!!.groups
             .flatMap(UpdatedTitleV2Group::titleGroups)
             .flatMap(OriginalTitleGroup::titles)
             .map(UpdatedTitle::title)
             .filter { it.language == langCode }
-            .map(Title::toSManga)
+            .distinctBy(Title::titleId)
-            .distinctBy(SManga::title)
 
-        return MangasPage(mangas, hasNextPage = false)
+        titleCache.putAll(directory.associateBy(Title::titleId))
+        titleCache.putAll(
+            result.success.webHomeViewV4.rankedTitles
+                .flatMap(RankedTitle::titles)
+                .filter { it.language == langCode }
+                .associateBy(Title::titleId),
+        )
+        titleCache.putAll(
+            result.success.webHomeViewV4.featuredTitleLists
+                .flatMap(FeaturedTitleList::featuredTitles)
+                .filter { it.language == langCode }
+                .associateBy(Title::titleId),
+        )
+
+        return parseDirectory(1)
+    }
+
+    override fun fetchSearchManga(
+        page: Int,
+        query: String,
+        filters: FilterList,
+    ): Observable<MangasPage> {
+        return if (page == 1) {
+            client.newCall(searchMangaRequest(page, query, filters))
+                .asObservableSuccess()
+                .map { searchMangaParse(it, query) }
+        } else {
+            Observable.just(parseDirectory(page))
+        }
     }
 
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
@@ -156,19 +187,12 @@ class MangaPlus(
             return pageListRequest(query.removePrefix(PREFIX_CHAPTER_ID_SEARCH))
         }
 
-        val newHeaders = headersBuilder()
+        return GET("$API_URL/title_list/allV2?format=json", headers)
-            .set("Referer", "$baseUrl/manga_list/all")
-            .set("X-Page", page.toString())
-            .build()
-
-        val apiUrl = "$API_URL/title_list/allV2".toHttpUrl().newBuilder()
-            .addQueryParameter("filter", query.trim())
-            .addQueryParameter("format", "json")
-
-        return GET(apiUrl.toString(), newHeaders)
     }
 
-    override fun searchMangaParse(response: Response): MangasPage {
+    override fun searchMangaParse(response: Response) = throw UnsupportedOperationException()
+
+    private fun searchMangaParse(response: Response, query: String): MangasPage {
         val result = response.asMangaPlusResponse()
 
         checkNotNull(result.success) {
@@ -187,7 +211,7 @@ class MangaPlus(
             checkNotNull(result.success.mangaViewer.titleId) { intl["chapter_expired"] }
 
             val titleId = result.success.mangaViewer.titleId
-            val cachedTitle = titleCache?.get(titleId)
+            val cachedTitle = titleCache[titleId]
 
             val title = cachedTitle?.toSManga() ?: run {
                 val titleRequest = mangaDetailsRequest(titleId.toString())
@@ -205,26 +229,17 @@ class MangaPlus(
             return MangasPage(listOfNotNull(title), hasNextPage = false)
         }
 
-        val filter = response.request.url.queryParameter("filter").orEmpty()
-
         val allTitlesList = result.success.allTitlesViewV2!!.allTitlesGroup
             .flatMap(AllTitlesGroup::titles)
             .filter { it.language == langCode }
 
-        titleCache = allTitlesList.associateBy(Title::titleId)
+        titleCache.putAll(allTitlesList.associateBy(Title::titleId))
-
+        directory = allTitlesList.filter { title ->
-        val searchResults = allTitlesList.filter { title ->
+            title.name.contains(query, ignoreCase = true) ||
-            title.name.contains(filter, ignoreCase = true) ||
+                title.author.orEmpty().contains(query, ignoreCase = true)
-                title.author.orEmpty().contains(filter, ignoreCase = true)
         }
 
-        val page = response.request.headers["X-Page"]!!.toInt()
+        return parseDirectory(1)
-        val pageList = searchResults
-            .drop((page - 1) * LISTING_ITEMS_PER_PAGE)
-            .take(LISTING_ITEMS_PER_PAGE)
-        val hasNextPage = (page + 1) * LISTING_ITEMS_PER_PAGE <= searchResults.size
-
-        return MangasPage(pageList.map(Title::toSManga), hasNextPage)
     }
 
     // Remove the '#' and map to the new url format used in website.
@@ -235,11 +250,7 @@ class MangaPlus(
     private fun mangaDetailsRequest(mangaUrl: String): Request {
         val titleId = mangaUrl.substringAfterLast("/")
 
-        val newHeaders = headersBuilder()
+        return GET("$APP_API_URL/title_detailV3?title_id=$titleId&lang=eng&os=android&os_ver=30&app_ver=${preferences.appVersion}&secret=${preferences.accountSecret}&format=json", headers)
-            .set("Referer", "$baseUrl/titles/$titleId")
-            .build()
-
-        return GET("$APP_API_URL/title_detailV3?title_id=$titleId&lang=eng&os=android&os_ver=30&app_ver=${preferences.appVersion}&secret=${preferences.accountSecret}&format=json", newHeaders)
     }
 
     override fun mangaDetailsParse(response: Response): SManga {
@@ -294,10 +305,6 @@ class MangaPlus(
     }
 
     private fun pageListRequest(chapterId: String): Request {
-        val newHeaders = headersBuilder()
-            .set("Referer", "$baseUrl/viewer/$chapterId")
-            .build()
-
         val url = "$APP_API_URL/manga_viewer".toHttpUrl().newBuilder()
             .addQueryParameter("chapter_id", chapterId)
             .addQueryParameter("split", if (preferences.splitImages) "yes" else "no")
@@ -312,7 +319,7 @@ class MangaPlus(
             .addQueryParameter("format", "json")
             .toString()
 
-        return GET(url, newHeaders)
+        return GET(url, headers)
     }
 
     override fun pageListParse(response: Response): List<Page> {
@@ -328,28 +335,15 @@ class MangaPlus(
             }
         }
 
-        val referer = response.request.header("Referer")!!
-
         return result.success.mangaViewer!!.pages
             .mapNotNull(MangaPlusPage::mangaPage)
             .mapIndexed { i, page ->
                 val encryptionKey = if (page.encryptionKey == null) "" else "#${page.encryptionKey}"
-                Page(i, referer, page.imageUrl + encryptionKey)
+                Page(i, imageUrl = page.imageUrl + encryptionKey)
             }
     }
 
-    override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl!!)
+    override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException()
-
-    override fun imageUrlParse(response: Response): String = ""
-
-    override fun imageRequest(page: Page): Request {
-        val newHeaders = headersBuilder()
-            .removeAll("Origin")
-            .set("Referer", page.url)
-            .build()
-
-        return GET(page.imageUrl!!, newHeaders)
-    }
 
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
         val qualityPref = ListPreference(screen.context).apply {
@@ -479,7 +473,7 @@ class MangaPlus(
             .substringBefore("/$TITLE_THUMBNAIL_PATH")
             .substringAfterLast("/")
             .toInt()
-        val title = titleCache?.get(titleId) ?: return response
+        val title = titleCache[titleId] ?: return response
 
         response.close()
         val thumbnailRequest = GET(title.portraitImageUrl, request.headers)
@@ -512,33 +506,27 @@ class MangaPlus(
         get() = getString("${SECRET_PREF_KEY}_$lang", SECRET_PREF_DEFAULT_VALUE)
 
     companion object {
-        private const val API_URL = "https://jumpg-webapi.tokyo-cdn.com/api"
-        private const val APP_API_URL = "https://jumpg-api.tokyo-cdn.com/api"
-        private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
-            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
-
-        private const val LISTING_ITEMS_PER_PAGE = 20
-
-        private const val QUALITY_PREF_KEY = "imageResolution"
-        private val QUALITY_PREF_ENTRY_VALUES = arrayOf("low", "high", "super_high")
-        private val QUALITY_PREF_DEFAULT_VALUE = QUALITY_PREF_ENTRY_VALUES[2]
-
-        private const val SPLIT_PREF_KEY = "splitImage"
-        private const val SPLIT_PREF_DEFAULT_VALUE = true
-
-        private const val VER_PREF_KEY = "appVer"
-        private const val VER_PREF_DEFAULT_VALUE = ""
-
-        private const val SECRET_PREF_KEY = "accountSecret"
-        private const val SECRET_PREF_DEFAULT_VALUE = ""
-
-        private const val NOT_FOUND_SUBJECT = "Not Found"
-
-        private const val TITLE_THUMBNAIL_PATH = "title_thumbnail_portrait_list"
-
         const val PREFIX_ID_SEARCH = "id:"
-        private val ID_SEARCH_PATTERN = "^id:(\\d+)$".toRegex()
         const val PREFIX_CHAPTER_ID_SEARCH = "chapter-id:"
-        private val CHAPTER_ID_SEARCH_PATTERN = "^chapter-id:(\\d+)$".toRegex()
     }
 }
+
+private const val API_URL = "https://jumpg-webapi.tokyo-cdn.com/api"
+private const val APP_API_URL = "https://jumpg-api.tokyo-cdn.com/api"
+private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
+
+private const val LISTING_ITEMS_PER_PAGE = 20
+
+private const val QUALITY_PREF_KEY = "imageResolution"
+private val QUALITY_PREF_ENTRY_VALUES = arrayOf("low", "high", "super_high")
+private val QUALITY_PREF_DEFAULT_VALUE = QUALITY_PREF_ENTRY_VALUES[2]
+
+private const val SPLIT_PREF_KEY = "splitImage"
+private const val SPLIT_PREF_DEFAULT_VALUE = true
+
+private const val VER_PREF_KEY = "appVer"
+private const val VER_PREF_DEFAULT_VALUE = ""
+
+private const val SECRET_PREF_KEY = "accountSecret"
+private const val SECRET_PREF_DEFAULT_VALUE = ""

src/all/mangaplus/src/eu/kanade/tachiyomi/extension/all/mangaplus/MangaPlusDto.kt

@@ -7,54 +7,68 @@ import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
 @Serializable
-data class MangaPlusResponse(
+class MangaPlusResponse(
     val success: SuccessResult? = null,
     val error: ErrorResult? = null,
 )
 
 @Serializable
-data class ErrorResult(val popups: List<Popup> = emptyList()) {
+class ErrorResult(val popups: List<Popup> = emptyList()) {
 
     fun langPopup(lang: Language): Popup? =
         popups.firstOrNull { it.language == lang }
 }
 
 @Serializable
-data class Popup(
+class Popup(
     val subject: String,
     val body: String,
     val language: Language? = Language.ENGLISH,
 )
 
 @Serializable
-data class SuccessResult(
+class SuccessResult(
     val isFeaturedUpdated: Boolean? = false,
-    val titleRankingView: TitleRankingView? = null,
+    val titleRankingViewV2: TitleRankingViewV2? = null,
     val titleDetailView: TitleDetailView? = null,
     val mangaViewer: MangaViewer? = null,
     val allTitlesViewV2: AllTitlesViewV2? = null,
-    val webHomeViewV3: WebHomeViewV3? = null,
+    val webHomeViewV4: WebHomeViewV4? = null,
 )
 
 @Serializable
-data class TitleRankingView(val titles: List<Title> = emptyList())
+class TitleRankingViewV2(val rankedTitles: List<RankedTitle> = emptyList())
 
 @Serializable
-data class AllTitlesViewV2(
+class RankedTitle(
+    val titles: List<Title> = emptyList(),
+)
+
+@Serializable
+class AllTitlesViewV2(
     @SerialName("AllTitlesGroup") val allTitlesGroup: List<AllTitlesGroup> = emptyList(),
 )
 
 @Serializable
-data class AllTitlesGroup(
+class AllTitlesGroup(
     val theTitle: String,
     val titles: List<Title> = emptyList(),
 )
 
 @Serializable
-data class WebHomeViewV3(val groups: List<UpdatedTitleV2Group> = emptyList())
+class WebHomeViewV4(
+    val groups: List<UpdatedTitleV2Group> = emptyList(),
+    val rankedTitles: List<RankedTitle> = emptyList(),
+    val featuredTitleLists: List<FeaturedTitleList> = emptyList(),
+)
 
 @Serializable
-data class TitleDetailView(
+class FeaturedTitleList(
+    val featuredTitles: List<Title> = emptyList(),
+)
+
+@Serializable
+class TitleDetailView(
     val title: Title,
     val titleImageUrl: String,
     val overview: String? = null,
@@ -155,7 +169,7 @@ data class TitleDetailView(
 }
 
 @Serializable
-data class TitleLabels(
+class TitleLabels(
     val releaseSchedule: ReleaseSchedule = ReleaseSchedule.DISABLED,
     val isSimulpub: Boolean = false,
     val planType: String = "standard",
@@ -185,7 +199,7 @@ enum class Rating {
 }
 
 @Serializable
-data class Label(val label: LabelCode? = LabelCode.WEEKLY_SHOUNEN_JUMP) {
+class Label(val label: LabelCode? = LabelCode.WEEKLY_SHOUNEN_JUMP) {
     val magazine: String?
         get() = when (label) {
             LabelCode.WEEKLY_SHOUNEN_JUMP -> "Weekly Shounen Jump"
@@ -236,21 +250,21 @@ enum class LabelCode {
 }
 
 @Serializable
-data class ChapterListGroup(
+class ChapterListGroup(
     val firstChapterList: List<Chapter> = emptyList(),
     val midChapterList: List<Chapter> = emptyList(),
     val lastChapterList: List<Chapter> = emptyList(),
 )
 
 @Serializable
-data class MangaViewer(
+class MangaViewer(
     val pages: List<MangaPlusPage> = emptyList(),
     val titleId: Int? = null,
     val titleName: String? = null,
 )
 
 @Serializable
-data class Title(
+class Title(
     val titleId: Int,
     val name: String,
     val author: String? = null,
@@ -280,22 +294,22 @@ enum class Language {
 }
 
 @Serializable
-data class UpdatedTitleV2Group(
+class UpdatedTitleV2Group(
     val groupName: String,
     val titleGroups: List<OriginalTitleGroup> = emptyList(),
 )
 
 @Serializable
-data class OriginalTitleGroup(
+class OriginalTitleGroup(
     val theTitle: String,
     val titles: List<UpdatedTitle> = emptyList(),
 )
 
 @Serializable
-data class UpdatedTitle(val title: Title)
+class UpdatedTitle(val title: Title)
 
 @Serializable
-data class Chapter(
+class Chapter(
     val titleId: Int,
     val chapterId: Int,
     val name: String,
@@ -317,10 +331,10 @@ data class Chapter(
 }
 
 @Serializable
-data class MangaPlusPage(val mangaPage: MangaPage? = null)
+class MangaPlusPage(val mangaPage: MangaPage? = null)
 
 @Serializable
-data class MangaPage(
+class MangaPage(
     val imageUrl: String,
     val width: Int,
     val height: Int,