Nekopost extension: Search fix (HTTP error 520) (#18931)

* format the code and change the throw exception

* fix nekopost search

* switch to HttpSource
use request/parse instead of fetch

* Remove commented code.

src/th/nekopost/build.gradle

@@ -6,7 +6,7 @@ ext {
     extName = 'Nekopost'
     pkgNameSuffix = 'th.nekopost'
     extClass = '.Nekopost'
-    extVersionCode = 9
+    extVersionCode = 10
     isNsfw = true
 }
 

src/th/nekopost/src/eu/kanade/tachiyomi/extension/th/nekopost/Nekopost.kt

@@ -2,38 +2,35 @@ package eu.kanade.tachiyomi.extension.th.nekopost
 
 import eu.kanade.tachiyomi.extension.th.nekopost.model.RawChapterInfo
 import eu.kanade.tachiyomi.extension.th.nekopost.model.RawProjectInfo
-import eu.kanade.tachiyomi.extension.th.nekopost.model.RawProjectNameListItem
+import eu.kanade.tachiyomi.extension.th.nekopost.model.RawProjectSearchSummary
 import eu.kanade.tachiyomi.extension.th.nekopost.model.RawProjectSummaryList
 import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.asObservableSuccess
+import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.source.model.FilterList
 import eu.kanade.tachiyomi.source.model.MangasPage
 import eu.kanade.tachiyomi.source.model.Page
 import eu.kanade.tachiyomi.source.model.SChapter
 import eu.kanade.tachiyomi.source.model.SManga
-import eu.kanade.tachiyomi.source.online.ParsedHttpSource
+import eu.kanade.tachiyomi.source.online.HttpSource
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
 import okhttp3.Headers
 import okhttp3.OkHttpClient
 import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
 import okhttp3.Response
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
-import rx.Observable
 import uy.kohesive.injekt.injectLazy
 import java.text.SimpleDateFormat
 import java.util.Locale
 
-class Nekopost : ParsedHttpSource() {
+class Nekopost : HttpSource() {
     private val json: Json by injectLazy()
     override val baseUrl: String = "https://www.nekopost.net/manga/"
 
-    private val latestMangaEndpoint: String =
+    private val latestMangaEndpoint: String = "https://api.osemocphoto.com/frontAPI/getLatestChapter/m"
-        "https://api.osemocphoto.com/frontAPI/getLatestChapter/m"
+    private val projectDataEndpoint: String = "https://api.osemocphoto.com/frontAPI/getProjectInfo"
-    private val projectDataEndpoint: String =
-        "https://api.osemocphoto.com/frontAPI/getProjectInfo"
     private val fileHost: String = "https://www.osemocphoto.com"
+    private val nekopostUrl = "https://www.nekopost.net"
 
     override val client: OkHttpClient = network.cloudflareClient
 
@@ -57,118 +54,96 @@ class Nekopost : ParsedHttpSource() {
         else -> SManga.UNKNOWN
     }
 
-    override fun latestUpdatesRequest(page: Int): Request = throw NotImplementedError("Unused")
+    override fun latestUpdatesRequest(page: Int): Request = throw UnsupportedOperationException("Not used.")
 
-    override fun latestUpdatesParse(response: Response): MangasPage =
+    override fun latestUpdatesParse(response: Response): MangasPage = throw UnsupportedOperationException("Not used.")
-        throw NotImplementedError("Unused")
 
-    override fun chapterListSelector(): String = throw NotImplementedError("Unused")
+    override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException("Not used.")
 
-    override fun chapterFromElement(element: Element): SChapter =
+    override fun imageUrlRequest(page: Page): Request = throw UnsupportedOperationException("Not used.")
-        throw NotImplementedError("Unused")
 
-    override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl)
+    override fun mangaDetailsRequest(manga: SManga): Request {
-
+        return GET("$projectDataEndpoint/${manga.url}", headers)
-    override fun imageUrlParse(document: Document): String = throw NotImplementedError("Unused")
-
-    override fun latestUpdatesFromElement(element: Element): SManga = throw Exception("Unused")
-
-    override fun latestUpdatesNextPageSelector(): String = throw Exception("Unused")
-
-    override fun latestUpdatesSelector(): String = throw Exception("Unused")
-
-    override fun mangaDetailsParse(document: Document): SManga = throw NotImplementedError("Unused")
-
-    override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
-        return client.newCall(GET("$projectDataEndpoint/${manga.url}", headers))
-            .asObservableSuccess()
-            .map { response ->
-                val responseBody = response.body
-                val projectInfo: RawProjectInfo = json.decodeFromString(responseBody.string())
-
-                manga.apply {
-                    projectInfo.projectInfo.let {
-                        url = it.projectId
-                        title = it.projectName
-                        artist = it.artistName
-                        author = it.authorName
-                        description = it.info
-                        status = getStatus(it.status)
-                        thumbnail_url =
-                            "$fileHost/collectManga/${it.projectId}/${it.projectId}_cover.jpg"
-                        initialized = true
-                    }
-
-                    genre = if (projectInfo.projectCategoryUsed != null) {
-                        projectInfo.projectCategoryUsed.joinToString(", ") { it.categoryName }
-                    } else {
-                        ""
-                    }
-                }
-            }
     }
 
-    override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
+    override fun mangaDetailsParse(response: Response): SManga {
-        return if (manga.status != SManga.LICENSED) {
+        val responseBody = response.body
-            client.newCall(GET("$projectDataEndpoint/${manga.url}", headers))
+        val projectInfo: RawProjectInfo = json.decodeFromString(responseBody.string())
-                .asObservableSuccess()
+        val manga = SManga.create()
-                .map { response ->
+        manga.apply {
-                    val responseBody = response.body
+            projectInfo.projectInfo.let {
-                    val projectInfo: RawProjectInfo = json.decodeFromString(responseBody.string())
+                url = it.projectId
+                title = it.projectName
+                artist = it.artistName
+                author = it.authorName
+                description = it.info
+                status = getStatus(it.status)
+                thumbnail_url = "$fileHost/collectManga/${it.projectId}/${it.projectId}_cover.jpg"
+                initialized = true
+            }
 
-                    manga.status = getStatus(projectInfo.projectInfo.status)
+            genre = if (projectInfo.projectCategoryUsed != null) {
+                projectInfo.projectCategoryUsed.joinToString(", ") { it.categoryName }
+            } else {
+                ""
+            }
+        }
+        return manga
+    }
 
-                    if (manga.status == SManga.LICENSED) {
+    override fun chapterListRequest(manga: SManga): Request {
-                        throw Exception("Licensed - No chapter to show")
+        val headers = Headers.headersOf("accept", "*/*", "content-type", "text/plain;charset=UTF-8", "origin", nekopostUrl)
-                    }
+        return GET("$projectDataEndpoint/${manga.url}", headers)
+    }
 
-                    projectInfo.projectChapterList!!.map { chapter ->
+    override fun chapterListParse(response: Response): List<SChapter> {
-                        SChapter.create().apply {
+        val responseBody = response.body.string()
-                            url =
+        val projectInfo: RawProjectInfo = json.decodeFromString(responseBody)
-                                "${manga.url}/${chapter.chapterId}/${manga.url}_${chapter.chapterId}.json"
+        val manga = SManga.create()
-                            name = chapter.chapterName
+        manga.status = getStatus(projectInfo.projectInfo.status)
-                            date_upload = SimpleDateFormat(
+
-                                "yyyy-MM-dd HH:mm:ss",
+        if (manga.status == SManga.LICENSED) {
-                                Locale("th"),
+            throw Exception("Licensed - No chapter to show")
-                            ).parse(chapter.createDate)?.time
+        }
-                                ?: 0L
+
-                            chapter_number = chapter.chapterNo.toFloat()
+        return projectInfo.projectChapterList!!.map { chapter ->
-                            scanlator = chapter.providerName
+            SChapter.create().apply {
-                        }
+                url = "${projectInfo.projectInfo.projectId.toInt()}/${chapter.chapterId}/${projectInfo.projectInfo.projectId.toInt()}_${chapter.chapterId}.json"
-                    }
+                name = chapter.chapterName
-                }
+                date_upload = SimpleDateFormat(
-        } else {
+                    "yyyy-MM-dd HH:mm:ss",
-            Observable.error(Exception("Licensed - No chapter to show"))
+                    Locale("th"),
+                ).parse(chapter.createDate)?.time ?: 0L
+                chapter_number = chapter.chapterNo.toFloat()
+                scanlator = chapter.providerName
+            }
         }
     }
 
-    override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
+    override fun pageListRequest(chapter: SChapter): Request {
-        return client.newCall(GET("$fileHost/collectManga/${chapter.url}", headers))
+        return GET("$fileHost/collectManga/${chapter.url}", headers)
-            .asObservableSuccess()
-            .map { response ->
-                val responseBody = response.body
-                val chapterInfo: RawChapterInfo = json.decodeFromString(responseBody.string())
-
-                chapterInfo.pageItem.map { page ->
-                    val imgUrl: String = if (page.pageName != null) {
-                        "$fileHost/collectManga/${chapterInfo.projectId}/${chapterInfo.chapterId}/${page.pageName}"
-                    } else {
-                        "$fileHost/collectManga/${chapterInfo.projectId}/${chapterInfo.chapterId}/${page.fileName}"
-                    }
-                    Page(
-                        index = page.pageNo,
-                        imageUrl = imgUrl,
-                    )
-                }
-            }
     }
 
-    override fun pageListParse(document: Document): List<Page> = throw NotImplementedError("Unused")
+    override fun pageListParse(response: Response): List<Page> {
+        val responseBody = response.body
+        val chapterInfo: RawChapterInfo = json.decodeFromString(responseBody.string())
 
+        return chapterInfo.pageItem.map { page ->
+            val imgUrl: String = if (page.pageName != null) {
+                "$fileHost/collectManga/${chapterInfo.projectId}/${chapterInfo.chapterId}/${page.pageName}"
+            } else {
+                "$fileHost/collectManga/${chapterInfo.projectId}/${chapterInfo.chapterId}/${page.fileName}"
+            }
+            Page(
+                index = page.pageNo,
+                imageUrl = imgUrl,
+            )
+        }
+    }
     override fun popularMangaRequest(page: Int): Request {
         if (page <= 1) existingProject.clear()
         // API has a bug that sometime it returns null on first page
-        return GET("$latestMangaEndpoint/${if (firstPageNulled) page else page - 1 }", headers)
+        return GET("$latestMangaEndpoint/${if (firstPageNulled) page else page - 1}", headers)
     }
 
     override fun popularMangaParse(response: Response): MangasPage {
@@ -176,18 +151,15 @@ class Nekopost : ParsedHttpSource() {
         val projectList: RawProjectSummaryList = json.decodeFromString(responseBody.string())
 
         val mangaList: List<SManga> = if (projectList.listChapter != null) {
-            projectList.listChapter
+            projectList.listChapter.filter { !existingProject.contains(it.projectId) }.map {
-                .filter { !existingProject.contains(it.projectId) }
+                SManga.create().apply {
-                .map {
+                    url = it.projectId
-                    SManga.create().apply {
+                    title = it.projectName
-                        url = it.projectId
+                    thumbnail_url = "$fileHost/collectManga/${it.projectId}/${it.projectId}_cover.jpg"
-                        title = it.projectName
+                    initialized = false
-                        thumbnail_url =
+                    status = 0
-                            "$fileHost/collectManga/${it.projectId}/${it.projectId}_cover.jpg"
-                        initialized = false
-                        status = 0
-                    }
                 }
+            }
         } else {
             firstPageNulled = true // API has a bug that sometime it returns null on first page
             return MangasPage(emptyList(), hasNextPage = false)
@@ -198,51 +170,27 @@ class Nekopost : ParsedHttpSource() {
         return MangasPage(mangaList, hasNextPage = true)
     }
 
-    override fun popularMangaFromElement(element: Element): SManga =
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
-        throw NotImplementedError("Unused")
+        val headers = Headers.headersOf("accept", "*/*", "content-type", "text/plain;charset=UTF-8", "origin", nekopostUrl)
-
+        val requestBody = "{\"keyword\":\"$query\"}".toRequestBody()
-    override fun popularMangaNextPageSelector(): String = throw Exception("Unused")
+        return POST("$nekopostUrl/api/explore/search", headers, requestBody)
-
-    override fun popularMangaSelector(): String = throw Exception("Unused")
-
-    override fun searchMangaFromElement(element: Element): SManga = throw Exception("Unused")
-
-    override fun searchMangaNextPageSelector(): String = throw Exception("Unused")
-
-    override fun fetchSearchManga(
-        page: Int,
-        query: String,
-        filters: FilterList,
-    ): Observable<MangasPage> {
-        return client.newCall(GET("$fileHost/dataJson/dataProjectName.json"))
-            .asObservableSuccess()
-            .map { response ->
-                val responseBody = response.body
-                val projectList: List<RawProjectNameListItem> =
-                    json.decodeFromString(responseBody.string())
-
-                val mangaList: List<SManga> = projectList.filter { project ->
-                    Regex(
-                        query,
-                        setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE),
-                    ).find(project.npName) != null
-                }.map { project ->
-                    SManga.create().apply {
-                        url = project.npProjectId
-                        title = project.npName
-                        status = getStatus(project.npStatus)
-                        initialized = false
-                    }
-                }
-
-                MangasPage(mangaList, false)
-            }
     }
 
-    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request =
+    override fun searchMangaParse(response: Response): MangasPage {
-        throw Exception("Unused")
+        val responseBody = response.body.string()
 
-    override fun searchMangaParse(response: Response): MangasPage = throw Exception("Unused")
+        val projectList: List<RawProjectSearchSummary> = json.decodeFromString(responseBody)
+        val mangaList: List<SManga> = projectList.filter { project ->
+            project.projectType == "m"
+        }.map { project ->
+            SManga.create().apply {
+                url = project.projectId.toString()
+                title = project.projectName
+                status = project.status
+                initialized = false
+            }
+        }
 
-    override fun searchMangaSelector(): String = throw Exception("Unused")
+        return MangasPage(mangaList, false)
+    }
 }

src/th/nekopost/src/eu/kanade/tachiyomi/extension/th/nekopost/model/RawProjectSearchSummary.kt

@@ -0,0 +1,19 @@
+package eu.kanade.tachiyomi.extension.th.nekopost.model
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class RawProjectSearchSummary(
+    val projectId: Int,
+    val projectName: String,
+    val projectType: String,
+    @SerialName("STATUS")
+    val status: Int,
+    val noChapter: Int,
+    val coverVersion: Int,
+    val info: String,
+    val views: Int,
+    @SerialName("lastUpdate")
+    val lastUpdateDate: String,
+)