[TH] Nekopost - Fix unable to read saved manga (#9459)

* Fix not found error and add all api types

* Update build.gradle

* Disable latest feature and use popular as latest instead

* Add logic to prevent duplicate projects

* Clear fetched list each time fetching from page 1

* Fix search logic

* Fix invalid genre joining
This commit is contained in:
Sittikorn Hirunpongthawat 2021-10-13 19:19:29 +07:00 committed by GitHub
parent cc792ccfb1
commit 9badbeb0db
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 295 additions and 319 deletions

View File

@ -5,7 +5,8 @@ ext {
extName = 'Nekopost'
pkgNameSuffix = 'th.nekopost'
extClass = '.Nekopost'
extVersionCode = 4
extVersionCode = 5
isNsfw = true
}
apply from: "$rootDir/common.gradle"

View File

@ -1,153 +0,0 @@
package eu.kanade.tachiyomi.extension.th.nekopost
data class RawMangaData(
val no_new_chapter: String,
val nc_chapter_id: String,
val np_project_id: String,
val np_name: String,
val np_name_link: String,
val nc_chapter_no: String,
val nc_chapter_name: String,
val nc_chapter_cover: String,
val nc_provider: String,
val np_group_dir: String,
val nc_created_date: String,
)
data class RawMangaDataList(
val code: String,
val listItem: Array<RawMangaData>?
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as RawMangaDataList
if (code != other.code) return false
if (listItem != null) {
if (other.listItem == null) return false
if (!listItem.contentEquals(other.listItem)) return false
} else if (other.listItem != null) return false
return true
}
override fun hashCode(): Int {
var result = code.hashCode()
result = 31 * result + (listItem?.contentHashCode() ?: 0)
return result
}
}
data class RawProjectData(
val np_status: String,
val np_project_id: String,
val np_type: String,
val np_name: String,
val np_name_link: String,
val np_flag_mature: String,
val np_info: String,
val np_view: String,
val np_comment: String,
val np_created_date: String,
val np_updated_date: String,
val author_name: String,
val artist_name: String,
val np_web: String,
val np_licenced_by: String,
)
data class RawProjectGenre(
val npc_name: String,
val npc_name_link: String,
)
data class RawChapterData(
val nc_chapter_id: String,
val nc_chapter_no: String,
val nc_chapter_name: String,
val nc_provider: String,
val cu_displayname: String,
val nc_created_date: String,
val nc_data_file: String,
val nc_owner_id: String,
)
data class RawMangaDetailedData(
val code: String,
val projectInfo: RawProjectData,
val projectCategoryUsed: Array<RawProjectGenre>?,
val projectChapterList: Array<RawChapterData>,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as RawMangaDetailedData
if (code != other.code) return false
if (projectInfo != other.projectInfo) return false
if (projectCategoryUsed != null) {
if (other.projectCategoryUsed == null) return false
if (!projectCategoryUsed.contentEquals(other.projectCategoryUsed)) return false
} else if (other.projectCategoryUsed != null) return false
if (!projectChapterList.contentEquals(other.projectChapterList)) return false
return true
}
override fun hashCode(): Int {
var result = code.hashCode()
result = 31 * result + projectInfo.hashCode()
result = 31 * result + (projectCategoryUsed?.contentHashCode() ?: 0)
result = 31 * result + projectChapterList.contentHashCode()
return result
}
}
data class RawPageData(
val pageNo: Int,
val fileName: String,
val width: Int,
val height: Int,
val pageCount: Int
)
data class RawChapterDetailedData(
val projectId: String,
val chapterId: Int,
val chapterNo: String,
val pageItem: Array<RawPageData>,
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as RawChapterDetailedData
if (projectId != other.projectId) return false
if (chapterId != other.chapterId) return false
if (chapterNo != other.chapterNo) return false
if (!pageItem.contentEquals(other.pageItem)) return false
return true
}
override fun hashCode(): Int {
var result = projectId.hashCode()
result = 31 * result + chapterId
result = 31 * result + chapterNo.hashCode()
result = 31 * result + pageItem.contentHashCode()
return result
}
}
data class MangaNameList(
val np_project_id: String,
val np_name: String,
val np_name_link: String,
val np_type: String,
val np_status: String,
val np_no_chapter: String,
)

View File

@ -1,6 +1,10 @@
package eu.kanade.tachiyomi.extension.th.nekopost
import com.google.gson.Gson
import eu.kanade.tachiyomi.extension.th.nekopost.model.RawChapterInfo
import eu.kanade.tachiyomi.extension.th.nekopost.model.RawProjectInfo
import eu.kanade.tachiyomi.extension.th.nekopost.model.RawProjectNameList
import eu.kanade.tachiyomi.extension.th.nekopost.model.RawProjectSummaryList
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.FilterList
@ -20,11 +24,14 @@ import java.text.SimpleDateFormat
import java.util.Locale
class Nekopost : ParsedHttpSource() {
override val baseUrl: String = "https://www.nekopost.net/manga/"
private val gson: Gson = Gson()
override val baseUrl: String = "https://www.nekopost.net/project"
private val mangaListUrl: String = "https://tuner.nekopost.net/ApiTest/getLatestChapterOffset/m/"
private val projectDataUrl: String = "https://tuner.nekopost.net/ApiTest/getProjectDetailFull/"
private val fileUrl: String = "https://fs.nekopost.net/"
private val latestMangaEndpoint: String =
"https://tuner.nekopost.net/ApiTest/getLatestChapterOffset/m"
private val projectDataEndpoint: String =
"https://tuner.nekopost.net/ApiTest/getProjectDetailFull"
private val fileHost: String = "https://fs.nekopost.net"
override val client: OkHttpClient = network.cloudflareClient
@ -32,34 +39,12 @@ class Nekopost : ParsedHttpSource() {
return super.headersBuilder().add("Referer", baseUrl)
}
private val existingProject: HashSet<String> = HashSet()
override val lang: String = "th"
override val name: String = "Nekopost"
override val supportsLatest: Boolean = true
private data class MangaListTracker(
var offset: Int = 0,
val list: HashSet<String> = HashSet()
)
private var latestMangaTracker = MangaListTracker()
private var popularMangaTracker = MangaListTracker()
data class ProjectRecord(
val project: SManga,
val project_id: String,
val chapter_list: HashSet<String> = HashSet(),
)
data class ChapterRecord(
val chapter: SChapter,
val chapter_id: String,
val project: ProjectRecord,
val pages_data: String,
)
private var projectUrlMap = HashMap<String, ProjectRecord>()
private var chapterList = HashMap<String, ChapterRecord>()
override val supportsLatest: Boolean = false
private fun getStatus(status: String) = when (status) {
"1" -> SManga.ONGOING
@ -68,132 +53,76 @@ class Nekopost : ParsedHttpSource() {
else -> SManga.UNKNOWN
}
private fun fetchMangas(page: Int, tracker: MangaListTracker): Observable<MangasPage> {
if (page == 1) {
tracker.list.clear()
tracker.offset = 0
}
override fun latestUpdatesRequest(page: Int): Request = throw NotImplementedError("Unused")
return client.newCall(latestUpdatesRequest(page + tracker.offset))
.asObservableSuccess()
.concatMap { response ->
latestUpdatesParse(response).let {
if (it.mangas.isEmpty() && it.hasNextPage) {
tracker.offset++
fetchLatestUpdates(page)
} else {
Observable.just(it)
}
}
}
}
private fun mangasRequest(page: Int): Request = GET("$mangaListUrl${page - 1}")
private fun mangasParse(response: Response, tracker: MangaListTracker): MangasPage {
val mangaData = Gson().fromJson(response.body!!.string(), RawMangaDataList::class.java)
return if (mangaData.listItem != null) {
val mangas: List<SManga> = mangaData.listItem.filter {
!tracker.list.contains(it.np_project_id)
}.map {
tracker.list.add(it.np_project_id)
SManga.create().apply {
url = it.np_project_id
title = it.np_name
thumbnail_url = "${fileUrl}collectManga/${it.np_project_id}/${it.np_project_id}_cover.jpg"
initialized = false
projectUrlMap[it.np_project_id] = ProjectRecord(
project = this,
project_id = it.np_project_id
)
}
}
MangasPage(mangas, true)
} else {
MangasPage(emptyList(), true)
}
}
override fun latestUpdatesParse(response: Response): MangasPage =
throw NotImplementedError("Unused")
override fun chapterListSelector(): String = throw NotImplementedError("Unused")
override fun chapterFromElement(element: Element): SChapter = throw NotImplementedError("Unused")
override fun chapterFromElement(element: Element): SChapter =
throw NotImplementedError("Unused")
override fun fetchImageUrl(page: Page): Observable<String> = Observable.just(page.imageUrl)
override fun imageUrlParse(document: Document): String = throw NotImplementedError("Unused")
override fun fetchLatestUpdates(page: Int): Observable<MangasPage> = fetchMangas(page, latestMangaTracker)
override fun latestUpdatesParse(response: Response): MangasPage = mangasParse(response, latestMangaTracker)
override fun latestUpdatesFromElement(element: Element): SManga = throw Exception("Unused")
override fun latestUpdatesNextPageSelector(): String = throw Exception("Unused")
override fun latestUpdatesRequest(page: Int): Request = mangasRequest(page)
override fun latestUpdatesSelector(): String = throw Exception("Unused")
override fun mangaDetailsParse(document: Document): SManga = throw NotImplementedError("Unused")
override fun fetchMangaDetails(sManga: SManga): Observable<SManga> {
val manga = projectUrlMap[sManga.url]!!
return client.newCall(GET("$projectDataUrl${manga.project_id}"))
override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
return client.newCall(GET("$projectDataEndpoint/${manga.url}"))
.asObservableSuccess()
.concatMap {
val mangaData = Gson().fromJson(it.body!!.string(), RawMangaDetailedData::class.java)
.map { response ->
val responseBody =
response.body ?: throw Error("Unable to fetch manga detail of ${manga.title}")
val projectInfo = gson.fromJson(responseBody.string(), RawProjectInfo::class.java)
Observable.just(
manga.project.apply {
mangaData.projectInfo.also { projectData ->
artist = projectData.artist_name
author = projectData.author_name
description = projectData.np_info
status = getStatus(projectData.np_status)
initialized = true
}
genre = mangaData.projectCategoryUsed?.joinToString(", ") { cat -> cat.npc_name }
?: ""
manga.apply {
projectInfo.projectData.let {
url = it.npProjectId
title = it.npName
artist = it.artistName
author = it.authorName
description = it.npInfo
status = getStatus(it.npStatus)
initialized = true
}
)
genre =
projectInfo.projectCategoryUsed.map { it.npcName }.joinToString(", ")
}
}
}
override fun fetchChapterList(sManga: SManga): Observable<List<SChapter>> {
val manga = projectUrlMap[sManga.url]!!
return if (manga.project.status != SManga.LICENSED) {
client.newCall(GET("$projectDataUrl${manga.project_id}"))
override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
return if (manga.status != SManga.LICENSED) {
client.newCall(GET("$projectDataEndpoint/${manga.url}"))
.asObservableSuccess()
.map {
val mangaData = Gson().fromJson(it.body!!.string(), RawMangaDetailedData::class.java)
.map { response ->
val responseBody =
response.body
?: throw Error("Unable to fetch manga detail of ${manga.title}")
val projectInfo =
gson.fromJson(responseBody.string(), RawProjectInfo::class.java)
mangaData.projectChapterList.map { chapter ->
val chapterUrl = "$baseUrl${manga.project_id}/${chapter.nc_chapter_no}"
manga.chapter_list.add(chapterUrl)
val createdChapter = SChapter.create().apply {
url = chapterUrl
name = chapter.nc_chapter_name
date_upload = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale("th")).parse(chapter.nc_created_date)?.time
projectInfo.projectChapterList.map { chapter ->
SChapter.create().apply {
url = "${manga.url}/${chapter.ncChapterId}/${chapter.ncDataFile}"
name = chapter.ncChapterName
date_upload = SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss",
Locale("th")
).parse(chapter.ncCreatedDate)?.time
?: 0L
chapter_number = chapter.nc_chapter_no.toFloat()
scanlator = chapter.cu_displayname
chapter_number = chapter.ncChapterNo.toFloat()
scanlator = chapter.cuDisplayname
}
chapterList[chapterUrl] = ChapterRecord(
chapter = createdChapter,
project = manga,
chapter_id = chapter.nc_chapter_id,
pages_data = chapter.nc_data_file,
)
createdChapter
}
}
} else {
@ -201,18 +130,20 @@ class Nekopost : ParsedHttpSource() {
}
}
override fun fetchPageList(sChapter: SChapter): Observable<List<Page>> {
val chapter = chapterList[sChapter.url]!!
return client.newCall(GET("${fileUrl}collectManga/${chapter.project.project_id}/${chapter.chapter_id}/${chapter.pages_data}"))
override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
return client.newCall(GET("$fileHost/collectManga/${chapter.url}"))
.asObservableSuccess()
.map {
val chapterData = Gson().fromJson(it.body!!.string(), RawChapterDetailedData::class.java)
.map { response ->
val responseBody =
response.body
?: throw Error("Unable to fetch page list of chapter ${chapter.chapter_number}")
val chapterInfo =
gson.fromJson(responseBody.string(), RawChapterInfo::class.java)
chapterData.pageItem.map { pageData ->
chapterInfo.pageItem.map { page ->
Page(
index = pageData.pageNo,
imageUrl = "${fileUrl}collectManga/${chapter.project.project_id}/${chapter.chapter_id}/${pageData.fileName}",
index = page.pageNo,
imageUrl = "$fileHost/collectManga/${chapterInfo.projectId}/${chapterInfo.chapterId}/${page.fileName}",
)
}
}
@ -220,52 +151,78 @@ class Nekopost : ParsedHttpSource() {
override fun pageListParse(document: Document): List<Page> = throw NotImplementedError("Unused")
override fun fetchPopularManga(page: Int): Observable<MangasPage> = fetchMangas(page, popularMangaTracker)
override fun popularMangaRequest(page: Int): Request {
if (page <= 1) existingProject.clear()
override fun popularMangaParse(response: Response): MangasPage = mangasParse(response, popularMangaTracker)
return GET("$latestMangaEndpoint/${page - 1}")
}
override fun popularMangaFromElement(element: Element): SManga = throw NotImplementedError("Unused")
override fun popularMangaParse(response: Response): MangasPage {
val responseBody = response.body ?: throw Error("Unable to fetch mangas")
val projectList = gson.fromJson(responseBody.string(), RawProjectSummaryList::class.java)
val mangaList: List<SManga> =
projectList.listItem
?.filter { !existingProject.contains(it.npProjectId) }
?.map {
SManga.create().apply {
url = it.npProjectId
title = it.npName
thumbnail_url =
"$fileHost/collectManga/${it.npProjectId}/${it.npProjectId}_cover.jpg"
initialized = false
status = 0
}
} ?: return MangasPage(emptyList(), hasNextPage = false)
mangaList.forEach { existingProject.add(it.url) }
return MangasPage(mangaList, hasNextPage = true)
}
override fun popularMangaFromElement(element: Element): SManga =
throw NotImplementedError("Unused")
override fun popularMangaNextPageSelector(): String = throw Exception("Unused")
override fun popularMangaRequest(page: Int): Request = mangasRequest(page)
override fun popularMangaSelector(): String = throw Exception("Unused")
override fun searchMangaFromElement(element: Element): SManga = throw Exception("Unused")
override fun searchMangaNextPageSelector(): String = throw Exception("Unused")
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
return client.newCall(GET("${fileUrl}dataJson/dataProjectName.json"))
override fun fetchSearchManga(
page: Int,
query: String,
filters: FilterList
): Observable<MangasPage> {
return client.newCall(GET("$fileHost/dataJson/dataProjectName.json"))
.asObservableSuccess()
.map {
val nameData = Gson().fromJson(it.body!!.string(), Array<MangaNameList>::class.java)
.map { response ->
val responseBody = response.body ?: throw Error("Unable to fetch title list")
val projectList =
gson.fromJson(responseBody.string(), RawProjectNameList::class.java)
val mangas: List<SManga> = nameData.filter { d -> Regex(query, setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE)).find(d.np_name) != null }
.map { matchedManga ->
if (!projectUrlMap.containsKey(matchedManga.np_project_id)) {
SManga.create().apply {
url = matchedManga.np_project_id
title = matchedManga.np_name
thumbnail_url = "${fileUrl}collectManga/${matchedManga.np_project_id}/${matchedManga.np_project_id}_cover.jpg"
initialized = false
projectUrlMap[matchedManga.np_project_id] = ProjectRecord(
project = this,
project_id = matchedManga.np_project_id
)
}
} else {
projectUrlMap[matchedManga.np_project_id]!!.project
}
val mangaList: List<SManga> = projectList.filter { project ->
Regex(
query,
setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE)
).find(project.npName) != null
}.map { project ->
SManga.create().apply {
url = project.npProjectId
title = project.npName
status = getStatus(project.npStatus)
initialized = false
}
}
MangasPage(mangas, true)
MangasPage(mangaList, false)
}
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = throw Exception("Unused")
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request =
throw Exception("Unused")
override fun searchMangaParse(response: Response): MangasPage = throw Exception("Unused")

View File

@ -0,0 +1,16 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawChapterInfo(
@SerializedName("chapterId")
val chapterId: Int,
@SerializedName("chapterNo")
val chapterNo: String,
@SerializedName("pageCount")
val pageCount: Int,
@SerializedName("pageItem")
val pageItem: List<RawPageItem>,
@SerializedName("projectId")
val projectId: String
)

View File

@ -0,0 +1,14 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawPageItem(
@SerializedName("fileName")
val fileName: String,
@SerializedName("height")
val height: Int,
@SerializedName("pageNo")
val pageNo: Int,
@SerializedName("width")
val width: Int
)

View File

@ -0,0 +1,10 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawProjectCategory(
@SerializedName("npc_name")
val npcName: String,
@SerializedName("npc_name_link")
val npcNameLink: String
)

View File

@ -0,0 +1,22 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawProjectChapter(
@SerializedName("cu_displayname")
val cuDisplayname: String,
@SerializedName("nc_chapter_id")
val ncChapterId: String,
@SerializedName("nc_chapter_name")
val ncChapterName: String,
@SerializedName("nc_chapter_no")
val ncChapterNo: String,
@SerializedName("nc_created_date")
val ncCreatedDate: String,
@SerializedName("nc_data_file")
val ncDataFile: String,
@SerializedName("nc_owner_id")
val ncOwnerId: String,
@SerializedName("nc_provider")
val ncProvider: String
)

View File

@ -0,0 +1,14 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawProjectInfo(
@SerializedName("code")
val code: String,
@SerializedName("projectCategoryUsed")
val projectCategoryUsed: List<RawProjectCategory>,
@SerializedName("projectChapterList")
val projectChapterList: List<RawProjectChapter>,
@SerializedName("projectInfo")
val projectData: RawProjectInfoData
)

View File

@ -0,0 +1,36 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawProjectInfoData(
@SerializedName("artist_name")
val artistName: String,
@SerializedName("author_name")
val authorName: String,
@SerializedName("np_comment")
val npComment: String,
@SerializedName("np_created_date")
val npCreatedDate: String,
@SerializedName("np_flag_mature")
val npFlagMature: String,
@SerializedName("np_info")
val npInfo: String,
@SerializedName("np_licenced_by")
val npLicencedBy: String,
@SerializedName("np_name")
val npName: String,
@SerializedName("np_name_link")
val npNameLink: String,
@SerializedName("np_project_id")
val npProjectId: String,
@SerializedName("np_status")
val npStatus: String,
@SerializedName("np_type")
val npType: String,
@SerializedName("np_updated_date")
val npUpdatedDate: String,
@SerializedName("np_view")
val npView: String,
@SerializedName("np_web")
val npWeb: String
)

View File

@ -0,0 +1,3 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
class RawProjectNameList : ArrayList<RawProjectNameListItem>()

View File

@ -0,0 +1,18 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawProjectNameListItem(
@SerializedName("np_name")
val npName: String,
@SerializedName("np_name_link")
val npNameLink: String,
@SerializedName("np_no_chapter")
val npNoChapter: String,
@SerializedName("np_project_id")
val npProjectId: String,
@SerializedName("np_status")
val npStatus: String,
@SerializedName("np_type")
val npType: String
)

View File

@ -0,0 +1,28 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawProjectSummary(
@SerializedName("nc_chapter_cover")
val ncChapterCover: String,
@SerializedName("nc_chapter_id")
val ncChapterId: String,
@SerializedName("nc_chapter_name")
val ncChapterName: String,
@SerializedName("nc_chapter_no")
val ncChapterNo: String,
@SerializedName("nc_created_date")
val ncCreatedDate: String,
@SerializedName("nc_provider")
val ncProvider: String,
@SerializedName("no_new_chapter")
val noNewChapter: String,
@SerializedName("np_group_dir")
val npGroupDir: String,
@SerializedName("np_name")
val npName: String,
@SerializedName("np_name_link")
val npNameLink: String,
@SerializedName("np_project_id")
val npProjectId: String
)

View File

@ -0,0 +1,10 @@
package eu.kanade.tachiyomi.extension.th.nekopost.model
import com.google.gson.annotations.SerializedName
data class RawProjectSummaryList(
@SerializedName("code")
val code: String,
@SerializedName("listItem")
val listItem: List<RawProjectSummary>?
)