Draff/TachiyomiSY-Plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup the mangadex list calls

Browse Source
This commit is contained in:
Jobobby04 2021-05-11 13:06:55 -04:00
parent ba9db7ceb9
commit 29e4392490
4 changed files with 48 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
app/src/main/java/exh/md/handlers/FollowsHandler.kt
View File

@ -19,6 +19,7 @@ import exh.md.handlers.serializers.MangaStatusResponse
import exh.md.handlers.serializers.UpdateReadingStatus
import exh.md.utils.FollowStatus
import exh.md.utils.MdUtil
import exh.md.utils.mdListCall
import exh.metadata.metadata.MangaDexSearchMetadata
import exh.util.under
import kotlinx.serialization.encodeToString
@ -197,34 +198,10 @@ class FollowsHandler(
*/
suspend fun fetchAllFollows(): List<Pair<SManga, MangaDexSearchMetadata>> {
return withIOContext {
val response = client.newCall(followsListRequest(0)).await()
if (response.code == 204) {
return@withIOContext emptyList()
val results = client.mdListCall<MangaResponse> {
followsListRequest(it)
}
val mangaListResponse = response.parseAs<MangaListResponse>(MdUtil.jsonParser)
val results = mangaListResponse.results.toMutableList()
if (results.isEmpty()) {
return@withIOContext emptyList()
}
var hasMoreResults = mangaListResponse.limit + mangaListResponse.offset under mangaListResponse.total
var lastOffset = mangaListResponse.offset
while (hasMoreResults) {
val offset = lastOffset + mangaListResponse.limit
val newResponse = client.newCall(followsListRequest(offset)).await()
if (newResponse.code != 204) {
val newMangaListResponse = newResponse.parseAs<MangaListResponse>(MdUtil.jsonParser)
results += newMangaListResponse.results
hasMoreResults = newMangaListResponse.limit + newMangaListResponse.offset under newMangaListResponse.total
lastOffset = newMangaListResponse.offset
} else {
hasMoreResults = false
}
}
val statuses = results.chunked(100)
.map {
client.newCall(mangaStatusListRequest(results)).await().parseAs<MangaStatusListResponse>().statuses

104
app/src/main/java/exh/md/handlers/MangaHandler.kt
View File

@ -3,7 +3,6 @@ package exh.md.handlers
import eu.kanade.tachiyomi.data.database.models.Track
import eu.kanade.tachiyomi.data.track.mdlist.MdList
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.network.await
import eu.kanade.tachiyomi.network.parseAs
import eu.kanade.tachiyomi.source.model.SChapter
@ -13,12 +12,11 @@ import eu.kanade.tachiyomi.source.model.toSChapter
import eu.kanade.tachiyomi.source.model.toSManga
import eu.kanade.tachiyomi.util.lang.runAsObservable
import eu.kanade.tachiyomi.util.lang.withIOContext
import exh.md.handlers.serializers.ChapterListResponse
import exh.md.handlers.serializers.ChapterResponse
import exh.md.handlers.serializers.GroupListResponse
import exh.md.utils.MdUtil
import exh.md.utils.mdListCall
import exh.metadata.metadata.MangaDexSearchMetadata
import exh.util.under
import kotlinx.coroutines.async
import okhttp3.CacheControl
import okhttp3.Headers
@ -66,95 +64,19 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l
}
fun fetchMangaDetailsObservable(manga: SManga, sourceId: Long): Observable<SManga> {
return client.newCall(mangaRequest(manga.toMangaInfo()))
.asObservableSuccess()
.flatMap { response ->
runAsObservable({
ApiMangaParser(client, lang).parseToManga(manga.toMangaInfo(), response, emptyList(), sourceId).toSManga()
return runAsObservable({
getMangaDetails(manga.toMangaInfo(), sourceId).toSManga()
})
}
}
fun fetchChapterListObservable(manga: SManga): Observable<List<SChapter>> {
return client.newCall(mangaFeedRequest(manga.toMangaInfo(), 0, lang))
.asObservableSuccess()
.map { response ->
if (response.code == 204) {
return@map emptyList()
}
val chapterListResponse = response.parseAs<ChapterListResponse>(MdUtil.jsonParser)
val results = chapterListResponse.results.toMutableList()
var hasMoreResults = chapterListResponse.limit + chapterListResponse.offset under chapterListResponse.total
var lastOffset = chapterListResponse.offset
while (hasMoreResults) {
val offset = lastOffset + chapterListResponse.limit
val newResponse = client.newCall(mangaFeedRequest(manga.toMangaInfo(), offset, lang)).execute()
if (newResponse.code != 204) {
val newChapterListResponse = newResponse
.parseAs<ChapterListResponse>(MdUtil.jsonParser)
results += newChapterListResponse.results
hasMoreResults = newChapterListResponse.limit + newChapterListResponse.offset under newChapterListResponse.total
lastOffset = newChapterListResponse.offset
} else {
hasMoreResults = false
}
}
val groupIds =
results.asSequence()
.map { chapter -> chapter.relationships }
.flatten()
.filter { it.type == "scanlation_group" }
.map { it.id }
.distinct()
.toList()
val groupMap = runCatching {
groupIds.chunked(100).flatMapIndexed { index, ids ->
val groupResponse = client.newCall(groupIdRequest(ids, 100 * index)).execute()
if (groupResponse.code != 204) {
groupResponse
.parseAs<GroupListResponse>(MdUtil.jsonParser)
.results
.map { group -> Pair(group.data.id, group.data.attributes.name) }
} else {
emptyList()
}
}.toMap()
}.getOrNull() ?: emptyMap()
ApiMangaParser(client, lang).chapterListParse(results, groupMap).map { it.toSChapter() }
}
}
fun fetchChapterListObservable(manga: SManga): Observable<List<SChapter>> = runAsObservable({
getChapterList(manga.toMangaInfo()).map { it.toSChapter() }
})
suspend fun getChapterList(manga: MangaInfo): List<ChapterInfo> {
return withIOContext {
val response = client.newCall(mangaFeedRequest(manga, 0, lang)).await()
if (response.code == 204) {
return@withIOContext emptyList()
}
val chapterListResponse = response.parseAs<ChapterListResponse>(MdUtil.jsonParser)
val results = chapterListResponse.results.toMutableList()
var hasMoreResults = chapterListResponse.limit + chapterListResponse.offset under chapterListResponse.total
var lastOffset = chapterListResponse.offset
while (hasMoreResults) {
val offset = lastOffset + chapterListResponse.limit
val newResponse = client.newCall(mangaFeedRequest(manga, offset, lang)).await()
if (newResponse.code != 204) {
val newChapterListResponse = newResponse
.parseAs<ChapterListResponse>(MdUtil.jsonParser)
results += newChapterListResponse.results
hasMoreResults = newChapterListResponse.limit + newChapterListResponse.offset under newChapterListResponse.total
lastOffset = newChapterListResponse.offset
} else {
hasMoreResults = false
}
val results = client.mdListCall<ChapterResponse> {
mangaFeedRequest(manga, it, lang)
}
val groupMap = getGroupMap(results)
@ -165,12 +87,10 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l
private suspend fun getGroupMap(results: List<ChapterResponse>): Map<String, String> {
val groupIds = results.asSequence()
.map { chapter -> chapter.relationships }
.flatten()
.flatMap { chapter -> chapter.relationships }
.filter { it.type == "scanlation_group" }
.map { it.id }
.distinct()
.toList()
.toSet()
return runCatching {
groupIds.chunked(100).flatMapIndexed { index, ids ->
@ -178,12 +98,12 @@ class MangaHandler(val client: OkHttpClient, val headers: Headers, private val l
if (response.code != 204) {
response
.parseAs<GroupListResponse>(MdUtil.jsonParser)
.results.map { group -> Pair(group.data.id, group.data.attributes.name) }
.results.map { group -> group.data.id to group.data.attributes.name }
} else {
emptyList()
}
}.toMap()
}.getOrNull() ?: emptyMap()
}.getOrNull().orEmpty()
}
suspend fun fetchRandomMangaId(): String {

11
app/src/main/java/exh/md/handlers/serializers/ListCallSerializer.kt Normal file
View File

@ -0,0 +1,11 @@
package exh.md.handlers.serializers
import kotlinx.serialization.Serializable
@Serializable
data class ListCallResponse<T>(
val limit: Int,
val offset: Int,
val total: Int,
val results: List<T>
)

21
app/src/main/java/exh/md/utils/MdUtil.kt
View File

@ -3,6 +3,7 @@ package exh.md.utils
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.data.track.mdlist.MdList
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.await
import eu.kanade.tachiyomi.network.parseAs
import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.model.SChapter
@ -10,6 +11,7 @@ import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.all.MangaDex
import exh.log.xLogD
import exh.md.handlers.serializers.AtHomeResponse
import exh.md.handlers.serializers.ListCallResponse
import exh.md.handlers.serializers.LoginBodyToken
import exh.md.handlers.serializers.MangaResponse
import exh.md.network.NoSessionException
@ -17,6 +19,7 @@ import exh.source.getMainSource
import exh.util.floor
import exh.util.nullIfBlank
import exh.util.nullIfZero
import exh.util.under
import kotlinx.serialization.SerializationException
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
@ -24,6 +27,7 @@ import kotlinx.serialization.json.Json
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import org.jsoup.parser.Parser
import tachiyomi.source.model.MangaInfo
import uy.kohesive.injekt.Injekt
@ -351,3 +355,20 @@ class MdUtil {
}
}
}
suspend inline fun <reified T> OkHttpClient.mdListCall(request: (offset: Int) -> Request): List<T> {
val results = mutableListOf<T>()
var offset = 0
do {
val response = newCall(request(offset)).await()
if (response.code == 204) {
break
}
val mangaListResponse = response.parseAs<ListCallResponse<T>>(MdUtil.jsonParser)
results += mangaListResponse.results
offset += mangaListResponse.limit
} while (offset under mangaListResponse.total)
return results
}