Add Pixiv Comic (#3199)

* working, with webview to load page

doubt:
1. wrong episode number (using list index instead of real chapter number)
2. should we add the unavailable chapter to show or not (start with ※)
3. webview approach (slow and might get in error, too uncontrollable)
4. differentiating tags (using #) and category might bring problem sinces the added #

* check converting from response to ubytearray to image

* works fine, keep logs

* get rid of logs and another small things

* add logo

* clean forgotten things

* lint check: fix comment

* Apply suggestions from code review

Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>

* some refactoring, rename extension name and package to Pixiv Comic

* delete unused dependency

* use serial name on model

* Apply suggestions from code review

Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>

* prioritizing search over filter, change manga and chapter parse to just store the id, add tag interceptor in the case of tag not found

---------

Co-authored-by: AwkwardPeak7 <48650614+AwkwardPeak7@users.noreply.github.com>
This commit is contained in:
sinkableShip 2024-05-26 14:50:49 +07:00 committed by Draff
parent 89f53742bb
commit 2c00628e87
10 changed files with 712 additions and 0 deletions

View File

@ -0,0 +1,7 @@
ext {
extName = 'Pixiv Comic'
extClass = '.PixivComic'
extVersionCode = 1
}
apply from: "$rootDir/common.gradle"

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

View File

@ -0,0 +1,315 @@
package eu.kanade.tachiyomi.extension.ja.pixivcomic
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.HttpSource
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import okhttp3.HttpUrl
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Request
import okhttp3.Response
import org.jsoup.Jsoup
import uy.kohesive.injekt.injectLazy
class PixivComic : HttpSource() {
override val lang: String = "ja"
override val supportsLatest = true
override val name = "Pixivコミック"
override val baseUrl = "https://comic.pixiv.net"
private val json: Json by injectLazy()
// since there's no page option for popular manga, we use this as storage storing manga id
private val alreadyLoadedPopularMangaIds = mutableSetOf<Int>()
// used to determine if popular manga has next page or not
private var popularMangaCountRequested = 0
/**
* the key can be any kind of string with minimum length of 1,
* the same key must be passed in [imageRequest] and [ShuffledImageInterceptor]
*/
private val key by lazy {
randomString()
}
private val timeAndHash by lazy {
getTimeAndHash()
}
override val client = network.cloudflareClient.newBuilder()
.addInterceptor(ShuffledImageInterceptor(key))
.addNetworkInterceptor(::tagInterceptor)
.build()
override fun headersBuilder() = super.headersBuilder()
.add("Referer", "$baseUrl/")
.add("X-Requested-With", "pixivcomic")
override fun popularMangaRequest(page: Int): Request {
if (page == 1) alreadyLoadedPopularMangaIds.clear()
popularMangaCountRequested = POPULAR_MANGA_COUNT_PER_PAGE * page
val url = apiBuilder()
.addPathSegments("rankings/popularity")
.addQueryParameter("label", "総合")
.addQueryParameter("count", popularMangaCountRequested.toString())
.build()
return GET(url, headers)
}
override fun popularMangaParse(response: Response): MangasPage {
val popular = json.decodeFromString<ApiResponse<Popular>>(response.body.string())
val mangas = popular.data.ranking.filterNot {
alreadyLoadedPopularMangaIds.contains(it.id)
}.map { manga ->
SManga.create().apply {
title = manga.title
thumbnail_url = manga.mainImageUrl
url = manga.id.toString()
alreadyLoadedPopularMangaIds.add(manga.id)
}
}
return MangasPage(mangas, popular.data.ranking.size == popularMangaCountRequested)
}
override fun latestUpdatesRequest(page: Int): Request {
val url = apiBuilder()
.addPathSegments("works/recent_updates")
.addQueryParameter("page", page.toString())
.build()
return GET(url, headers)
}
override fun latestUpdatesParse(response: Response): MangasPage {
val latest = json.decodeFromString<ApiResponse<Latest>>(response.body.string())
val mangas = latest.data.officialWorks.map { manga ->
SManga.create().apply {
title = manga.name
thumbnail_url = manga.image.main
url = manga.id.toString()
}
}
return MangasPage(mangas, latest.data.nextPageNumber != null)
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val apiBuilder = apiBuilder()
when {
// for searching with tags, all tags started with #
query.startsWith("#") -> {
val tag = query.removePrefix("#")
apiBuilder
.addPathSegment("tags")
.addPathSegment(tag)
.addPathSegments("works/v2")
.addQueryParameter("page", page.toString())
}
query.isNotBlank() -> {
apiBuilder
.addPathSegments("works/search/v2")
.addPathSegment(query)
.addQueryParameter("page", page.toString())
}
else -> {
var tagIsBlank = true
filters.forEach { filter ->
when (filter) {
is Tag -> {
if (filter.state.isNotBlank()) {
apiBuilder
.addPathSegment("tags")
.addPathSegment(filter.state.removePrefix("#"))
.addPathSegments("works/v2")
.addQueryParameter("page", page.toString())
.build()
tagIsBlank = false
}
}
is Category -> {
if (tagIsBlank) {
apiBuilder
.addPathSegment("categories")
.addPathSegment(filter.values[filter.state])
.addPathSegments("works")
.addQueryParameter("page", page.toString())
.build()
}
}
else -> {}
}
}
}
}
return GET(apiBuilder.build(), headers)
}
override fun searchMangaParse(response: Response) = latestUpdatesParse(response)
override fun getFilterList() = FilterList(TagHeader(), Tag(), CategoryHeader(), Category())
private class TagHeader : Filter.Header(TAG_HEADER_TEXT)
private class Tag : Filter.Text("Tag")
private class CategoryHeader : Filter.Header(CATEGORY_HEADER_TEXT)
private class Category : Filter.Select<String>("Category", categories)
override fun mangaDetailsRequest(manga: SManga): Request {
val url = apiBuilder()
.addPathSegments("works/v5")
.addPathSegment(manga.url)
.build()
return GET(url, headers)
}
override fun mangaDetailsParse(response: Response): SManga {
val manga = json.decodeFromString<ApiResponse<Manga>>(response.body.string())
val mangaInfo = manga.data.officialWork
return SManga.create().apply {
description = Jsoup.parse(mangaInfo.description).wholeText()
author = mangaInfo.author
val categories = mangaInfo.categories?.map { it.name } ?: listOf()
val tags = mangaInfo.tags?.map { "#${it.name}" } ?: listOf()
val genreString = categories.plus(tags).joinToString(", ")
genre = genreString
}
}
override fun getMangaUrl(manga: SManga): String {
val url = baseUrl.toHttpUrl().newBuilder()
.addPathSegment("works")
.addPathSegment(manga.url)
.build()
return url.toString()
}
override fun chapterListRequest(manga: SManga): Request {
val url = apiBuilder()
.addPathSegment("works")
.addPathSegment(manga.url)
.addPathSegments("episodes/v2")
.addQueryParameter("order", "desc")
.build()
return GET(url, headers)
}
override fun chapterListParse(response: Response): List<SChapter> {
val chapters = json.decodeFromString<ApiResponse<Chapters>>(response.body.string())
return chapters.data.episodes.filter { episodeInfo ->
episodeInfo.episode != null
}.mapIndexed { i, episodeInfo ->
SChapter.create().apply {
val episode = episodeInfo.episode!!
name = episode.numberingTitle.plus(": ${episode.subTitle}")
url = episode.id.toString()
date_upload = episode.readStartAt
chapter_number = i.toFloat()
}
}
}
override fun getChapterUrl(chapter: SChapter): String {
val url = baseUrl.toHttpUrl().newBuilder()
.addPathSegments("viewer/stories")
.addPathSegment(chapter.url)
.build()
return url.toString()
}
override fun pageListRequest(chapter: SChapter): Request {
val url = apiBuilder()
.addPathSegment("episodes")
.addPathSegment(chapter.url)
.addPathSegment("read_v4")
.build()
val header = headers.newBuilder()
.add("X-Client-Time", timeAndHash.first)
.add("X-Client-Hash", timeAndHash.second)
.build()
return GET(url, header)
}
override fun pageListParse(response: Response): List<Page> {
val shuffledPages = json.decodeFromString<ApiResponse<Pages>>(response.body.string())
return shuffledPages.data.readingEpisode.pages.mapIndexed { i, page ->
Page(i, imageUrl = page.url)
}
}
override fun imageUrlParse(response: Response): String {
throw UnsupportedOperationException()
}
override fun imageRequest(page: Page): Request {
val header = headers.newBuilder()
.add("X-Cobalt-Thumber-Parameter-GridShuffle-Key", key)
.build()
return GET(page.imageUrl!!, header)
}
private fun apiBuilder(): HttpUrl.Builder {
return baseUrl.toHttpUrl()
.newBuilder()
.addPathSegments("api/app")
}
companion object {
private const val POPULAR_MANGA_COUNT_PER_PAGE = 30
private const val TAG_HEADER_TEXT = "Can only filter 1 type (Category or Tag) at a time"
private const val CATEGORY_HEADER_TEXT = "This filter by Category is ignored if Tag isn't at blank"
private val categories = arrayOf(
"恋愛",
"動物",
"グルメ",
"ファンタジー",
"ホラー・ミステリー",
"アクション",
"エッセイ",
"ギャグ・コメディ",
"日常",
"ヒューマンドラマ",
"スポーツ",
"お仕事",
"BL",
"TL",
"百合",
"pixivコミック限定",
"映像化",
"コミカライズ",
"タテヨミ",
"読み切り",
"その他",
)
}
}

View File

@ -0,0 +1,99 @@
package eu.kanade.tachiyomi.extension.ja.pixivcomic
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
@Serializable
internal class ApiResponse<T>(
val data: T,
)
@Serializable
internal class Popular(
val ranking: List<RankingItem>,
) {
@Serializable
internal class RankingItem(
val id: Int,
val title: String,
@SerialName("main_image_url")
val mainImageUrl: String,
)
}
@Serializable
internal class Latest(
@SerialName("next_page_number")
val nextPageNumber: Int?,
@SerialName("official_works")
val officialWorks: List<OfficialWork>,
)
@Serializable
internal class Manga(
@SerialName("official_work")
val officialWork: OfficialWork,
)
@Serializable
internal class OfficialWork(
val id: Int,
val name: String,
val image: Image,
val author: String,
val description: String,
val categories: List<Category>?,
val tags: List<Tag>?,
) {
@Serializable
internal class Category(
val name: String,
)
@Serializable
internal class Tag(
val name: String,
)
@Serializable
internal class Image(
val main: String,
)
}
@Serializable
internal class Chapters(
val episodes: List<EpisodeInfo>,
) {
@Serializable
internal class EpisodeInfo(
val episode: Episode?,
)
@Serializable
internal class Episode(
val id: Int,
@SerialName("numbering_title")
val numberingTitle: String,
@SerialName("sub_title")
val subTitle: String,
@SerialName("read_start_at")
val readStartAt: Long,
)
}
@Serializable
internal class Pages(
@SerialName("reading_episode")
val readingEpisode: ReadingEpisode,
) {
@Serializable
internal class ReadingEpisode(
val pages: List<SinglePage>,
) {
@Serializable
internal class SinglePage(
val url: String,
)
}
}

View File

@ -0,0 +1,78 @@
package eu.kanade.tachiyomi.extension.ja.pixivcomic
import android.os.Build
import okhttp3.Interceptor
import okhttp3.Response
import java.security.MessageDigest
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
import java.util.TimeZone
import kotlin.math.abs
private const val TIME_SALT = "mAtW1X8SzGS880fsjEXlM73QpS1i4kUMBhyhdaYySk8nWz533nrEunaSplg63fzT"
private class NoSuchTagException(message: String) : Exception(message)
internal fun tagInterceptor(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (request.url.pathSegments.contains("tags") && response.code == 404) {
throw NoSuchTagException("The inputted tag doesn't exist")
}
return response
}
internal fun randomString(): String {
// the average length of key
val length = (30..40).random()
return buildString(length) {
val charPool = ('a'..'z') + ('A'..'Z') + (0..9)
for (i in 0 until length) {
append(charPool.random())
}
}
}
@OptIn(ExperimentalUnsignedTypes::class)
internal fun getTimeAndHash(): Pair<String, String> {
val timeFormatted = if (Build.VERSION.SDK_INT < 24) {
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH).format(Date())
.plus(getCurrentTimeZoneOffsetString())
} else {
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX", Locale.ENGLISH).format(Date())
}
val saltedTimeArray = timeFormatted.plus(TIME_SALT).toByteArray()
val saltedTimeHash = MessageDigest.getInstance("SHA-256")
.digest(saltedTimeArray).toUByteArray()
val hexadecimalTimeHash = saltedTimeHash.joinToString("") {
var hex = Integer.toHexString(it.toInt())
if (hex.length < 2) {
hex = "0$hex"
}
return@joinToString hex
}
return Pair(timeFormatted, hexadecimalTimeHash)
}
/**
* workaround to retrieve time zone offset for android with version lower than 24
*/
private fun getCurrentTimeZoneOffsetString(): String {
val timeZone = TimeZone.getDefault()
val offsetInMillis = timeZone.rawOffset
val hours = offsetInMillis / (1000 * 60 * 60)
val minutes = (offsetInMillis % (1000 * 60 * 60)) / (1000 * 60)
val sign = if (hours >= 0) "+" else "-"
val formattedHours = String.format("%02d", abs(hours))
val formattedMinutes = String.format("%02d", abs(minutes))
return "$sign$formattedHours:$formattedMinutes"
}

View File

@ -0,0 +1,213 @@
package eu.kanade.tachiyomi.extension.ja.pixivcomic
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.Canvas
import android.graphics.Paint
import okhttp3.Interceptor
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.Response
import okhttp3.ResponseBody
import okhttp3.ResponseBody.Companion.asResponseBody
import okio.Buffer
import java.io.InputStream
import java.security.MessageDigest
import kotlin.math.ceil
import kotlin.math.floor
private const val SHUFFLE_SALT = "4wXCKprMMoxnyJ3PocJFs4CYbfnbazNe"
private const val BYTES_PER_PIXEL = 4
private const val GRID_SIZE = 32
internal class ShuffledImageInterceptor(private val key: String) : Interceptor {
override fun intercept(chain: Interceptor.Chain): Response {
val request = chain.request()
val response = chain.proceed(request)
if (request.headers["X-Cobalt-Thumber-Parameter-GridShuffle-Key"] == null) {
return response
}
val imageBody = response.body.byteStream()
.toDeShuffledImage(key)
return response.newBuilder()
.body(imageBody)
.build()
}
@OptIn(ExperimentalUnsignedTypes::class)
private fun InputStream.toDeShuffledImage(key: String): ResponseBody {
// get the image color data
val shuffledImageBitmap = BitmapFactory.decodeStream(this)
val width = shuffledImageBitmap.width
val height = shuffledImageBitmap.height
val shuffledImageArray = UByteArray(width * height * BYTES_PER_PIXEL)
var index = 0
for (y in 0 until height) {
for (x in 0 until width) {
val pixel = shuffledImageBitmap.getPixel(x, y)
val alpha = pixel shr 24 and 0xff
val red = pixel shr 16 and 0xff
val green = pixel shr 8 and 0xff
val blue = pixel and 0xff
shuffledImageArray[index++] = alpha.toUByte()
shuffledImageArray[index++] = red.toUByte()
shuffledImageArray[index++] = green.toUByte()
shuffledImageArray[index++] = blue.toUByte()
}
}
// deShuffle the shuffled image
val deShuffledImageArray = deShuffleImage(shuffledImageArray, width, height, key)
// place it back together
val deShuffledImageBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
val canvas = Canvas(deShuffledImageBitmap)
index = 0
for (y in 0 until height) {
for (x in 0 until width) {
// it's rgba
val red = deShuffledImageArray[index++]
val green = deShuffledImageArray[index++]
val blue = deShuffledImageArray[index++]
val alpha = deShuffledImageArray[index++]
canvas.drawPoint(
x.toFloat(),
y.toFloat(),
Paint().apply {
setARGB(red.toInt(), green.toInt(), blue.toInt(), alpha.toInt())
},
)
}
}
return Buffer().run {
deShuffledImageBitmap.compress(Bitmap.CompressFormat.PNG, 100, outputStream())
asResponseBody("image/png".toMediaType())
}
}
@OptIn(ExperimentalUnsignedTypes::class)
private fun deShuffleImage(
shuffledImageArray: UByteArray,
width: Int,
height: Int,
key: String,
): UByteArray {
val verticalGridTotal = ceil(height.toFloat() / GRID_SIZE).toInt()
val horizontalGridTotal = floor(width.toFloat() / GRID_SIZE).toInt()
val grid2DArray = Array(verticalGridTotal) { Array(horizontalGridTotal) { it } }
val saltedKeyArray = SHUFFLE_SALT.plus(key).toByteArray()
val saltedKeyHash = MessageDigest.getInstance("SHA-256").digest(saltedKeyArray).toUByteArray()
val saltedKeyHashArray = saltedKeyHash.first16ByteBecome4UInt()
val hash = HashAlgorithm(saltedKeyHashArray)
for (i in 0 until 100) hash.next()
for (i in 0 until verticalGridTotal) {
val gridArray = grid2DArray[i]
for (j in (horizontalGridTotal - 1) downTo 1) {
val hashIndex = hash.next() % (j + 1).toUInt()
val grid = gridArray[j]
gridArray[j] = gridArray[hashIndex.toInt()]
gridArray[hashIndex.toInt()] = grid
}
}
for (i in 0 until verticalGridTotal) {
val gridArray = grid2DArray[i]
val indexOfIndexGridArray = gridArray.mapIndexed { index, _ ->
gridArray.indexOf(index)
}
grid2DArray[i] = indexOfIndexGridArray.toTypedArray()
}
val deShuffledImageArray = UByteArray(shuffledImageArray.size)
for (row in 0 until height) {
val verticalGridIndex = floor(row.toFloat() / GRID_SIZE).toInt()
val gridArray = grid2DArray[verticalGridIndex]
for (horizontalGridIndex in 0 until horizontalGridTotal) {
// places square grid to the places it supposed to be
val gridFrom = gridArray[horizontalGridIndex]
val gridToIndex = horizontalGridIndex * GRID_SIZE
val toIndex = (row * width + gridToIndex) * BYTES_PER_PIXEL
val gridFromIndex = gridFrom * GRID_SIZE
val fromIndex = (row * width + gridFromIndex) * BYTES_PER_PIXEL
val gridSizeBytes = GRID_SIZE * BYTES_PER_PIXEL
for (i in 0 until gridSizeBytes) {
deShuffledImageArray[toIndex + i] = shuffledImageArray[fromIndex + i]
}
// copy the small part of image that don't get shuffled (most right side of the image)
val horizontalIndex = horizontalGridTotal * GRID_SIZE
val startIndex = (row * width + horizontalIndex) * BYTES_PER_PIXEL
val lastIndex = (row * width + width) * BYTES_PER_PIXEL
for (i in startIndex until lastIndex) {
deShuffledImageArray[i] = shuffledImageArray[i]
}
}
}
return deShuffledImageArray
}
@OptIn(ExperimentalUnsignedTypes::class)
private fun UByteArray.first16ByteBecome4UInt(): UIntArray {
val binaries = this.copyOfRange(0, 16).map {
var binaryString = Integer.toBinaryString(it.toInt())
for (i in binaryString.length until 8) {
binaryString = "0$binaryString"
}
return@map binaryString
}
return UIntArray(4) { i ->
val binariesIndexStart = i * 4
val stringBuilder = StringBuilder()
for (index in binariesIndexStart + 3 downTo binariesIndexStart) {
stringBuilder.append(binaries[index])
}
Integer.parseUnsignedInt(stringBuilder.toString(), 2).toUInt()
}
}
@OptIn(ExperimentalUnsignedTypes::class)
private class HashAlgorithm(val hashArray: UIntArray) {
init {
if (hashArray.all { it == 0u }) {
hashArray[0] = 1u
}
}
fun next(): UInt {
val e = 9u * shiftOr((5u * hashArray[1]), 7)
val t = hashArray[1] shl 9
hashArray[2] = hashArray[2] xor hashArray[0]
hashArray[3] = hashArray[3] xor hashArray[1]
hashArray[1] = hashArray[1] xor hashArray[2]
hashArray[0] = hashArray[0] xor hashArray[3]
hashArray[2] = hashArray[2] xor t
hashArray[3] = shiftOr(this.hashArray[3], 11)
return e
}
private fun shiftOr(value: UInt, by: Int): UInt {
return (((value shl (by % 32))) or (value.toInt() ushr (32 - by)).toUInt())
}
}
}