Feat/pixiv deeplink (#9457)

* Pixiv: added deeplink and ID search (#9452)

Direct ID search is triggered by prefixing the item's ID with with `aid:` for artworks/illustrations, `sid:` for series, and `user:` for users. The former two are meant only for use in deeplinks, while the latter may also be useful for actual users and therefore has a more exposable name. (All of these prefixes are subject to change)

* Pixiv: bandaid fix for API not returning needed values (#9452)

Apparently depending on the circumstances, the API doesn't return the user ID to which a series belongs. This user ID is instead placed in the outer Illustration object. This very basic (and subject to a larger refactoring) fix ensures that the user ID is always present when needed to construct the link (it isn't required for anything else in the API)

* Pixiv: ensured that only exact matches to the deeplink patterns are handled specially (#9452)

The exact pattern is: `<type>:<ID consisting of digits>`. By ensuring that only digits and nothing else afterwards are allowed by the pattern matching (otherwise falling back to regular search), we further decrease the likelihood of users accidentally triggering this functionality (it sadly can't be entirely avoided, since deeplinks need to share an interface with the regular search queries)

* Pixiv: changed Deeplink system to use URL

Instead of complex parsing logic in the (deliberately lightweight and kotlin-wise handicapped) Deeplink Activity, the captured URL can just be passed to the search directly and handled there. The ability for the search to understand full Pixiv URLs is useful (and half-expected) either way, and it will not interfere with regular search function.

* Pixiv: fixed IndexOOB when query is empty/not a valid URI

* Pixiv: Applied suggestion to use OkHttp Urls

src/all/pixiv/AndroidManifest.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <application>
+        <activity
+            android:name=".all.pixiv.PixivUrlActivity"
+            android:excludeFromRecents="true"
+            android:exported="true"
+            android:theme="@android:style/Theme.NoDisplay">
+            <intent-filter android:autoVerify="true">
+                <action android:name="android.intent.action.VIEW" />
+
+                <category android:name="android.intent.category.DEFAULT" />
+                <category android:name="android.intent.category.BROWSABLE" />
+
+                <data android:host="pixiv.net" />
+                <data android:host="www.pixiv.net" />
+                <data android:scheme="http" />
+                <data android:scheme="https" />
+
+                <data android:pathPattern="/en/artworks/..*" />
+                <data android:pathPattern="/artworks/..*" />
+                <data android:pathPattern="/en/users/..*" />
+                <data android:pathPattern="/users/..*" />
+                <data android:pathPattern="/user/..*/series/..*" />
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>

src/all/pixiv/build.gradle

@@ -1,7 +1,7 @@
 ext {
     extName = 'Pixiv'
     extClass = '.PixivFactory'
-    extVersionCode = 9
+    extVersionCode = 10
     isNsfw = true
 }
 

src/all/pixiv/src/eu/kanade/tachiyomi/extension/all/pixiv/Pixiv.kt

@@ -9,6 +9,7 @@ import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.decodeFromJsonElement
 import okhttp3.Headers
 import okhttp3.HttpUrl
 import okhttp3.HttpUrl.Companion.toHttpUrl
@@ -38,14 +39,26 @@ class Pixiv(override val lang: String) : HttpSource() {
             client.newCall(request.url(url.build()).build()).execute()
     }
 
+    class PixivApiException(message: String? = null) : Exception(message, null)
+
     private inner class ApiCall(href: String?) : HttpCall(href) {
         init {
             url.addEncodedQueryParameter("lang", lang)
             request.addHeader("Accept", "application/json")
         }
 
-        inline fun <reified T> executeApi(): T =
-            json.decodeFromString<PixivApiResponse<T>>(execute().body.string()).body!!
+        /**
+         * Sends the previously constructed API call to the Pixiv API.
+         * If the server reports an error, A [PixivApiException] will be
+         * returned as a [Result.failure].
+         */
+        inline fun <reified T> executeApi(): Result<T> {
+            val resp = json.decodeFromString<PixivApiResponse>(execute().body.string())
+            if (resp.error) {
+                return Result.failure(PixivApiException(resp.message))
+            }
+            return Result.success(json.decodeFromJsonElement<T>(resp.body!!))
+        }
     }
 
     private var popularMangaNextPage = 1
@@ -59,13 +72,13 @@ class Pixiv(override val lang: String) : HttpSource() {
                 for (p in countUp(start = 1)) {
                     call.url.setEncodedQueryParameter("page", p.toString())
 
-                    val entries = call.executeApi<PixivRankings>().ranking!!
+                    val entries = call.executeApi<PixivRankings>().getOrThrow().ranking!!
                     if (entries.isEmpty()) break
 
                     val call = ApiCall("/touch/ajax/illust/details/many")
                     entries.forEach { call.url.addEncodedQueryParameter("illust_ids[]", it.illustId!!) }
 
-                    call.executeApi<PixivIllustsDetails>().illust_details!!.forEach { yield(it) }
+                    call.executeApi<PixivIllustsDetails>().getOrThrow().illust_details!!.forEach { yield(it) }
                 }
             }
                 .toSManga()
@@ -84,7 +97,39 @@ class Pixiv(override val lang: String) : HttpSource() {
     private var searchHash: Int? = null
     private lateinit var searchIterator: Iterator<SManga>
 
-    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+    override fun fetchSearchManga(
+        page: Int,
+        query: String,
+        filters: FilterList,
+    ): Observable<MangasPage> {
+        val target = PixivTarget.fromUri(query) /*?: PixivTarget.fromSearchQuery(query)*/
+
+        val singleResult = { manga: SManga? ->
+            Observable.just(
+                MangasPage(
+                    if (manga != null) {
+                        listOf(manga)
+                    } else {
+                        emptyList()
+                    },
+                    hasNextPage = false,
+                ),
+            )
+        }
+
+        // Deeplink selection of specific IDs: simply fetch the single object and return
+        when (target) {
+            is PixivTarget.Illustration ->
+                singleResult(getIllustCached(target.illustId)?.toSManga())
+            is PixivTarget.Series -> {
+                // TODO: caching!
+                val series = ApiCall("/touch/ajax/illust/series/${target.seriesId}")
+                    .executeApi<PixivSeriesDetails>().getOrNull()?.series
+                singleResult(series?.toSManga())
+            }
+            else -> null
+        }?.let { return it }
+
         val filters = filters.list as PixivFilters
         val hash = Pair(query, filters.toList()).hashCode()
 
@@ -94,7 +139,18 @@ class Pixiv(override val lang: String) : HttpSource() {
             lateinit var searchSequence: Sequence<PixivIllust>
             lateinit var predicates: List<(PixivIllust) -> Boolean>
 
-            if (query.isNotBlank()) {
+            // TODO: it would be useful to allow multiple user: tags in the query
+            if (target is PixivTarget.User) {
+                searchSequence = makeUserIdIllustSearchSequence(
+                    id = target.userId,
+                    type = filters.type,
+                )
+
+                predicates = buildList {
+                    filters.makeTagsPredicate()?.let(::add)
+                    filters.makeRatingPredicate()?.let(::add)
+                }
+            } else if (query.isNotBlank()) {
                 searchSequence = makeIllustSearchSequence(
                     word = query,
                     order = filters.order,
@@ -169,7 +225,7 @@ class Pixiv(override val lang: String) : HttpSource() {
         for (p in countUp(start = 1)) {
             call.url.setEncodedQueryParameter("p", p.toString())
 
-            val illusts = call.executeApi<PixivResults>().illusts!!
+            val illusts = call.executeApi<PixivResults>().getOrThrow().illusts!!
             if (illusts.isEmpty()) break
 
             for (illust in illusts) {
@@ -185,9 +241,6 @@ class Pixiv(override val lang: String) : HttpSource() {
         val searchUsers = HttpCall("/search_user.php?s_mode=s_usr")
             .apply { url.addQueryParameter("nick", nick) }
 
-        val fetchUserIllusts = ApiCall("/touch/ajax/user/illusts")
-            .apply { type?.let { url.setEncodedQueryParameter("type", it) } }
-
         for (p in countUp(start = 1)) {
             searchUsers.url.setEncodedQueryParameter("p", p.toString())
 
@@ -198,44 +251,71 @@ class Pixiv(override val lang: String) : HttpSource() {
             if (userIds.isEmpty()) break
 
             for (userId in userIds) {
-                fetchUserIllusts.url.setEncodedQueryParameter("id", userId)
-
-                for (p in countUp(start = 1)) {
-                    fetchUserIllusts.url.setEncodedQueryParameter("p", p.toString())
-
-                    val illusts = fetchUserIllusts.executeApi<PixivResults>().illusts!!
-                    if (illusts.isEmpty()) break
-
-                    yieldAll(illusts)
-                }
+                yieldAll(makeUserIdIllustSearchSequence(userId, type))
             }
         }
     }
+    private fun makeUserIdIllustSearchSequence(id: String, type: String?) = sequence<PixivIllust> {
+        val fetchUserIllusts = ApiCall("/touch/ajax/user/illusts")
+            .apply {
+                type?.let { url.setEncodedQueryParameter("type", it) }
+                url.setEncodedQueryParameter("id", id)
+            }
+
+        for (p in countUp(start = 1)) {
+            fetchUserIllusts.url.setEncodedQueryParameter("p", p.toString())
+
+            val illusts = fetchUserIllusts.executeApi<PixivResults>().getOrThrow().illusts!!
+            if (illusts.isEmpty()) break
+
+            yieldAll(illusts)
+        }
+    }
 
     override fun getFilterList() = FilterList(PixivFilters())
 
-    private fun Sequence<PixivIllust>.toSManga() = sequence<SManga> {
+    private fun List<PixivIllust>.toSManga() = asSequence().toSManga().toList()
+    private fun Sequence<PixivIllust>.toSManga() = sequence {
         val seriesIdsSeen = mutableSetOf<String>()
 
         forEach { illust ->
-            val series = illust.series
-
-            if (series == null) {
-                val manga = SManga.create()
-                manga.setUrlWithoutDomain("/artworks/${illust.id!!}")
-                manga.title = illust.title ?: "(null)"
-                manga.thumbnail_url = illust.url
-                yield(manga)
-            } else if (seriesIdsSeen.add(series.id!!)) {
-                val manga = SManga.create()
-                manga.setUrlWithoutDomain("/user/${series.userId!!}/series/${series.id}")
-                manga.title = series.title ?: "(null)"
-                manga.thumbnail_url = series.coverImage ?: illust.url
+            val manga = illust.toSManga()
+            if (seriesIdsSeen.add(manga.url)) {
                 yield(manga)
             }
         }
     }
 
+    private fun PixivSeries.toSearchResult() = PixivSearchResultSeries(
+        id = id,
+        title = title,
+        userId = userId,
+        coverImage = coverImage?.let { if (it.isString) it.content else null },
+    )
+    private fun PixivIllust.toSManga(): SManga {
+        if (series == null) {
+            val manga = SManga.create()
+            manga.setUrlWithoutDomain("/artworks/${id!!}")
+            manga.title = title ?: "(null)"
+            manga.thumbnail_url = url
+            return manga
+        } else {
+            val series = series.copy(userId = series.userId ?: author_details?.user_id)
+            val manga = series.toSManga().apply {
+                thumbnail_url = thumbnail_url ?: this@toSManga.url
+            }
+            return manga
+        }
+    }
+    private fun PixivSeries.toSManga() = toSearchResult().toSManga()
+    private fun PixivSearchResultSeries.toSManga(): SManga {
+        val manga = SManga.create()
+        manga.setUrlWithoutDomain("/user/${userId!!}/series/$id")
+        manga.title = title ?: "(null)"
+        manga.thumbnail_url = coverImage
+        return manga
+    }
+
     private var latestMangaNextPage = 1
     private lateinit var latestMangaIterator: Iterator<SManga>
 
@@ -247,7 +327,7 @@ class Pixiv(override val lang: String) : HttpSource() {
                 for (p in countUp(start = 1)) {
                     call.url.setEncodedQueryParameter("p", p.toString())
 
-                    val illusts = call.executeApi<PixivResults>().illusts!!
+                    val illusts = call.executeApi<PixivResults>().getOrThrow().illusts!!
                     if (illusts.isEmpty()) break
 
                     for (illust in illusts) {
@@ -269,14 +349,14 @@ class Pixiv(override val lang: String) : HttpSource() {
     }
 
     private val getIllustCached by lazy {
-        lruCached<String, PixivIllust>(25) { illustId ->
+        lruCached<String, PixivIllust?>(25) { illustId ->
             val call = ApiCall("/touch/ajax/illust/details?illust_id=$illustId")
-            return@lruCached call.executeApi<PixivIllustDetails>().illust_details!!
+            return@lruCached call.executeApi<PixivIllustDetails>().getOrNull()?.illust_details
         }
     }
 
     private val getSeriesIllustsCached by lazy {
-        lruCached<String, List<PixivIllust>>(25) { seriesId ->
+        lruCached<String, List<PixivIllust>?>(25) { seriesId ->
             val call = ApiCall("/touch/ajax/illust/series_content/$seriesId")
             var lastOrder = 0
 
@@ -284,7 +364,8 @@ class Pixiv(override val lang: String) : HttpSource() {
                 while (true) {
                     call.url.setEncodedQueryParameter("last_order", lastOrder.toString())
 
-                    val illusts = call.executeApi<PixivSeriesContents>().series_contents!!
+                    val illusts = call.executeApi<PixivSeriesContents>()
+                        .getOrElse { return@lruCached null }.series_contents!!
                     if (illusts.isEmpty()) break
 
                     addAll(illusts)
@@ -299,9 +380,9 @@ class Pixiv(override val lang: String) : HttpSource() {
 
         if (isSeries) {
             val series = ApiCall("/touch/ajax/illust/series/$id")
-                .executeApi<PixivSeriesDetails>().series!!
+                .executeApi<PixivSeriesDetails>().getOrThrow().series!!
 
-            val illusts = getSeriesIllustsCached(id)
+            val illusts = getSeriesIllustsCached(id)!!
 
             if (series.id != null && series.userId != null) {
                 manga.setUrlWithoutDomain("/user/${series.userId}/series/${series.id}")
@@ -321,7 +402,7 @@ class Pixiv(override val lang: String) : HttpSource() {
             val coverImage = series.coverImage?.let { if (it.isString) it.content else null }
             (coverImage ?: illusts.firstOrNull()?.url)?.let { manga.thumbnail_url = it }
         } else {
-            val illust = getIllustCached(id)
+            val illust = getIllustCached(id)!!
 
             illust.id?.let { manga.setUrlWithoutDomain("/artworks/$it") }
             illust.title?.let { manga.title = it }
@@ -343,8 +424,8 @@ class Pixiv(override val lang: String) : HttpSource() {
         val (id, isSeries) = parseSMangaUrl(manga.url)
 
         val illusts = when (isSeries) {
-            true -> getSeriesIllustsCached(id)
-            false -> listOf(getIllustCached(id))
+            true -> getSeriesIllustsCached(id)!!
+            false -> listOf(getIllustCached(id)!!)
         }
 
         val chapters = illusts.mapIndexed { i, illust ->
@@ -363,7 +444,7 @@ class Pixiv(override val lang: String) : HttpSource() {
         val illustId = chapter.url.substringAfterLast('/')
 
         val pages = ApiCall("/ajax/illust/$illustId/pages")
-            .executeApi<List<PixivIllustPage>>()
+            .executeApi<List<PixivIllustPage>>().getOrThrow()
             .mapIndexed { i, it -> Page(i, chapter.url, it.urls!!.original!!) }
 
         return Observable.just(pages)

src/all/pixiv/src/eu/kanade/tachiyomi/extension/all/pixiv/PixivConstants.kt

@@ -0,0 +1,3 @@
+package eu.kanade.tachiyomi.extension.all.pixiv
+
+internal val KNOWN_LOCALES = listOf("en")

src/all/pixiv/src/eu/kanade/tachiyomi/extension/all/pixiv/PixivLinkUtil.kt

@@ -0,0 +1,112 @@
+package eu.kanade.tachiyomi.extension.all.pixiv
+
+import android.net.Uri
+import okhttp3.HttpUrl
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
+
+interface PixivTargetCompanion<out T : PixivTarget> {
+    val SEARCH_PREFIX: String
+    fun fromSearchQuery(query: String): T? {
+        if (!query.startsWith(SEARCH_PREFIX)) return null
+        val id = query.removePrefix(SEARCH_PREFIX)
+        if (!id.matches("\\d+".toRegex())) return null
+        return fromSearchQueryId(id)
+    }
+
+    fun fromSearchQueryId(id: String): T?
+}
+
+sealed class PixivTarget {
+    companion object {
+        val BASE_URI = "https://www.pixiv.net".toHttpUrl()
+
+        fun fromSearchQuery(query: String) =
+            sequenceOf<PixivTargetCompanion<*>>(User, Series, Illustration)
+                .firstNotNullOfOrNull { it.fromSearchQuery(query) }
+
+        fun fromUri(uri: String) = uri.toHttpUrlOrNull()?.let { fromUri(it) }
+        fun fromUri(uri: Uri) = fromUri(uri.toString())
+        fun fromUri(uri: HttpUrl): PixivTarget? {
+            // if an absolute domain is specified, check if it matches. Tolerate relative urls as-is.
+            if (!(
+                uri.scheme in listOf(null, "http", "https") &&
+                    uri.host.let { "pixiv.net" == it.removePrefix("www.") }
+                )
+            ) {
+                return null
+            }
+
+            var pathSegments = uri.pathSegments.ifEmpty { null } ?: return null
+
+            if (KNOWN_LOCALES.contains(pathSegments[0])) {
+                pathSegments = pathSegments.subList(1, pathSegments.size)
+            }
+            if (pathSegments.size < 2) return null
+
+            with(pathSegments[0]) {
+                return when {
+                    equals("artworks") -> Illustration(pathSegments[1])
+                    equals("users") -> User(pathSegments[1])
+                    equals("user") &&
+                        (pathSegments.size >= 4 && pathSegments[2].equals("series")) ->
+                        Series(pathSegments[3], pathSegments[1])
+
+                    else -> null
+                }
+            }
+        }
+    }
+
+    abstract fun toHttpUrl(): HttpUrl
+    fun toUri() = Uri.parse(toHttpUrl().toString())
+
+    abstract fun toSearchQuery(): String
+
+    data class User(val userId: String) : PixivTarget() {
+        companion object : PixivTargetCompanion<User> {
+            override val SEARCH_PREFIX = "user:"
+            override fun fromSearchQueryId(id: String) = User(id)
+        }
+
+        override fun toHttpUrl() =
+            BASE_URI.newBuilder()
+                .addPathSegment("users")
+                .addPathSegment(userId).build()
+
+        override fun toSearchQuery(): String = SEARCH_PREFIX + userId
+    }
+
+    data class Illustration(val illustId: String) : PixivTarget() {
+        companion object : PixivTargetCompanion<Illustration> {
+            override val SEARCH_PREFIX = "aid:"
+            override fun fromSearchQueryId(id: String) = Illustration(id)
+        }
+
+        override fun toHttpUrl() =
+            BASE_URI.newBuilder()
+                .addPathSegment("artworks")
+                .addPathSegment(illustId).build()
+
+        override fun toSearchQuery(): String = SEARCH_PREFIX + illustId
+    }
+
+    data class Series(val seriesId: String, val authorUserId: String? = null) : PixivTarget() {
+        companion object : PixivTargetCompanion<Series> {
+            override val SEARCH_PREFIX = "sid:"
+            override fun fromSearchQueryId(id: String) = Series(id)
+        }
+
+        override fun toHttpUrl() =
+            BASE_URI.newBuilder()
+                .addPathSegment("user")
+                .addPathSegment(
+                    authorUserId
+                        ?: throw UnsupportedOperationException("TBD what should be done in this case"),
+                )
+                .addPathSegment("series")
+                .addPathSegment(seriesId).build()
+
+        override fun toSearchQuery(): String = SEARCH_PREFIX + seriesId
+    }
+}

src/all/pixiv/src/eu/kanade/tachiyomi/extension/all/pixiv/PixivTypes.kt

@@ -1,11 +1,14 @@
 package eu.kanade.tachiyomi.extension.all.pixiv
 
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.JsonElement
 import kotlinx.serialization.json.JsonPrimitive
 
 @Serializable
-internal data class PixivApiResponse<T>(
-    val body: T? = null,
+internal data class PixivApiResponse(
+    val error: Boolean = false,
+    val message: String? = null,
+    val body: JsonElement? = null,
 )
 
 @Serializable
@@ -58,6 +61,7 @@ internal data class PixivIllustPageUrls(
 
 @Serializable
 internal data class PixivAuthorDetails(
+    val user_id: String? = null,
     val user_name: String? = null,
 )
 
@@ -72,6 +76,9 @@ internal data class PixivSeries(
     val coverImage: JsonPrimitive? = null,
     val id: String? = null,
     val title: String? = null,
+    /**
+     * CAUTION: sometimes this isn't passed!
+     */
     val userId: String? = null,
 )
 
@@ -88,4 +95,5 @@ internal data class PixivRankings(
 @Serializable
 internal data class PixivRankingEntry(
     val illustId: String? = null,
+    val rank: Int? = null,
 )

src/all/pixiv/src/eu/kanade/tachiyomi/extension/all/pixiv/PixivUrlActivity.kt

@@ -0,0 +1,43 @@
+package eu.kanade.tachiyomi.extension.all.pixiv
+
+import android.app.Activity
+import android.content.ActivityNotFoundException
+import android.content.Intent
+import android.os.Bundle
+import android.util.Log
+import kotlin.system.exitProcess
+
+/**
+ * Springboard that accepts https://mangadex.com/title/xxx intents and redirects them to
+ * the main tachiyomi process. The idea is to not install the intent filter unless
+ * you have this extension installed, but still let the main tachiyomi app control
+ * things.
+ *
+ * Main goal was to make it easier to open manga in Tachiyomi in spite of the DDoS blocking
+ * the usual search screen from working.
+ */
+class PixivUrlActivity : Activity() {
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+
+        if (intent?.data != null) {
+            val mainIntent = Intent().apply {
+                action = "eu.kanade.tachiyomi.SEARCH"
+                putExtra("query", intent.data.toString())
+                putExtra("filter", packageName)
+            }
+
+            try {
+                startActivity(mainIntent)
+            } catch (e: ActivityNotFoundException) {
+                Log.e("PixivUrlActivity", e.toString())
+            }
+        } else {
+            Log.e("PixivUrlActivity", "Could not parse URI from intent $intent")
+        }
+
+        finish()
+        exitProcess(0)
+    }
+}