Add DeviantArt (#6694)

* Add DeviantArt

* Slight cleanup

* Use .absUrl(), remove not-null asserts

* Use less volatile selectors

* Use better selector for gallery name

* Remove not-null assert on subFolderGallery

* Remove autoVerify from manifest

* Remove unnecessary RSS request, simplify query parsing

* Fetch HQ image

* Account for gallery:{username} and gallery:{username}/all

* Reword search fallback message

* Allow parseDate() to accept null

src/all/deviantart/AndroidManifest.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <application>
+        <activity
+            android:name=".all.deviantart.DeviantArtUrlActivity"
+            android:excludeFromRecents="true"
+            android:exported="true"
+            android:theme="@android:style/Theme.NoDisplay">
+            <intent-filter>
+                <action android:name="android.intent.action.VIEW"/>
+                <category android:name="android.intent.category.DEFAULT"/>
+                <category android:name="android.intent.category.BROWSABLE"/>
+
+                <data android:scheme="http"/>
+                <data android:scheme="https"/>
+
+                <data android:host="www.deviantart.com"/>
+                <data android:host="deviantart.com"/>
+
+                <data android:pathPattern="/..*/gallery/..*"/>
+            </intent-filter>
+        </activity>
+    </application>
+</manifest>

src/all/deviantart/build.gradle

@@ -0,0 +1,8 @@
+ext {
+    extName = 'DeviantArt'
+    extClass = '.DeviantArt'
+    extVersionCode = 1
+    isNsfw = true
+}
+
+apply from: "$rootDir/common.gradle"

src/all/deviantart/src/eu/kanade/tachiyomi/extension/all/deviantart/DeviantArt.kt

@@ -0,0 +1,167 @@
+package eu.kanade.tachiyomi.extension.all.deviantart
+
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.source.model.FilterList
+import eu.kanade.tachiyomi.source.model.MangasPage
+import eu.kanade.tachiyomi.source.model.Page
+import eu.kanade.tachiyomi.source.model.SChapter
+import eu.kanade.tachiyomi.source.model.SManga
+import eu.kanade.tachiyomi.source.online.HttpSource
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.Request
+import okhttp3.Response
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import org.jsoup.parser.Parser
+import java.text.ParseException
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+class DeviantArt : HttpSource() {
+    override val name = "DeviantArt"
+    override val baseUrl = "https://deviantart.com"
+    override val lang = "all"
+    override val supportsLatest = false
+
+    private val backendBaseUrl = "https://backend.deviantart.com"
+    private fun backendBuilder() = backendBaseUrl.toHttpUrl().newBuilder()
+
+    private val dateFormat by lazy {
+        SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.ENGLISH)
+    }
+
+    private fun parseDate(dateStr: String?): Long {
+        return try {
+            dateFormat.parse(dateStr ?: "")!!.time
+        } catch (_: ParseException) {
+            0L
+        }
+    }
+
+    override fun popularMangaRequest(page: Int): Request {
+        throw UnsupportedOperationException(SEARCH_FORMAT_MSG)
+    }
+
+    override fun popularMangaParse(response: Response): MangasPage {
+        throw UnsupportedOperationException(SEARCH_FORMAT_MSG)
+    }
+
+    override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
+        require(query.startsWith("gallery:")) { SEARCH_FORMAT_MSG }
+        val querySegments = query.substringAfter(":").split("/")
+        val username = querySegments[0]
+        val folderId = querySegments.getOrElse(1) { "all" }
+        return GET("$baseUrl/$username/gallery/$folderId", headers)
+    }
+
+    override fun searchMangaParse(response: Response): MangasPage {
+        val manga = mangaDetailsParse(response)
+        return MangasPage(listOf(manga), false)
+    }
+
+    override fun latestUpdatesRequest(page: Int): Request {
+        throw UnsupportedOperationException()
+    }
+
+    override fun latestUpdatesParse(response: Response): MangasPage {
+        throw UnsupportedOperationException()
+    }
+
+    override fun mangaDetailsParse(response: Response): SManga {
+        val document = response.asJsoup()
+        val subFolderGallery = document.selectFirst("#sub-folder-gallery")
+        val manga = SManga.create().apply {
+            // If manga is sub-gallery then use sub-gallery name, else use gallery name
+            title = subFolderGallery?.selectFirst("._2vMZg + ._2vMZg")?.text()?.substringBeforeLast(" ")
+                ?: document.selectFirst(".ds-card-selected h2")!!.text()
+            author = document.title().substringBefore(" ")
+            description = subFolderGallery?.selectFirst(".legacy-journal")?.wholeText()
+            thumbnail_url = subFolderGallery?.selectFirst("img[property=contentUrl]")?.absUrl("src")
+        }
+        manga.setUrlWithoutDomain(response.request.url.toString())
+        return manga
+    }
+
+    override fun chapterListRequest(manga: SManga): Request {
+        val pathSegments = getMangaUrl(manga).toHttpUrl().pathSegments
+        val username = pathSegments[0]
+        val folderId = pathSegments[2]
+
+        val query = if (folderId == "all") {
+            "gallery:$username"
+        } else {
+            "gallery:$username/$folderId"
+        }
+
+        val url = backendBuilder()
+            .addPathSegment("rss.xml")
+            .addQueryParameter("q", query)
+            .build()
+
+        return GET(url, headers)
+    }
+
+    override fun chapterListParse(response: Response): List<SChapter> {
+        val document = response.asJsoupXml()
+        val chapterList = parseToChapterList(document).toMutableList()
+        var nextUrl = document.selectFirst("[rel=next]")?.absUrl("href")
+
+        while (nextUrl != null) {
+            val newRequest = GET(nextUrl, headers)
+            val newResponse = client.newCall(newRequest).execute()
+            val newDocument = newResponse.asJsoupXml()
+            val newChapterList = parseToChapterList(newDocument)
+            chapterList.addAll(newChapterList)
+
+            nextUrl = newDocument.selectFirst("[rel=next]")?.absUrl("href")
+        }
+
+        return indexChapterList(chapterList.toList())
+    }
+
+    private fun parseToChapterList(document: Document): List<SChapter> {
+        val items = document.select("item")
+        return items.map {
+            val chapter = SChapter.create()
+            chapter.setUrlWithoutDomain(it.selectFirst("link")!!.text())
+            chapter.apply {
+                name = it.selectFirst("title")!!.text()
+                date_upload = parseDate(it.selectFirst("pubDate")?.text())
+                scanlator = it.selectFirst("media|credit")?.text()
+            }
+        }
+    }
+
+    private fun indexChapterList(chapterList: List<SChapter>): List<SChapter> {
+        // DeviantArt allows users to arrange galleries arbitrarily so we will
+        // primitively index the list by checking the first and last dates
+        return if (chapterList.first().date_upload > chapterList.last().date_upload) {
+            chapterList.mapIndexed { i, chapter ->
+                chapter.apply { chapter_number = chapterList.size - i.toFloat() }
+            }
+        } else {
+            chapterList.mapIndexed { i, chapter ->
+                chapter.apply { chapter_number = i.toFloat() + 1 }
+            }
+        }
+    }
+
+    override fun pageListParse(response: Response): List<Page> {
+        val document = response.asJsoup()
+        val imageUrl = document.selectFirst("img[fetchpriority=high]")?.absUrl("src")
+        return listOf(Page(0, imageUrl = imageUrl))
+    }
+
+    override fun imageUrlParse(response: Response): String {
+        throw UnsupportedOperationException()
+    }
+
+    private fun Response.asJsoupXml(): Document {
+        return Jsoup.parse(body.string(), request.url.toString(), Parser.xmlParser())
+    }
+
+    companion object {
+        const val SEARCH_FORMAT_MSG = "Please enter a query in the format of gallery:{username} or gallery:{username}/{folderId}"
+    }
+}

src/all/deviantart/src/eu/kanade/tachiyomi/extension/all/deviantart/DeviantArtUrlActivity.kt

@@ -0,0 +1,37 @@
+package eu.kanade.tachiyomi.extension.all.deviantart
+
+import android.app.Activity
+import android.content.ActivityNotFoundException
+import android.content.Intent
+import android.os.Bundle
+import android.util.Log
+import kotlin.system.exitProcess
+
+class DeviantArtUrlActivity : Activity() {
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        val pathSegments = intent?.data?.pathSegments
+
+        if (pathSegments != null && pathSegments.size >= 3) {
+            val username = pathSegments[0]
+            val folderId = pathSegments[2]
+
+            val mainIntent = Intent().apply {
+                action = "eu.kanade.tachiyomi.SEARCH"
+                putExtra("query", "gallery:$username/$folderId")
+                putExtra("filter", packageName)
+            }
+
+            try {
+                startActivity(mainIntent)
+            } catch (e: ActivityNotFoundException) {
+                Log.e("DeviantArtUrlActivity", e.toString())
+            }
+        } else {
+            Log.e("DeviantArtUrlActivity", "Could not parse URI from intent $intent")
+        }
+
+        finish()
+        exitProcess(0)
+    }
+}