Draff/Tachiyomi-Extensions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite NHentai extension (#743)

Browse Source 
Rewrite NHentai extension
This commit is contained in:
Micael Valentim 2019-01-26 09:33:18 -03:00 committed by Carlos
parent 8b8eb09f28
commit fb356de90e
9 changed files with 203 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/all/nhentai/build.gradle
View File

@ -2,16 +2,11 @@ apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
ext {
appName = 'Tachiyomi: nhentai'
appName = 'Tachiyomi: NHentai'
pkgNameSuffix = 'all.nhentai'
extClass = '.NHJapanese; .NHEnglish; .NHChinese; .NHSpeechless; .NHCzech; .NHEsperanto; .NHMongolian; .NHSlovak; .NHArabic; .NHUkrainian'
extVersionCode = 3
extClass = '.NHEnglish; .NHJapanese; .NHChinese'
extVersionCode = 4
libVersion = '1.2'
}
dependencies {
compileOnly 'com.google.code.gson:gson:2.8.2'
compileOnly 'com.github.salomonbrys.kotson:kotson:2.5.0'
}
apply from: "$rootDir/common.gradle"

72
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/MetadataCopier.kt
View File

@ -1,72 +0,0 @@
package eu.kanade.tachiyomi.extension.all.nhentai
import eu.kanade.tachiyomi.source.model.SManga
import java.text.SimpleDateFormat
import java.util.*
private val ONGOING_SUFFIX = arrayOf(
"[ongoing]",
"(ongoing)",
"{ongoing}"
)
private val EX_DATE_FORMAT = SimpleDateFormat("yyyy-MM-dd HH:mm", Locale.US)
fun NHentaiMetadata.copyTo(manga: SManga) {
url?.let { manga.url = it }
mediaId?.let { mid ->
NHentaiMetadata.typeToExtension(thumbnailImageType)?.let {
manga.thumbnail_url = "https://t.nhentai.net/galleries/$mid/thumb.$it"
}
}
manga.title = englishTitle ?: japaneseTitle ?: shortTitle!!
//Set artist (if we can find one)
tags["artist"]?.let {
if (it.isNotEmpty()) manga.artist = it.joinToString(transform = Tag::name)
}
tags["category"]?.let {
if (it.isNotEmpty()) manga.genre = it.joinToString(transform = Tag::name)
}
//Try to automatically identify if it is ongoing, we try not to be too lenient here to avoid making mistakes
//We default to completed
manga.status = SManga.COMPLETED
englishTitle?.let { t ->
if (ONGOING_SUFFIX.any {
t.endsWith(it, ignoreCase = true)
}) manga.status = SManga.ONGOING
}
val titleDesc = StringBuilder()
englishTitle?.let { titleDesc += "English Title: $it\n" }
japaneseTitle?.let { titleDesc += "Japanese Title: $it\n" }
shortTitle?.let { titleDesc += "Short Title: $it\n" }
val detailsDesc = StringBuilder()
uploadDate?.let { detailsDesc += "Upload Date: ${EX_DATE_FORMAT.format(Date(it))}\n" }
pageImageTypes.size.let { detailsDesc += "Length: $it pages\n" }
favoritesCount?.let { detailsDesc += "Favorited: $it times\n" }
scanlator?.nullIfBlank()?.let { detailsDesc += "Scanlator: $it\n" }
val tagsDesc = buildTagsDescription(this)
manga.description = listOf(titleDesc.toString(), detailsDesc.toString(), tagsDesc.toString())
.filter(String::isNotBlank)
.joinToString(separator = "\n")
}
private fun buildTagsDescription(metadata: NHentaiMetadata)
= StringBuilder("Tags:\n").apply {
//BiConsumer only available in Java 8, we have to use destructuring here
metadata.tags.forEach { (namespace, tags) ->
if (tags.isNotEmpty()) {
val joinedTags = tags.joinToString(separator = " ", transform = { "<${it.name}>" })
this += "▪ $namespace: $joinedTags\n"
}
}
}

5
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHFilters.kt Normal file
View File

@ -0,0 +1,5 @@
package eu.kanade.tachiyomi.extension.all.nhentai
import eu.kanade.tachiyomi.source.model.Filter
class SortFilter : Filter.Select<String>("Sort", arrayOf("Date", "Popular"))

26
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHLangs.kt
View File

@ -1,29 +1,5 @@
package eu.kanade.tachiyomi.extension.all.nhentai
/**
* NHentai languages
*/
class NHJapanese : NHentai("ja", "japanese")
class NHEnglish : NHentai("en", "english")
class NHJapanese : NHentai("ja", "japanese")
class NHChinese : NHentai("zh", "chinese")
class NHSpeechless : NHentai("none", "speechless")
class NHCzech : NHentai("cs", "czech")
class NHEsperanto : NHentai("eo", "esperanto")
class NHMongolian : NHentai("mn", "mongolian")
class NHSlovak : NHentai("sk", "slovak")
class NHArabic : NHentai("ar", "arabic")
class NHUkrainian : NHentai("uk", "ukrainian")
fun getAllNHentaiLanguages() = listOf(
NHJapanese(),
NHEnglish(),
NHChinese(),
NHSpeechless(),
NHCzech(),
NHEsperanto(),
NHMongolian(),
NHSlovak(),
NHArabic(),
NHUkrainian()
)

17
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHUtil.kt
View File

@ -1,17 +0,0 @@
package eu.kanade.tachiyomi.extension.all.nhentai
/**
* Append Strings to StringBuilder with '+' operator
*/
operator fun StringBuilder.plusAssign(other: String) {
append(other)
}
/**
* Return null if String is blank, otherwise returns the original String
* @returns null if the String is blank, otherwise returns the original String
*/
fun String?.nullIfBlank(): String? = if (isNullOrBlank())
null
else
this

91
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHUtils.kt Normal file
View File

@ -0,0 +1,91 @@
package eu.kanade.tachiyomi.extension.all.nhentai
import org.jsoup.nodes.Document
import java.lang.StringBuilder
import java.text.SimpleDateFormat
class NHUtils {
companion object {
fun getArtists(document: Document): String {
val stringBuilder = StringBuilder()
val artists = document.select("#tags > div:nth-child(4) > span > a")
artists.forEach {
stringBuilder.append(cleanTag(it.text()))
if (it != artists.last())
stringBuilder.append(", ")
}
return stringBuilder.toString()
}
fun getGroups(document: Document): String? {
val stringBuilder = StringBuilder()
val groups = document.select("#tags > div:nth-child(5) > span > a")
groups.forEach {
stringBuilder.append(cleanTag(it.text()))
if (it != groups.last())
stringBuilder.append(", ")
}
return if (stringBuilder.toString().isEmpty()) null else stringBuilder.toString()
}
fun getTags(document: Document): String {
val stringBuilder = StringBuilder()
val parodies = document.select("#tags > div:nth-child(1) > span > a")
val characters = document.select("#tags > div:nth-child(2) > span > a")
val tags = document.select("#tags > div:nth-child(3) > span > a")
if (parodies.size > 0) {
stringBuilder.append("Parodies: ")
parodies.forEach {
stringBuilder.append(cleanTag(it.text()))
if (it != parodies.last())
stringBuilder.append(", ")
}
stringBuilder.append("\n\n")
}
if (characters.size > 0) {
stringBuilder.append("Characters: ")
characters.forEach {
stringBuilder.append(cleanTag(it.text()))
if (it != characters.last())
stringBuilder.append(", ")
}
stringBuilder.append("\n\n")
}
if (tags.size > 0) {
stringBuilder.append("Tags: ")
tags.forEach {
stringBuilder.append(cleanTag(it.text()))
if (it != tags.last())
stringBuilder.append(", ")
}
}
return stringBuilder.toString()
}
fun getTime(document: Document): Long {
val timeString = document.toString().substringAfter("datetime=\"").substringBefore("\">").replace("T", " ")
return SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSSZ").parse(timeString).time
}
private fun cleanTag(tag: String): String = tag.replace(Regex("\\(.*\\)"), "").trim()
}
}

300
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHentai.kt
View File

@ -1,216 +1,122 @@
package eu.kanade.tachiyomi.extension.all.nhentai
import android.net.Uri
import com.github.salomonbrys.kotson.*
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.Companion.getArtists
import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.Companion.getGroups
import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.Companion.getTags
import eu.kanade.tachiyomi.extension.all.nhentai.NHUtils.Companion.getTime
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.Request
import okhttp3.Response
import rx.Observable
/**
* NHentai source
*/
open class NHentai(override val lang: String, val nhLang: String) : HttpSource() {
override val name = "nhentai"
override val baseUrl = "https://nhentai.net"
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.net.URLEncoder
open class NHentai(override val lang: String, private val nhLang: String) : ParsedHttpSource() {
final override val baseUrl = "https://nhentai.net"
override val name = "NHentai"
override val supportsLatest = true
override val client = network.cloudflareClient
//TODO There is currently no way to get the most popular mangas
//TODO Instead, we delegate this to the latest updates thing to avoid confusing users with an empty screen
override fun fetchPopularManga(page: Int) = fetchLatestUpdates(page)
private val searchUrl = "$baseUrl/search"
override fun popularMangaRequest(page: Int)
= throw UnsupportedOperationException("This method should not be called!")
override fun chapterFromElement(element: Element) = throw UnsupportedOperationException("Not used")
override fun popularMangaParse(response: Response)
= throw UnsupportedOperationException("This method should not be called!")
override fun chapterListParse(response: Response): List<SChapter> {
val document = response.asJsoup()
val chapterList = mutableListOf<SChapter>()
val chapter = SChapter.create().apply {
name = "Chapter"
scanlator = getGroups(document)
date_upload = getTime(document)
setUrlWithoutDomain(response.request().url().encodedPath())
}
chapterList.add(chapter)
return chapterList
}
override fun chapterListRequest(manga: SManga): Request = GET("$baseUrl${manga.url}")
override fun chapterListSelector() = throw UnsupportedOperationException("Not used")
override fun getFilterList(): FilterList = FilterList(SortFilter())
override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("Not used")
override fun latestUpdatesFromElement(element: Element) = SManga.create().apply {
setUrlWithoutDomain(element.select("a").attr("href"))
title = element.select("a > div").text().replace("\"", "").trim()
}
override fun latestUpdatesNextPageSelector() = "#content > section.pagination > a.next"
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/language/$nhLang/?page=$page")
override fun latestUpdatesSelector() = "#content > div > div"
override fun mangaDetailsParse(document: Document) = SManga.create().apply {
title = document.select("#info > h1").text().replace("\"", "").trim()
thumbnail_url = document.select("#cover > a > img").attr("data-src")
status = SManga.COMPLETED
artist = getArtists(document)
author = artist
description = getTags(document)
}
override fun pageListParse(document: Document): List<Page> {
val pageElements = document.select("#thumbnail-container > div")
val pageList = mutableListOf<Page>()
pageElements.forEach {
Page(pageList.size).run {
this.imageUrl = it.select("a > img").attr("data-src").replace("t.nh", "i.nh").replace("t.", ".")
pageList.add(pageList.size, this)
}
}
return pageList
}
override fun pageListRequest(chapter: SChapter) = GET("$baseUrl${chapter.url}")
override fun popularMangaFromElement(element: Element) = SManga.create().apply {
setUrlWithoutDomain(element.select("a").attr("href"))
title = element.select("a > div").text().replace("\"", "").trim()
}
override fun popularMangaNextPageSelector() = "#content > section.pagination > a.next"
override fun popularMangaRequest(page: Int) = GET("$searchUrl/?q=+english&sort=popular")
override fun popularMangaSelector() = "#content > div > div"
override fun searchMangaFromElement(element: Element) = SManga.create().apply {
setUrlWithoutDomain(element.select("a").attr("href"))
title = element.select("a > div").text().replace("\"", "").trim()
}
override fun searchMangaNextPageSelector() = "#content > section.pagination > a.next"
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val uri = Uri.parse("$baseUrl/api/galleries/search").buildUpon()
uri.appendQueryParameter("query", "language:$nhLang $query")
uri.appendQueryParameter("page", page.toString())
val stringBuilder = StringBuilder()
stringBuilder.append(searchUrl)
stringBuilder.append("/?q=${URLEncoder.encode("$query +$nhLang", "UTF-8")}&")
filters.forEach {
if (it is UriFilter)
it.addToUri(uri)
}
return nhGet(uri.toString(), page)
}
override fun searchMangaParse(response: Response) = parseResultPage(response)
override fun fetchLatestUpdates(page: Int) = fetchSearchManga(page, "", getFilterList())
override fun latestUpdatesRequest(page: Int)
= throw UnsupportedOperationException("This method should not be called!")
override fun latestUpdatesParse(response: Response)
= throw UnsupportedOperationException("This method should not be called!")
override fun mangaDetailsParse(response: Response)
= parseGallery(jsonParser.parse(response.body()!!.string()).obj)
//Hack so we can use a different URL for fetching manga details and opening the details in the browser
override fun fetchMangaDetails(manga: SManga)
= client.newCall(urlToDetailsRequest(manga.url))
.asObservableSuccess()
.map { response ->
mangaDetailsParse(response).apply { initialized = true }
when (it) {
is SortFilter -> stringBuilder.append("sort=${it.values[it.state].toLowerCase()}&")
}
override fun mangaDetailsRequest(manga: SManga) = nhGet( baseUrl + manga.url )
fun urlToDetailsRequest(url: String) = nhGet("$baseUrl/api/gallery/${url.substringAfterLast('/')}")
fun parseResultPage(response: Response): MangasPage {
val res = jsonParser.parse(response.body()!!.string()).obj
res["error"]?.let {
throw RuntimeException("An error occurred while performing the search: $it")
}
val results = res.getAsJsonArray("result")?.map {
parseGallery(it.obj)
}
val numPages = res["num_pages"].nullInt
if (results != null && numPages != null)
return MangasPage(results, numPages > response.request().tag() as Int)
return MangasPage(emptyList(), false)
stringBuilder.append("page=$page")
return GET(stringBuilder.toString())
}
fun rawParseGallery(obj: JsonObject) = NHentaiMetadata().apply {
uploadDate = obj["upload_date"].nullLong
favoritesCount = obj["num_favorites"].nullLong
mediaId = obj["media_id"].nullString
obj["title"].nullObj?.let {
japaneseTitle = it["japanese"].nullString
shortTitle = it["pretty"].nullString
englishTitle = it["english"].nullString
}
obj["images"].nullObj?.let {
coverImageType = it["cover"]?.get("t").nullString
it["pages"].nullArray?.map {
it.nullObj?.get("t").nullString
}?.filterNotNull()?.let {
pageImageTypes.clear()
pageImageTypes.addAll(it)
}
thumbnailImageType = it["thumbnail"]?.get("t").nullString
}
scanlator = obj["scanlator"].nullString
id = obj["id"]?.asLong
obj["tags"].nullArray?.map {
val asObj = it.obj
Pair(asObj["type"].nullString, asObj["name"].nullString)
}?.apply {
tags.clear()
}?.forEach {
if (it.first != null && it.second != null)
tags.getOrPut(it.first!!) { mutableListOf() }.add(Tag(it.second!!, false))
}!!
}
fun parseGallery(obj: JsonObject) = SManga.create().apply {
rawParseGallery(obj).copyTo(this)
}
fun lazyLoadMetadata(url: String) =
client.newCall(urlToDetailsRequest(url))
.asObservableSuccess()
.map {
rawParseGallery(jsonParser.parse(it.body()!!.string()).obj)
}!!
override fun fetchChapterList(manga: SManga)
= Observable.just(listOf(SChapter.create().apply {
url = manga.url
name = "Chapter"
chapter_number = 1f
}))!!
override fun fetchPageList(chapter: SChapter)
= lazyLoadMetadata(chapter.url).map { metadata ->
if (metadata.mediaId == null) emptyList()
else
metadata.pageImageTypes.mapIndexed { index, s ->
val imageUrl = imageUrlFromType(metadata.mediaId!!, index + 1, s)
Page(index, imageUrl!!, imageUrl)
}
}!!
override fun fetchImageUrl(page: Page) = Observable.just(page.imageUrl!!)!!
fun imageUrlFromType(mediaId: String, page: Int, t: String) = NHentaiMetadata.typeToExtension(t)?.let {
"https://i.nhentai.net/galleries/$mediaId/$page.$it"
}
override fun chapterListParse(response: Response)
= throw UnsupportedOperationException("This method should not be called!")
override fun pageListParse(response: Response)
= throw UnsupportedOperationException("This method should not be called!")
override fun imageUrlParse(response: Response)
= throw UnsupportedOperationException("This method should not be called!")
override fun getFilterList() = FilterList(SortFilter())
private class SortFilter : UriSelectFilter("Sort", "sort", arrayOf(
Pair("date", "Date"),
Pair("popular", "Popularity")
), firstIsUnspecified = false)
private fun nhGet(url: String, tag: Any? = null) = GET(url)
.newBuilder()
//Requested by nhentai admins to use a custom user agent
.header("User-Agent",
"Mozilla/5.0 (X11; Linux x86_64) " +
"AppleWebKit/537.36 (KHTML, like Gecko) " +
"Chrome/56.0.2924.87 " +
"Safari/537.36 " +
"Tachiyomi/1.0")
.tag(tag).build()!!
/**
* Class that creates a select filter. Each entry in the dropdown has a name and a display name.
* If an entry is selected it is appended as a query parameter onto the end of the URI.
* If `firstIsUnspecified` is set to true, if the first entry is selected, nothing will be appended on the the URI.
*/
//vals: <name, display>
private open class UriSelectFilter(displayName: String, val uriParam: String, val vals: Array<Pair<String, String>>,
val firstIsUnspecified: Boolean = true,
defaultValue: Int = 0) :
Filter.Select<String>(displayName, vals.map { it.second }.toTypedArray(), defaultValue), UriFilter {
override fun addToUri(uri: Uri.Builder) {
if (state != 0 || !firstIsUnspecified)
uri.appendQueryParameter(uriParam, vals[state].first)
}
}
/**
* Represents a filter that is able to modify a URI.
*/
private interface UriFilter {
fun addToUri(uri: Uri.Builder)
}
companion object {
val jsonParser by lazy {
JsonParser()
}
}
}
override fun searchMangaSelector() = "#content > div > div"
}

44
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/NHentaiMetadata.kt
View File

@ -1,44 +0,0 @@
package eu.kanade.tachiyomi.extension.all.nhentai
/**
* NHentai metadata
*/
class NHentaiMetadata {
var id: Long? = null
var url: String?
get() = id?.let { "/g/$it" }
set(a) {
id = a?.substringAfterLast('/')?.toLong()
}
var uploadDate: Long? = null
var favoritesCount: Long? = null
var mediaId: String? = null
var japaneseTitle: String? = null
var englishTitle: String? = null
var shortTitle: String? = null
var coverImageType: String? = null
var pageImageTypes: MutableList<String> = mutableListOf()
var thumbnailImageType: String? = null
var scanlator: String? = null
val tags: MutableMap<String, MutableList<Tag>> = mutableMapOf()
companion object {
fun typeToExtension(t: String?) =
when (t) {
"p" -> "png"
"j" -> "jpg"
else -> null
}
}
}

7
src/all/nhentai/src/eu/kanade/tachiyomi/extension/all/nhentai/Tag.kt
View File

@ -1,7 +0,0 @@
package eu.kanade.tachiyomi.extension.all.nhentai
/**
* Simple tag model
*/
data class Tag(val name: String, val light: Boolean)