Update NewToki Extension to v1.2.19 (#5258)

* Mitigation about IP Ban by bunch of request on latest on manatoki.

RateLimit, And make default to not make bunch of request.
Split ManaToki into separate file as it became bigger.

* More rate limit

NewToki and ManaToki Shares IP bans.
Also when refresh the batch of mangas, It could be banned too.

So rate limit on ChapterList and MangaDetail request too. (And apply both of them)

* Fix lint about `java.util` and some format
This commit is contained in:
DitFranXX 2020-12-26 08:14:06 +09:00 committed by GitHub
parent ae0aad3c1f
commit 493c8d56c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 311 additions and 169 deletions

View File

@ -2,11 +2,15 @@ apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
ext {
extName = 'NewToki / ManaToki(ManaMoa)'
extName = 'NewToki / ManaToki'
pkgNameSuffix = 'ko.newtoki'
extClass = '.NewTokiFactory'
extVersionCode = 18
extVersionCode = 19
libVersion = '1.2'
}
dependencies {
implementation project(':lib-ratelimit')
}
apply from: "$rootDir/common.gradle"

View File

@ -0,0 +1,224 @@
package eu.kanade.tachiyomi.extension.ko.newtoki
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.CacheControl
import okhttp3.HttpUrl
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Element
import rx.Observable
import java.util.concurrent.TimeUnit
/*
* ManaToki Is too big to support in a Factory File., So split into separate file.
*/
class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domainNumber.net", "comic") {
// / ! DO NOT CHANGE THIS ! Only the site name changed from newtoki.
override val id by lazy { generateSourceId("NewToki", lang, versionId) }
override val supportsLatest by lazy { getExperimentLatest() }
override fun latestUpdatesSelector() = ".media.post-list"
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page")
override fun latestUpdatesNextPageSelector() = "nav.pg_wrap > .pg > strong"
override fun fetchLatestUpdates(page: Int): Observable<MangasPage> {
// if this is true, Handle Only 10 mangas with accurate Details per page. (Real Latest Page has 70 mangas.)
// Else, Parse from Latest page. which is incomplete.
val isParseWithDetail = getLatestWithDetail()
val reqPage = if (isParseWithDetail) ((page - 1) / 7 + 1) else page
return rateLimitedClient.newCall(latestUpdatesRequest(reqPage))
.asObservableSuccess()
.map { response ->
if (isParseWithDetail) latestUpdatesParseWithDetailPage(response, page)
else latestUpdatesParseWithLatestPage(response)
}
}
private fun latestUpdatesParseWithDetailPage(response: Response, page: Int): MangasPage {
val document = response.asJsoup()
// given cache time to prevent repeated lots of request in latest.
val cacheControl = CacheControl.Builder().maxAge(28, TimeUnit.DAYS).maxStale(28, TimeUnit.DAYS).build()
val rm = 70 * ((page - 1) / 7)
val min = (page - 1) * 10 - rm
val max = page * 10 - rm
val elements = document.select("${latestUpdatesSelector()} p > a").slice(min until max)
val mangas = elements.map { element ->
val url = element.attr("abs:href")
val manga = mangaDetailsParse(rateLimitedClient.newCall(GET(url, cache = cacheControl)).execute())
manga.url = getUrlPath(url)
manga
}
val hasNextPage = try {
!document.select(popularMangaNextPageSelector()).text().contains("10")
} catch (_: Exception) {
false
}
return MangasPage(mangas, hasNextPage)
}
private fun latestUpdatesParseWithLatestPage(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = document.select(latestUpdatesSelector()).map { element ->
latestUpdatesElementParse(element)
}
val hasNextPage = try {
!document.select(popularMangaNextPageSelector()).text().contains("10")
} catch (_: Exception) {
false
}
return MangasPage(mangas, hasNextPage)
}
private fun latestUpdatesElementParse(element: Element): SManga {
val linkElement = element.select("a.btn-primary")
val rawTitle = element.select(".post-subject > a").first().ownText().trim()
// TODO: Make Clear Regex.
val chapterRegex = Regex("""((?:\s+)(?:(?:(?:[0-9]+권)?(?:[0-9]+부)?(?:[0-9]*?시즌[0-9]*?)?)?(?:\s*)(?:(?:[0-9]+)(?:[-.](?:[0-9]+))?)?(?:\s*[~,]\s*)?(?:[0-9]+)(?:[-.](?:[0-9]+))?)(?:화))""")
val title = rawTitle.trim().replace(chapterRegex, "")
// val regexSpecialChapter = Regex("(부록|단편|외전|.+편)")
// val lastTitleWord = excludeChapterTitle.split(" ").last()
// val title = excludeChapterTitle.replace(lastTitleWord, lastTitleWord.replace(regexSpecialChapter, ""))
val manga = SManga.create()
manga.url = getUrlPath(linkElement.attr("href"))
manga.title = title
manga.thumbnail_url = element.select(".img-item > img").attr("src")
manga.initialized = false
return manga
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = HttpUrl.parse("$baseUrl/comic" + (if (page > 1) "/p$page" else ""))!!.newBuilder()
if (!query.isBlank()) {
url.addQueryParameter("stx", query)
return GET(url.toString())
}
filters.forEach { filter ->
when (filter) {
is SearchPublishTypeList -> {
if (filter.state > 0) {
url.addQueryParameter("publish", filter.values[filter.state])
}
}
is SearchJaumTypeList -> {
if (filter.state > 0) {
url.addQueryParameter("jaum", filter.values[filter.state])
}
}
is SearchGenreTypeList -> {
if (filter.state > 0) {
url.addQueryParameter("tag", filter.values[filter.state])
}
}
}
}
return GET(url.toString())
}
// [...document.querySelectorAll("form.form td")[2].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n')
private class SearchPublishTypeList : Filter.Select<String>(
"Publish",
arrayOf(
"전체",
"미분류",
"주간",
"격주",
"월간",
"격월/비정기",
"단편",
"단행본",
"완결"
)
)
// [...document.querySelectorAll("form.form td")[3].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n')
private class SearchJaumTypeList : Filter.Select<String>(
"Jaum",
arrayOf(
"전체",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"0-9",
"a-z"
)
)
// [...document.querySelectorAll("form.form td")[4].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n')
private class SearchGenreTypeList : Filter.Select<String>(
"Genre",
arrayOf(
"전체",
"17",
"BL",
"SF",
"TS",
"개그",
"게임",
"공포",
"도박",
"드라마",
"라노벨",
"러브코미디",
"로맨스",
"먹방",
"미스터리",
"백합",
"붕탁",
"성인",
"순정",
"스릴러",
"스포츠",
"시대",
"애니화",
"액션",
"역사",
"음악",
"이세계",
"일상",
"일상+치유",
"전생",
"추리",
"판타지",
"학원",
"호러"
)
)
override fun getFilterList() = FilterList(
Filter.Header("Filter can't use with query"),
SearchPublishTypeList(),
SearchJaumTypeList(),
SearchGenreTypeList()
)
}

View File

@ -7,7 +7,8 @@ import android.support.v7.preference.CheckBoxPreference
import android.support.v7.preference.EditTextPreference
import android.support.v7.preference.PreferenceScreen
import android.widget.Toast
import eu.kanade.tachiyomi.extension.BuildConfig
import eu.kanade.tachiyomi.extensions.BuildConfig
import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.ConfigurableSource
@ -40,6 +41,9 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
override val lang: String = "ko"
override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient
protected val rateLimitedClient: OkHttpClient = network.cloudflareClient.newBuilder()
.addNetworkInterceptor(RateLimitInterceptor(2, 5))
.build()
override fun popularMangaSelector() = "div#webtoon-list > ul > li"
@ -101,11 +105,13 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
// only exists on chapter with proper manga detail page.
val fullListButton = document.select(".comic-navbar .toon-nav a").last()
val list: List<SManga> = if (firstChapterButton?.text()?.contains("첫회보기") ?: false) { // Check this page is detail page
val list: List<SManga> = if (firstChapterButton?.text()?.contains("첫회보기")
?: false) { // Check this page is detail page
val details = mangaDetailsParse(document)
details.url = urlPath
listOf(details)
} else if (fullListButton?.text()?.contains("전체목록") ?: false) { // Check this page is chapter page
} else if (fullListButton?.text()?.contains("전체목록")
?: false) { // Check this page is chapter page
val url = fullListButton.attr("abs:href")
val details = mangaDetailsParse(client.newCall(GET(url)).execute())
details.url = getUrlPath(url)
@ -172,13 +178,30 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
if (name.contains("번외") || name.contains("특별편")) return -2f
val regex = Regex("([0-9]+)(?:[-.]([0-9]+))?(?:화)")
val (ch_primal, ch_second) = regex.find(name)!!.destructured
return (ch_primal + if (ch_second.isBlank()) "" else ".$ch_second").toFloatOrNull() ?: -1f
return (ch_primal + if (ch_second.isBlank()) "" else ".$ch_second").toFloatOrNull()
?: -1f
} catch (e: Exception) {
e.printStackTrace()
return -1f
}
}
override fun fetchMangaDetails(manga: SManga): Observable<SManga> {
return rateLimitedClient.newCall(mangaDetailsRequest(manga))
.asObservableSuccess()
.map { response ->
mangaDetailsParse(response).apply { initialized = true }
}
}
override fun fetchChapterList(manga: SManga): Observable<List<SChapter>> {
return rateLimitedClient.newCall(chapterListRequest(manga))
.asObservableSuccess()
.map { response ->
chapterListParse(response)
}
}
@SuppressLint("SimpleDateFormat")
private fun parseChapterDate(date: String): Long {
return try {
@ -209,8 +232,10 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
private val htmlDataRegex = Regex("""html_data\+='([^']+)'""")
override fun pageListParse(document: Document): List<Page> {
val script = document.select("script:containsData(html_data)").firstOrNull()?.data() ?: throw Exception("data script not found")
val loadScript = document.select("script:containsData(data_attribute)").firstOrNull()?.data() ?: throw Exception("load script not found")
val script = document.select("script:containsData(html_data)").firstOrNull()?.data()
?: throw Exception("data script not found")
val loadScript = document.select("script:containsData(data_attribute)").firstOrNull()?.data()
?: throw Exception("load script not found")
val dataAttr = "abs:data-" + loadScript.substringAfter("data_attribute: '").substringBefore("',")
return htmlDataRegex.findAll(script).map { it.groupValues[1] }
@ -218,7 +243,7 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
.flatMap { it.split(".") }
.joinToString("") { it.toIntOrNull(16)?.toChar()?.toString() ?: "" }
.let { Jsoup.parse(it) }
.select("img[src=/img/loading-image.gif]")
.select("img[src=/img/loading-image.gif], .view-img > img[itemprop]")
.mapIndexed { i, img -> Page(i, "", if (img.hasAttr(dataAttr)) img.attr(dataAttr) else img.attr("abs:content")) }
}
@ -275,9 +300,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
}
}
val latestWithDetailPref = androidx.preference.CheckBoxPreference(screen.context).apply {
key = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE
title = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE
summary = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY
setOnPreferenceChangeListener { _, newValue ->
try {
val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, newValue as Boolean).commit()
// Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show()
res
} catch (e: Exception) {
e.printStackTrace()
false
}
}
}
screen.addPreference(baseUrlPref)
if (name == "ManaToki") {
screen.addPreference(latestExperimentPref)
screen.addPreference(latestWithDetailPref)
}
}
@ -319,9 +362,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
}
}
val latestWithDetailPref = CheckBoxPreference(screen.context).apply {
key = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE
title = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE
summary = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY
setOnPreferenceChangeListener { _, newValue ->
try {
val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, newValue as Boolean).commit()
// Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show()
res
} catch (e: Exception) {
e.printStackTrace()
false
}
}
}
screen.addPreference(baseUrlPref)
if (name == "ManaToki") {
screen.addPreference(latestExperimentPref)
screen.addPreference(latestWithDetailPref)
}
}
@ -335,17 +396,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String
private fun getPrefBaseUrl(): String = preferences.getString(BASE_URL_PREF, defaultBaseUrl)!!
protected fun getExperimentLatest(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_PREF, false)
protected fun getLatestWithDetail(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, false)
companion object {
private const val RESTART_TACHIYOMI = "Restart Tachiyomi to apply new setting."
private const val BASE_URL_PREF_TITLE = "Override BaseUrl"
private const val BASE_URL_PREF = "overrideBaseUrl_v${BuildConfig.VERSION_NAME}"
private const val BASE_URL_PREF_SUMMARY = "For temporary uses. Update extension will erase this setting."
private const val RESTART_TACHIYOMI = "Restart Tachiyomi to apply new setting."
// Setting: Experimental Latest Fetcher
private const val EXPERIMENTAL_LATEST_PREF_TITLE = "Enable Latest (Experimental)"
private const val EXPERIMENTAL_LATEST_PREF = "fetchLatestExperiment"
private const val EXPERIMENTAL_LATEST_PREF_SUMMARY = "Fetch Latest Manga using Latest Chapters. May has duplicates, Also requires LOTS OF requests (70 per page)"
private const val EXPERIMENTAL_LATEST_PREF_SUMMARY = "Fetch Latest Manga using Latest Chapters. May has duplicates and May DB corruption on certain Tachiyomi builds"
// Setting: Experimental Latest Fetcher With Full Details (Optional)
private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE = "Fetch Latest with detail (Optional)"
private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF = "fetchLatestWithDetail"
private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY =
"Parse latest manga details with detail pages. This will reduce DB corruption on certain Tachiyomi builds.\n" +
"But makes chance of IP Ban, Also makes bunch of requests, For prevent IP ban, rate limit is set. so may slow,\n" +
"Still, It's experiment. Required to enable `Enable Latest (Experimental).`"
const val PREFIX_ID_SEARCH = "id:"
}

View File

@ -5,17 +5,12 @@ import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.SourceFactory
import eu.kanade.tachiyomi.source.model.Filter
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.CacheControl
import okhttp3.HttpUrl
import okhttp3.Request
import okhttp3.Response
import java.security.MessageDigest
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
import java.util.concurrent.TimeUnit.DAYS
/**
* Source changes domain names every few days (e.g. newtoki31.net to newtoki32.net)
@ -29,163 +24,11 @@ private val domainNumber = 32 + ((Date().time - SimpleDateFormat("yyyy-MM-dd", L
class NewTokiFactory : SourceFactory {
override fun createSources(): List<Source> = listOf(
NewTokiManga(),
ManaToki(domainNumber),
NewTokiWebtoon()
)
}
class NewTokiManga : NewToki("ManaToki", "https://manatoki$domainNumber.net", "comic") {
// / ! DO NOT CHANGE THIS ! Only the site name changed from newtoki.
override val id by lazy { generateSourceId("NewToki", lang, versionId) }
override val supportsLatest by lazy { getExperimentLatest() }
// this does 70 request per page....
override fun latestUpdatesSelector() = ".media.post-list p > a"
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page")
override fun latestUpdatesNextPageSelector() = "nav.pg_wrap > .pg > strong"
override fun latestUpdatesParse(response: Response): MangasPage {
val document = response.asJsoup()
// given cache time to prevent repeated lots of request in latest.
val cacheControl = CacheControl.Builder().maxAge(14, DAYS).maxStale(14, DAYS).build()
val mangas = document.select(latestUpdatesSelector()).map { element ->
val url = element.attr("abs:href")
val manga = mangaDetailsParse(client.newCall(GET(url, cache = cacheControl)).execute())
manga.url = getUrlPath(url)
manga
}
val hasNextPage = try {
!document.select(popularMangaNextPageSelector()).text().contains("10")
} catch (_: Exception) {
false
}
return MangasPage(mangas, hasNextPage)
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = HttpUrl.parse("$baseUrl/comic" + (if (page > 1) "/p$page" else ""))!!.newBuilder()
if (!query.isBlank()) {
url.addQueryParameter("stx", query)
return GET(url.toString())
}
filters.forEach { filter ->
when (filter) {
is SearchPublishTypeList -> {
if (filter.state > 0) {
url.addQueryParameter("publish", filter.values[filter.state])
}
}
is SearchJaumTypeList -> {
if (filter.state > 0) {
url.addQueryParameter("jaum", filter.values[filter.state])
}
}
is SearchGenreTypeList -> {
if (filter.state > 0) {
url.addQueryParameter("tag", filter.values[filter.state])
}
}
}
}
return GET(url.toString())
}
// [...document.querySelectorAll("form.form td")[2].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n')
private class SearchPublishTypeList : Filter.Select<String>(
"Publish",
arrayOf(
"전체",
"미분류",
"주간",
"격주",
"월간",
"격월/비정기",
"단편",
"단행본",
"완결"
)
)
// [...document.querySelectorAll("form.form td")[3].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n')
private class SearchJaumTypeList : Filter.Select<String>(
"Jaum",
arrayOf(
"전체",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"0-9",
"a-z"
)
)
// [...document.querySelectorAll("form.form td")[4].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n')
private class SearchGenreTypeList : Filter.Select<String>(
"Genre",
arrayOf(
"전체",
"17",
"BL",
"SF",
"TS",
"개그",
"게임",
"공포",
"도박",
"드라마",
"라노벨",
"러브코미디",
"로맨스",
"먹방",
"미스터리",
"백합",
"붕탁",
"성인",
"순정",
"스릴러",
"스포츠",
"시대",
"애니화",
"액션",
"역사",
"음악",
"이세계",
"일상",
"일상+치유",
"전생",
"추리",
"판타지",
"학원",
"호러"
)
)
override fun getFilterList() = FilterList(
Filter.Header("Filter can't use with query"),
SearchPublishTypeList(),
SearchJaumTypeList(),
SearchGenreTypeList()
)
}
class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "webtoon") {
// / ! DO NOT CHANGE THIS ! Prevent to treating as a new site
override val id by lazy { generateSourceId("NewToki (Webtoon)", lang, versionId) }