diff --git a/src/ko/newtoki/build.gradle b/src/ko/newtoki/build.gradle index 14f32704b..e0bbe5d22 100644 --- a/src/ko/newtoki/build.gradle +++ b/src/ko/newtoki/build.gradle @@ -2,11 +2,15 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' ext { - extName = 'NewToki / ManaToki(ManaMoa)' + extName = 'NewToki / ManaToki' pkgNameSuffix = 'ko.newtoki' extClass = '.NewTokiFactory' - extVersionCode = 18 + extVersionCode = 19 libVersion = '1.2' } +dependencies { + implementation project(':lib-ratelimit') +} + apply from: "$rootDir/common.gradle" diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt new file mode 100644 index 000000000..63c0d162d --- /dev/null +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt @@ -0,0 +1,224 @@ +package eu.kanade.tachiyomi.extension.ko.newtoki + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservableSuccess +import eu.kanade.tachiyomi.source.model.Filter +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.MangasPage +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.util.asJsoup +import okhttp3.CacheControl +import okhttp3.HttpUrl +import okhttp3.Request +import okhttp3.Response +import org.jsoup.nodes.Element +import rx.Observable +import java.util.concurrent.TimeUnit + +/* + * ManaToki Is too big to support in a Factory File., So split into separate file. + */ + +class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domainNumber.net", "comic") { + // / ! DO NOT CHANGE THIS ! Only the site name changed from newtoki. + override val id by lazy { generateSourceId("NewToki", lang, versionId) } + override val supportsLatest by lazy { getExperimentLatest() } + + override fun latestUpdatesSelector() = ".media.post-list" + override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page") + override fun latestUpdatesNextPageSelector() = "nav.pg_wrap > .pg > strong" + override fun fetchLatestUpdates(page: Int): Observable { + // if this is true, Handle Only 10 mangas with accurate Details per page. (Real Latest Page has 70 mangas.) + // Else, Parse from Latest page. which is incomplete. + val isParseWithDetail = getLatestWithDetail() + val reqPage = if (isParseWithDetail) ((page - 1) / 7 + 1) else page + return rateLimitedClient.newCall(latestUpdatesRequest(reqPage)) + .asObservableSuccess() + .map { response -> + if (isParseWithDetail) latestUpdatesParseWithDetailPage(response, page) + else latestUpdatesParseWithLatestPage(response) + } + } + + private fun latestUpdatesParseWithDetailPage(response: Response, page: Int): MangasPage { + val document = response.asJsoup() + + // given cache time to prevent repeated lots of request in latest. + val cacheControl = CacheControl.Builder().maxAge(28, TimeUnit.DAYS).maxStale(28, TimeUnit.DAYS).build() + + val rm = 70 * ((page - 1) / 7) + val min = (page - 1) * 10 - rm + val max = page * 10 - rm + val elements = document.select("${latestUpdatesSelector()} p > a").slice(min until max) + val mangas = elements.map { element -> + val url = element.attr("abs:href") + val manga = mangaDetailsParse(rateLimitedClient.newCall(GET(url, cache = cacheControl)).execute()) + manga.url = getUrlPath(url) + manga + } + + val hasNextPage = try { + !document.select(popularMangaNextPageSelector()).text().contains("10") + } catch (_: Exception) { + false + } + + return MangasPage(mangas, hasNextPage) + } + + private fun latestUpdatesParseWithLatestPage(response: Response): MangasPage { + val document = response.asJsoup() + + val mangas = document.select(latestUpdatesSelector()).map { element -> + latestUpdatesElementParse(element) + } + + val hasNextPage = try { + !document.select(popularMangaNextPageSelector()).text().contains("10") + } catch (_: Exception) { + false + } + + return MangasPage(mangas, hasNextPage) + } + + private fun latestUpdatesElementParse(element: Element): SManga { + val linkElement = element.select("a.btn-primary") + val rawTitle = element.select(".post-subject > a").first().ownText().trim() + + // TODO: Make Clear Regex. + val chapterRegex = Regex("""((?:\s+)(?:(?:(?:[0-9]+권)?(?:[0-9]+부)?(?:[0-9]*?시즌[0-9]*?)?)?(?:\s*)(?:(?:[0-9]+)(?:[-.](?:[0-9]+))?)?(?:\s*[~,]\s*)?(?:[0-9]+)(?:[-.](?:[0-9]+))?)(?:화))""") + val title = rawTitle.trim().replace(chapterRegex, "") + // val regexSpecialChapter = Regex("(부록|단편|외전|.+편)") + // val lastTitleWord = excludeChapterTitle.split(" ").last() + // val title = excludeChapterTitle.replace(lastTitleWord, lastTitleWord.replace(regexSpecialChapter, "")) + + val manga = SManga.create() + manga.url = getUrlPath(linkElement.attr("href")) + manga.title = title + manga.thumbnail_url = element.select(".img-item > img").attr("src") + manga.initialized = false + return manga + } + + override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { + val url = HttpUrl.parse("$baseUrl/comic" + (if (page > 1) "/p$page" else ""))!!.newBuilder() + + if (!query.isBlank()) { + url.addQueryParameter("stx", query) + return GET(url.toString()) + } + + filters.forEach { filter -> + when (filter) { + is SearchPublishTypeList -> { + if (filter.state > 0) { + url.addQueryParameter("publish", filter.values[filter.state]) + } + } + + is SearchJaumTypeList -> { + if (filter.state > 0) { + url.addQueryParameter("jaum", filter.values[filter.state]) + } + } + + is SearchGenreTypeList -> { + if (filter.state > 0) { + url.addQueryParameter("tag", filter.values[filter.state]) + } + } + } + } + + return GET(url.toString()) + } + + // [...document.querySelectorAll("form.form td")[2].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') + private class SearchPublishTypeList : Filter.Select( + "Publish", + arrayOf( + "전체", + "미분류", + "주간", + "격주", + "월간", + "격월/비정기", + "단편", + "단행본", + "완결" + ) + ) + + // [...document.querySelectorAll("form.form td")[3].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') + private class SearchJaumTypeList : Filter.Select( + "Jaum", + arrayOf( + "전체", + "ㄱ", + "ㄴ", + "ㄷ", + "ㄹ", + "ㅁ", + "ㅂ", + "ㅅ", + "ㅇ", + "ㅈ", + "ㅊ", + "ㅋ", + "ㅌ", + "ㅍ", + "ㅎ", + "0-9", + "a-z" + ) + ) + + // [...document.querySelectorAll("form.form td")[4].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') + private class SearchGenreTypeList : Filter.Select( + "Genre", + arrayOf( + "전체", + "17", + "BL", + "SF", + "TS", + "개그", + "게임", + "공포", + "도박", + "드라마", + "라노벨", + "러브코미디", + "로맨스", + "먹방", + "미스터리", + "백합", + "붕탁", + "성인", + "순정", + "스릴러", + "스포츠", + "시대", + "애니화", + "액션", + "역사", + "음악", + "이세계", + "일상", + "일상+치유", + "전생", + "추리", + "판타지", + "학원", + "호러" + ) + ) + + override fun getFilterList() = FilterList( + Filter.Header("Filter can't use with query"), + SearchPublishTypeList(), + SearchJaumTypeList(), + SearchGenreTypeList() + ) +} diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt index 4dce2efff..5b19de4f3 100644 --- a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt @@ -7,7 +7,8 @@ import android.support.v7.preference.CheckBoxPreference import android.support.v7.preference.EditTextPreference import android.support.v7.preference.PreferenceScreen import android.widget.Toast -import eu.kanade.tachiyomi.extension.BuildConfig +import eu.kanade.tachiyomi.extensions.BuildConfig +import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.ConfigurableSource @@ -40,6 +41,9 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String override val lang: String = "ko" override val supportsLatest = true override val client: OkHttpClient = network.cloudflareClient + protected val rateLimitedClient: OkHttpClient = network.cloudflareClient.newBuilder() + .addNetworkInterceptor(RateLimitInterceptor(2, 5)) + .build() override fun popularMangaSelector() = "div#webtoon-list > ul > li" @@ -101,11 +105,13 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String // only exists on chapter with proper manga detail page. val fullListButton = document.select(".comic-navbar .toon-nav a").last() - val list: List = if (firstChapterButton?.text()?.contains("첫회보기") ?: false) { // Check this page is detail page + val list: List = if (firstChapterButton?.text()?.contains("첫회보기") + ?: false) { // Check this page is detail page val details = mangaDetailsParse(document) details.url = urlPath listOf(details) - } else if (fullListButton?.text()?.contains("전체목록") ?: false) { // Check this page is chapter page + } else if (fullListButton?.text()?.contains("전체목록") + ?: false) { // Check this page is chapter page val url = fullListButton.attr("abs:href") val details = mangaDetailsParse(client.newCall(GET(url)).execute()) details.url = getUrlPath(url) @@ -172,13 +178,30 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String if (name.contains("번외") || name.contains("특별편")) return -2f val regex = Regex("([0-9]+)(?:[-.]([0-9]+))?(?:화)") val (ch_primal, ch_second) = regex.find(name)!!.destructured - return (ch_primal + if (ch_second.isBlank()) "" else ".$ch_second").toFloatOrNull() ?: -1f + return (ch_primal + if (ch_second.isBlank()) "" else ".$ch_second").toFloatOrNull() + ?: -1f } catch (e: Exception) { e.printStackTrace() return -1f } } + override fun fetchMangaDetails(manga: SManga): Observable { + return rateLimitedClient.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .map { response -> + mangaDetailsParse(response).apply { initialized = true } + } + } + + override fun fetchChapterList(manga: SManga): Observable> { + return rateLimitedClient.newCall(chapterListRequest(manga)) + .asObservableSuccess() + .map { response -> + chapterListParse(response) + } + } + @SuppressLint("SimpleDateFormat") private fun parseChapterDate(date: String): Long { return try { @@ -209,8 +232,10 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String private val htmlDataRegex = Regex("""html_data\+='([^']+)'""") override fun pageListParse(document: Document): List { - val script = document.select("script:containsData(html_data)").firstOrNull()?.data() ?: throw Exception("data script not found") - val loadScript = document.select("script:containsData(data_attribute)").firstOrNull()?.data() ?: throw Exception("load script not found") + val script = document.select("script:containsData(html_data)").firstOrNull()?.data() + ?: throw Exception("data script not found") + val loadScript = document.select("script:containsData(data_attribute)").firstOrNull()?.data() + ?: throw Exception("load script not found") val dataAttr = "abs:data-" + loadScript.substringAfter("data_attribute: '").substringBefore("',") return htmlDataRegex.findAll(script).map { it.groupValues[1] } @@ -218,7 +243,7 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String .flatMap { it.split(".") } .joinToString("") { it.toIntOrNull(16)?.toChar()?.toString() ?: "" } .let { Jsoup.parse(it) } - .select("img[src=/img/loading-image.gif]") + .select("img[src=/img/loading-image.gif], .view-img > img[itemprop]") .mapIndexed { i, img -> Page(i, "", if (img.hasAttr(dataAttr)) img.attr(dataAttr) else img.attr("abs:content")) } } @@ -275,9 +300,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String } } + val latestWithDetailPref = androidx.preference.CheckBoxPreference(screen.context).apply { + key = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + title = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + summary = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY + + setOnPreferenceChangeListener { _, newValue -> + try { + val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, newValue as Boolean).commit() + // Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() + res + } catch (e: Exception) { + e.printStackTrace() + false + } + } + } + screen.addPreference(baseUrlPref) if (name == "ManaToki") { screen.addPreference(latestExperimentPref) + screen.addPreference(latestWithDetailPref) } } @@ -319,9 +362,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String } } + val latestWithDetailPref = CheckBoxPreference(screen.context).apply { + key = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + title = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + summary = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY + + setOnPreferenceChangeListener { _, newValue -> + try { + val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, newValue as Boolean).commit() + // Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() + res + } catch (e: Exception) { + e.printStackTrace() + false + } + } + } + screen.addPreference(baseUrlPref) if (name == "ManaToki") { screen.addPreference(latestExperimentPref) + screen.addPreference(latestWithDetailPref) } } @@ -335,17 +396,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String private fun getPrefBaseUrl(): String = preferences.getString(BASE_URL_PREF, defaultBaseUrl)!! protected fun getExperimentLatest(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_PREF, false) + protected fun getLatestWithDetail(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, false) companion object { + private const val RESTART_TACHIYOMI = "Restart Tachiyomi to apply new setting." + private const val BASE_URL_PREF_TITLE = "Override BaseUrl" private const val BASE_URL_PREF = "overrideBaseUrl_v${BuildConfig.VERSION_NAME}" private const val BASE_URL_PREF_SUMMARY = "For temporary uses. Update extension will erase this setting." - private const val RESTART_TACHIYOMI = "Restart Tachiyomi to apply new setting." // Setting: Experimental Latest Fetcher private const val EXPERIMENTAL_LATEST_PREF_TITLE = "Enable Latest (Experimental)" private const val EXPERIMENTAL_LATEST_PREF = "fetchLatestExperiment" - private const val EXPERIMENTAL_LATEST_PREF_SUMMARY = "Fetch Latest Manga using Latest Chapters. May has duplicates, Also requires LOTS OF requests (70 per page)" + private const val EXPERIMENTAL_LATEST_PREF_SUMMARY = "Fetch Latest Manga using Latest Chapters. May has duplicates and May DB corruption on certain Tachiyomi builds" + + // Setting: Experimental Latest Fetcher With Full Details (Optional) + private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE = "Fetch Latest with detail (Optional)" + private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF = "fetchLatestWithDetail" + private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY = + "Parse latest manga details with detail pages. This will reduce DB corruption on certain Tachiyomi builds.\n" + + "But makes chance of IP Ban, Also makes bunch of requests, For prevent IP ban, rate limit is set. so may slow,\n" + + "Still, It's experiment. Required to enable `Enable Latest (Experimental).`" const val PREFIX_ID_SEARCH = "id:" } diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt index e09c7d36a..7c8cf74a7 100644 --- a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt @@ -5,17 +5,12 @@ import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceFactory import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.util.asJsoup -import okhttp3.CacheControl import okhttp3.HttpUrl import okhttp3.Request -import okhttp3.Response import java.security.MessageDigest import java.text.SimpleDateFormat import java.util.Date import java.util.Locale -import java.util.concurrent.TimeUnit.DAYS /** * Source changes domain names every few days (e.g. newtoki31.net to newtoki32.net) @@ -29,163 +24,11 @@ private val domainNumber = 32 + ((Date().time - SimpleDateFormat("yyyy-MM-dd", L class NewTokiFactory : SourceFactory { override fun createSources(): List = listOf( - NewTokiManga(), + ManaToki(domainNumber), NewTokiWebtoon() ) } -class NewTokiManga : NewToki("ManaToki", "https://manatoki$domainNumber.net", "comic") { - // / ! DO NOT CHANGE THIS ! Only the site name changed from newtoki. - override val id by lazy { generateSourceId("NewToki", lang, versionId) } - override val supportsLatest by lazy { getExperimentLatest() } - - // this does 70 request per page.... - override fun latestUpdatesSelector() = ".media.post-list p > a" - override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page") - override fun latestUpdatesNextPageSelector() = "nav.pg_wrap > .pg > strong" - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - - // given cache time to prevent repeated lots of request in latest. - val cacheControl = CacheControl.Builder().maxAge(14, DAYS).maxStale(14, DAYS).build() - val mangas = document.select(latestUpdatesSelector()).map { element -> - val url = element.attr("abs:href") - val manga = mangaDetailsParse(client.newCall(GET(url, cache = cacheControl)).execute()) - manga.url = getUrlPath(url) - manga - } - - val hasNextPage = try { - !document.select(popularMangaNextPageSelector()).text().contains("10") - } catch (_: Exception) { - false - } - - return MangasPage(mangas, hasNextPage) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = HttpUrl.parse("$baseUrl/comic" + (if (page > 1) "/p$page" else ""))!!.newBuilder() - - if (!query.isBlank()) { - url.addQueryParameter("stx", query) - return GET(url.toString()) - } - - filters.forEach { filter -> - when (filter) { - is SearchPublishTypeList -> { - if (filter.state > 0) { - url.addQueryParameter("publish", filter.values[filter.state]) - } - } - - is SearchJaumTypeList -> { - if (filter.state > 0) { - url.addQueryParameter("jaum", filter.values[filter.state]) - } - } - - is SearchGenreTypeList -> { - if (filter.state > 0) { - url.addQueryParameter("tag", filter.values[filter.state]) - } - } - } - } - - return GET(url.toString()) - } - - // [...document.querySelectorAll("form.form td")[2].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') - private class SearchPublishTypeList : Filter.Select( - "Publish", - arrayOf( - "전체", - "미분류", - "주간", - "격주", - "월간", - "격월/비정기", - "단편", - "단행본", - "완결" - ) - ) - - // [...document.querySelectorAll("form.form td")[3].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') - private class SearchJaumTypeList : Filter.Select( - "Jaum", - arrayOf( - "전체", - "ㄱ", - "ㄴ", - "ㄷ", - "ㄹ", - "ㅁ", - "ㅂ", - "ㅅ", - "ㅇ", - "ㅈ", - "ㅊ", - "ㅋ", - "ㅌ", - "ㅍ", - "ㅎ", - "0-9", - "a-z" - ) - ) - - // [...document.querySelectorAll("form.form td")[4].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') - private class SearchGenreTypeList : Filter.Select( - "Genre", - arrayOf( - "전체", - "17", - "BL", - "SF", - "TS", - "개그", - "게임", - "공포", - "도박", - "드라마", - "라노벨", - "러브코미디", - "로맨스", - "먹방", - "미스터리", - "백합", - "붕탁", - "성인", - "순정", - "스릴러", - "스포츠", - "시대", - "애니화", - "액션", - "역사", - "음악", - "이세계", - "일상", - "일상+치유", - "전생", - "추리", - "판타지", - "학원", - "호러" - ) - ) - - override fun getFilterList() = FilterList( - Filter.Header("Filter can't use with query"), - SearchPublishTypeList(), - SearchJaumTypeList(), - SearchGenreTypeList() - ) -} - class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "webtoon") { // / ! DO NOT CHANGE THIS ! Prevent to treating as a new site override val id by lazy { generateSourceId("NewToki (Webtoon)", lang, versionId) }