diff --git a/src/ko/newtoki/build.gradle b/src/ko/newtoki/build.gradle index d3ca11203..b46173f88 100644 --- a/src/ko/newtoki/build.gradle +++ b/src/ko/newtoki/build.gradle @@ -4,8 +4,9 @@ apply plugin: 'kotlin-android' ext { extName = 'NewToki / ManaToki' pkgNameSuffix = 'ko.newtoki' - extClass = '.NewTokiFactory' - extVersionCode = 23 + extClass = '.TokiFactory' + extVersionCode = 24 + isNsfw = true } apply from: "$rootDir/common.gradle" diff --git a/src/ko/newtoki/domain_log.md b/src/ko/newtoki/domain_log.md new file mode 100644 index 000000000..4214bc5ea --- /dev/null +++ b/src/ko/newtoki/domain_log.md @@ -0,0 +1,56 @@ +1. Visit https://t.me/s/newtoki3 and load all messages. + +2. Run + ```js + $$(".tgme_widget_message_service_date").map(e => e.innerText) + ``` + +3. Paste the dates into a spreadsheet. + +4. Remove duplicates. + +The number is changed to 154 on 2022-09-21. + +| Date | Days | Average | +| ---------: | ---: | ------: | +| 2022-09-21 | 12 | 12.0 | +| 2022-09-09 | 7 | 9.5 | +| 2022-09-02 | 8 | 9.0 | +| 2022-08-25 | 6 | 8.3 | +| 2022-08-19 | 11 | 8.8 | +| 2022-08-08 | 10 | 9.0 | +| 2022-07-29 | 8 | 8.9 | +| 2022-07-21 | 9 | 8.9 | +| 2022-07-12 | 7 | 8.7 | +| 2022-07-05 | 11 | 8.9 | +| 2022-06-24 | 6 | 8.6 | +| 2022-06-18 | 8 | 8.6 | +| 2022-06-10 | 3 | 8.2 | +| 2022-06-07 | 11 | 8.4 | +| 2022-05-27 | 6 | 8.2 | +| 2022-05-21 | 11 | 8.4 | +| 2022-05-10 | 11 | 8.5 | +| 2022-04-29 | 6 | 8.4 | +| 2022-04-23 | 8 | 8.4 | +| 2022-04-15 | 6 | 8.3 | +| 2022-04-09 | 10 | 8.3 | +| 2022-03-30 | 11 | 8.5 | +| 2022-03-19 | 9 | 8.5 | +| 2022-03-10 | 14 | 8.7 | +| 2022-02-24 | 12 | 8.8 | +| 2022-02-12 | 15 | 9.1 | +| 2022-01-28 | 9 | 9.1 | +| 2022-01-19 | 9 | 9.1 | +| 2022-01-10 | 17 | 9.3 | +| 2021-12-24 | 7 | 9.3 | +| 2021-12-17 | 6 | 9.2 | +| 2021-12-11 | 8 | 9.1 | +| 2021-12-03 | 9 | 9.1 | +| 2021-11-24 | 8 | 9.1 | +| 2021-11-16 | 11 | 9.1 | +| 2021-11-05 | 7 | 9.1 | +| 2021-10-29 | 9 | 9.1 | +| 2021-10-20 | 12 | 9.2 | +| 2021-10-08 | 8 | 9.1 | +| 2021-09-30 | 15 | 9.3 | +| 2021-09-15 | | | \ No newline at end of file diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/DomainNumber.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/DomainNumber.kt new file mode 100644 index 000000000..4817a3bb3 --- /dev/null +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/DomainNumber.kt @@ -0,0 +1,89 @@ +package eu.kanade.tachiyomi.extension.ko.newtoki + +import android.util.Log +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.util.asJsoup +import okhttp3.Interceptor +import okhttp3.Response +import java.io.IOException + +/** + * Source changes domain names every few days (e.g. newtoki31.net to newtoki32.net) + * The domain name was newtoki32 on 2019-11-14, this attempts to match the rate at which the domain changes + * + * Since 2020-09-20, They changed manga side to Manatoki. + * It was merged after shutdown of ManaMoa. + * This is by the head of Manamoa, as they decided to move to Newtoki. + * + * Updated on 2022-09-21, see `domain_log.md`. + * To avoid going too fast and to utilize redirections, + * the number is decremented by 1 initially, + * and increments every 9.2 days which is a bit slower than the average. + */ +val fallbackDomainNumber get() = (154 - 1) + ((System.currentTimeMillis() - 1663723150_000) / 794880_000).toInt() + +var domainNumber = "" + get() { + val currentValue = field + if (currentValue.isNotEmpty()) return currentValue + + val prefValue = newTokiPreferences.domainNumber + if (prefValue.isNotEmpty()) { + field = prefValue + return prefValue + } + + val fallback = fallbackDomainNumber.toString() + domainNumber = fallback + return fallback + } + set(value) { + for (preference in arrayOf(manaTokiPreferences, newTokiPreferences)) { + preference.domainNumber = value + } + field = value + } + +object DomainInterceptor : Interceptor { + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + + val response = try { + chain.proceed(request) + } catch (e: IOException) { + if (chain.call().isCanceled()) throw e + + val newDomainNumber = try { + val document = chain.proceed(GET("https://t.me/s/newtoki3")).asJsoup() + val description = document.selectFirst("meta[property=og:description]").attr("content") + numberRegex.find(description)!!.value + } catch (_: Throwable) { + fallbackDomainNumber.toString() + } + domainNumber = newDomainNumber + + val url = request.url + val newHost = numberRegex.replaceFirst(url.host, newDomainNumber) + val newUrl = url.newBuilder().host(newHost).build() + try { + chain.proceed(request.newBuilder().url(newUrl).build()) + } catch (e: IOException) { + Log.e("NewToki", "failed to fetch $newUrl", e) + throw IOException(editDomainNumber(), e) + } + } + + if (response.priorResponse == null) return response + + val newUrl = response.request.url + if ("captcha" in newUrl.toString()) throw IOException(solveCaptcha()) + + val newHost = newUrl.host + if (newHost.startsWith(MANATOKI_PREFIX) || newHost.startsWith(NEWTOKI_PREFIX)) { + numberRegex.find(newHost)?.run { domainNumber = value } + } + return response + } + + private val numberRegex by lazy { Regex("""\d+""") } +} diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt index 4ac184856..96002caf0 100644 --- a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt @@ -1,97 +1,32 @@ package eu.kanade.tachiyomi.extension.ko.newtoki import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.util.asJsoup -import okhttp3.CacheControl import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Request -import okhttp3.Response import org.jsoup.nodes.Element -import rx.Observable -import java.util.concurrent.TimeUnit /* * ManaToki is too big to support in a Factory File., So split into separate file. */ -class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domainNumber.net", "comic") { +object ManaToki : NewToki("ManaToki", "comic") { // / ! DO NOT CHANGE THIS ! Only the site name changed from newtoki. - override val id by lazy { generateSourceId("NewToki", lang, versionId) } - override val supportsLatest by lazy { getExperimentLatest() } + override val id = MANATOKI_ID - override fun latestUpdatesSelector() = ".media.post-list" - override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page") - override fun latestUpdatesNextPageSelector() = "nav.pg_wrap > .pg > strong" - override fun fetchLatestUpdates(page: Int): Observable { - // if this is true, Handle Only 10 mangas with accurate Details per page. (Real Latest Page has 70 mangas.) - // Else, Parse from Latest page. which is incomplete. - val isParseWithDetail = getLatestWithDetail() - val reqPage = if (isParseWithDetail) ((page - 1) / 7 + 1) else page - return rateLimitedClient.newCall(latestUpdatesRequest(reqPage)) - .asObservableSuccess() - .map { response -> - if (isParseWithDetail) latestUpdatesParseWithDetailPage(response, page) - else latestUpdatesParseWithLatestPage(response) - } - } + override val baseUrl get() = "https://$MANATOKI_PREFIX$domainNumber.net" - private fun latestUpdatesParseWithDetailPage(response: Response, page: Int): MangasPage { - val document = response.asJsoup() + override val preferences = manaTokiPreferences - // given cache time to prevent repeated lots of request in latest. - val cacheControl = CacheControl.Builder().maxAge(28, TimeUnit.DAYS).maxStale(28, TimeUnit.DAYS).build() + private val chapterRegex by lazy { Regex(""" [ \d,~.-]+화$""") } - val rm = 70 * ((page - 1) / 7) - val min = (page - 1) * 10 - rm - val max = page * 10 - rm - val elements = document.select("${latestUpdatesSelector()} p > a").slice(min until max) - val mangas = elements.map { element -> - val url = element.attr("abs:href") - val manga = mangaDetailsParse(rateLimitedClient.newCall(GET(url, cache = cacheControl)).execute()) - manga.url = getUrlPath(url) - manga - } - - val hasNextPage = try { - !document.select(popularMangaNextPageSelector()).text().contains("10") - } catch (_: Exception) { - false - } - - return MangasPage(mangas, hasNextPage) - } - - private fun latestUpdatesParseWithLatestPage(response: Response): MangasPage { - val document = response.asJsoup() - - val mangas = document.select(latestUpdatesSelector()).map { element -> - latestUpdatesElementParse(element) - } - - val hasNextPage = try { - !document.select(popularMangaNextPageSelector()).text().contains("10") - } catch (_: Exception) { - false - } - - return MangasPage(mangas, hasNextPage) - } - - private fun latestUpdatesElementParse(element: Element): SManga { + fun latestUpdatesElementParse(element: Element): SManga { val linkElement = element.select("a.btn-primary") val rawTitle = element.select(".post-subject > a").first().ownText().trim() - // TODO: Make Clear Regex. - val chapterRegex = Regex("""((?:\s+)(?:(?:(?:[0-9]+권)?(?:[0-9]+부)?(?:[0-9]*?시즌[0-9]*?)?)?(?:\s*)(?:(?:[0-9]+)(?:[-.](?:[0-9]+))?)?(?:\s*[~,]\s*)?(?:[0-9]+)(?:[-.](?:[0-9]+))?)(?:화))""") val title = rawTitle.trim().replace(chapterRegex, "") - // val regexSpecialChapter = Regex("(부록|단편|외전|.+편)") - // val lastTitleWord = excludeChapterTitle.split(" ").last() - // val title = excludeChapterTitle.replace(lastTitleWord, lastTitleWord.replace(regexSpecialChapter, "")) val manga = SManga.create() manga.url = getUrlPath(linkElement.attr("href")) @@ -104,8 +39,6 @@ class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domai override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { val url = ("$baseUrl/comic" + (if (page > 1) "/p$page" else "")).toHttpUrl().newBuilder() - val genres = mutableListOf() - filters.forEach { filter -> when (filter) { is SearchPublishTypeList -> { @@ -121,20 +54,17 @@ class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domai } is SearchGenreTypeList -> { - filter.state.forEach { - if (it.state) { - genres.add(it.id) - } - } + val genres = filter.state.filter { it.state }.joinToString(",") { it.id } + url.addQueryParameter("tag", genres) } is SearchSortTypeList -> { - url.addQueryParameter("sst", listOf("wr_datetime", "wr_hit", "wr_good", "as_update")[filter.state]) + val state = filter.state ?: return@forEach + url.addQueryParameter("sst", arrayOf("wr_datetime", "wr_hit", "wr_good", "as_update")[state.index]) + url.addQueryParameter("sod", if (state.ascending) "asc" else "desc") } - is SearchOrderTypeList -> { - url.addQueryParameter("sod", listOf("desc", "asc")[filter.state]) - } + else -> {} } } @@ -142,14 +72,12 @@ class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domai url.addQueryParameter("stx", query) // Remove some filter QueryParams that not working with query - url.setQueryParameter("publish", null) - url.setQueryParameter("jaum", null) - - return GET(url.toString()) + url.removeAllQueryParameters("publish") + url.removeAllQueryParameters("jaum") + url.removeAllQueryParameters("tag") } - url.addQueryParameter("tag", genres.joinToString(",")) - return GET(url.toString()) + return GET(url.toString(), headers) } private class SearchCheckBox(name: String, val id: String = name) : Filter.CheckBox(name) @@ -227,7 +155,7 @@ class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domai ).map { SearchCheckBox(it) } ) - private class SearchSortTypeList : Filter.Select( + private class SearchSortTypeList : Filter.Sort( "Sort", arrayOf( "기본(날짜순)", @@ -237,20 +165,12 @@ class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domai ) ) - private class SearchOrderTypeList : Filter.Select( - "Order", - arrayOf( - "Descending", - "Ascending" - ) - ) - override fun getFilterList() = FilterList( - Filter.Header("Some filters can't use with query"), + SearchSortTypeList(), + Filter.Separator(), + Filter.Header(ignoredForTextSearch()), SearchPublishTypeList(), SearchJaumTypeList(), - SearchSortTypeList(), - SearchOrderTypeList(), SearchGenreTypeList() ) } diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt index 0144468d9..8fcea9398 100644 --- a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt @@ -1,10 +1,7 @@ package eu.kanade.tachiyomi.extension.ko.newtoki -import android.annotation.SuppressLint -import android.app.Application import android.content.SharedPreferences -import android.widget.Toast -import eu.kanade.tachiyomi.AppInfo +import android.util.Log import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.network.interceptor.rateLimit @@ -16,41 +13,50 @@ import eu.kanade.tachiyomi.source.model.SChapter import eu.kanade.tachiyomi.source.model.SManga import eu.kanade.tachiyomi.source.online.ParsedHttpSource import eu.kanade.tachiyomi.util.asJsoup +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient -import okhttp3.Request import okhttp3.Response import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.nodes.Element import rx.Observable -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get -import java.net.URI -import java.net.URISyntaxException import java.text.SimpleDateFormat import java.util.Calendar +import java.util.Locale +import java.util.concurrent.TimeUnit /** * NewToki Source + * + * Based on https://github.com/gnuboard/gnuboard5 **/ -open class NewToki(override val name: String, private val defaultBaseUrl: String, private val boardName: String) : ConfigurableSource, ParsedHttpSource() { - override val baseUrl by lazy { getPrefBaseUrl() } +abstract class NewToki(override val name: String, private val boardName: String) : ConfigurableSource, ParsedHttpSource() { + override val lang: String = "ko" override val supportsLatest = true - override val client: OkHttpClient = network.cloudflareClient - protected val rateLimitedClient: OkHttpClient by lazy { - network.cloudflareClient.newBuilder() - .rateLimit(1, getRateLimitPeriod()) - .build() + + override val client: OkHttpClient by lazy { + buildClient(withRateLimit = false) } + protected val rateLimitedClient: OkHttpClient by lazy { + buildClient(withRateLimit = true) + } + + private fun buildClient(withRateLimit: Boolean) = + network.cloudflareClient.newBuilder() + .apply { if (withRateLimit) rateLimit(1, preferences.rateLimitPeriod.toLong()) } + .addInterceptor(DomainInterceptor) // not rate-limited + .connectTimeout(10, TimeUnit.SECONDS) // fail fast + .build() + override fun popularMangaSelector() = "div#webtoon-list > ul > li" override fun popularMangaFromElement(element: Element): SManga { val linkElement = element.getElementsByTag("a").first() val manga = SManga.create() - manga.setUrlWithoutDomain(linkElement.attr("href").substringBefore("?")) + manga.url = getUrlPath(linkElement.attr("href")) manga.title = element.select("span.title").first().ownText() manga.thumbnail_url = linkElement.getElementsByTag("img").attr("src") return manga @@ -59,17 +65,16 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String override fun popularMangaNextPageSelector() = "ul.pagination > li:last-child:not(.disabled)" // Do not add page parameter if page is 1 to prevent tracking. - override fun popularMangaRequest(page: Int) = GET("$baseUrl/$boardName" + if (page > 1) "/p$page" else "") + override fun popularMangaRequest(page: Int) = GET("$baseUrl/$boardName" + if (page > 1) "/p$page" else "", headers) override fun searchMangaSelector() = popularMangaSelector() override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element) override fun searchMangaNextPageSelector() = popularMangaNextPageSelector() - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = GET("$baseUrl/$boardName" + (if (page > 1) "/p$page" else "") + "?stx=$query") override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { return if (query.startsWith(PREFIX_ID_SEARCH)) { val realQuery = query.removePrefix(PREFIX_ID_SEARCH) val urlPath = "/$boardName/$realQuery" - rateLimitedClient.newCall(GET("$baseUrl$urlPath")) + rateLimitedClient.newCall(GET("$baseUrl$urlPath", headers)) .asObservableSuccess() .map { response -> // the id is matches any of 'post' from their CMS board. @@ -95,7 +100,7 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String } fullListButton?.text()?.contains("전체목록") == true -> { // Check this page is chapter page val url = fullListButton.attr("abs:href") - val details = mangaDetailsParse(rateLimitedClient.newCall(GET(url)).execute()) + val details = mangaDetailsParse(rateLimitedClient.newCall(GET(url, headers)).execute()) details.url = getUrlPath(url) listOf(details) } @@ -113,10 +118,11 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String val description = descriptionElement.map { it.text().trim() } + val prefix = if (isCleanPath(document.location())) "" else needMigration() val manga = SManga.create() manga.title = title - manga.description = description.joinToString("\n") + manga.description = description.joinToString("\n", prefix = prefix) manga.thumbnail_url = thumbnail descriptionElement.forEach { val text = it.text() @@ -148,7 +154,7 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String val rawName = linkElement.ownText().trim() val chapter = SChapter.create() - chapter.url = getUrlWithoutDomainWithFallback(linkElement.attr("href")) + chapter.setUrlWithoutDomain(linkElement.attr("href")) chapter.chapter_number = parseChapterNumber(rawName) chapter.name = rawName chapter.date_upload = parseChapterDate(element.select(".wr-date").last().text().trim()) @@ -160,21 +166,30 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String if (name.contains("[단편]")) return 1f // `특별` means `Special`, so It can be buggy. so pad `편`(Chapter) to prevent false return if (name.contains("번외") || name.contains("특별편")) return -2f - val regex = Regex("([0-9]+)(?:[-.]([0-9]+))?(?:화)") + val regex = chapterNumberRegex val (ch_primal, ch_second) = regex.find(name)!!.destructured return (ch_primal + if (ch_second.isBlank()) "" else ".$ch_second").toFloatOrNull() ?: -1f } catch (e: Exception) { - e.printStackTrace() + Log.e("NewToki", "failed to parse chapter number '$name'", e) return -1f } } + private fun mangaDetailsParseWithTitleCheck(manga: SManga, document: Document) = + mangaDetailsParse(document).apply { + // TODO: don't throw when there is download folder rename feature + if (manga.description.isNullOrEmpty() && manga.title != title) { + throw Exception(titleNotMatch(title)) + } + } + override fun fetchMangaDetails(manga: SManga): Observable { return rateLimitedClient.newCall(mangaDetailsRequest(manga)) .asObservableSuccess() .map { response -> - mangaDetailsParse(response).apply { initialized = true } + val document = response.asJsoup() + mangaDetailsParseWithTitleCheck(manga, document).apply { initialized = true } } } @@ -182,11 +197,19 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String return rateLimitedClient.newCall(chapterListRequest(manga)) .asObservableSuccess() .map { response -> - chapterListParse(response) + val document = response.asJsoup() + val title = mangaDetailsParseWithTitleCheck(manga, document).title + document.select(chapterListSelector()).map { + chapterFromElement(it).apply { + name = name.removePrefix(title).trimStart() + } + } } } - @SuppressLint("SimpleDateFormat") + // not thread-safe + private val dateFormat by lazy { SimpleDateFormat("yyyy.MM.dd", Locale.ENGLISH) } + private fun parseChapterDate(date: String): Long { return try { if (date.contains(":")) { @@ -205,16 +228,14 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String calendar.timeInMillis } else { - SimpleDateFormat("yyyy.MM.dd").parse(date)?.time ?: 0 + dateFormat.parse(date)?.time ?: 0 } } catch (e: Exception) { - e.printStackTrace() + Log.e("NewToki", "failed to parse chapter date '$date'", e) 0 } } - private val htmlDataRegex = Regex("""html_data\+='([^']+)'""") - override fun pageListParse(document: Document): List { val script = document.select("script:containsData(html_data)").firstOrNull()?.data() ?: throw Exception("data script not found") @@ -231,177 +252,37 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String .mapIndexed { i, img -> Page(i, "", if (img.hasAttr(dataAttr)) img.attr(dataAttr) else img.attr("abs:content")) } } - override fun latestUpdatesSelector() = popularMangaSelector() - override fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element) - override fun latestUpdatesRequest(page: Int) = popularMangaRequest(page) - override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector() + override fun latestUpdatesSelector() = ".media.post-list" + override fun latestUpdatesFromElement(element: Element) = ManaToki.latestUpdatesElementParse(element) + override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page", headers) + override fun latestUpdatesNextPageSelector() = ".pg_end" // We are able to get the image URL directly from the page list override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("This method should not be called!") override fun getFilterList() = FilterList() - private val preferences: SharedPreferences by lazy { - Injekt.get().getSharedPreferences("source_$id", 0x0000) - } + abstract val preferences: SharedPreferences override fun setupPreferenceScreen(screen: androidx.preference.PreferenceScreen) { - val baseUrlPref = androidx.preference.EditTextPreference(screen.context).apply { - key = BASE_URL_PREF_TITLE - title = BASE_URL_PREF_TITLE - summary = BASE_URL_PREF_SUMMARY - this.setDefaultValue(defaultBaseUrl) - dialogTitle = BASE_URL_PREF_TITLE - dialogMessage = "Default: $defaultBaseUrl" - - setOnPreferenceChangeListener { _, newValue -> - try { - val res = preferences.edit().putString(BASE_URL_PREF, newValue as String).commit() - Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() - res - } catch (e: Exception) { - e.printStackTrace() - false - } - } - } - - val latestExperimentPref = androidx.preference.CheckBoxPreference(screen.context).apply { - key = EXPERIMENTAL_LATEST_PREF_TITLE - title = EXPERIMENTAL_LATEST_PREF_TITLE - summary = EXPERIMENTAL_LATEST_PREF_SUMMARY - - setOnPreferenceChangeListener { _, newValue -> - try { - val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_PREF, newValue as Boolean).commit() - Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() - res - } catch (e: Exception) { - e.printStackTrace() - false - } - } - } - - val latestWithDetailPref = androidx.preference.CheckBoxPreference(screen.context).apply { - key = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE - title = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE - summary = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY - - setOnPreferenceChangeListener { _, newValue -> - try { - val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, newValue as Boolean).commit() - // Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() - res - } catch (e: Exception) { - e.printStackTrace() - false - } - } - } - - val rateLimitPeriodPref = androidx.preference.EditTextPreference(screen.context).apply { - key = RATE_LIMIT_PERIOD_PREF_TITLE - title = RATE_LIMIT_PERIOD_PREF_TITLE - summary = RATE_LIMIT_PERIOD_PREF_SUMMARY - this.setDefaultValue(defaultRateLimitPeriod.toString()) - dialogTitle = RATE_LIMIT_PERIOD_PREF_TITLE - dialogMessage = "Min 1 to Max 9, Invalid value will treat as $defaultRateLimitPeriod. Only Integer.\nDefault: $defaultRateLimitPeriod" - - setOnPreferenceChangeListener { _, newValue -> - try { - // Make sure to validate the value. - val p = (newValue as String).toLongOrNull(10) - var value = p ?: defaultRateLimitPeriod - if (p == null || value !in 1..9) { - Toast.makeText(screen.context, RATE_LIMIT_PERIOD_PREF_WARNING_INVALID_VALUE, Toast.LENGTH_LONG).show() - value = defaultRateLimitPeriod - } - val res = preferences.edit().putLong(RATE_LIMIT_PERIOD_PREF, value).commit() - Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() - res - } catch (e: Exception) { - e.printStackTrace() - false - } - } - } - - screen.addPreference(baseUrlPref) - if (name == "ManaToki") { - screen.addPreference(latestExperimentPref) - screen.addPreference(latestWithDetailPref) - } - screen.addPreference(rateLimitPeriodPref) + getPreferencesInternal(screen.context).map(screen::addPreference) } protected fun getUrlPath(orig: String): String { - return try { - URI(orig).path - } catch (e: URISyntaxException) { - orig - } + val url = baseUrl.toHttpUrl().resolve(orig) ?: return orig + val pathSegments = url.pathSegments + return "/${pathSegments[0]}/${pathSegments[1]}" } - // This is just replicate of original method but with fallback. - protected fun getUrlWithoutDomainWithFallback(orig: String): String { - return try { - val uri = URI(orig) - var out = uri.path - if (uri.query != null) { - out += "?" + uri.query - } - if (uri.fragment != null) { - out += "#" + uri.fragment - } - out - } catch (e: URISyntaxException) { - // fallback method. may not work. - orig.substringAfter(baseUrl) - } - } - - private fun getPrefBaseUrl(): String = preferences.getString(BASE_URL_PREF, defaultBaseUrl)!! - protected fun getExperimentLatest(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_PREF, false) - protected fun getLatestWithDetail(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, false) - private fun getRateLimitPeriod(): Long = try { // Check again as preference is bit weirdly buggy. - val v = preferences.getLong(RATE_LIMIT_PERIOD_PREF, defaultRateLimitPeriod) - if (v in 1..9) v else defaultRateLimitPeriod - } catch (e: Exception) { - defaultRateLimitPeriod + private fun isCleanPath(absUrl: String): Boolean { + val url = absUrl.toHttpUrl() + return url.pathSegments.size == 2 && url.querySize == 0 && url.fragment == null } companion object { - private const val RESTART_TACHIYOMI = "Restart Tachiyomi to apply new setting." - - private const val BASE_URL_PREF_TITLE = "Override BaseUrl" - private val BASE_URL_PREF = "overrideBaseUrl_v${AppInfo.getVersionName()}" - private const val BASE_URL_PREF_SUMMARY = "For temporary uses. Update extension will erase this setting." - - // Setting: Experimental Latest Fetcher - private const val EXPERIMENTAL_LATEST_PREF_TITLE = "Enable Latest (Experimental)" - private const val EXPERIMENTAL_LATEST_PREF = "fetchLatestExperiment" - private const val EXPERIMENTAL_LATEST_PREF_SUMMARY = "Fetch Latest Manga using Latest Chapters. May has duplicates and May DB corruption on certain Tachiyomi builds" - - // Setting: Experimental Latest Fetcher With Full Details (Optional) - private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE = "Fetch Latest with detail (Optional)" - private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF = "fetchLatestWithDetail" - private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY = - "Parse latest manga details with detail pages. This will reduce DB corruption on certain Tachiyomi builds.\n" + - "But makes chance of IP Ban, Also makes bunch of requests, For prevent IP ban, rate limit is set. so may slow,\n" + - "Still, It's experiment. Required to enable `Enable Latest (Experimental).`" - - // Settings: Rate Limit Period - private const val defaultRateLimitPeriod: Long = 2L - private const val RATE_LIMIT_PERIOD_PREF_TITLE = "Rate Limit Request Period Seconds" - private const val RATE_LIMIT_PERIOD_PREF = "rateLimitPeriod" - private const val RATE_LIMIT_PERIOD_PREF_SUMMARY = - "As Source is using Temporary IP ban system to who makes bunch of request, Some of requests are rate limited\n" + - "If you want to reduce limit, Use this option.\n" + - "Invalid value will treat as default $defaultRateLimitPeriod seconds.\n" + - "(Valid: Min 1 to Max 9)" - private const val RATE_LIMIT_PERIOD_PREF_WARNING_INVALID_VALUE = "Invalid value detected. Treating as $defaultRateLimitPeriod..." - const val PREFIX_ID_SEARCH = "id:" + + private val chapterNumberRegex by lazy { Regex("([0-9]+)(?:[-.]([0-9]+))?화") } + private val htmlDataRegex by lazy { Regex("""html_data\+='([^']+)'""") } } } diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiWebtoon.kt similarity index 66% rename from src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt rename to src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiWebtoon.kt index aaa76d0a4..e913b1012 100644 --- a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiWebtoon.kt @@ -1,37 +1,18 @@ package eu.kanade.tachiyomi.extension.ko.newtoki import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.Source -import eu.kanade.tachiyomi.source.SourceFactory import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Request -import java.security.MessageDigest -import java.text.SimpleDateFormat -import java.util.Date -import java.util.Locale -/** - * Source changes domain names every few days (e.g. newtoki31.net to newtoki32.net) - * The domain name was newtoki32 on 2019-11-14, this attempts to match the rate at which the domain changes - * - * Since 2020-09-20, They changed manga side to Manatoki. - * It was merged after shutdown of ManaMoa. - * This is by the head of Manamoa, as they decided to move to Newtoki. - */ -private val domainNumber = 32 + ((Date().time - SimpleDateFormat("yyyy-MM-dd", Locale.US).parse("2019-11-14")!!.time) / 595000000) - -class NewTokiFactory : SourceFactory { - override fun createSources(): List = listOf( - ManaToki(domainNumber), - NewTokiWebtoon() - ) -} - -class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "webtoon") { +object NewTokiWebtoon : NewToki("NewToki", "webtoon") { // / ! DO NOT CHANGE THIS ! Prevent to treating as a new site - override val id by lazy { generateSourceId("NewToki (Webtoon)", lang, versionId) } + override val id = NEWTOKI_ID + + override val baseUrl get() = "https://$NEWTOKI_PREFIX$domainNumber.com" + + override val preferences = newTokiPreferences override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { val url = ("$baseUrl/webtoon" + (if (page > 1) "/p$page" else "")).toHttpUrl().newBuilder() @@ -44,12 +25,12 @@ class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "w } is SearchSortTypeList -> { - url.addQueryParameter("sst", listOf("as_update", "wr_hit", "wr_good")[filter.state]) + val state = filter.state ?: return@forEach + url.addQueryParameter("sst", arrayOf("as_update", "wr_hit", "wr_good")[state.index]) + url.addQueryParameter("sod", if (state.ascending) "asc" else "desc") } - is SearchOrderTypeList -> { - url.addQueryParameter("sod", listOf("desc", "asc")[filter.state]) - } + else -> {} } } @@ -76,11 +57,13 @@ class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "w url.addQueryParameter("tag", filter.values[filter.state]) } } + + else -> {} } } } - return GET(url.toString()) + return GET(url.toString(), headers) } private class SearchTargetTypeList : Filter.Select("Type", arrayOf("전체", "일반웹툰", "성인웹툰", "BL/GL", "완결웹툰")) @@ -143,7 +126,7 @@ class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "w ) ) - private class SearchSortTypeList : Filter.Select( + private class SearchSortTypeList : Filter.Sort( "Sort", arrayOf( "기본(업데이트순)", @@ -152,28 +135,13 @@ class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "w ) ) - private class SearchOrderTypeList : Filter.Select( - "Order", - arrayOf( - "Descending", - "Ascending" - ) - ) - override fun getFilterList() = FilterList( SearchTargetTypeList(), SearchSortTypeList(), - SearchOrderTypeList(), Filter.Separator(), - Filter.Header("Under 3 Filters can't use with query"), + Filter.Header(ignoredForTextSearch()), SearchYoilTypeList(), SearchJaumTypeList(), SearchGenreTypeList() ) } - -fun generateSourceId(name: String, lang: String, versionId: Int): Long { - val key = "${name.lowercase()}/$lang/$versionId" - val bytes = MessageDigest.getInstance("MD5").digest(key.toByteArray()) - return (0..7).map { bytes[it].toLong() and 0xff shl 8 * (7 - it) }.reduce(Long::or) and Long.MAX_VALUE -} diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/Preferences.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/Preferences.kt new file mode 100644 index 000000000..176a64118 --- /dev/null +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/Preferences.kt @@ -0,0 +1,74 @@ +package eu.kanade.tachiyomi.extension.ko.newtoki + +import android.app.Application +import android.content.Context +import android.content.SharedPreferences +import androidx.preference.EditTextPreference +import androidx.preference.ListPreference +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get + +const val MANATOKI_ID = 2526381983439079467L // "NewToki/ko/1" +const val NEWTOKI_ID = 1977818283770282459L // "NewToki (Webtoon)/ko/1" + +const val MANATOKI_PREFIX = "manatoki" +const val NEWTOKI_PREFIX = "newtoki" + +val manaTokiPreferences = getSharedPreferences(MANATOKI_ID) +val newTokiPreferences = getSharedPreferences(NEWTOKI_ID) + +fun getPreferencesInternal(context: Context) = arrayOf( + + EditTextPreference(context).apply { + key = DOMAIN_NUMBER_PREF + title = domainNumberTitle() + summary = domainNumberSummary() + setOnPreferenceChangeListener { _, newValue -> + val value = newValue as String + if (value.isEmpty() || value != value.trim()) { + false + } else { + domainNumber = value + true + } + } + }, + + ListPreference(context).apply { + key = RATE_LIMIT_PERIOD_PREF + title = rateLimitTitle() + summary = "%s\n" + requiresAppRestart() + + val values = Array(RATE_LIMIT_PERIOD_MAX) { (it + 1).toString() } + entries = Array(RATE_LIMIT_PERIOD_MAX) { rateLimitEntry(values[it]) } + entryValues = values + + setDefaultValue(RATE_LIMIT_PERIOD_DEFAULT) + }, +) + +var SharedPreferences.domainNumber: String + get() = getString(DOMAIN_NUMBER_PREF, "")!! + set(value) = edit().putString(DOMAIN_NUMBER_PREF, value).apply() + +val SharedPreferences.rateLimitPeriod: Int + get() = getString(RATE_LIMIT_PERIOD_PREF, RATE_LIMIT_PERIOD_DEFAULT)!!.toInt().coerceIn(1, RATE_LIMIT_PERIOD_MAX) + +/** + * Don't use the following legacy keys: + * - "Override BaseUrl" + * - "overrideBaseUrl_v${AppInfo.getVersionName()}" + * - "Enable Latest (Experimental)" + * - "fetchLatestExperiment" + * - "Fetch Latest with detail (Optional)" + * - "fetchLatestWithDetail" + * - "Rate Limit Request Period Seconds" + */ + +private const val DOMAIN_NUMBER_PREF = "domainNumber" +private const val RATE_LIMIT_PERIOD_PREF = "rateLimitPeriod" +private const val RATE_LIMIT_PERIOD_DEFAULT = 2.toString() +private const val RATE_LIMIT_PERIOD_MAX = 9 + +private fun getSharedPreferences(id: Long): SharedPreferences = + Injekt.get().getSharedPreferences("source_$id", 0x0000) diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/Strings.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/Strings.kt new file mode 100644 index 000000000..2a9271479 --- /dev/null +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/Strings.kt @@ -0,0 +1,76 @@ +package eu.kanade.tachiyomi.extension.ko.newtoki + +import android.os.Build +import android.os.LocaleList +import java.util.Locale + +private val useKorean by lazy { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + LocaleList.getDefault().getFirstMatch(arrayOf("ko", "en"))?.language == "ko" + } else { + Locale.getDefault().language == "ko" + } +} + +// region Prompts + +fun solveCaptcha() = when { + useKorean -> "WebView에서 캡챠 풀기" + else -> "Solve Captcha with WebView" +} + +fun titleNotMatch(realTitle: String) = when { + useKorean -> "이 만화를 찾으시려면 '$realTitle'으로 검색하세요" + else -> "Find this manga by searching '$realTitle'" +} + +fun needMigration() = when { + useKorean -> "이 항목은 URL 포맷이 틀립니다. 중복된 항목을 피하려면 동일한 소스로 이전하세요.\n\n" + else -> "This entry has wrong URL format. Please migrate to the same source to avoid duplicates.\n\n" +} + +// endregion + +// region Filters + +fun ignoredForTextSearch() = when { + useKorean -> "검색에서 다음 필터 항목은 무시됩니다" + else -> "The following filters are ignored for text search" +} + +// endregion + +// region Preferences + +fun domainNumberTitle() = when { + useKorean -> "도메인 번호" + else -> "Domain number" +} + +fun domainNumberSummary() = when { + useKorean -> "도메인 번호는 자동으로 갱신됩니다" + else -> "This number is updated automatically" +} + +fun editDomainNumber() = when { + useKorean -> "확장기능 설정에서 도메인 번호를 수정해 주세요" + else -> "Please edit domain number in extension settings" +} + +fun rateLimitTitle() = when { + useKorean -> "요청 제한" + else -> "Rate limit" +} + +fun rateLimitEntry(period: String) = when { + useKorean -> "${period}초마다 요청" + else -> "1 request every $period seconds" +} + +// taken from app strings +fun requiresAppRestart() = when { + useKorean -> "설정을 적용하려면 앱을 재시작하세요" + else -> "Requires app restart to take effect" +} + +// endregion diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/TokiFactory.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/TokiFactory.kt new file mode 100644 index 000000000..a60bd25e2 --- /dev/null +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/TokiFactory.kt @@ -0,0 +1,7 @@ +package eu.kanade.tachiyomi.extension.ko.newtoki + +import eu.kanade.tachiyomi.source.SourceFactory + +class TokiFactory : SourceFactory { + override fun createSources() = listOf(ManaToki, NewTokiWebtoon) +}