diff --git a/src/en/bookwalker/assets/webview-script.js b/src/en/bookwalker/assets/webview-script.js new file mode 100644 index 000000000..519eef32d --- /dev/null +++ b/src/en/bookwalker/assets/webview-script.js @@ -0,0 +1,189 @@ +//__INJECT_WEBVIEW_INTERFACE = { reportImage: console.log } +const webviewInterface = __INJECT_WEBVIEW_INTERFACE; + +const checkLoadedTimer = setInterval(() => { + if (document.getElementById('pageSliderCounter')?.innerText) { + webviewInterface.reportViewerLoaded(); + clearInterval(checkLoadedTimer); + } +}, 10); + +const checkMessageTimer = setInterval(() => { + const messageElt = document.querySelector('#messageDialog .message'); + if (messageElt) { + webviewInterface.reportFailedToLoad(messageElt.textContent); + clearInterval(checkMessageTimer); + } +}, 2000); + +// In order for image extraction to work, the viewer needs to be in horizontal mode. +// That setting is stored in localStorage, so we intercept calls to localStorage.getItem() +// in order to fake the value we want. +// There is a potential timing concern here if this JavaScript runs too late, but it +// seems that the value is read later in the loading process so it should be okay. +localStorage.getItem = function(key) { + let result = Storage.prototype.getItem.apply(this, [key]); + if (key === '/NFBR_Settings/NFBR.SettingData') { + try { + const data = JSON.parse(result); + data.viewerPageTransitionAxis = 'horizontal'; + result = JSON.stringify(data); + } + catch (e) {} + } + return result; +}; + +function getCurrentPageIndex() { + return +document.getElementById('pageSliderCounter').innerText.split('/')[0] - 1; +} + +function getLastPageIndex() { + return +document.getElementById('pageSliderCounter').innerText.split('/')[1] - 1; +} + +const alreadyFetched = []; + +// The goal here is to capture the full-size processed image data just before it +// gets resized and drawn to the viewport. +const baseDrawImage = CanvasRenderingContext2D.prototype.drawImage; +CanvasRenderingContext2D.prototype.drawImage = function(image, sx, sy, sWidth, sHeight /* , ... */) { + baseDrawImage.apply(this, arguments); + + // It's important that the screen size is small enough that only one page + // appears at a time so that this page number is accurate. + // Otherwise, pages could end up out of order, skipped, or duplicated. + const pageIdx = getCurrentPageIndex(); + if (alreadyFetched[pageIdx]) { + // We already found this page, no need to do the processing again + return; + } + + const current = document.querySelector('.currentScreen'); + // It can render pages on the opposite side of the spread even in one-page mode, + // so to make sure we're not grabbing the wrong image, we want to check that + // the current page is the side that the image is being drawn to. + if (current.contains(this.canvas)) { + // imageData should be a Uint8ClampedArray containing RGBA row-major bitmap image data. + // We don't create the final JPEGs right here with Canvas.toBlob/OffscreenCanvas.convertToBlob + // because in testing, that was _extremely_ slow to run in the Webview, taking upwards of + // 15 seconds per image. Doing that conversion on the JVM side is near-instantaneous. + let imageData; + + if (image instanceof ImageBitmap) { + const ctx = new OffscreenCanvas(sWidth, sHeight) + .getContext('2d', { willReadFrequently: true }); + ctx.drawImage(image, sx, sy, sWidth, sHeight); + imageData = ctx.getImageData(sx, sy, sWidth, sHeight).data; + } + else if (image instanceof HTMLCanvasElement && image.matches('canvas.dummy[width][height]')) { + const ctx = image.getContext('2d', { willReadFrequently: true }); + imageData = ctx.getImageData(sx, sy, sWidth, sHeight).data; + } + else { + // Other misc images can sometimes be drawn. We don't care about those. + return; + } + console.log("intercepted image"); + + alreadyFetched[pageIdx] = true; + + // The WebView interface only allows communicating with strings, + // so we need to convert our ArrayBuffer to a string for transport. + const textData = new TextDecoder("windows-1252").decode(imageData); + console.log("sending encoded data"); + webviewInterface.reportImage(pageIdx, textData, sWidth, sHeight); + } +} + +// JS UTILITIES +const leftArrowKeyCode = 37; +const rightArrowKeyCode = 39; + +function fireKeyboardEvent(elt, keyCode) { + elt.dispatchEvent(new window.KeyboardEvent('keydown', { keyCode })); +} + +window.__INJECT_JS_UTILITIES = { + fetchPageData(targetPageIndex) { + alreadyFetched[targetPageIndex] = false; + + const lastPageIndex = getLastPageIndex(); + + if (targetPageIndex > lastPageIndex) { + // This generally occurs when reading a preview chapter. + webviewInterface.reportImageDoesNotExist( + targetPageIndex, + "You have reached the end of the preview.", + ); + return; + } + + const renderer = document.getElementById('renderer'); + const slider = document.getElementById('pageSliderBar'); + const isLTR = Boolean(slider.querySelector('.ui-slider-range-min')); + + const [forwardsKeyCode, backwardsKeyCode] = isLTR + ? [rightArrowKeyCode, leftArrowKeyCode] + : [leftArrowKeyCode, rightArrowKeyCode]; + + if (getCurrentPageIndex() === targetPageIndex) { + // The image may have already loaded, but we need to shuffle around for it to get reported. + // Otherwise, we can get stuck waiting for the image to be reported forever and + // eventually time out. + console.log('already at correct page'); + if (targetPageIndex === lastPageIndex) { + fireKeyboardEvent(renderer, backwardsKeyCode); + fireKeyboardEvent(renderer, forwardsKeyCode); + } + else { + fireKeyboardEvent(renderer, forwardsKeyCode); + fireKeyboardEvent(renderer, backwardsKeyCode); + } + return; + } + + function invertIfRTL(value) { + if (isLTR) { + return value; + } + return 1 - value; + } + + const { x, width, y, height } = slider.getBoundingClientRect(); + const options = { + clientX: x + width * invertIfRTL(targetPageIndex / lastPageIndex), + clientY: y + height / 2, + bubbles: true, + }; + slider.dispatchEvent(new MouseEvent('mousedown', options)); + slider.dispatchEvent(new MouseEvent('mouseup', options)); + + // That should have gotten us most of the way there but since the clicks aren't always + // perfectly accurate, we may need to make some adjustments to get the rest of the way. + // This mostly comes up for longer chapters and volumes that have a large number of pages, + // since the small webview makes the slider pretty short. + + function adjustPage() { + const distance = targetPageIndex - getCurrentPageIndex(); + console.log("current", getCurrentPageIndex(), "target", targetPageIndex, "distance", distance) + if (distance !== 0) { + const keyCode = distance > 0 ? forwardsKeyCode : backwardsKeyCode; + for (let i = 0; i < Math.abs(distance); i++) { + renderer.dispatchEvent(new KeyboardEvent('keydown', { keyCode })); + } + } + + console.log("final location", getCurrentPageIndex()); + + // Sometimes, particularly when the page has just loaded, the adjustment doesn't work. + // If that happens, retry the adjustment after a brief delay. + if (getCurrentPageIndex() !== targetPageIndex) { + console.log("retrying page adjustment in 100ms...") + setTimeout(adjustPage, 100); + } + } + + adjustPage(); + } +} diff --git a/src/en/bookwalker/build.gradle b/src/en/bookwalker/build.gradle new file mode 100644 index 000000000..6050cc24f --- /dev/null +++ b/src/en/bookwalker/build.gradle @@ -0,0 +1,8 @@ +ext { + extName = 'BookWalker Global' + extClass = '.BookWalker' + extVersionCode = 1 + isNsfw = true +} + +apply from: "$rootDir/common.gradle" diff --git a/src/en/bookwalker/res/mipmap-hdpi/ic_launcher.png b/src/en/bookwalker/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 000000000..aa40647af Binary files /dev/null and b/src/en/bookwalker/res/mipmap-hdpi/ic_launcher.png differ diff --git a/src/en/bookwalker/res/mipmap-mdpi/ic_launcher.png b/src/en/bookwalker/res/mipmap-mdpi/ic_launcher.png new file mode 100644 index 000000000..b00bd253f Binary files /dev/null and b/src/en/bookwalker/res/mipmap-mdpi/ic_launcher.png differ diff --git a/src/en/bookwalker/res/mipmap-xhdpi/ic_launcher.png b/src/en/bookwalker/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 000000000..8c3f67d00 Binary files /dev/null and b/src/en/bookwalker/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/src/en/bookwalker/res/mipmap-xxhdpi/ic_launcher.png b/src/en/bookwalker/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 000000000..edd86fb22 Binary files /dev/null and b/src/en/bookwalker/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/src/en/bookwalker/res/mipmap-xxxhdpi/ic_launcher.png b/src/en/bookwalker/res/mipmap-xxxhdpi/ic_launcher.png new file mode 100644 index 000000000..58c98a8e4 Binary files /dev/null and b/src/en/bookwalker/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalker.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalker.kt new file mode 100644 index 000000000..9e894f667 --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalker.kt @@ -0,0 +1,821 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker + +import android.app.Application +import android.text.Editable +import android.text.TextWatcher +import android.util.Log +import androidx.preference.EditTextPreference +import androidx.preference.ListPreference +import androidx.preference.PreferenceScreen +import androidx.preference.SwitchPreferenceCompat +import eu.kanade.tachiyomi.extension.en.bookwalker.dto.HoldBookEntityDto +import eu.kanade.tachiyomi.extension.en.bookwalker.dto.HoldBooksInfoDto +import eu.kanade.tachiyomi.extension.en.bookwalker.dto.SeriesDto +import eu.kanade.tachiyomi.extension.en.bookwalker.dto.SingleDto +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.POST +import eu.kanade.tachiyomi.network.await +import eu.kanade.tachiyomi.source.ConfigurableSource +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.MangasPage +import eu.kanade.tachiyomi.source.model.Page +import eu.kanade.tachiyomi.source.model.SChapter +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.source.online.ParsedHttpSource +import eu.kanade.tachiyomi.util.asJsoup +import keiyoushi.utils.getPreferencesLazy +import keiyoushi.utils.parseAs +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.launch +import kotlinx.serialization.json.Json +import kotlinx.serialization.modules.SerializersModule +import kotlinx.serialization.modules.plus +import kotlinx.serialization.modules.polymorphic +import kotlinx.serialization.modules.subclass +import okhttp3.Call +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MultipartBody +import okhttp3.Request +import okhttp3.Response +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import rx.Observable +import rx.Single +import uy.kohesive.injekt.injectLazy +import java.util.regex.PatternSyntaxException + +class BookWalker : ConfigurableSource, ParsedHttpSource(), BookWalkerPreferences { + + override val name = "BookWalker Global" + + override val baseUrl = "https://global.bookwalker.jp" + + override val lang = "en" + + override val supportsLatest = true + + override val client = network.client.newBuilder() + .addInterceptor(BookWalkerImageRequestInterceptor(this)) + .build() + + // The UA should be a desktop UA because all research was done with a desktop UA, and the site + // renders differently with a mobile user agent. + // However, we're not overriding headersBuilder because we don't want to show the desktop site + // when a user opens a WebView to e.g. log in or make a purchase. + // We just need the desktop site when making requests from extension logic. + val callHeaders = headers.newBuilder() + .set("User-Agent", USER_AGENT_DESKTOP) + .build() + + private val json = Json { + ignoreUnknownKeys = true + serializersModule += SerializersModule { + polymorphic(HoldBookEntityDto::class) { + subclass(SingleDto::class) + subclass(SeriesDto::class) + } + } + } + + val app by injectLazy() + + private val preferences by getPreferencesLazy() + + override val showLibraryInPopular + get() = preferences.getBoolean(PREF_SHOW_LIBRARY_IN_POPULAR, false) + + override val shouldValidateLogin + get() = preferences.getBoolean(PREF_VALIDATE_LOGGED_IN, true) + + override val imageQuality + get() = ImageQualityPref.fromKey( + preferences.getString(ImageQualityPref.PREF_KEY, ImageQualityPref.defaultOption.key)!!, + ) + + override val filterChapters + get() = FilterChaptersPref.fromKey( + preferences.getString( + FilterChaptersPref.PREF_KEY, + FilterChaptersPref.defaultOption.key, + )!!, + ) + + override val attemptToReadPreviews + get() = preferences.getBoolean(PREF_ATTEMPT_READ_PREVIEWS, false) + + override val excludeCategoryFilters + get() = Regex( + preferences.getString( + PREF_CATEGORY_EXCLUDE_REGEX, + categoryExcludeRegexDefault, + )!!, + RegexOption.IGNORE_CASE, + ) + + override val excludeGenreFilters + get() = Regex( + preferences.getString(PREF_GENRE_EXCLUDE_REGEX, genreExcludeRegexDefault)!!, + RegexOption.IGNORE_CASE, + ) + + override fun setupPreferenceScreen(screen: PreferenceScreen) { + fun regularExpressionPref(block: EditTextPreference.() -> Unit): EditTextPreference { + fun validateRegex(regex: String): String? { + return try { + Regex(regex) + null + } catch (e: PatternSyntaxException) { + e.message + } + } + + return EditTextPreference(screen.context).apply { + dialogMessage = "Enter a regular expression. " + + "Sub-string matches will be counted as matches. Matches are case-insensitive." + + setOnBindEditTextListener { field -> + field.addTextChangedListener( + object : TextWatcher { + override fun beforeTextChanged(s: CharSequence, start: Int, count: Int, after: Int) {} + override fun onTextChanged(s: CharSequence, start: Int, before: Int, count: Int) {} + + override fun afterTextChanged(s: Editable) { + validateRegex(s.toString())?.let { field.error = it } + } + }, + ) + } + + setOnPreferenceChangeListener { _, new -> + validateRegex(new as String) == null + } + + block() + } + } + + SwitchPreferenceCompat(screen.context).apply { + key = PREF_VALIDATE_LOGGED_IN + title = "Validate Login" + summary = "Validate that you are logged in before allowing certain actions. This is " + + "recommended to avoid confusing behavior when your login session expires.\n" + + "If you are using this extension as an anonymous user, disable this option." + + setDefaultValue(true) + }.also(screen::addPreference) + + ListPreference(screen.context).apply { + key = ImageQualityPref.PREF_KEY + title = "Image Quality" + summary = "%s" + + entries = arrayOf( + "Automatic", + "Medium", + "High", + ) + + entryValues = arrayOf( + ImageQualityPref.DEVICE.key, + ImageQualityPref.MEDIUM.key, + ImageQualityPref.HIGH.key, + ) + + setDefaultValue(ImageQualityPref.defaultOption.key) + }.also(screen::addPreference) + + SwitchPreferenceCompat(screen.context).apply { + key = PREF_SHOW_LIBRARY_IN_POPULAR + title = "Show My Library in Popular" + summary = "Show your library instead of popular manga when browsing \"Popular\"." + + setDefaultValue(false) + }.also(screen::addPreference) + + ListPreference(screen.context).apply { + key = FilterChaptersPref.PREF_KEY + title = "Filter Shown Chapters" + summary = "Choose what types of chapters to show." + + entries = arrayOf( + "Show owned and free chapters", + "Show obtainable chapters", + "Show all chapters", + ) + + entryValues = arrayOf( + FilterChaptersPref.OWNED.key, + FilterChaptersPref.OBTAINABLE.key, + FilterChaptersPref.ALL.key, + ) + + setDefaultValue(FilterChaptersPref.defaultOption.key) + }.also(screen::addPreference) + + SwitchPreferenceCompat(screen.context).apply { + key = PREF_ATTEMPT_READ_PREVIEWS + title = "Show Previews When Available" + summary = "Determines whether attempting to read an un-owned chapter should show the " + + "preview. Even when disabled, you will still be able to read free chapters you " + + "have not \"purchased\"." + + setDefaultValue(true) + }.also(screen::addPreference) + + regularExpressionPref { + key = PREF_CATEGORY_EXCLUDE_REGEX + title = "Exclude Category Filters" + summary = "Hide certain categories from being listed in the search filters. " + + "This will not hide manga with those categories from search results." + + setDefaultValue(categoryExcludeRegexDefault) + }.also(screen::addPreference) + + regularExpressionPref { + key = PREF_GENRE_EXCLUDE_REGEX + title = "Exclude Genre Filters" + summary = "Hide certain genres from being listed in the search filters. " + + "This will not hide manga with those genres from search results." + + setDefaultValue(genreExcludeRegexDefault) + }.also(screen::addPreference) + } + + private val filterInfo by lazy { BookWalkerFilters(this) } + + override fun getFilterList(): FilterList { + filterInfo.fetchIfNecessaryInBackground() + + fun Iterable.prependAll(): List { + return mutableListOf(allFilter).apply { addAll(this@prependAll) } + } + + return FilterList( + SelectOneFilter( + "Sort By", + "order", + listOf( + FilterInfo("Relevancy", "score"), + FilterInfo("Popularity", "rank"), + FilterInfo("Release Date", "release"), + FilterInfo("Title", "title"), + ), + ), + SelectOneFilter( + "Categories", + QUERY_PARAM_CATEGORY, + filterInfo.categories + ?.filterNot { excludeCategoryFilters.containsMatchIn(it.name) } + ?.prependAll() ?: fallbackFilters, + ), + SelectMultipleFilter( + "Genre", + QUERY_PARAM_GENRE, + filterInfo.genres + ?.filterNot { excludeGenreFilters.containsMatchIn(it.name) } + ?: fallbackFilters, + ), + // Author filter disabled for now, since the performance/UX in-app is pretty bad +// SelectMultipleFilter( +// "Author", +// QUERY_PARAM_AUTHOR, +// filterInfo.authors ?: fallbackFilters, +// ), + SelectOneFilter( + "Publisher", + QUERY_PARAM_PUBLISHER, + filterInfo.publishers?.prependAll() ?: fallbackFilters, + ), + OthersFilter(), + PriceFilter(), + ExcludeFilter(), + ) + } + + override fun popularMangaRequest(page: Int): Request { + filterInfo.fetchIfNecessaryInBackground() + + if (showLibraryInPopular) { + return POST( + "$baseUrl/prx/holdBooks-api/hold-book-list/", + callHeaders, + MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart("holdBook-series", "1") + .build(), + ) + } + // /categories/2/ - manga + // np=0 - display by series + // order=rank - sort by popularity + return GET("$baseUrl/categories/2/?order=rank&np=0&page=$page", callHeaders) + } + + override fun popularMangaParse(response: Response): MangasPage { + if (showLibraryInPopular) { + val manga = response.parseAs().holdBookList.entities + .map { + when (it) { + is SeriesDto -> SManga.create().apply { + url = "/series/${it.seriesId}/" + title = it.seriesName.cleanTitle() + thumbnail_url = it.imageUrl + } + is SingleDto -> SManga.create().apply { + url = it.detailUrl.substring(baseUrl.length) + title = it.title.cleanTitle() + thumbnail_url = it.imageUrl + author = it.authors.joinToString { a -> a.authorName } + } + } + } + return MangasPage(manga, false) + } + return super.popularMangaParse(response) + } + + override fun popularMangaNextPageSelector(): String = + ".pager-area .next > a" + + override fun popularMangaSelector(): String = + ".book-list-area .o-tile" + + override fun popularMangaFromElement(element: Element): SManga { + val titleElt = element.select(".a-tile-ttl a") + + return SManga.create().apply { + url = titleElt.attr("href").substring(baseUrl.length) + title = titleElt.attr("title").cleanTitle() + thumbnail_url = element.select(".a-tile-thumb-img > img") + .attr("data-srcset").getHighestQualitySrcset() + } + } + + override fun latestUpdatesRequest(page: Int): Request { + filterInfo.fetchIfNecessaryInBackground() + + // qcat=2 - only show manga + // np=0 - display by series + return GET("$baseUrl/new/?order=release&qcat=2&np=0&page=$page", callHeaders) + } + + override fun latestUpdatesNextPageSelector(): String? = popularMangaNextPageSelector() + override fun latestUpdatesSelector(): String = popularMangaSelector() + override fun latestUpdatesFromElement(element: Element): SManga = popularMangaFromElement(element) + + override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { + filterInfo.fetchIfNecessaryInBackground() + + val urlBuilder = "$baseUrl/search/".toHttpUrl().newBuilder().apply { + addQueryParameter("np", "0") // display by series + addQueryParameter("page", page.toString()) + addQueryParameter("word", query) + + filters.list + .filterIsInstance() + .flatMap { it.getQueryParams() } + .forEach { + // special case since sorting by relevance doesn't work without search terms + if (query.isEmpty() && it.first == "order" && it.second == "score") { + addQueryParameter(it.first, "rank") // sort by popularity + } else { + addQueryParameter(it.first, it.second) + } + } + } + + return GET(urlBuilder.build(), callHeaders) + } + + override fun searchMangaNextPageSelector(): String? = popularMangaNextPageSelector() + override fun searchMangaSelector(): String = popularMangaSelector() + override fun searchMangaFromElement(element: Element): SManga = popularMangaFromElement(element) + + override fun fetchMangaDetails(manga: SManga): Observable { + // Manga with "/series/" in their URL are actual series. + // Manga without are individual chapters/volumes (referred to here as "singles"). + return if (manga.url.startsWith("/series/")) { + fetchSeriesMangaDetails(manga) + } else { + fetchSingleMangaDetails(manga) + } + } + + override fun mangaDetailsParse(document: Document): SManga = + throw UnsupportedOperationException() + + private fun fetchSeriesMangaDetails(manga: SManga): Observable { + return rxSingle { + val seriesPage = client.newCall( + GET("$baseUrl${manga.url}?order=release&np=1", callHeaders), + ).awaitSuccess() + .asJsoup() + .let { validateLogin(it) } + + val mangaDetails = SManga.create().apply { + updateDetailsFromSeriesPage(seriesPage) + } + + // It generally doesn't matter which chapter we take the description from, but for + // series that release in volumes we want the earliest one, which will _usually_ be the + // last one on the page. With that said, it's not worth it to paginate in order to find + // the earliest volume, and volume releases don't usually have 60+ volumes anyways. + val chapterUrl = seriesPage + .select(".o-tile .a-tile-ttl a").last() + ?.attr("href") + ?: return@rxSingle mangaDetails + + val chapterPage = client.newCall(GET(chapterUrl, callHeaders)).await().asJsoup() + + mangaDetails.apply { + description = getDescriptionFromChapterPage(chapterPage) + } + }.toObservable() + } + + private fun fetchSingleMangaDetails(manga: SManga): Observable { + return rxSingle { + val document = client.newCall(GET(baseUrl + manga.url, callHeaders)) + .awaitSuccess() + .asJsoup() + + SManga.create().apply { + title = getTitleFromChapterPage(document)?.cleanTitle().orEmpty() + + description = getDescriptionFromChapterPage(document) + // From the browse pages we can't distinguish between a true one-shot and a + // serial manga with only one chapter, but we can detect if there's a series + // reference in the chapter page. If there is, we should let the user know that + // they may need to take some action in the future to correct the error. + if (document.selectFirst(".product-detail th:contains(Series Title)") != null) { + description = ( + "WARNING: This entry is being treated as a one-shot but appears to " + + "have an associated series. If another chapter is released in " + + "the future, you will likely need to migrate this to itself." + + "\n\n$description" + ).trim() + } + } + }.toObservable() + } + + private fun SManga.updateDetailsFromSeriesPage(document: Document) = run { + // Take the thumbnail from the first chapter that is not on pre-order. + // Pre-order chapters often just have a gray rectangle with "NOW PRINTING" as their + // thumbnail, which doesn't look very pretty for the catalog. + thumbnail_url = document + .select(".o-tile:not(:has(.a-ribbon-pre-order)) .a-tile-thumb-img > img") + .attr("data-srcset") + .getHighestQualitySrcset() + title = document.selectFirst(".title-main-inner")!!.ownText().cleanTitle() + author = getAvailableFilterNames(document, "side-author").joinToString() + genre = getAvailableFilterNames(document, "side-genre").joinToString() + + val statusIndicators = document.select("ul.side-others > li > a").map { it.ownText() } + + status = + if (statusIndicators.any { it.startsWith("Completed") }) { + if (statusIndicators.any { it.startsWith("Pre-Order") }) { + SManga.PUBLISHING_FINISHED + } else { + SManga.COMPLETED + } + } else { + SManga.ONGOING + } + } + + private fun getTitleFromChapterPage(document: Document): String? { + return document.selectFirst(".detail-book-title-box h1[itemprop='name']")?.ownText() + } + + private fun getDescriptionFromChapterPage(document: Document): String { + return buildString { + append(document.select(".synopsis-lead").text()) + append("\n\n") + append(document.select(".synopsis-text").text()) + }.trim() + } + + override fun fetchChapterList(manga: SManga): Observable> { + return rxSingle { + if (!manga.url.startsWith("/series/")) { + val document = client.newCall(GET(baseUrl + manga.url, callHeaders)) + .awaitSuccess() + .asJsoup() + + return@rxSingle listOfNotNull( + chapterFromChapterPage(document)?.apply { + url = manga.url + }, + ) + } + + suspend fun getDocumentForPage(page: Int): Document { + return client.newCall( + GET("$baseUrl${manga.url}?order=release&page=$page", callHeaders), + ).awaitSuccess().asJsoup() + } + + val firstPage = validateLogin(getDocumentForPage(1)) + val publishers = getAvailableFilterNames(firstPage, "side-publisher") + .joinToString() + + val pageCount = firstPage.selectFirst(".pager-area li:has(+ .next) > a") + ?.ownText()?.toIntOrNull() + ?: 1 + + val laterPages = (2..pageCount).map { n -> async { getDocumentForPage(n) } } + .awaitAll() + + (listOf(firstPage) + laterPages).flatMap { document -> + document.select(chapterListSelector()).map { + chapterFromElement(it).apply { + scanlator = publishers + } + } + } + }.toObservable() + } + + override fun chapterListSelector(): String { + return when (filterChapters) { + FilterChaptersPref.OWNED -> + ".book-list-area .o-tile:has(.a-read-btn-s, .a-free-btn-s)" + FilterChaptersPref.OBTAINABLE -> + ".book-list-area .o-tile:not(:has(.a-ribbon-bundle, .a-ribbon-pre-order))" + else -> // preorders shown, still not showing bundles since those aren't chapters + ".book-list-area .o-tile:not(:has(.a-ribbon-bundle))" + } + } + + override fun chapterFromElement(element: Element): SChapter = SChapter.create().apply { + val statusSuffix = + if (element.selectFirst("[data-action-label='Read']") != null) { + "" // user is currently able to read the chapter + } else if (element.selectFirst(".a-label-free") != null) { + " $FREE_ICON" // it's free to read but the user technically doesn't "own" it + } else if (element.selectFirst(".a-cart-btn-s, .a-cart-btn-s--on") != null) { + " $PURCHASE_ICON" + } else if (element.selectFirst(".a-ribbon-pre-order") != null) { + " $PREORDER_ICON" + } else { + // Some content does not fall into any of the above bins. It seems to mostly be + // bonus content you get from purchasing other items, but there may be exceptions. + " $UNKNOWN_ICON" + } + + val titleElt = element.select(".a-tile-ttl a") + val title = titleElt.attr("title").cleanTitle() + val chapterNumber = title.parseChapterNumber() + + url = titleElt.attr("href").substring(baseUrl.length) + name = WORD_JOINER + (chapterNumber?.first ?: title) + statusSuffix + chapter_number = chapterNumber?.second ?: -1f + // scanlator set by caller + } + + private fun chapterFromChapterPage(document: Document): SChapter? = SChapter.create().apply { + // See chapterFromElement for info on these statuses + val statusSuffix = + if (document.selectFirst(".a-read-on-btn") != null) { + "" + } else if (document.selectFirst(".a-cart-btn:contains(Free)") != null) { + " $FREE_ICON" + } else if (document.selectFirst(".a-cart-btn:contains(Cart)") != null) { + if (!filterChapters.includes(FilterChaptersPref.OBTAINABLE)) { + return null + } + " $PURCHASE_ICON" + } else if (document.selectFirst(".a-order-btn") != null) { + if (!filterChapters.includes(FilterChaptersPref.ALL)) { + return null + } + " $PREORDER_ICON" + } else { + if (!filterChapters.includes(FilterChaptersPref.ALL)) { + return null + } + " $UNKNOWN_ICON" + } + + val title = getTitleFromChapterPage(document).orEmpty().cleanTitle() + + val chapterNumber = title.parseChapterNumber() + + // No need to set URL, that will be handled by the caller. + name = WORD_JOINER + (chapterNumber?.first ?: title) + statusSuffix + scanlator = document.select(".product-detail tr:has(th:contains(Publisher)) > td").text() + chapter_number = chapterNumber?.second ?: -1f + } + + override fun fetchPageList(chapter: SChapter): Observable> { + return rxSingle { + val document = client.newCall(GET(baseUrl + chapter.url, callHeaders)) + .awaitSuccess() + .asJsoup() + .let { validateLogin(it) } + + val pagesText = document + .select(".product-detail tr:has(th:contains(Page count)) > td").text() + + val pagesCount = Regex("\\d+").find(pagesText)?.value?.toIntOrNull() + ?: throw Error("Could not determine number of pages in chapter") + + if (pagesCount == 0) { + throw Error("The page count is 0. If this chapter was just released, wait a bit for the page count to update.") + } + + val isFreeChapter = document.selectFirst(".a-cart-btn:contains(Free)") != null + val readerUrl = document.selectFirst("a.a-read-on-btn")?.attr("href") + ?: if (attemptToReadPreviews || isFreeChapter) { + document.selectFirst(".free-preview > a")?.attr("href") + ?: throw Error("No preview available") + } else { + throw Error("You don't own this chapter, or you aren't logged in") + } + + // We need to use the full cooperative URL every time we try to load a chapter since + // the reader page relies on transient cookies set in the cooperative flow. + // This call is simply being used to ensure the user is logged in. + // Note that this is not fool-proof since the app may cache the page list, so sometimes + // the best we can do is detect that the user is not logged in when loading the page + // and fail to load the image at that point. + tryCooperativeRedirect(readerUrl, "You must log in again. Open in WebView and click the shopping cart.") + + IntRange(0, pagesCount - 1).map { + // The page index query parameter exists only to prevent the app from trying to + // be smart about caching by page URLs, since the URL is the same for all the pages. + // It doesn't do anything, and in fact gets stripped back out in imageRequest. + Page( + it, + imageUrl = readerUrl.toHttpUrl().newBuilder() + .setQueryParameter(PAGE_INDEX_QUERY_PARAM, it.toString()) + .build() + .toString(), + ) + } + }.toObservable() + } + + override fun pageListParse(document: Document): List = + throw UnsupportedOperationException() + + override fun imageUrlParse(document: Document): String = + throw UnsupportedOperationException() + + override fun imageRequest(page: Page): Request { + // This URL doesn't actually contain the image. It will be intercepted, and the actual image + // will be extracted from a webview of the URL being sent here. + val imageUrl = page.imageUrl!!.toHttpUrl() + return GET( + imageUrl.newBuilder() + .removeAllQueryParameters(PAGE_INDEX_QUERY_PARAM) + .build() + .toString(), + callHeaders.newBuilder() + .set(HEADER_IS_REQUEST_FROM_EXTENSION, "true") + .set(HEADER_PAGE_INDEX, imageUrl.queryParameter(PAGE_INDEX_QUERY_PARAM)!!) + .build(), + ) + } + + private suspend fun validateLogin(document: Document): Document { + if (!shouldValidateLogin) { + return document + } + val signInBtn = document.selectFirst(".logout-nav-area .btn-sign-in a") + if (signInBtn != null) { + // Sometimes just clicking on the button will sign the user in without needing to input + // credentials, so we'll try to log in automatically. + val signInUrl = signInBtn.attr("href") + val redirectedPage = tryCooperativeRedirect(signInUrl) + return client.newCall(GET(redirectedPage, callHeaders)).awaitSuccess().asJsoup() + } + return document + } + + private suspend fun tryCooperativeRedirect(url: String, message: String = "Logged out, check website in WebView"): HttpUrl { + return client.newCall(GET(url, callHeaders)).await().use { + val redirectUrl = it.request.url + + if (redirectUrl.host == "member.bookwalker.jp" && redirectUrl.pathSegments.contains("login")) { + throw Exception(message) + } + + Log.d("bookwalker", "Successfully redirected to $redirectUrl") + redirectUrl + } + } + + private suspend fun Call.awaitSuccess(): Response { + return await().also { + if (!it.isSuccessful) { + it.close() + throw Exception("HTTP Error ${it.code}") + } + } + } + + private fun rxSingle(dispatcher: CoroutineDispatcher = Dispatchers.IO, block: suspend CoroutineScope.() -> T): Single { + return Single.create { sub -> + CoroutineScope(dispatcher).launch { + try { + sub.onSuccess(block()) + } catch (e: Throwable) { + sub.onError(e) + } + } + } + } + + private fun getAvailableFilterNames(doc: Document, filterClassName: String): List { + return doc.select("ul.$filterClassName > li > a > span").map { it.ownText() } + } + + private fun String.cleanTitle(): String { + return replace(CLEAN_TITLE_PATTERN, "").trim() + } + + private fun String.getHighestQualitySrcset(): String? { + val srcsetPairs = split(',').map { + val parts = it.trim().split(' ') + Pair(parts[1].trimEnd('x').toIntOrNull(), parts[0]) + } + return srcsetPairs.maxByOrNull { it.first ?: 0 }?.second + } + + private fun String.parseChapterNumber(): Pair? { + for (pattern in CHAPTER_NUMBER_PATTERNS) { + val match = pattern.find(this) + if (match != null) { + return Pair( + match.groups[0]!!.value.replaceFirstChar(Char::titlecase), + match.groups[1]!!.value.toFloat(), + ) + } + } + // Cannot parse chapter number + return null + } + + companion object { + + private val allFilter = FilterInfo("All", "") + private val fallbackFilters = listOf(FilterInfo("Press reset to load filters", "")) + + private val categoryExcludeRegexDefault = arrayOf( + "audiobooks", + "bookshelf skin", + "int'l manga", // already present in genres + ).joinToString("|") + + private val genreExcludeRegexDefault = arrayOf( + "coupon", + "bundle set", + "bonus items", + "completed series", // already present in others + "\\d{4}", // the genre list is bloated with things like "fall anime 2019" + "kc simulpub", // this is a specific publisher; "Simulpub Release" is separate + "kodansha promotion", + "shonen jump", + "media do", + "youtuber recommendations", + ).joinToString("|") + + private val CLEAN_TITLE_PATTERN = Regex( + listOf( + "(manga)", + "(comic)", + "", + "(serial)", + "", + // Deliberately not stripping tags like "(Light Novels)" + ).joinToString("|", transform = Regex::escape), + RegexOption.IGNORE_CASE, + ) + + private val CHAPTER_NUMBER_PATTERNS = listOf( + // All must have exactly one capture group for the chapter number + Regex("""vol\.?\s*([0-9.]+)""", RegexOption.IGNORE_CASE), + Regex("""volume\s+([0-9.]+)""", RegexOption.IGNORE_CASE), + Regex("""chapter\s+([0-9.]+)""", RegexOption.IGNORE_CASE), + Regex("""#([0-9.]+)""", RegexOption.IGNORE_CASE), + ) + + // Word joiners (zero-width non-breaking spaces) are used to avoid series titles from + // getting automatically stripped out from the start of chapter names. + private const val WORD_JOINER = "\u2060" + private const val PURCHASE_ICON = "\uD83D\uDCB5" // dollar bill emoji + private const val PREORDER_ICON = "\uD83D\uDD51" // two-o-clock emoji + private const val FREE_ICON = "\uD83C\uDF81" // wrapped present emoji + private const val UNKNOWN_ICON = "\u2753" // question mark emoji + + private const val PAGE_INDEX_QUERY_PARAM = "nocache_pagenum" + } +} diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerChapterReader.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerChapterReader.kt new file mode 100644 index 000000000..b7da78fa6 --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerChapterReader.kt @@ -0,0 +1,366 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker + +import android.annotation.SuppressLint +import android.app.Application +import android.graphics.Bitmap +import android.util.Log +import android.webkit.JavascriptInterface +import android.webkit.WebView +import android.webkit.WebViewClient +import eu.kanade.tachiyomi.extension.en.bookwalker.BookWalkerChapterReader.ImageResult.NotReady.get +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flatMapConcat +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withTimeoutOrNull +import uy.kohesive.injekt.injectLazy +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.charset.Charset +import java.util.Timer +import kotlin.concurrent.schedule +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine +import kotlin.math.max +import kotlin.time.Duration.Companion.seconds + +class BookWalkerChapterReader(val readerUrl: String, private val prefs: BookWalkerPreferences) { + + /** + * For when the reader is working properly but there's still an error fetching data. + * Currently this is only used when trying to fetch past the available portion of a preview. + */ + class NonFatalReaderException(message: String) : Exception(message) + + private val app by injectLazy() + + // We need to be careful to avoid CoroutineScope.launch since that will disconnect thrown + // exceptions inside the launched code from the calling coroutine and cause the entire + // application to crash rather than just showing an error when an image fails to load. + + // WebView interaction _must_ occur on the UI thread, but everything else should happen on an + // IO thread to avoid stalling the application. + + private fun evaluateOnUiThread(block: suspend CoroutineScope.() -> T): Deferred = + CoroutineScope(Dispatchers.Main.immediate).async { block() } + + private fun evaluateOnIOThread(block: suspend CoroutineScope.() -> T): Deferred = + CoroutineScope(Dispatchers.IO).async { block() } + + private val isDestroyed = MutableStateFlow(false) + + /** + * Calls [block] with the reader's active WebView as its argument and returns the result. + * [block] is guaranteed to run on a thread that can safely interact with the WebView. + */ + private suspend fun usingWebView(block: suspend (webview: WebView) -> T): T { + if (isDestroyed.value) { + throw Exception("Reader was destroyed") + } + return webview.await().let { + evaluateOnUiThread { + block(it) + }.await() + } + } + + private val webview = evaluateOnUiThread { + suspendCoroutine { cont -> + var cancelled = false + val timer = Timer().schedule(WEBVIEW_STARTUP_TIMEOUT.inWholeMilliseconds) { + // Don't destroy the webview here, that's the responsibility of the caller. + cancelled = true + cont.resumeWithException(Exception("WebView didn't load within $WEBVIEW_STARTUP_TIMEOUT")) + } + + Log.d("bookwalker", "Creating Webview...") + WebView(app).apply { + // The aspect ratio needs to be thinner than 3:2 to avoid two pages rendering at + // once, which would break the image-fetching logic. 1:1 works fine. + // We grab the image before it gets resized to fit the viewport so the image size + // doesn't directly correlate with screen size, but the size of the screen still + // affects the size of source image that the reader tries to render. + // The available resolutions vary per series, but in general, the largest resolution + // is typically on the order of 2k pixels vertical, with reduced-resolution variants + // on each factor of two (1/2, 1/4, etc.) for smaller screens. + val size = when (prefs.imageQuality) { + ImageQualityPref.DEVICE -> max( + app.resources.displayMetrics.heightPixels, + app.resources.displayMetrics.widthPixels, + ) + // "Medium" doesn't necessarily mean we'll use the 1/2x variant, just that we'll + // use the variant that BookWalker thinks is appropriate for a 1000px screen + // (which typically is the 1/2x variant for manga with high native resolutions). + ImageQualityPref.MEDIUM -> 1000 + // A 2000x2000px WebView consistently captured the largest variant in testing, + // but just in case some series can have a higher max resolution, 3000px is used + // for the "high" image quality option. In theory we could go higher (like 10k) + // and it wouldn't affect the image size, but there start to be performance + // issues when the BookWalker viewer tries to draw onto huge canvases. + ImageQualityPref.HIGH -> 3000 + } + Log.d("bookwalker", "WebView size $size") + // Note: The BookWalker viewer is DPI-aware, so even though the innerWidth/Height + // values in JavaScript may not match the layout() call, everything works properly. + layout(0, 0, size, size) + + @SuppressLint("SetJavaScriptEnabled") + settings.javaScriptEnabled = true + settings.domStorageEnabled = true + // Mobile vs desktop doesn't matter much, but the mobile layout has a longer page + // slider which allows for more accuracy when trying to jump to a particular page. + settings.userAgentString = USER_AGENT_MOBILE + + webViewClient = object : WebViewClient() { + override fun onPageFinished(view: WebView, url: String) { + super.onPageFinished(view, url) + + timer.cancel() + + if (cancelled) { + Log.d("bookwalker", "WebView loaded $url after being destroyed") + return + } + + Log.d("bookwalker", "WebView loaded $url") + + if (url.contains("member.bookwalker.jp")) { + cont.resumeWithException(Exception("Logged out, check website in WebView")) + return + } + + cont.resume(view) + } + } + +// webChromeClient = object : WebChromeClient() { +// override fun onConsoleMessage(consoleMessage: ConsoleMessage): Boolean { +// Log.println( +// when (consoleMessage.messageLevel()!!) { +// ConsoleMessage.MessageLevel.TIP -> Log.VERBOSE +// ConsoleMessage.MessageLevel.DEBUG -> Log.DEBUG +// ConsoleMessage.MessageLevel.LOG -> Log.INFO +// ConsoleMessage.MessageLevel.WARNING -> Log.WARN +// ConsoleMessage.MessageLevel.ERROR -> Log.ERROR +// }, +// "bookwalker.console", +// "${consoleMessage.sourceId()}:${consoleMessage.lineNumber()} ${consoleMessage.message()}", +// ) +// +// return super.onConsoleMessage(consoleMessage) +// } +// } + + this.addJavascriptInterface(jsInterface, INTERFACE_NAME) + + // Adding the below line makes a console error go away, but it doesn't seem to affect functionality. + // webview.addJavascriptInterface(object {}, "Notification") + + loadUrl(readerUrl) + } + }.also { + it.evaluateJavascript( + injectionScriptReplacements.asIterable() + .fold(webviewInjectionScript) { script, replacement -> + script.replace(replacement.key, replacement.value) + }, + null, + ) + } + } + + private suspend fun evaluateJavascript(script: String): String? { + return usingWebView { webview -> + suspendCoroutine { cont -> + webview.evaluateJavascript(script) { + cont.resume(it) + } + } + } + } + + suspend fun destroy() { + Log.d("bookwalker", "Destroy called") + try { + usingWebView { + it.destroy() + isDestroyed.value = true + } + } catch (e: Exception) { + // OK, the webview was probably already destroyed +// Log.d("bookwalker", "Destroy error: $e") + } + } + + /** + * Returns a flow which transparently forwards the original flow except that if the WebView is + * destroyed while waiting for data (or was already destroyed), suspending calls to obtain the + * data will throw. + * + * If the WebView was already destroyed when the suspending call was made but data from the + * original flow is immediately received, it is not deterministic whether it will return the + * data or throw. + */ + private fun Flow.throwOnDestroyed(): Flow { + return merge( + this.map { false to it }, + isDestroyed.filter { it }.map { true to null }, + ).map { + if (it.first) { + throw Exception("Reader was destroyed") + } + // Can't use !! here because T might be nullable + @Suppress("UNCHECKED_CAST") + it.second as T + } + } + + private val isViewerReady = MutableStateFlow(false) + + private suspend fun waitForViewer() { + webview.await() + isViewerReady.filter { it }.throwOnDestroyed().first() + } + + private sealed class ImageResult { + object NotReady : ImageResult() + class Found(val data: Deferred) : ImageResult() + class NotFound(val error: Throwable) : ImageResult() + + suspend fun Flow.get(): ByteArray { + @OptIn(FlowPreview::class) + return flatMapConcat { + when (it) { + is NotReady -> emptyFlow() + is Found -> flow { emit(it.data.await()) } + is NotFound -> throw it.error + } + }.first() + } + } + + private val imagesMap = mutableMapOf>() + + private val navigationMutex = Mutex() + + /** + * Retrieves JPEG image data for the requested page (0-indexed) + */ + suspend fun getPage(index: Int): ByteArray { + val state = synchronized(imagesMap) { + imagesMap.getOrPut(index) { MutableStateFlow(ImageResult.NotReady) } + } + + waitForViewer() + + // Attempting to fetch two pages concurrently doesn't work. + val imgData = navigationMutex.withLock { + evaluateJavascript("(() => $JS_UTILS_NAME.fetchPageData($index))()") + + val result = withTimeoutOrNull(IMAGE_FETCH_TIMEOUT) { + Log.d("bookwalker", "Waiting on image index $index ($state)") + state.throwOnDestroyed().get() + } ?: throw Exception("Timed out waiting for image $index to load") + + // Stop holding onto the image in the map so it can get collected. Due to caching by the + // app, it is unlikely that the same reader will need to fetch the same image twice, and + // even if it does, the image may still be stored in memory at the webview level. + state.value = ImageResult.NotReady + + result + } + + Log.d("bookwalker", "Retrieved data for image $index (${imgData.size} bytes)") + return imgData + } + + private val jsInterface = object { + @JavascriptInterface + fun reportViewerLoaded() { + Log.d("bookwalker", "Viewer loaded") + isViewerReady.value = true + } + + @JavascriptInterface + fun reportFailedToLoad(message: String) { + Log.e("bookwalker", "Failed to load BookWalker viewer: $message") + // launch should be safe here, destroy() should not throw (except for truly critical errors) + CoroutineScope(Dispatchers.IO).launch { destroy() } + } + + @JavascriptInterface + fun reportImage(index: Int, imageDataAsString: String, width: Int, height: Int) { + // Byte arrays cannot be directly transferred efficiently, so strings are our + // best choice for transporting large images out of the Webview. + // See https://stackoverflow.com/a/45506857 + val data = imageDataAsString.toByteArray(Charset.forName("windows-1252")) + Log.d("bookwalker", "received image $index (${data.size} bytes, ${width}x$height)") + + val state = synchronized(imagesMap) { + imagesMap.getOrPut(index) { MutableStateFlow(ImageResult.NotReady) } + } + + // The raw bitmap data is very large, so we want to compress it down as soon as possible + // and store that rather than keeping uncompressed images in memory. + state.value = ImageResult.Found( + evaluateOnIOThread { + val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + bitmap.copyPixelsFromBuffer(ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN)) + + ByteArrayOutputStream().apply { + bitmap.compress(Bitmap.CompressFormat.JPEG, 90, this) + }.toByteArray() + }, + ) + } + + @JavascriptInterface + fun reportImageDoesNotExist(index: Int, reason: String) { + val state = synchronized(imagesMap) { + imagesMap.getOrPut(index) { MutableStateFlow(ImageResult.NotReady) } + } + + state.value = ImageResult.NotFound(NonFatalReaderException(reason)) + } + } + + companion object { + private val webviewInjectionScript by lazy { + this::class.java.getResource("/assets/webview-script.js")?.readText() + ?: throw Error("Failed to retrieve webview injection script") + } + + private const val INTERFACE_NAME = "BOOKWALKER_EXT_COMMS" + private const val JS_UTILS_NAME = "BOOKWALKER_EXT_UTILS" + + private val injectionScriptReplacements = mapOf( + "__INJECT_WEBVIEW_INTERFACE" to INTERFACE_NAME, + "__INJECT_JS_UTILITIES" to JS_UTILS_NAME, + ) + + // Sometimes the webview just fails to load for some reason and we need to retry, so this + // timeout should be kept as short as possible. 15 seconds seems like a decent upper bound. + private val WEBVIEW_STARTUP_TIMEOUT = 15.seconds + + // Images can take a while to load especially if the viewer is in a poor location and needs + // to track to a completely different part of the chapter, but if it's been 15 seconds + // since an image was requested with no response, it usually means something is broken. + // Note that the image fetch timer only starts after the webview loads. + private val IMAGE_FETCH_TIMEOUT = 15.seconds + } +} diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerConstants.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerConstants.kt new file mode 100644 index 000000000..f2a3b27f7 --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerConstants.kt @@ -0,0 +1,52 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker + +const val PREF_VALIDATE_LOGGED_IN = "validateLoggedIn" +const val PREF_SHOW_LIBRARY_IN_POPULAR = "showLibraryInPopular" +const val PREF_ATTEMPT_READ_PREVIEWS = "attemptReadPreviews" +const val PREF_CATEGORY_EXCLUDE_REGEX = "categoryExcludeRegex" +const val PREF_GENRE_EXCLUDE_REGEX = "genreExcludeRegex" + +enum class ImageQualityPref(val key: String) { + DEVICE("device"), + MEDIUM("medium"), + HIGH("high"), + ; + + companion object { + const val PREF_KEY = "imageResolution" + val defaultOption = DEVICE + fun fromKey(key: String) = values().find { it.key == key } ?: defaultOption + } +} + +enum class FilterChaptersPref(val key: String) { + OWNED("owned"), + OBTAINABLE("obtainable"), + ALL("all"), + ; + + fun includes(other: FilterChaptersPref): Boolean { + return this >= other + } + + companion object { + const val PREF_KEY = "filterChapters" + val defaultOption = OBTAINABLE + fun fromKey(key: String) = values().find { it.key == key } ?: defaultOption + } +} + +const val QUERY_PARAM_CATEGORY = "qcat" +const val QUERY_PARAM_GENRE = "qtag" + +// const val QUERY_PARAM_AUTHOR = "qaut" +const val QUERY_PARAM_PUBLISHER = "qcom" + +const val HEADER_IS_REQUEST_FROM_EXTENSION = "x-is-bookwalker-extension" +const val HEADER_PAGE_INDEX = "x-page-index" + +const val USER_AGENT_DESKTOP = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36" + +const val USER_AGENT_MOBILE = "Mozilla/5.0 (Linux; Android 10; Pixel 4) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36" diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerFilters.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerFilters.kt new file mode 100644 index 000000000..61b3a8716 --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerFilters.kt @@ -0,0 +1,160 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker + +import android.util.Log +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.await +import eu.kanade.tachiyomi.source.model.Filter +import eu.kanade.tachiyomi.util.asJsoup +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlin.math.max + +class BookWalkerFilters(private val bookwalker: BookWalker) { + var categories: List? = null + private set + var genres: List? = null + private set + + // Author filter disabled for now, since the performance/UX in-app is pretty bad +// var authors: List? = null +// private set + var publishers: List? = null + private set + + private val fetchMutex = Mutex() + private var hasObtainedFilters = false + + fun fetchIfNecessaryInBackground() { + CoroutineScope(Dispatchers.IO).launch { + try { + fetchIfNecessary() + } catch (e: Exception) { + Log.e("bookwalker", e.toString()) + } + } + } + + // Lock applied so that we don't try to make additional new requests before the first set of + // requests have finished. + private suspend fun fetchIfNecessary() = fetchMutex.withLock { + // In theory the list of filters could change while the app is alive, but in practice that + // seems fairly unlikely, and we can save a lot of unnecessary calls by assuming it won't. + if (hasObtainedFilters) { + return + } + + coroutineScope { + listOf( + async { if (categories == null) categories = fetchFilters("categories") }, + async { if (genres == null) genres = fetchFilters("genre") }, +// async { if (authors == null) authors = fetchFilters("authors") }, + async { if (publishers == null) publishers = fetchFilters("publishers") }, + ).awaitAll() + + hasObtainedFilters = true + } + } + + private suspend fun fetchFilters(entityName: String): List { + val entityPath = "/$entityName/" + val response = bookwalker.client.newCall( + GET(bookwalker.baseUrl + entityPath, bookwalker.callHeaders), + ).await() + val document = response.asJsoup() + return document.select(".link-list > li > a").map { + FilterInfo( + it.text(), + it.attr("href").removePrefix(entityPath).trimEnd('/'), + ) + } + } +} + +class FilterInfo(name: String, val id: String) : Filter.CheckBox(name) { + override fun toString(): String { + return name + } +} + +interface QueryParamFilter { + fun getQueryParams(): List> +} + +class SelectOneFilter( + name: String, + private val queryParam: String, + options: List, +) : QueryParamFilter, Filter.Select( + name, + options.toTypedArray(), + max(0, options.indexOfFirst { it.id == "2" }), // Default to manga +) { + override fun getQueryParams(): List> { + return listOf(queryParam to values[state].id) + } +} + +class SelectMultipleFilter( + name: String, + private val queryParam: String, + options: List, +) : QueryParamFilter, Filter.Group(name, options) { + override fun getQueryParams(): List> { + return listOf( + queryParam to state.filter { it.state }.joinToString(",") { it.id }, + ) + } +} + +class OthersFilter : QueryParamFilter, Filter.Group( + "Others", + listOf( + FilterInfo("On Sale", "qspp"), + FilterInfo("Coin Boost", "qcon"), + FilterInfo("Pre-Order", "qcos"), + FilterInfo("Completed", "qcpl"), + FilterInfo("Bonus Item", "qspe"), + FilterInfo("Exclude Purchased", "qseq"), + ), +) { + override fun getQueryParams(): List> { + return state.filter { it.state }.map { it.id to "1" } + } +} + +class ExcludeFilter : QueryParamFilter, Filter.Text("Exclude search terms (comma-separated)") { + override fun getQueryParams(): List> { + return state.split(',') + .map { it.trim() } + .filter { it.isNotEmpty() } + .map { "qnot[]" to it.replace(" ", "+") } + } +} + +class TextFilter( + name: String, + private val queryParam: String, + defaultValue: String = "", +) : QueryParamFilter, Filter.Text(name, defaultValue) { + override fun getQueryParams(): List> { + return listOf(queryParam to state) + } +} + +class PriceFilter : QueryParamFilter, Filter.Group( + "Price", + listOf( + TextFilter("Min Price ($)", "qpri_min"), + TextFilter("Max Price ($)", "qpri_max"), + ), +) { + override fun getQueryParams(): List> { + return state.filter { it.state.isNotEmpty() }.flatMap { it.getQueryParams() } + } +} diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerImageRequestInterceptor.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerImageRequestInterceptor.kt new file mode 100644 index 000000000..a6e74472f --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerImageRequestInterceptor.kt @@ -0,0 +1,96 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker + +import android.util.Log +import kotlinx.coroutines.runBlocking +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import java.util.LinkedHashMap +import kotlin.time.Duration.Companion.minutes + +class BookWalkerImageRequestInterceptor(private val prefs: BookWalkerPreferences) : Interceptor { + + private val readerQueue = object : LinkedHashMap>( + MAX_ACTIVE_READERS + 1, + 1f, + true, + ) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry>): Boolean { + if (size > MAX_ACTIVE_READERS) { + eldest.value.expire() + return true + } + return false + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + + if (request.headers[HEADER_IS_REQUEST_FROM_EXTENSION] != "true") { + return chain.proceed(request) + } + + val readerUrl = request.url.toString() + val pageIndex = request.headers[HEADER_PAGE_INDEX]!!.toInt() + + Log.d("bookwalker", "Intercepting request for page $pageIndex") + + val reader = synchronized(readerQueue) { + readerQueue.getOrPut(readerUrl) { + Expiring(BookWalkerChapterReader(readerUrl, prefs), READER_EXPIRATION_TIME) { + disposeReader(this) + } + } + } + + val imageData = try { + runBlocking { + reader.contents.getPage(pageIndex) + } + } catch (e: BookWalkerChapterReader.NonFatalReaderException) { + // Just re-throw, this isn't worth killing the webview over. + Log.e("bookwalker", e.toString()) + throw e + } catch (e: Exception) { + // If there's some other exception, we can generally assume that the + // webview is broken in some way and should be re-created. + reader.expire() + Log.e("bookwalker", e.toString()) + throw e + } + + return Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(imageData.toResponseBody(IMAGE_MEDIA_TYPE)) + .build() + } + + private fun disposeReader(reader: BookWalkerChapterReader) { +// Log.d("bookwalker", "Disposing reader ${reader.readerUrl}") + synchronized(readerQueue) { + readerQueue.remove(reader.readerUrl) + } + runBlocking { + reader.destroy() + } + } + + companion object { + // Having at most two chapter readers at once should be enough to avoid thrashing on chapter + // transitions without keeping an excessive number of webviews running in the background. + // In theory there could also be a download happening at the same time, but 2 is probably fine. + private const val MAX_ACTIVE_READERS = 2 + + // We don't get events when a user exits the reader or a download finishes, so we want to + // make sure to clean up unused readers after a period of time. + private val READER_EXPIRATION_TIME = 1.minutes + + private val IMAGE_MEDIA_TYPE = "image/jpeg".toMediaType() + } +} diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerPreferences.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerPreferences.kt new file mode 100644 index 000000000..aca4f5d57 --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/BookWalkerPreferences.kt @@ -0,0 +1,11 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker + +interface BookWalkerPreferences { + val showLibraryInPopular: Boolean + val shouldValidateLogin: Boolean + val imageQuality: ImageQualityPref + val filterChapters: FilterChaptersPref + val attemptToReadPreviews: Boolean + val excludeCategoryFilters: Regex + val excludeGenreFilters: Regex +} diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/Expiring.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/Expiring.kt new file mode 100644 index 000000000..cc79792c0 --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/Expiring.kt @@ -0,0 +1,53 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.yield +import kotlin.time.Duration + +class Expiring( + private val obj: T, + private val expirationTime: Duration, + val onExpire: T.() -> Unit, +) { + + private var lastAccessed = System.currentTimeMillis() + + private var expired = false + + val contents + get() = run { + lastAccessed = System.currentTimeMillis() + obj + } + + private var expirationJob: Job + + fun expire() { + if (!expired) { + expired = true + expirationJob.cancel() + onExpire(obj) + } + } + + init { + expirationJob = CoroutineScope(Dispatchers.IO).launch { + while (true) { + yield() + val targetTime = lastAccessed + expirationTime.inWholeMilliseconds + val currentTime = System.currentTimeMillis() + val difference = targetTime - currentTime + if (difference > 0) { + delay(difference) + } else { + break + } + } + expire() + } + } +} diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/HoldBooksInfoDto.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/HoldBooksInfoDto.kt new file mode 100644 index 000000000..cddc6d38f --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/HoldBooksInfoDto.kt @@ -0,0 +1,18 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker.dto + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonClassDiscriminator + +@Serializable +class HoldBooksInfoDto( + val holdBookList: HoldBookListDto, +) + +@Serializable +class HoldBookListDto( + val entities: List, +) + +@Serializable +@JsonClassDiscriminator("type") +sealed class HoldBookEntityDto diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/SeriesDto.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/SeriesDto.kt new file mode 100644 index 000000000..d53300d9b --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/SeriesDto.kt @@ -0,0 +1,12 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker.dto + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +@SerialName("series") +class SeriesDto( + val seriesId: Int, + val seriesName: String, + val imageUrl: String, +) : HoldBookEntityDto() diff --git a/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/SingleDto.kt b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/SingleDto.kt new file mode 100644 index 000000000..97cb428a1 --- /dev/null +++ b/src/en/bookwalker/src/eu/kanade/tachiyomi/extension/en/bookwalker/dto/SingleDto.kt @@ -0,0 +1,18 @@ +package eu.kanade.tachiyomi.extension.en.bookwalker.dto + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +@SerialName("normal") +class SingleDto( + val detailUrl: String, + val title: String, + val imageUrl: String, + val authors: List, +) : HoldBookEntityDto() + +@Serializable +class AuthorDto( + val authorName: String, +)