Mangadex fixes and improvements (#187)

* -added language support
-fixed minor laguage paging issue
-fixed some webtoons not working
-adjusted browse by letter to be a dropdown

* -added tristate to search with r18

* switched to es-419 for latin america

* removed unneeded get all mangadex languages

* commented out tristate for as cookie for r18 doesnt work anymore when not logged in

* switched to SourceFactory

* removed empty line

* fixed missing info from manga details

* Moved english to its own package to prevent having to migrate sources

* optimized image url parsing to not load each page then each image.

src/all/mangadex/build.gradle

@@ -0,0 +1,17 @@
+apply plugin: 'com.android.application'
+apply plugin: 'kotlin-android'
+
+ext {
+    appName = 'Tachiyomi: MangaDex'
+    pkgNameSuffix = "all.mangadex"
+    extClass = '.MangadexFactory'
+    extVersionCode = 4
+    extVersionSuffix = 4
+    libVersion = '1.2'
+}
+
+dependencies {
+
+}
+
+apply from: "$rootDir/common.gradle"

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/Mangadex.kt

@@ -1,16 +1,19 @@
-package eu.kanade.tachiyomi.extension.en.mangadex
+package eu.kanade.tachiyomi.extension.all.mangadex
 
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.model.*
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import okhttp3.HttpUrl
+import okhttp3.OkHttpClient
 import okhttp3.Request
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
+import rx.Observable
 import java.net.URLEncoder
 import java.text.SimpleDateFormat
 
-class Mangadex : ParsedHttpSource() {
+open class Mangadex(override val lang: String, private val internalLang: String, val pageStart: Int) : ParsedHttpSource() {
 
     override val name = "MangaDex"
 
@@ -18,44 +21,40 @@ class Mangadex : ParsedHttpSource() {
 
     override val supportsLatest = true
 
-    override val lang = "en"
+    override val client = clientBuilder(ALL)
 
-    private val internalLang = "gb"
+    private fun clientBuilder(r18Toggle: Int): OkHttpClient = network.cloudflareClient.newBuilder()
-
-    override val client = network.cloudflareClient.newBuilder()
             .addNetworkInterceptor { chain ->
                 val newReq = chain
                         .request()
                         .newBuilder()
-                        .addHeader("Cookie", cookiesHeader)
+                        .addHeader("Cookie", cookiesHeader(r18Toggle))
                         .build()
-
                 chain.proceed(newReq)
             }.build()!!
 
-    private val cookiesHeader by lazy {
+    private fun cookiesHeader(r18Toggle: Int): String {
         val cookies = mutableMapOf<String, String>()
-        cookies.put("mangadex_h_toggle", "1")
+        cookies.put("mangadex_h_toggle", r18Toggle.toString())
-        buildCookies(cookies)
+        return buildCookies(cookies)
     }
 
-    private fun buildCookies(cookies: Map<String, String>)
+    private fun buildCookies(cookies: Map<String, String>) = cookies.entries.joinToString(separator = "; ", postfix = ";") {
-            = cookies.entries.map {
         "${URLEncoder.encode(it.key, "UTF-8")}=${URLEncoder.encode(it.value, "UTF-8")}"
-    }.joinToString(separator = "; ", postfix = ";")
+    }
 
     override fun popularMangaSelector() = ".table-responsive tbody tr"
 
     override fun latestUpdatesSelector() = ".table-responsive tbody tr a.manga_title[href*=manga]"
 
     override fun popularMangaRequest(page: Int): Request {
-        val pageStr = if (page != 1) "//" + ((page * 100) - 100) else ""
+        val pageStr = if (page != 1) "/" + ((page * 100) - 100) else ""
         return GET("$baseUrl/titles$pageStr", headers)
     }
 
     override fun latestUpdatesRequest(page: Int): Request {
-        val pageStr = if (page != 1) ((page * 20) - 20) else ""
+        val pageStr = if (page != 1) "/" + ((page * 20)) else ""
-        return GET("$baseUrl/1/$page", headers)
+        return GET("$baseUrl/$pageStart$pageStr", headers)
     }
 
     override fun popularMangaFromElement(element: Element): SManga {
@@ -83,6 +82,32 @@ class Mangadex : ParsedHttpSource() {
 
     override fun searchMangaNextPageSelector() = ".pagination li:not(.disabled) span[title*=last page]:not(disabled)"
 
+    override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
+        return getSearchClient(filters).newCall(searchMangaRequest(page, query, filters))
+                .asObservableSuccess()
+                .map { response ->
+                    searchMangaParse(response)
+                }
+    }
+
+    /* get search client based off r18 filter.  This will always return default client builder now until r18 solution is found or login is add
+     */
+    private fun getSearchClient(filters: FilterList): OkHttpClient {
+        filters.forEach { filter ->
+            when (filter) {
+                is R18 -> {
+                    return when {
+                        filter.isExcluded() -> clientBuilder(NO_R18)
+                        filter.isIncluded() -> clientBuilder(ONLY_R18)
+                        else -> clientBuilder(ALL)
+
+                    }
+                }
+            }
+        }
+        return clientBuilder(ALL)
+    }
+
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
         val byGenre = filters.find { it is GenreList }
         val genres = mutableListOf<String>()
@@ -96,8 +121,9 @@ class Mangadex : ParsedHttpSource() {
         }
         //do browse by letter if set
         val byLetter = filters.find { it is ByLetter }
-        if (byLetter != null && (byLetter as ByLetter).state != 0) {
+
-            val s = byLetter.values[byLetter.state]
+        if (byLetter != null && (byLetter as ByLetter).state.first().state != 0) {
+            val s = byLetter.state.first().values[byLetter.state.first().state]
             val pageStr = if (page != 1) (((page - 1) * 100)).toString() else "0"
             val url = HttpUrl.parse("$baseUrl/titles/")!!.newBuilder().addPathSegment(s).addPathSegment(pageStr)
             return GET(url.toString(), headers)
@@ -124,15 +150,14 @@ class Mangadex : ParsedHttpSource() {
 
     override fun mangaDetailsParse(document: Document): SManga {
         val manga = SManga.create()
-        val imageElement = document.select(".table-condensed").first()
+        val infoElement = document.select(".row.edit").first()
-        val infoElement = document.select(".table.table-condensed.edit").first()
         val genreElement = infoElement.select("tr:eq(3) td .genre")
 
         manga.author = infoElement.select("tr:eq(1) td").first()?.text()
         manga.artist = infoElement.select("tr:eq(2) td").first()?.text()
         manga.status = parseStatus(infoElement.select("tr:eq(5) td").first()?.text())
         manga.description = infoElement.select("tr:eq(7) td").first()?.text()
-        manga.thumbnail_url = imageElement.select("img").first()?.attr("src").let { baseUrl + "/" + it }
+        manga.thumbnail_url = infoElement.select("img").first()?.attr("src").let { baseUrl + "/" + it }
         var genres = mutableListOf<String>()
         genreElement?.forEach { genres.add(it.text()) }
         manga.genre = genres.joinToString(", ")
@@ -162,30 +187,21 @@ class Mangadex : ParsedHttpSource() {
     override fun pageListParse(document: Document): List<Page> {
         val pages = mutableListOf<Page>()
         val url = document.baseUri()
-        val select = document.select("#jump_page")
+
-        //if its a regular manga get the pages from the drop down selector
+        val dataUrl = document.select("script").last().html().substringAfter("dataurl = '").substringBefore("';")
-        if (select.isNotEmpty()) {
+        val imageUrl = document.select("script").last().html().substringAfter("page_array = [").substringBefore("];")
-            select.first().select("option").forEach {
+        val listImageUrls = imageUrl.replace("'", "").split(",")
-                pages.add(Page(pages.size, url + "/" + it.attr("value")))
+        val server = document.select("script").last().html().substringAfter("server = '").substringBefore("';")
-            }
+
-        } else {
+        listImageUrls.filter { it.isNotBlank() }.forEach {
-            //webtoon get all the image urls on the one page
+            val url = "$server$dataUrl/$it"
-            document.select(".edit.webtoon").forEach {
+            pages.add(Page(pages.size, "", getImageUrl(url)))
-                pages.add(Page(pages.size, "", it.attr("src")))
-            }
         }
 
         return pages
     }
 
-    override fun imageUrlParse(document: Document): String {
+    override fun imageUrlParse(document: Document): String = ""
-        val attr = document.select("#current_page").first().attr("src")
-        //some images are hosted elsewhere
-        if (attr.startsWith("http")) {
-            return attr
-        }
-        return baseUrl + attr
-    }
 
     private fun parseStatus(status: String?) = when {
         status == null -> SManga.UNKNOWN
@@ -195,19 +211,28 @@ class Mangadex : ParsedHttpSource() {
         else -> SManga.UNKNOWN
     }
 
+    fun getImageUrl(attr: String): String {
+        //some images are hosted elsewhere
+        if (attr.startsWith("http")) {
+            return attr
+        }
+        return baseUrl + attr
+    }
+
     private class TextField(name: String, val key: String) : Filter.Text(name)
     private class Genre(val id: String, name: String) : Filter.CheckBox(name)
     private class GenreList(genres: List<Genre>) : Filter.Group<Genre>("Genres", genres)
-    private class ByLetter : Filter.Select<String>("Browse By Letter", arrayOf("", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"))
+    private class R18(name: String) : Filter.TriState(name)
-
+    private class ByLetter(letters: List<Letters>) : Filter.Group<Letters>("Browse by Letter only", letters)
+    private class Letters : Filter.Select<String>("Letter", arrayOf("", "~", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"))
 
     override fun getFilterList() = FilterList(
             TextField("Author", "author"),
             TextField("Artist", "artist"),
+            //R18("Show R18+"),
             GenreList(getGenreList()),
-            Filter.Header("Note: Browsing by Letter"),
+            ByLetter(listOf(Letters()))
-            Filter.Header("Ignores other Search Fields"),
+    )
-            ByLetter())
 
 
     private fun getGenreList() = listOf(
@@ -252,4 +277,10 @@ class Mangadex : ParsedHttpSource() {
             Genre("39", "[no chapters]"),
             Genre("40", "Game")
     )
+
+    companion object {
+        const val NO_R18 = 0
+        const val ALL = 1
+        const val ONLY_R18 = 2
+    }
 }

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangadexFactory.kt

@@ -0,0 +1,11 @@
+package eu.kanade.tachiyomi.extension.all.mangadex
+
+import eu.kanade.tachiyomi.source.Source
+import eu.kanade.tachiyomi.source.SourceFactory
+
+/**
+ * Created by Carlos on 2/8/2018.
+ */
+class MangadexFactory : SourceFactory {
+    override fun createSources(): List<Source> = getAllMangaDexLanguages()
+}

src/all/mangadex/src/eu/kanade/tachiyomi/extension/all/mangadex/MangadexLanguages.kt

@@ -0,0 +1,36 @@
+package eu.kanade.tachiyomi.extension.all.mangadex
+
+/**
+ * Mangadex languages
+ */
+
+class MangaDexPolish : Mangadex("pl", "pl", 3)
+class MangaDexItalian : Mangadex("it", "it", 6)
+class MangaDexRussian : Mangadex("ru", "ru", 7)
+class MangaDexGerman : Mangadex("de", "de", 8)
+class MangaDexFrench : Mangadex("fr", "fr", 10)
+class MangaDexVietnamese : Mangadex("vi", "vn", 12)
+class MangaDexSpanishSpain : Mangadex("es", "es", 15)
+class MangaDexPortuguese : Mangadex("pt", "br", 16)
+class MangaDexSwedish : Mangadex("sv", "se", 18)
+class MangaDexTurkish : Mangadex("tr", "tr", 26)
+class MangaDexIndonesian : Mangadex("id", "id", 27)
+class MangaDexSpanishLTAM : Mangadex("es-419", "mx", 29)
+class MangaDexCatalan : Mangadex("ca", "ct", 33)
+
+fun getAllMangaDexLanguages() = listOf(
+        MangaDexEnglish(),
+        MangaDexPolish(),
+        MangaDexItalian(),
+        MangaDexRussian(),
+        MangaDexGerman(),
+        MangaDexFrench(),
+        MangaDexVietnamese(),
+        MangaDexSpanishSpain(),
+        MangaDexPortuguese(),
+        MangaDexSwedish(),
+        MangaDexTurkish(),
+        MangaDexIndonesian(),
+        MangaDexSpanishLTAM(),
+        MangaDexCatalan()
+)

src/all/mangadex/src/eu/kanade/tachiyomi/extension/en/mangadex/MangaDexEnglish.kt

@@ -0,0 +1,3 @@
+package eu.kanade.tachiyomi.extension.all.mangadex
+
+class MangaDexEnglish : Mangadex("en", "gb", 1)

src/en/mangadex/build.gradle

@@ -1,19 +0,0 @@
-apply plugin: 'com.android.application'
-apply plugin: 'kotlin-android'
-
-
-ext {
-    appName = 'Tachiyomi: MangaDex'
-    pkgNameSuffix = "en.mangadex"
-    extClass = '.Mangadex'
-    extVersionCode = 3
-    extVersionSuffix = 3
-    libVersion = '1.2'
-}
-
-dependencies {
-    provided "com.google.code.gson:gson:2.8.0"
-    provided "com.github.salomonbrys.kotson:kotson:2.5.0"
-}
-
-apply from: "$rootDir/common.gradle"