Add Tsumino captcha display and merge branch 'master' of upstream

# Conflicts:
#	.github/readme-images/app-icon.png
#	.github/readme-images/screens.png
#	.travis.yml
#	README.md
#	app/build.gradle
#	app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt
#	app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt
#	app/src/main/java/eu/kanade/tachiyomi/source/online/YamlHttpSource.kt
#	app/src/main/java/eu/kanade/tachiyomi/source/online/YamlHttpSourceMappings.kt
#	app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryController.kt
#	app/src/main/java/eu/kanade/tachiyomi/ui/main/MainActivity.kt
#	app/src/main/java/eu/kanade/tachiyomi/util/DynamicConcurrentMergeOperator.java

app/src/main/AndroidManifest.xml

@@ -179,6 +179,7 @@
                     android:scheme="https"/>
             </intent-filter>
         </activity>
+        <activity android:name="exh.captcha.SolveCaptchaActivity" />
 
     </application>
 

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferenceKeys.kt

@@ -141,5 +141,7 @@ object PreferenceKeys {
 
     const val eh_enableExHentai = "enable_exhentai"
 
+    const val eh_ts_aspNetCookie = "eh_ts_aspNetCookie"
+
     const val eh_showSettingsUploadWarning = "eh_showSettingsUploadWarning1"
 }

app/src/main/java/eu/kanade/tachiyomi/data/preference/PreferencesHelper.kt

@@ -219,6 +219,8 @@ class PreferencesHelper(val context: Context) {
 
     fun eh_lenientSync() = rxPrefs.getBoolean(Keys.eh_lenientSync, false)
 
+    fun eh_ts_aspNetCookie() = rxPrefs.getString(Keys.eh_ts_aspNetCookie, "")
+
     fun eh_showSettingsUploadWarning() = rxPrefs.getBoolean(Keys.eh_showSettingsUploadWarning, true)
     // <-- EH
 }

app/src/main/java/eu/kanade/tachiyomi/source/SourceManager.kt

@@ -28,6 +28,7 @@ open class SourceManager(private val context: Context) {
 
     init {
         createInternalSources().forEach { registerSource(it) }
+
         //Recreate sources when they change
         val prefEntries = arrayOf(
                 prefs.enableExhentai(),
@@ -39,8 +40,7 @@ open class SourceManager(private val context: Context) {
         ).map { it.asObservable() }
 
         Observable.merge(prefEntries).skip(prefEntries.size - 1).subscribe {
-            sourcesMap.clear()
+            createEHSources().forEach { registerSource(it) }
-            createSources()
         }
     }
 
@@ -87,7 +87,7 @@ open class SourceManager(private val context: Context) {
         exSrcs += PervEden(PERV_EDEN_IT_SOURCE_ID, PervEdenLang.it)
         exSrcs += NHentai(context)
         exSrcs += HentaiCafe()
-        exSrcs += Tsumino()
+        exSrcs += Tsumino(context)
         return exSrcs
     }
 }

app/src/main/java/eu/kanade/tachiyomi/source/online/english/Tsumino.kt

@@ -1,8 +1,12 @@
 package eu.kanade.tachiyomi.source.online.english
 
+import android.content.Context
 import android.net.Uri
 import com.github.salomonbrys.kotson.*
 import com.google.gson.JsonParser
+import eu.kanade.tachiyomi.data.preference.PreferencesHelper
+import eu.kanade.tachiyomi.data.preference.getOrDefault
+import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.source.model.*
@@ -10,21 +14,25 @@ import eu.kanade.tachiyomi.source.online.LewdSource
 import eu.kanade.tachiyomi.source.online.ParsedHttpSource
 import eu.kanade.tachiyomi.util.asJsoup
 import exh.TSUMINO_SOURCE_ID
+import exh.captcha.CaptchaCompletionVerifier
+import exh.captcha.SolveCaptchaActivity
 import exh.metadata.EMULATED_TAG_NAMESPACE
 import exh.metadata.models.Tag
 import exh.metadata.models.TsuminoMetadata
 import exh.metadata.models.TsuminoMetadata.Companion.BASE_URL
 import exh.util.urlImportFetchSearchManga
-import okhttp3.FormBody
+import okhttp3.*
-import okhttp3.Request
-import okhttp3.Response
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import rx.Observable
+import rx.schedulers.Schedulers
+import uy.kohesive.injekt.injectLazy
 import java.text.SimpleDateFormat
 import java.util.*
 
-class Tsumino: ParsedHttpSource(), LewdSource<TsuminoMetadata, Document> {
+class Tsumino(private val context: Context): ParsedHttpSource(), LewdSource<TsuminoMetadata, Document>, CaptchaCompletionVerifier {
+    private val preferences: PreferencesHelper by injectLazy()
+
     override val id = TSUMINO_SOURCE_ID
     
     override val lang = "en"
@@ -228,6 +236,8 @@ class Tsumino: ParsedHttpSource(), LewdSource<TsuminoMetadata, Document> {
     override fun fetchChapterList(manga: SManga) = lazyLoadMeta(queryFromUrl(manga.url),
             client.newCall(mangaDetailsRequest(manga)).asObservableSuccess().map { it.asJsoup() }
     ).map {
+        trickTsumino(it.tmId)
+
         listOf(
                 SChapter.create().apply {
                     url = "/Read/View/${it.tmId}"
@@ -239,27 +249,97 @@ class Tsumino: ParsedHttpSource(), LewdSource<TsuminoMetadata, Document> {
                 }
         )
     }
-    
+
+    fun trickTsumino(id: String?) {
+        if(id == null) return
+
+        //Make one call to /Read/View (ASP session cookie)
+        val rvReq = GET("$BASE_URL/Read/View/$id")
+        val resp = client.newCall(rvReq).execute()
+
+        // Make 5 requests to the first 5 pages of the book in reader process
+        var chain: Observable<Any> = Observable.just(0)
+        for(i in 1 .. 5) {
+            chain = chain.flatMap {
+                val req = GET("$BASE_URL/Read/Process/$id/$i")
+                client.newCall(req).asObservableSuccess()
+            }
+        }
+
+        chain.observeOn(Schedulers.io())
+                .subscribeOn(Schedulers.io())
+                .subscribe()
+    }
+
+    override val client: OkHttpClient
+        get() = super.client.newBuilder()
+                .cookieJar(CookieJar.NO_COOKIES)
+                .addNetworkInterceptor {
+                    val cAspNetCookie = preferences.eh_ts_aspNetCookie().getOrDefault()
+
+                    var request = it.request()
+
+                    if(cAspNetCookie.isNotBlank()) {
+                        request = it.request()
+                                .newBuilder()
+                                .header("Cookie", "ASP.NET_SessionId=$cAspNetCookie")
+                                .build()
+                    }
+
+                    val response = it.proceed(request)
+
+                    val newCookie = response.headers("Set-Cookie").map(String::trim).find {
+                        it.startsWith(ASP_NET_COOKIE_NAME)
+                    }
+
+                    if(newCookie != null) {
+                        val res = newCookie.substringAfter('=')
+                                .substringBefore(';')
+                                .trim()
+
+                        preferences.eh_ts_aspNetCookie().set(res)
+                    }
+
+                    response
+                }.build()
+
     override fun fetchPageList(chapter: SChapter): Observable<List<Page>> {
         val id = chapter.url.substringAfterLast('/')
         val call = POST("$BASE_URL/Read/Load", body = FormBody.Builder().add("q", id).build())
         return client.newCall(call).asObservableSuccess().map {
             val parsed = jsonParser.parse(it.body()!!.string()).obj
             val pageUrls = parsed["reader_page_urls"].array
-            
+
             val imageUrl = Uri.parse("$BASE_URL/Image/Object")
             pageUrls.mapIndexed { index, obj ->
                 val newImageUrl = imageUrl.buildUpon().appendQueryParameter("name", obj.string)
                 Page(index, chapter.url + "#${index + 1}", newImageUrl.toString())
             }
-        }
+        }.doOnError {
+                    val aspNetCookie = preferences.eh_ts_aspNetCookie().getOrDefault()
+
+                    val cookiesMap = if(aspNetCookie.isNotBlank())
+                        mapOf(ASP_NET_COOKIE_NAME to aspNetCookie)
+                    else
+                        emptyMap()
+
+                    SolveCaptchaActivity.launch(context,
+                            this,
+                            cookiesMap,
+                            CAPTCHA_SCRIPT,
+                            "$BASE_URL/Read/Auth/$id")
+                }
     }
-    
+
+    override fun verify(url: String): Boolean {
+        return Uri.parse(url).pathSegments.getOrNull(1) == "View"
+    }
+
     override fun pageListParse(document: Document) = throw UnsupportedOperationException("Unused method called!")
     override fun imageUrlParse(document: Document) = throw UnsupportedOperationException("Unused method called!")
-    
+
     data class AdvSearchEntry(val type: Int, val text: String, val exclude: Boolean)
-    
+
     override fun getFilterList() = FilterList(
             Filter.Header("Separate tags with commas"),
             Filter.Header("Prepend with dash to exclude"),
@@ -271,15 +351,15 @@ class Tsumino: ParsedHttpSource(), LewdSource<TsuminoMetadata, Document> {
             ParodyFilter(),
             CharactersFilter(),
             UploaderFilter(),
-            
+
             Filter.Separator(),
-            
+
             SortFilter(),
             LengthFilter(),
             MinimumRatingFilter(),
             ExcludeParodiesFilter()
     )
-    
+
     class TagFilter : AdvSearchEntryFilter("Tags", 1)
     class CategoryFilter : AdvSearchEntryFilter("Categories", 2)
     class CollectionFilter : AdvSearchEntryFilter("Collections", 3)
@@ -289,17 +369,25 @@ class Tsumino: ParsedHttpSource(), LewdSource<TsuminoMetadata, Document> {
     class CharactersFilter : AdvSearchEntryFilter("Characters", 7)
     class UploaderFilter : AdvSearchEntryFilter("Uploaders", 8)
     open class AdvSearchEntryFilter(name: String, val type: Int) : Filter.Text(name)
-    
+
     class SortFilter : Filter.Select<SortType>("Sort by", SortType.values())
     class LengthFilter : Filter.Select<LengthType>("Length", LengthType.values())
     class MinimumRatingFilter : Filter.Select<String>("Minimum rating", (0 .. 5).map { "$it stars" }.toTypedArray())
     class ExcludeParodiesFilter : Filter.CheckBox("Exclude parodies")
-    
+
     companion object {
         val jsonParser by lazy {
             JsonParser()
         }
-        
+
         val TM_DATE_FORMAT = SimpleDateFormat("yyyy MMM dd", Locale.US)
+
+        private val ASP_NET_COOKIE_NAME = "ASP.NET_SessionId"
+
+        private val CAPTCHA_SCRIPT = """
+            |try{ document.querySelector('.tsumino-nav-btn').remove(); } catch(e) {}
+            |try{ document.querySelector('.tsumino-nav-title').href = '#' ;} catch(e) {}
+            |try{ document.querySelector('.tsumino-nav-items').remove() ;} catch(e) {}
+            """.trimMargin()
     }
 }

app/src/main/java/exh/captcha/SolveCaptchaActivity.kt

@@ -1,23 +1,120 @@
 package exh.captcha
 
+import android.content.Context
+import android.content.Intent
+import android.os.Build
 import android.os.Bundle
 import android.support.v7.app.AppCompatActivity
+import android.webkit.CookieManager
+import android.webkit.CookieSyncManager
+import android.webkit.WebView
+import android.webkit.WebViewClient
+import eu.kanade.tachiyomi.R
+import eu.kanade.tachiyomi.source.Source
+import eu.kanade.tachiyomi.source.SourceManager
+import kotlinx.android.synthetic.main.eh_activity_captcha.*
+import uy.kohesive.injekt.injectLazy
+import java.net.URL
 
 class SolveCaptchaActivity : AppCompatActivity() {
+    private val sourceManager: SourceManager by injectLazy()
+
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
 
-        val sourceId = intent.getIntExtra(SOURCE_ID_EXTRA, -1)
+        setContentView(R.layout.eh_activity_captcha)
-        val source = sourc
 
-        if(sourceId == -1) {
+        val sourceId = intent.getLongExtra(SOURCE_ID_EXTRA, -1)
+        val source = if(sourceId != -1L)
+            sourceManager.get(sourceId) as? CaptchaCompletionVerifier
+        else null
+
+        val cookies: HashMap<String, String>?
+                = intent.getSerializableExtra(COOKIES_EXTRA) as? HashMap<String, String>
+
+        val script: String? = intent.getStringExtra(SCRIPT_EXTRA)
+
+        val url: String? = intent.getStringExtra(URL_EXTRA)
+
+        if(source == null || cookies == null || url == null) {
             finish()
             return
         }
+
+        toolbar.title = source.name + ": Solve captcha"
+
+        val parsedUrl = URL(url)
+
+        val cm = CookieManager.getInstance()
+
+        fun continueLoading() {
+            cookies.forEach { (t, u) ->
+                val cookieString = t + "=" + u + "; domain=" + parsedUrl.host
+                cm.setCookie(url, cookieString)
+            }
+
+            if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP)
+                CookieSyncManager.createInstance(this).sync()
+
+            webview.settings.javaScriptEnabled = true
+            webview.settings.domStorageEnabled = true
+
+            webview.webViewClient = object : WebViewClient() {
+                override fun onPageFinished(view: WebView, url: String) {
+                    super.onPageFinished(view, url)
+
+                    if(source.verify(url)) {
+                        finish()
+                    } else {
+                        view.loadUrl("javascript:(function() {$script})();")
+                    }
+                }
+            }
+
+            webview.loadUrl(url)
+        }
+
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
+            cm.removeAllCookies { continueLoading() }
+        } else {
+            cm.removeAllCookie()
+            continueLoading()
+        }
+
+        setSupportActionBar(toolbar)
+        
+        supportActionBar?.setDisplayHomeAsUpEnabled(true)
+    }
+
+    override fun onSupportNavigateUp(): Boolean {
+        finish()
+        return true
     }
 
     companion object {
         const val SOURCE_ID_EXTRA = "source_id_extra"
+        const val COOKIES_EXTRA = "cookies_extra"
+        const val SCRIPT_EXTRA = "script_extra"
+        const val URL_EXTRA = "url_extra"
+
+        fun launch(context: Context,
+                   source: CaptchaCompletionVerifier,
+                   cookies: Map<String, String>,
+                   script: String,
+                   url: String) {
+            val intent = Intent(context, SolveCaptchaActivity::class.java).apply {
+                putExtra(SOURCE_ID_EXTRA, source.id)
+                putExtra(COOKIES_EXTRA, HashMap(cookies))
+                putExtra(SCRIPT_EXTRA, script)
+                putExtra(URL_EXTRA, url)
+            }
+
+            context.startActivity(intent)
+        }
     }
 }
 
+interface CaptchaCompletionVerifier : Source {
+    fun verify(url: String): Boolean
+}
+

app/src/main/java/exh/metadata/models/TsuminoMetadata.kt

@@ -121,7 +121,7 @@ open class TsuminoMetadata : RealmObject(), SearchableGalleryMetadata {
     }
     
     companion object {
-        val BASE_URL = "https://www.tsumino.com"
+        val BASE_URL = "http://www.tsumino.com"
         
         fun tmIdFromUrl(url: String)
             = Uri.parse(url).pathSegments[2]

app/src/main/java/exh/ui/login/LoginController.kt

@@ -14,6 +14,7 @@ import eu.kanade.tachiyomi.data.preference.PreferencesHelper
 import eu.kanade.tachiyomi.source.SourceManager
 import eu.kanade.tachiyomi.source.online.all.EHentai
 import eu.kanade.tachiyomi.ui.base.controller.NucleusController
+import eu.kanade.tachiyomi.util.launchUI
 import exh.EXH_SOURCE_ID
 import exh.uconfig.WarnConfigureDialogController
 import kotlinx.android.synthetic.main.eh_activity_login.view.*
@@ -49,9 +50,9 @@ class LoginController : NucleusController<LoginPresenter>() {
 
             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
                 CookieManager.getInstance().removeAllCookies {
-                    Observable.fromCallable {
+                    launchUI {
                         startWebview(view)
-                    }.subscribeOn(AndroidSchedulers.mainThread()).subscribe()
+                    }
                 }
             } else {
                 CookieManager.getInstance().removeAllCookie()
@@ -67,7 +68,7 @@ class LoginController : NucleusController<LoginPresenter>() {
 
             webview.loadUrl("https://forums.e-hentai.org/index.php?act=Login")
 
-            webview.setWebViewClient(object : WebViewClient() {
+            webview.webViewClient = object : WebViewClient() {
                 override fun onPageFinished(view: WebView, url: String) {
                     super.onPageFinished(view, url)
                     Timber.d(url)
@@ -88,7 +89,7 @@ class LoginController : NucleusController<LoginPresenter>() {
                         }
                     }
                 }
-            })
+            }
         }
     }
 

app/src/main/res/layout/eh_activity_captcha.xml

@@ -1,22 +1,27 @@
 <?xml version="1.0" encoding="utf-8"?>
-<RelativeLayout
+<android.support.design.widget.CoordinatorLayout
     xmlns:android="http://schemas.android.com/apk/res/android" android:layout_width="match_parent"
-    android:layout_height="match_parent">
+    android:layout_height="match_parent"
-
+    android:fitsSystemWindows="true">
-    <android.support.v7.widget.Toolbar
+    <RelativeLayout
-        android:id="@+id/toolbar"
         android:layout_width="match_parent"
-        android:layout_height="wrap_content"
+        android:layout_height="match_parent">
-        android:layout_alignParentStart="true"
-        android:layout_alignParentTop="true"
-        android:background="?attr/colorPrimary"
-        android:minHeight="?attr/actionBarSize"
-        android:theme="?attr/actionBarTheme" />
 
-    <WebView
+        <android.support.v7.widget.Toolbar
-        android:id="@+id/webview"
+            android:id="@+id/toolbar"
-        android:layout_width="match_parent"
+            android:layout_width="match_parent"
-        android:layout_height="match_parent"
+            android:layout_height="wrap_content"
-        android:layout_below="@+id/toolbar"
+            android:layout_alignParentStart="true"
-        android:layout_centerHorizontal="true" />
+            android:layout_alignParentTop="true"
-</RelativeLayout>
+            android:background="?attr/colorPrimary"
+            android:minHeight="?attr/actionBarSize"
+            android:theme="?attr/actionBarTheme" />
+
+        <WebView
+            android:id="@+id/webview"
+            android:layout_width="match_parent"
+            android:layout_height="match_parent"
+            android:layout_below="@+id/toolbar"
+            android:layout_centerHorizontal="true" />
+    </RelativeLayout>
+</android.support.design.widget.CoordinatorLayout>