lots of MMRCMS changes (#527)

* lots of MMRCMS changes

Fix http://www.on-manga.com
Fix covers not loading for some extensions when browsing
Rewrote generator in kotlin since shell script required unix/mac and installing outside packages
fixed #506 mangawww
removed 4 manga
fixed #480 fallen angels scans conflict
fixed hentai shark

* clean up gradle file

src/all/mmrcms/build.gradle

@@ -5,8 +5,8 @@ ext {
     appName = 'Tachiyomi: My Manga Reader CMS (Many sources)'
     pkgNameSuffix = 'all.mmrcms'
     extClass = '.MyMangaReaderCMSSources'
-    extVersionCode = 10
+    extVersionCode = 11
-    extVersionSuffix = 10
+    extVersionSuffix = 11
     libVersion = '1.2'
 }
 

src/all/mmrcms/genSources.sh

@@ -1,367 +0,0 @@
-#!/usr/bin/env bash
-echo "My Manga Reader CMS source generator by: nulldev"
-# CMS: https://getcyberworks.com/product/manga-reader-cms/
-
-# Print a message out to stderr
-function echoErr() {
-	echo "ERROR: $@" >&2
-}
-
-# Require that a command exists before continuing
-function require() {
-	command -v $1 >/dev/null 2>&1 || { echoErr "This script requires $1 but it's not installed."; exit 1; }
-}
-
-# Define commands that this script depends on
-require xmllint
-require jq
-require perl
-require wget
-require curl
-require grep
-require sed
-
-# Show help/usage info
-function printHelp() {
-	echo "Usage: ./genSources.sh [options]"
-	echo ""
-	echo "Options:"
-	echo "--help: Show this help page"
-	echo "--dry-run: Perform a dry run (make no changes)"
-	echo "--list: List currently available sources"
-	echo "--out <file>: Explicitly specify output file"
-}
-# Target file
-TARGET="src/eu/kanade/tachiyomi/extension/all/mmrcms/GeneratedSources.kt"
-# String containing processed URLs (used to detect duplicate URLs)
-PROCESSED=""
-
-# Parse CLI args
-while [ $# -gt 0 ]
-do
-	case "$1" in
-		--help)
-			printHelp
-			exit 0
-			;;
-		--dry-run) OPT_DRY_RUN=true
-			;;
-		--list)
-			OPT_DRY_RUN=true
-			OPT_LIST=true
-			;;
-		--out)
-			TARGET="$2"
-			shift
-			;;
-		--*)
-			echo "Invalid option $1!"
-			printHelp
-			exit -1
-			;;
-		*)
-			echo "Invalid argument $1!"
-			printHelp
-			exit -1
-			;;
-	esac
-	shift
-done
-
-# Change target if performing dry run
-if [ "$OPT_DRY_RUN" = true ] ; then
-	# Do not warn if dry running because of list
-	if ! [ "$OPT_LIST" = true ] ; then
-		echo "Performing a dry run, no changes will be made!"
-	fi
-	TARGET="/dev/null"
-else
-	# Delete old sources
-	rm "$TARGET"
-fi
-
-# Variable used to store output while processing
-QUEUED_SOURCES="["
-
-# lang, name, baseUrl
-function gen() {
-	PROCESSED="$PROCESSED$3\n"
-	if [ "$OPT_LIST" = true ] ; then
-		echo "- $(echo "$1" | awk '{print toupper($0)}'): $2"
-	else
-		echo "Generating source: $2"
-		QUEUED_SOURCES="$QUEUED_SOURCES"$'\n'"$(genSource "$1" "$2" "$3")"
-		# genSource runs in a subprocess, so we check for bad exit code and exit current process if necessary
-		[ $? -ne 0 ] && exit -1;
-	fi
-}
-
-# Find and get the item URL from an HTML page
-function getItemUrl() {
-	grep -oP "(?<=showURL = \")(.*)(?=SELECTION)" "$1"
-}
-
-# Strip all scripts and Cloudflare email protection from page
-# We strip Cloudflare email protection as titles like 'IDOLM@STER' can trigger it and break the parser
-function stripScripts() {
-	perl -0pe 's/<script.*?>[\s\S]*?< *?\/ *?script *?>//g' |\
-		perl -0pe 's/<span class="__cf_email__".*?>[\s\S]*?< *?\/ *?span *?>/???@???/g'
-}
-
-# Verify that a response is valid
-function verifyResponse() {
-	[ "${1##*$'\n'}" -eq "200" ] && [[ "$1" != *"Whoops, looks like something went wrong"* ]]
-}
-
-# Get the available tags from the manga list page
-function parseTagsFromMangaList() {
-	xmllint --xpath "//div[contains(@class, 'tag-links')]//a" --html "$1" 2>/dev/null |\
-		sed 's/<\/a>/"},\n/g; s/">/", "name": "/g;' |\
-		perl -pe 's/<a.*?\/tag\//            {"id": "/gi;' |\
-		sed '/^</d'
-}
-
-# Get the available categories from the manga list page
-function parseCategoriesFromMangaList() {
-	xmllint --xpath "//li//a[contains(@class, 'category')]" --html "$1" 2>/dev/null |\
-		sed 's/<\/a>/"},\n/g; s/" class="category">/", "name": "/g;' |\
-		perl -pe 's/<a.*?\?cat=/            {"id": "/gi;'
-}
-
-# Get the available categories from the advanced search page
-function parseCategoriesFromAdvancedSearch() {
-	xmllint --xpath "//select[@name='categories[]']/option" --html "$1" 2>/dev/null |\
-		sed 's/<\/option>/"},\n/g; s/<option value="/            {"id": "/g; s/">/", "name": "/g;'
-}
-
-# Unescape HTML entities
-function unescapeHtml() {
-	echo "$1" | perl -C -MHTML::Entities -pe 'decode_entities($_);'
-}
-
-# Remove the last character from a string, often used to remove the trailing comma
-function stripLastComma() {
-	echo "${1::-1}"
-}
-
-# lang, name, baseUrl
-function genSource() {
-	# Allocate temp files
-	DL_TMP="$(mktemp)"
-	PG_TMP="$(mktemp)"
-
-	# Fetch categories from advanced search
-	wget "$3/advanced-search" -O "$DL_TMP" --no-check-certificate
-	# Find manga/comic URL
-	ITEM_URL="$(getItemUrl "$DL_TMP")"
-	# Remove scripts
-	cat "$DL_TMP" | stripScripts > "$PG_TMP"
-	# Find and transform categories
-	CATEGORIES="$(parseCategoriesFromAdvancedSearch "$PG_TMP")"
-	# Get item url from home page if not on advanced search page!
-	if [[ -z "${ITEM_URL// }" ]]; then
-		# Download home page
-		wget "$3" -O "$DL_TMP" --no-check-certificate
-		# Extract item url again
-		ITEM_URL="$(getItemUrl "$DL_TMP")"
-		# Still missing?
-		if [[ -z "${ITEM_URL// }" ]]; then
-			echoErr "Could not get item URL!"
-			exit -1
-		fi
-	fi
-
-	# Calculate location of manga list page
-	LIST_URL_PREFIX="manga"
-	# Get last path item in item URL and set as URL prefix
-	if [[ $ITEM_URL =~ .*\/([^\\]+)\/ ]]; then
-		LIST_URL_PREFIX="${BASH_REMATCH[1]}"
-	fi
-	# Download manga list page
-	wget "$3/$LIST_URL_PREFIX-list" -O "$DL_TMP" --no-check-certificate
-	# Remove scripts
-	cat "$DL_TMP" | stripScripts > "$PG_TMP"
-
-	# Get categories from manga list page if we couldn't from advanced search
-	if [[ -z "${CATEGORIES// }" ]]; then
-		# Parse
-		CATEGORIES="$(parseCategoriesFromMangaList "$PG_TMP")"
-		# Check again
-		if [[ -z "${CATEGORIES// }" ]]; then
-			echoErr "Could not get categories!"
-			exit -1
-		fi
-	fi
-
-	# Get tags from manga list page
-	TAGS="$(parseTagsFromMangaList "$PG_TMP")"
-	if [[ -z "${TAGS// }" ]]; then
-		TAGS="null"
-	else
-		TAGS="$(stripLastComma "$TAGS")"
-		TAGS=$'[\n'"$TAGS"$'\n        ]'
-	fi
-
-	# Unescape HTML entities
-	CATEGORIES="$(unescapeHtml "$CATEGORIES")"
-	# Check if latest manga is supported
-	LATEST_RESP=$(curl -k --write-out \\n%{http_code} --silent --output - "$3/filterList?page=1&sortBy=last_release&asc=false")
-	SUPPORTS_LATEST="false"
-	if verifyResponse "$LATEST_RESP"; then
-		SUPPORTS_LATEST="true"
-	fi
-	# Remove leftover html pages
-	rm "$DL_TMP"
-	rm "$PG_TMP"
-
-	# Cleanup categories
-	CATEGORIES="$(stripLastComma "$CATEGORIES")"
-
-	echo "    {"
-	echo "        \"language\": \"$1\","
-	echo "        \"name\": \"$2\","
-	echo "        \"base_url\": \"$3\","
-	echo "        \"supports_latest\": $SUPPORTS_LATEST,"
-	echo "        \"item_url\": \"$ITEM_URL\","
-	echo "        \"categories\": ["
-	echo "$CATEGORIES"
-	echo "        ],"
-	echo "        \"tags\": $TAGS"
-	echo "    },"
-}
-
-# Source list
-gen "ar" "مانجا اون لاين" "http://www.on-manga.com"
-# Went offline
-# gen "ar" "Manga FYI" "http://mangafyi.com/manga/arabic"
-gen "en" "Read Comics Online" "http://readcomicsonline.ru"
-gen "en" "Fallen Angels Scans" "http://manga.fascans.com"
-# Went offline
-# gen "en" "MangaRoot" "http://mangaroot.com"
-gen "en" "Mangawww Reader" "http://mangawww.com"
-# Went offline
-# gen "en" "MangaForLife" "http://manga4ever.com"
-# Went offline
-# gen "en" "Manga Spoil" "http://mangaspoil.com"
-gen "en" "MangaBlue" "http://mangablue.com"
-# Went offline
-# gen "en" "Manga Forest" "https://mangaforest.com"
-# Went offline
-# gen "en" "DManga" "http://dmanga.website"
-gen "en" "Chibi Manga Reader" "http://www.cmreader.info"
-gen "en" "ZXComic" "http://zxcomic.com"
-# Went offline
-# gen "en" "DB Manga" "http://dbmanga.com"
-# Went offline
-# gen "en" "Mangacox" "http://mangacox.com"
-
-# Protected by CloudFlare
-# gen "en" "GO Manhwa" "http://gomanhwa.xyz"
-# Went offline
-# gen "en" "KoManga" "https://komanga.net"
-# Went offline
-# gen "en" "Manganimecan" "http://manganimecan.com"
-# Went offline
-# gen "en" "Hentai2Manga" "http://hentai2manga.com"
-gen "en" "White Cloud Pavilion" "http://www.whitecloudpavilion.com/manga/free"
-gen "en" "4 Manga" "http://4-manga.com"
-gen "en" "XYXX.INFO" "http://xyxx.info"
-gen "en" "MangaTreat Scans" "http://www.mangatreat.com"
-gen "en" "Isekai Manga Reader" "https://isekaimanga.club"
-gen "es" "My-mangas.com" "https://my-mangas.com"
-gen "es" "SOS Scanlation" "https://sosscanlation.com"
-# Went offline
-# gen "fa" "TrinityReader" "http://trinityreader.pw"
-gen "fr" "Manga-LEL" "https://www.manga-lel.com"
-gen "fr" "Manga Etonnia" "https://www.etonnia.com"
-gen "fr" "Scan FR" "http://www.scan-fr.io"
-# Went offline
-# gen "fr" "ScanFR.com" "http://scanfr.com"
-# Went offline
-# gen "fr" "Manga FYI" "http://mangafyi.com/manga/french"
-# Went offline
-# gen "fr" "scans-manga" "http://scans-manga.com"
-gen "fr" "Henka no Kaze" "http://henkanokazelel.esy.es/upload"
-# Went offline
-# gen "fr" "Tous Vos Scans" "http://www.tous-vos-scans.com"
-# Went offline
-# gen "id" "Manga Desu" "http://mangadesu.net"
-# Went offline
-# gen "id" "Komik Mangafire.ID" "http://go.mangafire.id"
-gen "id" "MangaOnline" "https://mangaonline.web.id"
-# Went offline
-# gen "id" "MangaNesia" "https://manganesia.com"
-gen "id" "Komikid" "http://www.komikid.com"
-# Now uses wpmanga
-# gen "id" "MangaID" "https://mangaid.me"
-# Went offline
-# gen "id" "Manga Seru" "http://www.mangaseru.top"
-# Went offline
-# gen "id" "Manga FYI" "http://mangafyi.com/manga/indonesian"
-gen "id" "Bacamangaku" "http://www.bacamangaku.com"
-# Went offline
-# gen "id" "Indo Manga Reader" "http://indomangareader.com"
-# Protected by Cloudflare
-# gen "it" "Kingdom Italia Reader" "http://kireader.altervista.org"
-# Went offline
-# gen "ja" "IchigoBook" "http://ichigobook.com"
-# Went offline
-# gen "ja" "Mangaraw Online" "http://mangaraw.online"
-gen "ja" "Mangazuki RAWS" "https://raws.mangazuki.co"
-gen "ja" "RAW MANGA READER" "https://rawmanga.site"
-# Went offline
-# gen "ja" "MangaRAW" "https://www.mgraw.com"
-gen "ja" "マンガ/漫画 マガジン/雑誌 raw" "http://netabare-manga-raw.com"
-gen "pl" "ToraScans" "http://torascans.pl"
-gen "pt" "Comic Space" "https://www.comicspace.com.br"
-gen "pt" "Mangás Yuri" "https://mangasyuri.net"
-gen "pl" "Dracaena" "http://dracaena.webd.pl/czytnik"
-gen "pl" "Nikushima" "http://azbivo.webd.pro"
-gen "ru" "NAKAMA" "http://nakama.ru"
-gen "ru" "Anigai clan" "http://anigai.ru"
-# Went offline
-# gen "tr" "MangAoi" "http://mangaoi.com"
-gen "tr" "MangaHanta" "http://mangahanta.com"
-gen "tr" "ManhuaTR" "http://www.manhua-tr.com"
-gen "vi" "Fallen Angels Scans" "http://truyen.fascans.com"
-# Blocks bots (like this one)
-# gen "tr" "Epikmanga" "http://www.epikmanga.com"
-# NOTE: THIS SOURCE CONTAINS A CUSTOM LANGUAGE SYSTEM (which will be ignored)!
-gen "other" "HentaiShark" "https://www.hentaishark.com"
-
-if ! [ "$OPT_LIST" = true ] ; then
-	# Remove last comma from output
-	QUEUED_SOURCES="$(stripLastComma "$QUEUED_SOURCES")"
-	# Format, minify and split JSON output into chunks of 5000 chars
-	OUTPUT="$(echo -e "$QUEUED_SOURCES\n]" | jq -c . | fold -s -w5000)"
-	# Write file header
-	echo -e "package eu.kanade.tachiyomi.extension.all.mmrcms\n" >> "$TARGET"
-	echo -e "// GENERATED FILE, DO NOT MODIFY!" >> "$TARGET"
-	echo -e "// Generated on $(date)\n" >> "$TARGET"
-	# Convert split lines into variables
-	COUNTER=0
-	CONCAT="val SOURCES: String get() = "
-	TOTAL_LINES="$(echo "$OUTPUT" | wc -l)"
-	while read -r line; do
-		COUNTER=$[$COUNTER +1]
-		VARNAME="MMRSOURCE_$COUNTER"
-		echo "private val $VARNAME = \"\"\"$line\"\"\"" >> "$TARGET"
-		CONCAT="$CONCAT$VARNAME"
-		if [ "$COUNTER" -ne "$TOTAL_LINES" ]; then
-			CONCAT="$CONCAT + "
-		fi
-	done <<< "$OUTPUT"
-	echo "$CONCAT" >> "$TARGET"
-fi
-
-# Detect and warn about duplicate sources
-DUPES="$(echo -e "$PROCESSED" | sort | uniq -d)"
-if [[ ! -z "$DUPES" ]]; then
-	echo
-	echo "----> WARNING, DUPLICATE SOURCES DETECTED! <----"
-	echo "Listing duplicates:"
-	echo "$DUPES"
-	echo
-fi
-
-echo "Done!"

src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/Generator.kt

@@ -0,0 +1,308 @@
+package eu.kanade.tachiyomi.extension.all.mmrcms
+
+import android.annotation.TargetApi
+import android.os.Build
+import com.google.gson.Gson
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import java.io.PrintWriter
+import java.security.cert.CertificateException
+import java.time.ZonedDateTime
+import java.time.format.DateTimeFormatter
+import javax.net.ssl.SSLContext
+import javax.net.ssl.TrustManager
+import javax.net.ssl.X509TrustManager
+
+/**
+ * This class generates the sources for MMRCMS.
+ * Credit to nulldev for writing the original shell script
+ *
+# CMS: https://getcyberworks.com/product/manga-reader-cms/
+ */
+
+class Generator {
+
+    @TargetApi(Build.VERSION_CODES.O)
+    fun generate() {
+        val buffer = StringBuffer()
+        val dateTime = ZonedDateTime.now()
+        val formattedDate = dateTime.format(DateTimeFormatter.RFC_1123_DATE_TIME)
+        buffer.append("package eu.kanade.tachiyomi.extension.all.mmrcms")
+        buffer.append("\n\n// GENERATED FILE, DO NOT MODIFY!\n//Generated $formattedDate\n\n")
+        var number = 1
+        sources.forEach {
+            try {
+                var map = mutableMapOf<String, Any>()
+                map["language"] = it.first
+                map["name"] = it.second
+                map["base_url"] = it.third
+                map["supports_latest"] = supportsLatest(it.third)
+
+                val advancedSearchDocument = getDocument("${it.third}/advanced-search", false)
+
+                var parseCategories = mutableListOf<Map<String, String>>()
+                if (advancedSearchDocument != null) {
+                    parseCategories = parseCategories(advancedSearchDocument)
+                }
+
+                val homePageDocument = getDocument("${it.third}")!!
+
+                val itemUrl = getItemUrl(homePageDocument)
+
+                var prefix = itemUrl.substringAfterLast("/").substringBeforeLast("/")
+
+                val mangaListDocument = getDocument("${it.third}/$prefix-list")!!
+
+                if (parseCategories.isEmpty()) {
+                    parseCategories = parseCategories(mangaListDocument)
+                }
+                map["item_url"] = itemUrl
+                map["categories"] = parseCategories
+                val tags = parseTags(mangaListDocument)
+                map["tags"] = "null"
+                if (tags.size in 1..49) {
+                    map["tags"] = tags
+                }
+
+
+                val toJson = Gson().toJson(map)
+
+
+                buffer.append("private const val MMRSOURCE_$number = \"\"\"$toJson\"\"\"\n")
+                number++
+
+            } catch (e: Exception) {
+                println("error generating source ${it.second} ${e.printStackTrace()}")
+            }
+        }
+
+        buffer.append("val SOURCES: List<String> get() = listOf(")
+        for (i in 1 until number) {
+            buffer.append("MMRSOURCE_$i")
+            when (i) {
+                number - 1 -> {
+                    buffer.append(")\n")
+                }
+                else -> {
+                    buffer.append(", ")
+                }
+            }
+        }
+        if (!DRY_RUN) {
+            val writer = PrintWriter(relativePath)
+            writer.write(buffer.toString())
+            writer.close()
+
+        } else {
+            val writer = PrintWriter(relativePathTest)
+            writer.write(buffer.toString())
+            writer.close()
+        }
+    }
+
+    private fun getDocument(url: String, printStackTrace: Boolean = true): Document? {
+        try {
+
+            val response = getOkHttpClient().newCall(Request.Builder().url(url).build()).execute()
+            if (response.code() == 200) {
+                return Jsoup.parse(response.body()?.string())
+            }
+        } catch (e: Exception) {
+            if (printStackTrace) {
+                e.printStackTrace()
+            }
+        }
+        return null
+    }
+
+    private fun parseTags(mangaListDocument: Document): MutableList<Map<String, String>> {
+        val elements = mangaListDocument.select("div.tag-links a")
+
+        if (elements.isEmpty()) {
+            return mutableListOf()
+        }
+        var array = mutableListOf<Map<String, String>>()
+        elements.forEach {
+            var map = mutableMapOf<String, String>()
+            map["id"] = it.attr("href").substringAfterLast("/")
+            map["name"] = it.text()
+            array.add(map)
+        }
+        return array
+
+    }
+
+    private fun getItemUrl(document: Document): String {
+        return document.toString().substringAfter("showURL = \"").substringBefore("/SELECTION\";")
+    }
+
+    private fun supportsLatest(third: String): Boolean {
+        getDocument("$third/filterList?page=1&sortBy=last_release&asc=false", false) ?: return false
+        return true
+    }
+
+    private fun parseCategories(document: Document): MutableList<Map<String, String>> {
+        var array = mutableListOf<Map<String, String>>()
+        var elements = document.select("select[name^=categories] option")
+        if (elements.size == 0) {
+            return mutableListOf()
+        }
+        var id = 1
+        elements.forEach {
+            var map = mutableMapOf<String, String>()
+            map["id"] = id.toString()
+            map["name"] = it.text()
+            array.add(map)
+            id++
+        }
+        return array
+    }
+
+
+    @Throws(Exception::class)
+    private fun getOkHttpClient(): OkHttpClient {
+        val trustAllCerts = arrayOf<TrustManager>(object : X509TrustManager {
+            @Throws(CertificateException::class)
+            override fun checkClientTrusted(chain: Array<java.security.cert.X509Certificate>, authType: String) {
+            }
+
+            @Throws(CertificateException::class)
+            override fun checkServerTrusted(chain: Array<java.security.cert.X509Certificate>, authType: String) {
+            }
+
+            override fun getAcceptedIssuers(): Array<java.security.cert.X509Certificate> {
+                return arrayOf()
+            }
+        })
+
+        // Install the all-trusting trust manager
+
+        val sc = SSLContext.getInstance("SSL")
+        sc.init(null, trustAllCerts, java.security.SecureRandom())
+        val sslSocketFactory = sc.socketFactory
+
+        // Create all-trusting host name verifier
+        // Install the all-trusting host verifier
+
+        val builder = OkHttpClient.Builder()
+        builder.sslSocketFactory(sslSocketFactory, trustAllCerts[0] as X509TrustManager)
+        builder.hostnameVerifier { _, _ -> true }
+        return builder.build()
+    }
+
+
+    companion object {
+        const val DRY_RUN = false
+        val sources = listOf(
+                Triple("ar", "مانجا اون لاين", "http://www.on-manga.com"),
+                //Went offline
+                //Triple("ar", "Manga FYI", "http://mangafyi.com/manga/arabic"
+                Triple("en", "Read Comics Online", "http://readcomicsonline.ru"),
+                Triple("en", "Fallen Angels", "http://manga.fascans.com"),
+                //Went offline
+                //Triple("en", "MangaRoot", "http://mangaroot.com"),
+                Triple("en", "Mangawww Reader", "http://mangawww.club"),
+                //Went offline
+                //Triple("en", "MangaForLife", "http://manga4ever.com"),
+                //Went offline
+                //Triple("en", "Manga Spoil", "http://mangaspoil.com"),
+                Triple("en", "MangaBlue", "http://mangablue.com"),
+                //Went offline
+                //Triple("en", "Manga Forest", "https://mangaforest.com"),
+                //Went offline
+                //Triple("en", "DManga", "http://dmanga.website"
+                Triple("en", "Chibi Manga Reader", "http://www.cmreader.info"),
+                Triple("en", "ZXComic", "http://zxcomic.com"),
+                //Went offline
+                //Triple("en", "DB Manga", "http://dbmanga.com"),
+                //Went offline
+                //Triple("en", "Mangacox", "http://mangacox.com"),
+
+                //Protected by CloudFlare
+                //Triple("en", "GO Manhwa", "http://gomanhwa.xyz"
+                //Went offline
+                //Triple("en", "KoManga", "https://komanga.net"
+                //Went offline
+                //Triple("en", "Manganimecan", "http://manganimecan.com"),
+                //Went offline
+                //Triple("en", "Hentai2Manga", "http://hentai2manga.com"),
+                Triple("en", "White Cloud Pavilion", "http://www.whitecloudpavilion.com/manga/free"),
+                //Went offline
+                //Triple("en", "4 Manga", "http://4-manga.com"),
+                Triple("en", "XYXX.INFO", "http://xyxx.info"),
+                Triple("en", "MangaTreat Scans", "http://www.mangatreat.com"),
+                Triple("en", "Isekai Manga Reader", "https://isekaimanga.club"),
+                Triple("es", "My-mangas.com", "https://my-mangas.com"),
+                Triple("es", "SOS Scanlation", "https://sosscanlation.com"),
+                //Went offline
+                //Triple("fa", "TrinityReader", "http://trinityreader.pw"
+                Triple("fr", "Manga-LEL", "https://www.manga-lel.com"),
+                Triple("fr", "Manga Etonnia", "https://www.etonnia.com"),
+                Triple("fr", "Scan FR", "http://www.scan-fr.io"),
+                //Went offline
+                //Triple("fr", "ScanFR.com"),, "http://scanfr.com"),
+                //Went offline
+                //Triple("fr", "Manga FYI", "http://mangafyi.com/manga/french"
+                //Went offline
+                //Triple("fr", "scans-manga", "http://scans-manga.com"),
+                Triple("fr", "Henka no Kaze", "http://henkanokazelel.esy.es/upload"),
+                //Went offline
+                //Triple("fr", "Tous Vos Scans", "http://www.tous-vos-scans.com"),
+                //Went offline
+                //Triple("id", "Manga Desu", "http://mangadesu.net"
+                //Went offline
+                //Triple("id", "Komik Mangafire.ID", "http://go.mangafire.id"
+                Triple("id", "MangaOnline", "https://mangaonline.web.id"),
+                //Went offline
+                //Triple("id", "MangaNesia", "https://manganesia.com"),
+                Triple("id", "Komikid", "http://www.komikid.com"),
+                //Now uses wpmanga
+                //Triple("id", "MangaID", "https://mangaid.me"
+                //Went offline
+                //Triple("id", "Manga Seru", "http://www.mangaseru.top"
+                //Went offline
+                //Triple("id", "Manga FYI", "http://mangafyi.com/manga/indonesian"
+                Triple("id", "Bacamangaku", "http://www.bacamangaku.com"),
+                //Went offline
+                //Triple("id", "Indo Manga Reader", "http://indomangareader.com"),
+                //Protected by Cloudflare
+                //Triple("it", "Kingdom Italia Reader", "http://kireader.altervista.org"),
+                //Went offline
+                //Triple("ja", "IchigoBook", "http://ichigobook.com"),
+                //Went offline
+                //Triple("ja", "Mangaraw Online", "http://mangaraw.online"
+                Triple("ja", "Mangazuki RAWS", "https://raws.mangazuki.co"),
+                Triple("ja", "RAW MANGA READER", "https://rawmanga.site"),
+                //Went offline
+                //Triple("ja", "MangaRAW", "https://www.mgraw.com"),
+                Triple("ja", "マンガ/漫画 マガジン/雑誌 raw", "http://netabare-manga-raw.com"),
+                Triple("pl", "ToraScans", "http://torascans.pl"),
+                Triple("pt", "Comic Space", "https://www.comicspace.com.br"),
+                Triple("pt", "Mangás Yuri", "https://mangasyuri.net"),
+                Triple("pl", "Dracaena", "http://dracaena.webd.pl/czytnik"),
+                Triple("pl", "Nikushima", "http://azbivo.webd.pro"),
+                Triple("ru", "NAKAMA", "http://nakama.ru"),
+                Triple("ru", "Anigai clan", "http://anigai.ru"),
+                //Went offline
+                //Triple("tr", "MangAoi", "http://mangaoi.com"),
+                Triple("tr", "MangaHanta", "http://mangahanta.com"),
+                //WEnt offline
+                //Triple("tr", "ManhuaTR", "http://www.manhua-tr.com"),
+                Triple("vi", "Fallen Angels Scans", "http://truyen.fascans.com"),
+                //Blocks bots (like this one)
+                //Triple("tr", "Epikmanga", "http://www.epikmanga.com"),
+                //NOTE: THIS SOURCE CONTAINS A CUSTOM LANGUAGE SYSTEM (which will be ignored)!
+                Triple("other", "HentaiShark", "https://www.hentaishark.com"))
+
+        val relativePath = System.getProperty("user.dir") + "/src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/GeneratedSources.kt"
+        val relativePathTest = System.getProperty("user.dir") + "/src/all/mmrcms/TestGeneratedSources.kt"
+
+
+        @JvmStatic
+        fun main(args: Array<String>) {
+            Generator().generate()
+        }
+    }
+}

src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/MyMangaReaderCMSSource.kt

@@ -9,6 +9,7 @@ import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.source.model.*
 import eu.kanade.tachiyomi.source.online.HttpSource
 import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
 import org.jsoup.nodes.Element
@@ -27,6 +28,8 @@ class MyMangaReaderCMSSource(override val lang: String,
     private val itemUrlPath = Uri.parse(itemUrl).pathSegments.first()
     private val parsedBaseUrl = Uri.parse(baseUrl)
 
+    override val client: OkHttpClient = network.cloudflareClient
+
     override fun popularMangaRequest(page: Int) = GET("$baseUrl/filterList?page=$page&sortBy=views&asc=false")
     override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
         //Query overrides everything
@@ -57,7 +60,7 @@ class MyMangaReaderCMSSource(override val lang: String,
                             title = it["value"].string
 
                             // Guess thumbnails
-                            thumbnail_url = "$baseUrl/uploads/manga/$segment/cover/cover_250x350.jpg"
+                            // thumbnail_url = "$baseUrl/uploads/manga/$segment/cover/cover_250x350.jpg"
                         }
                     }, false)
         } else {
@@ -69,26 +72,39 @@ class MyMangaReaderCMSSource(override val lang: String,
 
     private fun internalMangaParse(response: Response): MangasPage {
         val document = response.asJsoup()
-        return MangasPage(document.getElementsByClass("col-sm-6").map {
+
+        return MangasPage(document.select("div[class^=col-sm]").map {
             SManga.create().apply {
                 val urlElement = it.getElementsByClass("chart-title")
+                if (urlElement.size == 0) {
+                    url = getUrlWithoutBaseUrl(it.select("a").attr("href"))
+                    title = it.select("div.caption").text()
+                } else {
                     url = getUrlWithoutBaseUrl(urlElement.attr("href"))
                     title = urlElement.text().trim()
-                thumbnail_url = coverGuess(it.select(".media-left img").attr("src"))
+                }
+
+                val cover = it.select(".media-left img").attr("src")
+                thumbnail_url =
+                        if (cover.isEmpty()) {
+                            coverGuess(it.select("img").attr("src"), url)
+                        } else {
+                            coverGuess(cover, url)
+                        }
             }
         }, document.select(".pagination a[rel=next]").isNotEmpty())
     }
 
     // Guess thumbnails on broken websites
 
-    private fun coverGuess(url: String?): String {
+    private fun coverGuess(url: String?, mangaUrl: String): String {
         // Guess thumbnails on broken websites
         if (url != null && url.isNotBlank()) {
-            if( url.startsWith("//")){
+            if (url.startsWith("//")) {
                 return "$baseUrl/uploads/manga/${url.substringBeforeLast("/cover/").substringAfter("/manga/")}/cover/cover_250x350.jpg"
             }
             if (url.endsWith("no-image.png")) {
-                return "$baseUrl/uploads/manga/${url?.substringAfterLast('/')}/cover/cover_250x350.jpg"
+                return "$baseUrl/uploads/manga/${mangaUrl?.substringAfterLast('/')}/cover/cover_250x350.jpg"
             }
             return url
         }
@@ -120,7 +136,7 @@ class MyMangaReaderCMSSource(override val lang: String,
     override fun mangaDetailsParse(response: Response) = SManga.create().apply {
         val document = response.asJsoup()
         title = document.getElementsByClass("widget-title").text().trim()
-        thumbnail_url = coverGuess(document.select(".row .img-responsive").attr("src"))
+        thumbnail_url = coverGuess(document.select(".row .img-responsive").attr("src"), document.location())
         description = document.select(".row .well p").text().trim()
 
         var cur: String? = null

src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/MyMangaReaderCMSSources.kt

@@ -2,14 +2,13 @@ package eu.kanade.tachiyomi.extension.all.mmrcms
 
 import com.github.salomonbrys.kotson.array
 import com.github.salomonbrys.kotson.bool
-import com.github.salomonbrys.kotson.nullArray
 import com.github.salomonbrys.kotson.string
 import com.google.gson.JsonArray
 import com.google.gson.JsonObject
 import com.google.gson.JsonParser
 import eu.kanade.tachiyomi.source.SourceFactory
 
-class MyMangaReaderCMSSources: SourceFactory {
+class MyMangaReaderCMSSources : SourceFactory {
     /**
      * Create a new copy of the sources
      * @return The created sources
@@ -17,11 +16,10 @@ class MyMangaReaderCMSSources: SourceFactory {
     override fun createSources() = parseSources(SOURCES)
 
     /**
-     * Parse a JSON array of sources into a list of `MyMangaReaderCMSSource`s
+     * Parse a List of JSON sources into a list of `MyMangaReaderCMSSource`s
      *
-     * Example JSON array:
+     * Example JSON :
      * ```
-     * [
      *     {
      *         "language": "en",
      *         "name": "Example manga reader",
@@ -37,28 +35,28 @@ class MyMangaReaderCMSSources: SourceFactory {
      *             {"id": "adventure", "name": "Adventure"}
      *         ]
      *     }
-     * ]
+     *
-     * ```
      *
      * Sources that do not supports tags may use `null` instead of a list of json objects
      *
-     * @param sourceString The JSON array of sources to parse
+     * @param sourceString The List of JSON strings 1 entry = one source
      * @return The list of parsed sources
      */
-    private fun parseSources(sourceString: String): List<MyMangaReaderCMSSource> {
+    private fun parseSources(sourceString: List<String>): List<MyMangaReaderCMSSource> {
         val parser = JsonParser()
-        val array = parser.parse(sourceString).array
+        return sourceString.map {
+            val jsonObject = parser.parse(it) as JsonObject
 
-        return array.map {
+            val language = jsonObject["language"].string
-            it as JsonObject
+            val name = jsonObject["name"].string
-
+            val baseUrl = jsonObject["base_url"].string
-            val language = it["language"].string
+            val supportsLatest = jsonObject["supports_latest"].bool
-            val name = it["name"].string
+            val itemUrl = jsonObject["item_url"].string
-            val baseUrl = it["base_url"].string
+            val categories = mapToPairs(jsonObject["categories"].array)
-            val supportsLatest = it["supports_latest"].bool
+            var tags = emptyList<Pair<String, String>>()
-            val itemUrl = it["item_url"].string
+            if (jsonObject["tags"].isJsonArray) {
-            val categories = mapToPairs(it["categories"].array)
+                tags = jsonObject["tags"].asJsonArray.let { mapToPairs(it) }
-            val tags = it["tags"].nullArray?.let { mapToPairs(it) }
+            }
 
             MyMangaReaderCMSSource(
                     language,
@@ -82,8 +80,7 @@ class MyMangaReaderCMSSources: SourceFactory {
      * @param array The array to process
      * @return The new list of pairs
      */
-    private fun mapToPairs(array: JsonArray): List<Pair<String, String>>
+    private fun mapToPairs(array: JsonArray): List<Pair<String, String>> = array.map {
-            = array.map {
         it as JsonObject
 
         it["id"].string to it["name"].string