From 318f335bf866ef45bb3c3cf74f9a5b8965172a04 Mon Sep 17 00:00:00 2001 From: Andy Bao Date: Sat, 17 Mar 2018 16:54:00 -0400 Subject: [PATCH] Add support for My Manga Reader CMS sources (many, many sources) (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add My Manga Reader CMS sources and generator Currently supported sources: - EN: Read Comics Online - EN: Fallen Angels Scans - EN: MangaRoot - EN: Mangawww Reader - EN: MangaForLife - ES: My-mangas.com - FA: TrinityReader - ID: Manga Desu - JA: IchigoBook - TR: MangAoi * Add more sources Code cleanup Added thumbnail guesser to keyword search Fix build Currently supported sources: - AR: مانجا اون لاين - EN: Read Comics Online - EN: Fallen Angels Scans - EN: MangaRoot - EN: Mangawww Reader - EN: MangaForLife - EN: Manga Mofo - EN: H-Manga.moe - EN: MangaBlue - EN: Manga Forest - EN: DManga - ES: My-mangas.com - FA: TrinityReader - FR: Manga-LEL - FR: Manga Etonnia - FR: Tous Vos Scans - ID: Manga Desu - ID: Komik Mangafire.ID - ID: MangaOnline - ID: MangaNesia - ID: KOMIK.CO.ID - ID: MangaID - ID: Indo Manga Reader - JA: IchigoBook - JA: Mangaraw Online - PL: Candy Scans - PT: Comic Space - PT: Mangás Yuri - RU: NAKAMA - TR: MangAoi - TR: MangaHanta * Disable latest updates for sources do not support it * Latest updates support scanner no longer generates false positives * Fix source generator being included in APK Remove sources that went offline Currently supported sources: - AR: مانجا اون لاين - EN: Read Comics Online - EN: Fallen Angels Scans - EN: MangaRoot - EN: Mangawww Reader - EN: MangaForLife - EN: Manga Mofo - EN: H-Manga.moe - EN: MangaBlue - EN: Manga Forest - EN: DManga - ES: My-mangas.com - FA: TrinityReader - FR: Manga-LEL - FR: Manga Etonnia - FR: Tous Vos Scans - ID: Manga Desu - ID: MangaOnline - ID: KOMIK.CO.ID - ID: MangaID - JA: Mangaraw Online - PL: Candy Scans - PT: Mangás Yuri - RU: NAKAMA - TR: MangAoi - TR: MangaHanta * Code cleanup Remove dead sources Fix announcements being recognized as chapters in some sources Currently supported sources: - AR: مانجا اون لاين - EN: Read Comics Online - EN: Fallen Angels Scans - EN: Mangawww Reader - EN: MangaForLife - EN: Manga Mofo - EN: H-Manga.moe - EN: MangaBlue - EN: Manga Forest - EN: DManga - ES: My-mangas.com - FA: TrinityReader - FR: Manga-LEL - FR: Manga Etonnia - FR: Tous Vos Scans - ID: Manga Desu - ID: MangaOnline - ID: KOMIK.CO.ID - ID: MangaID - JA: Mangaraw Online - PL: Candy Scans - PT: Mangás Yuri - RU: NAKAMA - TR: MangAoi - TR: MangaHanta * Remove logging from source (as logging library is not available) * Fix HTML entities not being escaped Add some new sources and remove obsolete sources Currently supported sources: - AR: مانجا اون لاين - EN: Read Comics Online - EN: Fallen Angels Scans - EN: Mangawww Reader - EN: MangaForLife - EN: Manga Spoil - EN: H-Manga.moe - EN: DManga - EN: Chibi Manga Reader - EN: ZXComic - ES: My-mangas.com - FA: TrinityReader - FR: Manga-LEL - FR: Manga Etonnia - ID: Manga Desu - ID: MangaOnline - ID: KOMIK.CO.ID - ID: MangaID - ID: Manga Seru - JA: Mangaraw Online - JA: Mangazuki RAWS - PL: Candy Scans - PT: Mangás Yuri - RU: NAKAMA - TR: MangAoi - TR: MangaHanta - OTHER: HentaiShark * Remove offline sources * Extend HttpSource instead of ParsedHttpSource * Update sources Currently supported sources: - AR: مانجا اون لاين - EN: Read Comics Online - EN: Fallen Angels Scans - EN: MangaForLife - EN: Manga Spoil - EN: DManga - EN: Chibi Manga Reader - EN: ZXComic - EN: DB Manga - EN: Mangacox - EN: GO Manhwa - EN: Hentai2Manga - ES: My-mangas.com - ES: SOS Scanlation - FA: TrinityReader - FR: Manga-LEL - FR: Scan FR - ID: Manga Desu - ID: Komikid - ID: MangaID - ID: Manga Seru - JA: Mangaraw Online - JA: Mangazuki RAWS - JA: MangaRAW - PL: Candy Scans - PT: Mangás Yuri - RU: NAKAMA - RU: AkaiYuhiMun team - TR: MangAoi - TR: MangaHanta - TR: ManhuaTR - OTHER: HentaiShark * Change extension name and remove dead sources Currently supported sources: - AR: مانجا اون لاين - EN: Read Comics Online - EN: Fallen Angels Scans - EN: MangaForLife - EN: Manga Spoil - EN: DManga - EN: Chibi Manga Reader - EN: ZXComic - EN: Mangacox - EN: Hentai2Manga - ES: My-mangas.com - ES: SOS Scanlation - FA: TrinityReader - FR: Manga-LEL - FR: Scan FR - ID: Manga Desu - ID: Komikid - ID: MangaID - ID: Manga Seru - JA: Mangaraw Online - JA: Mangazuki RAWS - JA: MangaRAW - PL: Candy Scans - PT: Mangás Yuri - RU: NAKAMA - TR: MangAoi - TR: MangaHanta - TR: ManhuaTR - OTHER: HentaiShark * Add tag searching support Remove dead sources Enable dead sources that are now online Add some new sources Sources are now parsed from JSON (still hardcoded) Currently supported sources: - AR: مانجا اون لاين - AR: Manga FYI - EN: Read Comics Online - EN: Fallen Angels Scans - EN: Mangawww Reader - EN: MangaForLife - EN: Manga Spoil - EN: DManga - EN: Chibi Manga Reader - EN: ZXComic - EN: Mangacox - EN: KoManga - EN: Manganimecan - EN: Hentai2Manga - EN: White Cloud Pavilion - EN: 4 Manga - EN: XYXX.INFO - ES: My-mangas.com - ES: SOS Scanlation - FR: Manga-LEL - FR: Manga Etonnia - FR: Scan FR - FR: ScanFR.com - FR: Manga FYI - FR: Mugiwara - FR: scans-manga - ID: Manga Desu - ID: MangaOnline - ID: Komikid - ID: MangaID - ID: Manga Seru - ID: Manga FYI - JA: Mangazuki RAWS - JA: MangaRAW - PL: Candy Scans - PL: ToraScans - PT: Comic Space - PT: Mangás Yuri - RU: NAKAMA - TR: MangAoi - TR: MangaHanta - TR: ManhuaTR - VI: Fallen Angels Scans - OTHER: HentaiShark * Update source categories and tags * Add icon Remove dead source --- src/all/mmrcms/build.gradle | 18 + src/all/mmrcms/genSources.sh | 346 ++++++++++++++++++ .../mmrcms/res/mipmap-hdpi/ic_launcher.png | Bin 0 -> 2053 bytes .../mmrcms/res/mipmap-mdpi/ic_launcher.png | Bin 0 -> 1566 bytes .../mmrcms/res/mipmap-xhdpi/ic_launcher.png | Bin 0 -> 3038 bytes .../mmrcms/res/mipmap-xxhdpi/ic_launcher.png | Bin 0 -> 4435 bytes .../mmrcms/res/mipmap-xxxhdpi/ic_launcher.png | Bin 0 -> 6836 bytes src/all/mmrcms/res/web_hi_res_512.png | Bin 0 -> 19117 bytes .../extension/all/mmrcms/GeneratedSources.kt | 47 +++ .../all/mmrcms/MyMangaReaderCMSSource.kt | 286 +++++++++++++++ .../all/mmrcms/MyMangaReaderCMSSources.kt | 93 +++++ 11 files changed, 790 insertions(+) create mode 100644 src/all/mmrcms/build.gradle create mode 100755 src/all/mmrcms/genSources.sh create mode 100644 src/all/mmrcms/res/mipmap-hdpi/ic_launcher.png create mode 100644 src/all/mmrcms/res/mipmap-mdpi/ic_launcher.png create mode 100644 src/all/mmrcms/res/mipmap-xhdpi/ic_launcher.png create mode 100644 src/all/mmrcms/res/mipmap-xxhdpi/ic_launcher.png create mode 100644 src/all/mmrcms/res/mipmap-xxxhdpi/ic_launcher.png create mode 100644 src/all/mmrcms/res/web_hi_res_512.png create mode 100644 src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/GeneratedSources.kt create mode 100644 src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/MyMangaReaderCMSSource.kt create mode 100644 src/all/mmrcms/src/eu/kanade/tachiyomi/extension/all/mmrcms/MyMangaReaderCMSSources.kt diff --git a/src/all/mmrcms/build.gradle b/src/all/mmrcms/build.gradle new file mode 100644 index 000000000..3e6bab3a6 --- /dev/null +++ b/src/all/mmrcms/build.gradle @@ -0,0 +1,18 @@ +apply plugin: 'com.android.application' +apply plugin: 'kotlin-android' + +ext { + appName = 'Tachiyomi: My Manga Reader CMS (Many sources)' + pkgNameSuffix = 'all.mmrcms' + extClass = '.MyMangaReaderCMSSources' + extVersionCode = 1 + extVersionSuffix = 1 + libVersion = '1.2' +} + +dependencies { + provided "com.google.code.gson:gson:2.8.1" + provided "com.github.salomonbrys.kotson:kotson:2.5.0" +} + +apply from: "$rootDir/common.gradle" diff --git a/src/all/mmrcms/genSources.sh b/src/all/mmrcms/genSources.sh new file mode 100755 index 000000000..d6e4179c1 --- /dev/null +++ b/src/all/mmrcms/genSources.sh @@ -0,0 +1,346 @@ +#!/usr/bin/env bash +echo "My Manga Reader CMS source generator by: nulldev" +# CMS: https://getcyberworks.com/product/manga-reader-cms/ + +# Print a message out to stderr +function echoErr() { + echo "ERROR: $@" >&2 +} + +# Require that a command exists before continuing +function require() { + command -v $1 >/dev/null 2>&1 || { echoErr "This script requires $1 but it's not installed."; exit 1; } +} + +# Define commands that this script depends on +require xmllint +require jq +require perl +require wget +require curl +require grep +require sed + +# Show help/usage info +function printHelp() { + echo "Usage: ./genSources.sh [options]" + echo "" + echo "Options:" + echo "--help: Show this help page" + echo "--dry-run: Perform a dry run (make no changes)" + echo "--list: List currently available sources" + echo "--out : Explicitly specify output file" +} +# Target file +TARGET="src/eu/kanade/tachiyomi/extension/all/mmrcms/GeneratedSources.kt" +# String containing processed URLs (used to detect duplicate URLs) +PROCESSED="" + +# Parse CLI args +while [ $# -gt 0 ] +do + case "$1" in + --help) + printHelp + exit 0 + ;; + --dry-run) OPT_DRY_RUN=true + ;; + --list) + OPT_DRY_RUN=true + OPT_LIST=true + ;; + --out) + TARGET="$2" + shift + ;; + --*) + echo "Invalid option $1!" + printHelp + exit -1 + ;; + *) + echo "Invalid argument $1!" + printHelp + exit -1 + ;; + esac + shift +done + +# Change target if performing dry run +if [ "$OPT_DRY_RUN" = true ] ; then + # Do not warn if dry running because of list + if ! [ "$OPT_LIST" = true ] ; then + echo "Performing a dry run, no changes will be made!" + fi + TARGET="/dev/null" +else + # Delete old sources + rm "$TARGET" +fi + +# Variable used to store output while processing +QUEUED_SOURCES="[" + +# lang, name, baseUrl +function gen() { + PROCESSED="$PROCESSED$3\n" + if [ "$OPT_LIST" = true ] ; then + echo "- $(echo "$1" | awk '{print toupper($0)}'): $2" + else + echo "Generating source: $2" + QUEUED_SOURCES="$QUEUED_SOURCES"$'\n'"$(genSource "$1" "$2" "$3")" + # genSource runs in a subprocess, so we check for bad exit code and exit current process if necessary + [ $? -ne 0 ] && exit -1; + fi +} + +# Find and get the item URL from an HTML page +function getItemUrl() { + grep -oP "(?<=showURL = \")(.*)(?=SELECTION)" "$1" +} + +# Strip all scripts and Cloudflare email protection from page +# We strip Cloudflare email protection as titles like 'IDOLM@STER' can trigger it and break the parser +function stripScripts() { + perl -0pe 's/[\s\S]*?< *?\/ *?script *?>//g' |\ + perl -0pe 's/[\s\S]*?< *?\/ *?span *?>/???@???/g' +} + +# Verify that a response is valid +function verifyResponse() { + [ "${1##*$'\n'}" -eq "200" ] && [[ "$1" != *"Whoops, looks like something went wrong"* ]] +} + +# Get the available tags from the manga list page +function parseTagsFromMangaList() { + xmllint --xpath "//div[contains(@class, 'tag-links')]//a" --html "$1" 2>/dev/null |\ + sed 's/<\/a>/"},\n/g; s/">/", "name": "/g;' |\ + perl -pe 's//dev/null |\ + sed 's/<\/a>/"},\n/g; s/" class="category">/", "name": "/g;' |\ + perl -pe 's//dev/null |\ + sed 's/<\/option>/"},\n/g; s/