feat: db changes to accommodate new cross device syncing logic. (#450)

* feat: db changes to accommodate new syncing logic.

Using timestamp to sync is a bit skewed due to system clock etc and therefore there was a lot of issues with it such as removing a manga that shouldn't have been removed. Marking chapters as unread even though it was marked as a read. Hopefully by using versioning system it should eliminate those issues.

* chore: add new line.

* chore: remove isSyncing from Chapter/Manga model.

* chore: remove isSyncing leftover.

* chore: remove isSyncing.

* refactor: remove isSync guard.

Just use it directly to 1 now since we don't have the isSyncing field in Manga or Chapter.

* Lint and stuff

* Add missing ,

---------

Co-authored-by: AntsyLich <59261191+AntsyLich@users.noreply.github.com>
(cherry picked from commit 4ae9dbe52487185ef9ee25f58fabe5025bb2278b)

# Conflicts:
#	app/build.gradle.kts
#	app/src/main/java/eu/kanade/tachiyomi/data/backup/create/creators/MangaBackupCreator.kt
#	app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupChapter.kt
#	app/src/main/java/eu/kanade/tachiyomi/data/backup/models/BackupManga.kt
#	data/src/main/java/tachiyomi/data/chapter/ChapterRepositoryImpl.kt
#	data/src/main/sqldelight/tachiyomi/migrations/2.sqm
#	domain/src/main/java/tachiyomi/domain/manga/model/MangaUpdate.kt
This commit is contained in:
KaiserBh 2024-03-10 06:45:41 +11:00 committed by Jobobby04
parent a4d88515fb
commit cdb07c893b
21 changed files with 200 additions and 56 deletions

View File

@ -26,7 +26,7 @@ android {
defaultConfig {
applicationId = "eu.kanade.tachiyomi.sy"
versionCode = 65
versionCode = 66
versionName = "1.10.5"
buildConfigField("String", "COMMIT_COUNT", "\"${getCommitCount()}\"")

View File

@ -134,6 +134,7 @@ private fun Manga.toBackupManga(/* SY --> */customMangaInfo: CustomMangaInfo?/*
updateStrategy = this.updateStrategy,
lastModifiedAt = this.lastModifiedAt,
favoriteModifiedAt = this.favoriteModifiedAt,
version = this.version,
// SY -->
).also { backupManga ->
customMangaInfo?.let {

View File

@ -4,6 +4,7 @@ import kotlinx.serialization.Serializable
import kotlinx.serialization.protobuf.ProtoNumber
import tachiyomi.domain.chapter.model.Chapter
@Suppress("MagicNumber")
@Serializable
data class BackupChapter(
// in 1.x some of these values have different names
@ -21,6 +22,7 @@ data class BackupChapter(
@ProtoNumber(9) var chapterNumber: Float = 0F,
@ProtoNumber(10) var sourceOrder: Long = 0,
@ProtoNumber(11) var lastModifiedAt: Long = 0,
@ProtoNumber(12) var version: Long = 0,
) {
fun toChapterImpl(): Chapter {
return Chapter.create().copy(
@ -35,36 +37,40 @@ data class BackupChapter(
dateUpload = this@BackupChapter.dateUpload,
sourceOrder = this@BackupChapter.sourceOrder,
lastModifiedAt = this@BackupChapter.lastModifiedAt,
version = this@BackupChapter.version,
)
}
}
val backupChapterMapper =
{ _: Long,
_: Long,
url: String,
name: String,
scanlator: String?,
read: Boolean,
bookmark: Boolean,
lastPageRead: Long,
chapterNumber: Double,
source_order: Long,
dateFetch: Long,
dateUpload: Long,
lastModifiedAt: Long,
->
BackupChapter(
url = url,
name = name,
chapterNumber = chapterNumber.toFloat(),
scanlator = scanlator,
read = read,
bookmark = bookmark,
lastPageRead = lastPageRead,
dateFetch = dateFetch,
dateUpload = dateUpload,
sourceOrder = source_order,
lastModifiedAt = lastModifiedAt,
)
}
val backupChapterMapper = {
_: Long,
_: Long,
url: String,
name: String,
scanlator: String?,
read: Boolean,
bookmark: Boolean,
lastPageRead: Long,
chapterNumber: Double,
sourceOrder: Long,
dateFetch: Long,
dateUpload: Long,
lastModifiedAt: Long,
version: Long,
_: Long,
->
BackupChapter(
url = url,
name = name,
chapterNumber = chapterNumber.toFloat(),
scanlator = scanlator,
read = read,
bookmark = bookmark,
lastPageRead = lastPageRead,
dateFetch = dateFetch,
dateUpload = dateUpload,
sourceOrder = sourceOrder,
lastModifiedAt = lastModifiedAt,
version = version,
)
}

View File

@ -6,7 +6,10 @@ import kotlinx.serialization.protobuf.ProtoNumber
import tachiyomi.domain.manga.model.CustomMangaInfo
import tachiyomi.domain.manga.model.Manga
@Suppress("DEPRECATION")
@Suppress(
"DEPRECATION",
"MagicNumber",
)
@Serializable
data class BackupManga(
// in 1.x some of these values have different names
@ -40,6 +43,7 @@ data class BackupManga(
@ProtoNumber(106) var lastModifiedAt: Long = 0,
@ProtoNumber(107) var favoriteModifiedAt: Long? = null,
@ProtoNumber(108) var excludedScanlators: List<String> = emptyList(),
@ProtoNumber(109) var version: Long = 0,
// SY specific values
@ProtoNumber(600) var mergedMangaReferences: List<BackupMergedMangaReference> = emptyList(),
@ -76,6 +80,7 @@ data class BackupManga(
updateStrategy = this@BackupManga.updateStrategy,
lastModifiedAt = this@BackupManga.lastModifiedAt,
favoriteModifiedAt = this@BackupManga.favoriteModifiedAt,
version = this@BackupManga.version,
)
}
}

View File

@ -105,7 +105,7 @@ class MangaRestorer(
}
private suspend fun restoreExistingManga(manga: Manga, dbManga: Manga): Manga {
return if (manga.lastModifiedAt > dbManga.lastModifiedAt) {
return if (manga.version > dbManga.version) {
updateManga(dbManga.copyFrom(manga).copy(id = dbManga.id))
} else {
updateManga(manga.copyFrom(dbManga).copy(id = dbManga.id))
@ -124,6 +124,7 @@ class MangaRestorer(
ogStatus = newer.status,
// SY <--
initialized = this.initialized || newer.initialized,
version = newer.version,
)
}
@ -150,6 +151,8 @@ class MangaRestorer(
dateAdded = manga.dateAdded,
mangaId = manga.id,
updateStrategy = manga.updateStrategy.let(UpdateStrategyColumnAdapter::encode),
version = manga.version,
isSyncing = 1,
)
}
return manga
@ -161,6 +164,7 @@ class MangaRestorer(
return manga.copy(
initialized = manga.description != null,
id = insertManga(manga),
version = manga.version,
)
}
@ -207,7 +211,7 @@ class MangaRestorer(
}
private fun Chapter.forComparison() =
this.copy(id = 0L, mangaId = 0L, dateFetch = 0L, dateUpload = 0L, lastModifiedAt = 0L)
this.copy(id = 0L, mangaId = 0L, dateFetch = 0L, dateUpload = 0L, lastModifiedAt = 0L, version = 0L)
private suspend fun insertNewChapters(chapters: List<Chapter>) {
handler.await(true) {
@ -224,6 +228,7 @@ class MangaRestorer(
chapter.sourceOrder,
chapter.dateFetch,
chapter.dateUpload,
chapter.version,
)
}
}
@ -245,6 +250,8 @@ class MangaRestorer(
dateFetch = null,
dateUpload = null,
chapterId = chapter.id,
version = chapter.version,
isSyncing = 0,
)
}
}
@ -277,6 +284,7 @@ class MangaRestorer(
coverLastModified = manga.coverLastModified,
dateAdded = manga.dateAdded,
updateStrategy = manga.updateStrategy,
version = manga.version,
)
mangasQueries.selectLastInsertedRowId()
}

View File

@ -21,6 +21,8 @@ interface Chapter : SChapter, Serializable {
var source_order: Int
var last_modified: Long
var version: Long
}
fun Chapter.toDomainChapter(): DomainChapter? {
@ -39,5 +41,6 @@ fun Chapter.toDomainChapter(): DomainChapter? {
chapterNumber = chapter_number.toDouble(),
scanlator = scanlator,
lastModifiedAt = last_modified,
version = version,
)
}

View File

@ -28,6 +28,8 @@ class ChapterImpl : Chapter {
override var last_modified: Long = 0
override var version: Long = 0
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false

View File

@ -254,6 +254,7 @@ class EHentaiUpdateHelper(context: Context) {
scanlator = null,
sourceOrder = -1,
lastModifiedAt = 0,
version = 0,
)
}
}

View File

@ -32,13 +32,15 @@ private val mapper = { cursor: SqlCursor ->
calculate_interval = cursor.getLong(20)!!,
last_modified_at = cursor.getLong(21)!!,
favorite_modified_at = cursor.getLong(22),
totalCount = cursor.getLong(23)!!,
readCount = cursor.getDouble(24)!!,
latestUpload = cursor.getLong(25)!!,
chapterFetchedAt = cursor.getLong(26)!!,
lastRead = cursor.getLong(27)!!,
bookmarkCount = cursor.getDouble(28)!!,
category = cursor.getLong(29)!!,
version = cursor.getLong(23)!!,
is_syncing = cursor.getLong(24)!!,
totalCount = cursor.getLong(25)!!,
readCount = cursor.getDouble(26)!!,
latestUpload = cursor.getLong(27)!!,
chapterFetchedAt = cursor.getLong(28)!!,
lastRead = cursor.getLong(29)!!,
bookmarkCount = cursor.getDouble(30)!!,
category = cursor.getLong(31)!!,
)
}

View File

@ -17,6 +17,9 @@ object ChapterMapper {
dateFetch: Long,
dateUpload: Long,
lastModifiedAt: Long,
version: Long,
@Suppress("UNUSED_PARAMETER")
isSyncing: Long,
): Chapter = Chapter(
id = id,
mangaId = mangaId,
@ -31,5 +34,6 @@ object ChapterMapper {
chapterNumber = chapterNumber,
scanlator = scanlator,
lastModifiedAt = lastModifiedAt,
version = version
)
}

View File

@ -29,6 +29,7 @@ class ChapterRepositoryImpl(
chapter.sourceOrder,
chapter.dateFetch,
chapter.dateUpload,
chapter.version,
)
val lastInsertId = chaptersQueries.selectLastInsertedRowId().executeAsOne()
chapter.copy(id = lastInsertId)
@ -64,6 +65,8 @@ class ChapterRepositoryImpl(
dateFetch = chapterUpdate.dateFetch,
dateUpload = chapterUpdate.dateUpload,
chapterId = chapterUpdate.id,
version = chapterUpdate.version,
isSyncing = 0,
)
}
}

View File

@ -6,6 +6,7 @@ import tachiyomi.domain.manga.model.Manga
import tachiyomi.view.LibraryView
object MangaMapper {
@Suppress("LongParameterList")
fun mapManga(
id: Long,
source: Long,
@ -33,6 +34,9 @@ object MangaMapper {
calculateInterval: Long,
lastModifiedAt: Long,
favoriteModifiedAt: Long?,
version: Long,
@Suppress("UNUSED_PARAMETER")
isSyncing: Long,
): Manga = Manga(
id = id,
source = source,
@ -58,8 +62,10 @@ object MangaMapper {
initialized = initialized,
lastModifiedAt = lastModifiedAt,
favoriteModifiedAt = favoriteModifiedAt,
version = version,
)
@Suppress("LongParameterList")
fun mapLibraryManga(
id: Long,
source: Long,
@ -87,6 +93,8 @@ object MangaMapper {
calculateInterval: Long,
lastModifiedAt: Long,
favoriteModifiedAt: Long?,
version: Long,
isSyncing: Long,
totalCount: Long,
readCount: Double,
latestUpload: Long,
@ -121,6 +129,8 @@ object MangaMapper {
calculateInterval,
lastModifiedAt,
favoriteModifiedAt,
version,
isSyncing,
),
category = category,
totalChapters = totalCount,
@ -156,6 +166,7 @@ object MangaMapper {
fetchInterval = libraryView.calculate_interval.toInt(),
lastModifiedAt = libraryView.last_modified_at,
favoriteModifiedAt = libraryView.favorite_modified_at,
version = libraryView.version,
),
category = libraryView.category,
totalChapters = libraryView.totalCount,

View File

@ -119,6 +119,7 @@ class MangaRepositoryImpl(
coverLastModified = manga.coverLastModified,
dateAdded = manga.dateAdded,
updateStrategy = manga.updateStrategy,
version = manga.version,
)
mangasQueries.selectLastInsertedRowId()
}
@ -168,6 +169,8 @@ class MangaRepositoryImpl(
dateAdded = value.dateAdded,
mangaId = value.id,
updateStrategy = value.updateStrategy?.let(UpdateStrategyColumnAdapter::encode),
version = value.version,
isSyncing = 0,
)
}
}

View File

@ -14,6 +14,8 @@ CREATE TABLE chapters(
date_fetch INTEGER NOT NULL,
date_upload INTEGER NOT NULL,
last_modified_at INTEGER NOT NULL DEFAULT 0,
version INTEGER NOT NULL DEFAULT 0,
is_syncing INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY(manga_id) REFERENCES mangas (_id)
ON DELETE CASCADE
);
@ -30,6 +32,22 @@ BEGIN
WHERE _id = new._id;
END;
CREATE TRIGGER update_chapter_and_manga_version AFTER UPDATE ON chapters
WHEN new.is_syncing = 0 AND (
new.read != old.read OR
new.bookmark != old.bookmark OR
new.last_page_read != old.last_page_read
)
BEGIN
-- Update the chapter version
UPDATE chapters SET version = version + 1
WHERE _id = new._id;
-- Update the manga version
UPDATE mangas SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;
getChapterById:
SELECT *
FROM chapters
@ -97,9 +115,14 @@ removeChaptersWithIds:
DELETE FROM chapters
WHERE _id IN :chapterIds;
resetIsSyncing:
UPDATE chapters
SET is_syncing = 0
WHERE is_syncing = 1;
insert:
INSERT INTO chapters(manga_id, url, name, scanlator, read, bookmark, last_page_read, chapter_number, source_order, date_fetch, date_upload, last_modified_at)
VALUES (:mangaId, :url, :name, :scanlator, :read, :bookmark, :lastPageRead, :chapterNumber, :sourceOrder, :dateFetch, :dateUpload, 0);
INSERT INTO chapters(manga_id, url, name, scanlator, read, bookmark, last_page_read, chapter_number, source_order, date_fetch, date_upload, last_modified_at, version, is_syncing)
VALUES (:mangaId, :url, :name, :scanlator, :read, :bookmark, :lastPageRead, :chapterNumber, :sourceOrder, :dateFetch, :dateUpload, 0, :version, 0);
update:
UPDATE chapters
@ -113,7 +136,9 @@ SET manga_id = coalesce(:mangaId, manga_id),
chapter_number = coalesce(:chapterNumber, chapter_number),
source_order = coalesce(:sourceOrder, source_order),
date_fetch = coalesce(:dateFetch, date_fetch),
date_upload = coalesce(:dateUpload, date_upload)
date_upload = coalesce(:dateUpload, date_upload),
version = coalesce(:version, version),
is_syncing = coalesce(:isSyncing, is_syncing)
WHERE _id = :chapterId;
selectLastInsertedRowId:

View File

@ -26,7 +26,9 @@ CREATE TABLE mangas(
update_strategy INTEGER AS UpdateStrategy NOT NULL DEFAULT 0,
calculate_interval INTEGER DEFAULT 0 NOT NULL,
last_modified_at INTEGER NOT NULL DEFAULT 0,
favorite_modified_at INTEGER
favorite_modified_at INTEGER,
version INTEGER NOT NULL DEFAULT 0,
is_syncing INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX library_favorite_index ON mangas(favorite) WHERE favorite = 1;
@ -49,6 +51,16 @@ BEGIN
WHERE _id = new._id;
END;
CREATE TRIGGER update_manga_version AFTER UPDATE ON mangas
BEGIN
UPDATE mangas SET version = version + 1
WHERE _id = new._id AND new.is_syncing = 0 AND (
new.url != old.url OR
new.description != old.description OR
new.favorite != old.favorite
);
END;
getMangaById:
SELECT *
FROM mangas
@ -112,6 +124,11 @@ resetViewerFlags:
UPDATE mangas
SET viewer = 0;
resetIsSyncing:
UPDATE mangas
SET is_syncing = 0
WHERE is_syncing = 1;
getSourceIdsWithNonLibraryManga:
SELECT source, COUNT(*) AS manga_count
FROM mangas
@ -135,8 +152,8 @@ WHERE favorite = 0 AND source IN :sourceIdsAND AND _id NOT IN (
);
insert:
INSERT INTO mangas(source, url, artist, author, description, genre, title, status, thumbnail_url, favorite, last_update, next_update, initialized, viewer, chapter_flags, cover_last_modified, date_added, update_strategy, calculate_interval, last_modified_at)
VALUES (:source, :url, :artist, :author, :description, :genre, :title, :status, :thumbnailUrl, :favorite, :lastUpdate, :nextUpdate, :initialized, :viewerFlags, :chapterFlags, :coverLastModified, :dateAdded, :updateStrategy, :calculateInterval, 0);
INSERT INTO mangas(source, url, artist, author, description, genre, title, status, thumbnail_url, favorite, last_update, next_update, initialized, viewer, chapter_flags, cover_last_modified, date_added, update_strategy, calculate_interval, last_modified_at, version)
VALUES (:source, :url, :artist, :author, :description, :genre, :title, :status, :thumbnailUrl, :favorite, :lastUpdate, :nextUpdate, :initialized, :viewerFlags, :chapterFlags, :coverLastModified, :dateAdded, :updateStrategy, :calculateInterval, 0, :version);
update:
UPDATE mangas SET
@ -158,7 +175,9 @@ UPDATE mangas SET
cover_last_modified = coalesce(:coverLastModified, cover_last_modified),
date_added = coalesce(:dateAdded, date_added),
update_strategy = coalesce(:updateStrategy, update_strategy),
calculate_interval = coalesce(:calculateInterval, calculate_interval)
calculate_interval = coalesce(:calculateInterval, calculate_interval),
version = coalesce(:version, version),
is_syncing = coalesce(:isSyncing, is_syncing)
WHERE _id = :mangaId;
selectLastInsertedRowId:

View File

@ -2,25 +2,22 @@ CREATE TABLE mangas_categories(
_id INTEGER NOT NULL PRIMARY KEY,
manga_id INTEGER NOT NULL,
category_id INTEGER NOT NULL,
last_modified_at INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY(category_id) REFERENCES categories (_id)
ON DELETE CASCADE,
FOREIGN KEY(manga_id) REFERENCES mangas (_id)
ON DELETE CASCADE
);
CREATE TRIGGER update_last_modified_at_mangas_categories
AFTER UPDATE ON mangas_categories
FOR EACH ROW
CREATE TRIGGER insert_manga_category_update_version AFTER INSERT ON mangas_categories
BEGIN
UPDATE mangas_categories
SET last_modified_at = strftime('%s', 'now')
WHERE _id = new._id;
UPDATE mangas
SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;
insert:
INSERT INTO mangas_categories(manga_id, category_id, last_modified_at)
VALUES (:mangaId, :categoryId, 0);
INSERT INTO mangas_categories(manga_id, category_id)
VALUES (:mangaId, :categoryId);
deleteMangaCategoryByMangaId:
DELETE FROM mangas_categories

View File

@ -0,0 +1,46 @@
-- Mangas table
ALTER TABLE mangas ADD COLUMN version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE mangas ADD COLUMN is_syncing INTEGER NOT NULL DEFAULT 0;
-- Chapters table
ALTER TABLE chapters ADD COLUMN version INTEGER NOT NULL DEFAULT 0;
ALTER TABLE chapters ADD COLUMN is_syncing INTEGER NOT NULL DEFAULT 0;
-- Mangas triggers
DROP TRIGGER IF EXISTS update_manga_version;
CREATE TRIGGER update_manga_version AFTER UPDATE ON mangas
BEGIN
UPDATE mangas SET version = version + 1
WHERE _id = new._id AND new.is_syncing = 0 AND (
new.url != old.url OR
new.description != old.description OR
new.favorite != old.favorite
);
END;
-- Chapters triggers
DROP TRIGGER IF EXISTS update_chapter_and_manga_version;
CREATE TRIGGER update_chapter_and_manga_version AFTER UPDATE ON chapters
WHEN new.is_syncing = 0 AND (
new.read != old.read OR
new.bookmark != old.bookmark OR
new.last_page_read != old.last_page_read
)
BEGIN
-- Update the chapter version
UPDATE chapters SET version = version + 1
WHERE _id = new._id;
-- Update the manga version
UPDATE mangas SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;
-- manga_categories table
DROP TRIGGER IF EXISTS insert_manga_category_update_version;
CREATE TRIGGER insert_manga_category_update_version AFTER INSERT ON mangas_categories
BEGIN
UPDATE mangas
SET version = version + 1
WHERE _id = new.manga_id AND (SELECT is_syncing FROM mangas WHERE _id = new.manga_id) = 0;
END;

View File

@ -14,6 +14,7 @@ data class Chapter(
val chapterNumber: Double,
val scanlator: String?,
val lastModifiedAt: Long,
val version: Long,
) {
val isRecognizedNumber: Boolean
get() = chapterNumber >= 0f
@ -43,6 +44,7 @@ data class Chapter(
chapterNumber = -1.0,
scanlator = null,
lastModifiedAt = 0,
version = 1,
)
}
}

View File

@ -13,6 +13,7 @@ data class ChapterUpdate(
val dateUpload: Long? = null,
val chapterNumber: Double? = null,
val scanlator: String? = null,
val version: Long? = null,
)
fun Chapter.toChapterUpdate(): ChapterUpdate {
@ -29,5 +30,6 @@ fun Chapter.toChapterUpdate(): ChapterUpdate {
dateUpload,
chapterNumber,
scanlator,
version,
)
}

View File

@ -33,6 +33,7 @@ data class Manga(
val initialized: Boolean,
val lastModifiedAt: Long,
val favoriteModifiedAt: Long?,
val version: Long,
) : Serializable {
// SY -->
@ -159,6 +160,7 @@ data class Manga(
initialized = false,
lastModifiedAt = 0L,
favoriteModifiedAt = null,
version = 0L,
)
// SY -->

View File

@ -23,6 +23,7 @@ data class MangaUpdate(
val thumbnailUrl: String? = null,
val updateStrategy: UpdateStrategy? = null,
val initialized: Boolean? = null,
val version: Long? = null,
// SY -->
val filteredScanlators: List<String>? = null,
// SY <--
@ -52,5 +53,6 @@ fun Manga.toMangaUpdate(): MangaUpdate {
// SY <--
updateStrategy = updateStrategy,
initialized = initialized,
version = version,
)
}