chore: improve google drive sync. (#1200)
improve google drive sync, removes the lock, change to protobuf, and potentially fix deviceId not being unique, since it wasn't appState...
This commit is contained in:
KaiserBh
GitHub
parent
d29a4ff381
commit
c2eece0fff
app/src/main/java/eu/kanade
@ -29,7 +29,7 @@ class SyncPreferences(
|
||||
)
|
||||
|
||||
fun uniqueDeviceID(): String {
|
||||
val uniqueIDPreference = preferenceStore.getString("unique_device_id", "")
|
||||
val uniqueIDPreference = preferenceStore.getString(Preference.appStateKey("unique_device_id"), "")
|
||||
|
||||
// Retrieve the current value of the preference
|
||||
var uniqueID = uniqueIDPreference.get()
|
||||
|
||||
@ -10,7 +10,6 @@ import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeToken
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
|
||||
import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse
|
||||
import com.google.api.client.http.ByteArrayContent
|
||||
import com.google.api.client.http.InputStreamContent
|
||||
import com.google.api.client.http.javanet.NetHttpTransport
|
||||
import com.google.api.client.json.JsonFactory
|
||||
@ -20,11 +19,10 @@ import com.google.api.services.drive.DriveScopes
|
||||
import com.google.api.services.drive.model.File
|
||||
import eu.kanade.domain.sync.SyncPreferences
|
||||
import eu.kanade.tachiyomi.data.backup.models.Backup
|
||||
import kotlinx.coroutines.delay
|
||||
import eu.kanade.tachiyomi.data.backup.models.BackupSerializer
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.decodeFromStream
|
||||
import kotlinx.serialization.json.encodeToStream
|
||||
import kotlinx.serialization.protobuf.ProtoBuf
|
||||
import logcat.LogPriority
|
||||
import logcat.logcat
|
||||
import tachiyomi.core.common.i18n.stringResource
|
||||
@ -37,7 +35,6 @@ import uy.kohesive.injekt.api.get
|
||||
import java.io.IOException
|
||||
import java.io.PipedInputStream
|
||||
import java.io.PipedOutputStream
|
||||
import java.time.Instant
|
||||
import java.util.zip.GZIPInputStream
|
||||
import java.util.zip.GZIPOutputStream
|
||||
|
||||
@ -64,12 +61,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
|
||||
|
||||
private val appName = context.stringResource(MR.strings.app_name)
|
||||
|
||||
private val remoteFileName = "${appName}_sync_data.gz"
|
||||
|
||||
private val lockFileName = "${appName}_sync.lock"
|
||||
private val remoteFileName = "${appName}_sync.proto.gz"
|
||||
|
||||
private val googleDriveService = GoogleDriveService(context)
|
||||
|
||||
private val protoBuf: ProtoBuf = Injekt.get()
|
||||
|
||||
override suspend fun doSync(syncData: SyncData): Backup? {
|
||||
beforeSync()
|
||||
|
||||
@ -107,64 +104,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
|
||||
}
|
||||
|
||||
private suspend fun beforeSync() {
|
||||
try {
|
||||
googleDriveService.refreshToken()
|
||||
val drive = googleDriveService.driveService
|
||||
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
|
||||
|
||||
var backoff = 1000L
|
||||
var retries = 0 // Retry counter
|
||||
val maxRetries = 10 // Maximum number of retries
|
||||
|
||||
while (retries < maxRetries) {
|
||||
val lockFiles = findLockFile(drive)
|
||||
logcat(LogPriority.DEBUG) { "Found ${lockFiles.size} lock file(s)" }
|
||||
|
||||
when {
|
||||
lockFiles.isEmpty() -> {
|
||||
logcat(LogPriority.DEBUG) { "No lock file found, creating a new one" }
|
||||
createLockFile(drive)
|
||||
break
|
||||
}
|
||||
lockFiles.size == 1 -> {
|
||||
val lockFile = lockFiles.first()
|
||||
val createdTime = Instant.parse(lockFile.createdTime.toString())
|
||||
val ageMinutes = java.time.Duration.between(createdTime, Instant.now()).toMinutes()
|
||||
logcat(LogPriority.DEBUG) { "Lock file age: $ageMinutes minutes" }
|
||||
if (ageMinutes <= 3) {
|
||||
logcat(LogPriority.DEBUG) { "Lock file is new, proceeding with sync" }
|
||||
break
|
||||
} else {
|
||||
logcat(LogPriority.DEBUG) { "Lock file is old, deleting and creating a new one" }
|
||||
deleteLockFile(drive)
|
||||
createLockFile(drive)
|
||||
break
|
||||
}
|
||||
}
|
||||
else -> {
|
||||
logcat(LogPriority.DEBUG) { "Multiple lock files found, applying backoff" }
|
||||
delay(backoff) // Apply backoff strategy
|
||||
backoff = (backoff * 2).coerceAtMost(16000L)
|
||||
logcat(LogPriority.DEBUG) { "Backoff increased to $backoff milliseconds" }
|
||||
}
|
||||
}
|
||||
retries++ // Increment retry counter
|
||||
logcat(LogPriority.DEBUG) { "Loop iteration complete, retry count: $retries, backoff time: $backoff" }
|
||||
}
|
||||
|
||||
if (retries >= maxRetries) {
|
||||
logcat(LogPriority.ERROR) { "Max retries reached, exiting sync process" }
|
||||
throw Exception(context.stringResource(SYMR.strings.error_before_sync_gdrive) + ": Max retries reached.")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
logcat(LogPriority.ERROR, throwable = e) { "Error in GoogleDrive beforeSync" }
|
||||
throw Exception(context.stringResource(SYMR.strings.error_before_sync_gdrive) + ": ${e.message}", e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun pullSyncData(): SyncData? {
|
||||
val drive = googleDriveService.driveService ?:
|
||||
throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
|
||||
val drive = googleDriveService.driveService
|
||||
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
|
||||
|
||||
val fileList = getAppDataFileList(drive)
|
||||
if (fileList.isEmpty()) {
|
||||
@ -178,7 +123,10 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
|
||||
try {
|
||||
drive.files().get(gdriveFileId).executeMediaAsInputStream().use { inputStream ->
|
||||
GZIPInputStream(inputStream).use { gzipInputStream ->
|
||||
return Json.decodeFromStream(SyncData.serializer(), gzipInputStream)
|
||||
val byteArray = gzipInputStream.readBytes()
|
||||
val backup = protoBuf.decodeFromByteArray(BackupSerializer, byteArray)
|
||||
val deviceId = fileList[0].appProperties["deviceId"] ?: ""
|
||||
return SyncData(deviceId = deviceId, backup = backup)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
@ -192,29 +140,40 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
|
||||
?: throw Exception(context.stringResource(SYMR.strings.google_drive_not_signed_in))
|
||||
|
||||
val fileList = getAppDataFileList(drive)
|
||||
val backup = syncData.backup ?: return
|
||||
|
||||
val byteArray = protoBuf.encodeToByteArray(BackupSerializer, backup)
|
||||
if (byteArray.isEmpty()) {
|
||||
throw IllegalStateException(context.stringResource(MR.strings.empty_backup_error))
|
||||
}
|
||||
|
||||
PipedOutputStream().use { pos ->
|
||||
PipedInputStream(pos).use { pis ->
|
||||
withIOContext {
|
||||
// Start a coroutine or a background thread to write JSON to the PipedOutputStream
|
||||
launch {
|
||||
GZIPOutputStream(pos).use { gzipOutputStream ->
|
||||
Json.encodeToStream(SyncData.serializer(), syncData, gzipOutputStream)
|
||||
gzipOutputStream.write(byteArray)
|
||||
}
|
||||
}
|
||||
|
||||
val mediaContent = InputStreamContent("application/octet-stream", pis)
|
||||
|
||||
if (fileList.isNotEmpty()) {
|
||||
val fileId = fileList[0].id
|
||||
val mediaContent = InputStreamContent("application/gzip", pis)
|
||||
drive.files().update(fileId, null, mediaContent).execute()
|
||||
val fileMetadata = File().apply {
|
||||
name = remoteFileName
|
||||
mimeType = "application/octet-stream"
|
||||
appProperties = mapOf("deviceId" to syncData.deviceId)
|
||||
}
|
||||
drive.files().update(fileId, fileMetadata, mediaContent).execute()
|
||||
logcat(LogPriority.DEBUG) { "Updated existing sync data file in Google Drive with file ID: $fileId" }
|
||||
} else {
|
||||
val fileMetadata = File().apply {
|
||||
name = remoteFileName
|
||||
mimeType = "application/gzip"
|
||||
mimeType = "application/octet-stream"
|
||||
parents = listOf("appDataFolder")
|
||||
appProperties = mapOf("deviceId" to syncData.deviceId)
|
||||
}
|
||||
val mediaContent = InputStreamContent("application/gzip", pis)
|
||||
val uploadedFile = drive.files().create(fileMetadata, mediaContent)
|
||||
.setFields("id")
|
||||
.execute()
|
||||
@ -228,12 +187,12 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
|
||||
private fun getAppDataFileList(drive: Drive): MutableList<File> {
|
||||
try {
|
||||
// Search for the existing file by name in the appData folder
|
||||
val query = "mimeType='application/gzip' and name = '$remoteFileName'"
|
||||
val query = "mimeType='application/x-gzip' and name = '$remoteFileName'"
|
||||
val fileList = drive.files()
|
||||
.list()
|
||||
.setSpaces("appDataFolder")
|
||||
.setQ(query)
|
||||
.setFields("files(id, name, createdTime)")
|
||||
.setFields("files(id, name, createdTime, appProperties)")
|
||||
.execute()
|
||||
.files
|
||||
logcat { "AppData folder file list: $fileList" }
|
||||
@ -245,62 +204,6 @@ class GoogleDriveSyncService(context: Context, json: Json, syncPreferences: Sync
|
||||
}
|
||||
}
|
||||
|
||||
private fun createLockFile(drive: Drive) {
|
||||
try {
|
||||
val fileMetadata = File().apply {
|
||||
name = lockFileName
|
||||
mimeType = "text/plain"
|
||||
parents = listOf("appDataFolder")
|
||||
}
|
||||
|
||||
// Create an empty content to upload as the lock file
|
||||
val emptyContent = ByteArrayContent.fromString("text/plain", "")
|
||||
|
||||
val file = drive.files().create(fileMetadata, emptyContent)
|
||||
.setFields("id, name, createdTime")
|
||||
.execute()
|
||||
|
||||
logcat { "Created lock file with ID: ${file.id}" }
|
||||
} catch (e: Exception) {
|
||||
logcat(LogPriority.ERROR, throwable = e) { "Error creating lock file" }
|
||||
throw Exception(e.message, e)
|
||||
}
|
||||
}
|
||||
|
||||
private fun findLockFile(drive: Drive): MutableList<File> {
|
||||
return try {
|
||||
val query = "mimeType='text/plain' and name = '$lockFileName'"
|
||||
val fileList = drive.files()
|
||||
.list()
|
||||
.setSpaces("appDataFolder")
|
||||
.setQ(query)
|
||||
.setFields("files(id, name, createdTime)")
|
||||
.execute().files
|
||||
logcat { "Lock file search result: $fileList" }
|
||||
fileList
|
||||
} catch (e: Exception) {
|
||||
logcat(LogPriority.ERROR, throwable = e) { "Error finding lock file" }
|
||||
mutableListOf()
|
||||
}
|
||||
}
|
||||
|
||||
private fun deleteLockFile(drive: Drive) {
|
||||
try {
|
||||
val lockFiles = findLockFile(drive)
|
||||
|
||||
if (lockFiles.isNotEmpty()) {
|
||||
for (file in lockFiles) {
|
||||
drive.files().delete(file.id).execute()
|
||||
logcat { "Deleted lock file with ID: ${file.id}" }
|
||||
}
|
||||
} else {
|
||||
logcat { "No lock file found to delete." }
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
logcat(LogPriority.ERROR, throwable = e) { "Error deleting lock file" }
|
||||
throw Exception(context.stringResource(SYMR.strings.error_deleting_google_drive_lock_file), e)
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun deleteSyncDataFromGoogleDrive(): DeleteSyncDataStatus {
|
||||
val drive = googleDriveService.driveService
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user