nullptr/Wallenc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sync): добавил механизм синхронизации хранилищ и управление группами

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Роман Пытков
2026-05-12 23:46:31 +03:00
parent d6bfdff077
commit f38b3dfbb4
27 changed files with 1819 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

118
domain-vault/src/main/java/com/github/nullptroma/wallenc/infrastructure/storages/encrypt/EncryptedStorageAccessor.kt
View File

@@ -1,5 +1,6 @@
package com.github.nullptroma.wallenc.infrastructure.storages.encrypt
import com.github.nullptroma.wallenc.infrastructure.utils.CloseHandledStreamExtension.Companion.onClosed
import com.github.nullptroma.wallenc.infrastructure.utils.CloseHandledStreamExtension.Companion.onClosing
import com.github.nullptroma.wallenc.domain.common.impl.CommonDirectory
import com.github.nullptroma.wallenc.domain.common.impl.CommonFile
@@ -12,6 +13,10 @@ import com.github.nullptroma.wallenc.domain.interfaces.IDirectory
import com.github.nullptroma.wallenc.domain.interfaces.IFile
import com.github.nullptroma.wallenc.domain.interfaces.IMetaInfo
import com.github.nullptroma.wallenc.domain.interfaces.IStorageAccessor
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncLock
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.DisposableHandle
import kotlinx.coroutines.flow.Flow
@@ -21,8 +26,12 @@ import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import java.io.InputStream
import java.io.OutputStream
import java.time.Instant
import java.util.UUID
import kotlin.io.path.Path
import kotlin.io.path.pathString
@@ -33,6 +42,7 @@ class EncryptedStorageAccessor(
private val systemHiddenDirName: String,
private val scope: CoroutineScope
) : IStorageAccessor, DisposableHandle {
private val syncActorId = UUID.randomUUID().toString()
private val _size = MutableStateFlow<Long?>(null)
override val size: StateFlow<Long?> = _size
@@ -50,6 +60,7 @@ class EncryptedStorageAccessor(
private val dataEncryptor = Encryptor(key.toAesKey())
private val pathEncryptor: EncryptorWithStaticIv? = if(pathIv != null) EncryptorWithStaticIv(key.toAesKey(), pathIv) else null
private val syncLockMutex = Mutex()
private var systemHiddenFilesIsActual = false
init {
@@ -239,6 +250,7 @@ class EncryptedStorageAccessor(
override suspend fun touchFile(path: String) {
source.touchFile(encryptPath(path))
appendSyncEntry(path = path, operation = StorageSyncOperation.UPSERT)
}
override suspend fun touchDir(path: String) {
@@ -247,11 +259,16 @@ class EncryptedStorageAccessor(
override suspend fun delete(path: String) {
source.delete(encryptPath(path))
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
}
override suspend fun openWrite(path: String): OutputStream {
val stream = source.openWrite(encryptPath(path))
return dataEncryptor.encryptStream(stream)
return dataEncryptor.encryptStream(stream).onClosed {
scope.launch {
appendSyncEntry(path = path, operation = StorageSyncOperation.UPSERT)
}
}
}
override suspend fun openRead(path: String): InputStream {
@@ -261,6 +278,7 @@ class EncryptedStorageAccessor(
override suspend fun moveToTrash(path: String) {
source.moveToTrash(encryptPath(path))
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
}
override fun dispose() {
@@ -280,6 +298,98 @@ class EncryptedStorageAccessor(
}
}
override suspend fun readSyncJournal(): List<StorageSyncJournalEntry> {
val bytes = openReadSystemFile(SYNC_JOURNAL_FILENAME).use { it.readBytes() }
if (bytes.isEmpty()) {
return emptyList()
}
return runCatching {
val javaType = jackson.typeFactory.constructCollectionType(
List::class.java,
StorageSyncJournalEntry::class.java,
)
@Suppress("UNCHECKED_CAST")
(jackson.readValue(bytes, javaType) as List<StorageSyncJournalEntry>)
}.getOrElse {
emptyList()
}
}
override suspend fun appendSyncJournal(entries: List<StorageSyncJournalEntry>) {
if (entries.isEmpty()) {
return
}
val next = readSyncJournal().toMutableList().apply { addAll(entries) }
openWriteSystemFile(SYNC_JOURNAL_FILENAME).use { out ->
jackson.writeValue(out, next)
}
}
override suspend fun rewriteSyncJournal(entries: List<StorageSyncJournalEntry>) {
openWriteSystemFile(SYNC_JOURNAL_FILENAME).use { out ->
jackson.writeValue(out, entries)
}
}
override suspend fun readSyncLock(): StorageSyncLock? {
val bytes = openReadSystemFile(SYNC_LOCK_FILENAME).use { it.readBytes() }
if (bytes.isEmpty()) {
return null
}
return runCatching {
jackson.readValue(bytes, StorageSyncLock::class.java)
}.getOrNull()
}
override suspend fun tryAcquireSyncLock(holderId: String, leaseUntil: Instant): Boolean {
return syncLockMutex.withLock {
val current = readSyncLock()
if (current != null && current.holderId != holderId) {
return@withLock false
}
val next = StorageSyncLock(
holderId = holderId,
leaseUntil = leaseUntil,
updatedAt = Instant.now(),
)
openWriteSystemFile(SYNC_LOCK_FILENAME).use { out ->
jackson.writeValue(out, next)
}
readSyncLock()?.holderId == holderId
}
}
override suspend fun releaseSyncLock(holderId: String) {
syncLockMutex.withLock {
val current = readSyncLock() ?: return@withLock
if (current.holderId != holderId) {
return@withLock
}
openWriteSystemFile(SYNC_LOCK_FILENAME).use { out ->
out.write(ByteArray(0))
}
}
}
private suspend fun appendSyncEntry(path: String, operation: StorageSyncOperation) {
val cleanedPath = if (path.startsWith("/")) path else "/$path"
val entries = readSyncJournal()
val nextSequence = (entries.maxOfOrNull { it.revision.sequence } ?: 0L) + 1L
appendSyncJournal(
listOf(
StorageSyncJournalEntry(
path = cleanedPath,
operation = operation,
revision = StorageSyncRevision(
sequence = nextSequence,
actorId = syncActorId,
createdAt = Instant.now(),
),
),
),
)
}
private fun Iterable<IFile>.filterSystemHiddenFiles(): List<IFile> {
return this.filter { file ->
!file.metaInfo.path.contains(
@@ -296,4 +406,10 @@ class EncryptedStorageAccessor(
}
}
private companion object {
private const val SYNC_JOURNAL_FILENAME = "sync-journal.json"
private const val SYNC_LOCK_FILENAME = "sync-lock.json"
private val jackson = com.fasterxml.jackson.module.kotlin.jacksonObjectMapper()
.apply { findAndRegisterModules() }
}
}

166
domain-vault/src/main/java/com/github/nullptroma/wallenc/infrastructure/storages/local/LocalStorageAccessor.kt
View File

@@ -12,6 +12,10 @@ import com.github.nullptroma.wallenc.domain.interfaces.IDirectory
import com.github.nullptroma.wallenc.domain.interfaces.IFile
import com.github.nullptroma.wallenc.domain.interfaces.IMetaInfo
import com.github.nullptroma.wallenc.domain.interfaces.IStorageAccessor
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncLock
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.flow.Flow
@@ -26,9 +30,14 @@ import kotlinx.coroutines.withContext
import java.io.File
import java.io.InputStream
import java.io.OutputStream
import java.nio.ByteBuffer
import java.nio.channels.FileChannel
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardOpenOption
import java.time.Clock
import java.time.Instant
import java.util.UUID
import kotlin.io.path.Path
import kotlin.io.path.absolute
import kotlin.io.path.fileSize
@@ -39,6 +48,7 @@ class LocalStorageAccessor(
filesystemBasePath: String,
private val ioDispatcher: CoroutineDispatcher
) : IStorageAccessor {
private val syncActorId = UUID.randomUUID().toString()
private val _filesystemBasePath: Path = Path(filesystemBasePath).normalize().absolute()
private val _size = MutableStateFlow<Long?>(null)
@@ -478,8 +488,16 @@ class LocalStorageAccessor(
}
override suspend fun touchFile(path: String): Unit = withContext(ioDispatcher) {
touchFileInternal(path, recordJournal = true)
}
private suspend fun touchFileInternal(path: String, recordJournal: Boolean) {
createFile(path)
if (recordJournal) {
appendSyncEntry(path = path, operation = StorageSyncOperation.UPSERT)
}
// перебор все каталогов и обновление их времени модификации
var parent = Path(path).parent
while(parent != null) {
@@ -502,17 +520,19 @@ class LocalStorageAccessor(
else pair.file.delete()
pair.metaFile.delete()
scanSizeAndNumOfFiles()
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
}
}
override suspend fun openWrite(path: String): OutputStream = withContext(ioDispatcher) {
touchFile(path)
touchFileInternal(path, recordJournal = false)
val pair = LocalStorageFilePair.from(_filesystemBasePath, path)
?: throw Exception("Файла нет") // TODO
return@withContext pair.file.outputStream().onClosed {
CoroutineScope(ioDispatcher).launch {
touchFile(path)
touchFileInternal(path, recordJournal = false)
scanSizeAndNumOfFiles()
appendSyncEntry(path = path, operation = StorageSyncOperation.UPSERT)
}
}
}
@@ -528,6 +548,7 @@ class LocalStorageAccessor(
?: throw Exception("Файла нет") // TODO
val newMeta = pair.meta.copy(isDeleted = true)
writeMeta(pair.metaFile, newMeta)
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
}
override suspend fun openReadSystemFile(name: String): InputStream = withContext(ioDispatcher) {
@@ -554,11 +575,152 @@ class LocalStorageAccessor(
return@withContext file.outputStream()
}
override suspend fun readSyncJournal(): List<StorageSyncJournalEntry> = withContext(ioDispatcher) {
val bytes = openReadSystemFile(SYNC_JOURNAL_FILENAME).use { it.readBytes() }
if (bytes.isEmpty()) {
return@withContext emptyList()
}
return@withContext runCatching {
jackson.readValue<List<StorageSyncJournalEntry>>(bytes)
}.getOrElse {
emptyList()
}
}
override suspend fun appendSyncJournal(entries: List<StorageSyncJournalEntry>) = withContext(ioDispatcher) {
if (entries.isEmpty()) {
return@withContext
}
val next = readSyncJournal().toMutableList().apply {
addAll(entries)
}
openWriteSystemFile(SYNC_JOURNAL_FILENAME).use { out ->
jackson.writeValue(out, next)
}
}
override suspend fun rewriteSyncJournal(entries: List<StorageSyncJournalEntry>) = withContext(ioDispatcher) {
openWriteSystemFile(SYNC_JOURNAL_FILENAME).use { out ->
jackson.writeValue(out, entries)
}
}
override suspend fun readSyncLock(): StorageSyncLock? = withContext(ioDispatcher) {
val bytes = openReadSystemFile(SYNC_LOCK_FILENAME).use { it.readBytes() }
if (bytes.isEmpty()) {
return@withContext null
}
return@withContext runCatching {
jackson.readValue<StorageSyncLock>(bytes)
}.getOrNull()
}
override suspend fun tryAcquireSyncLock(holderId: String, leaseUntil: Instant): Boolean = withContext(ioDispatcher) {
val lockPath = systemLockPath()
Files.createDirectories(lockPath.parent)
FileChannel.open(
lockPath,
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.WRITE,
).use { channel ->
val fileLock = channel.lock()
try {
val current = readSyncLockFromChannel(channel)
if (current != null && current.holderId != holderId) {
return@withContext false
}
writeSyncLockToChannel(
channel = channel,
lock = StorageSyncLock(
holderId = holderId,
leaseUntil = leaseUntil,
updatedAt = Instant.now(),
),
)
return@withContext true
} finally {
fileLock.release()
}
}
}
override suspend fun releaseSyncLock(holderId: String) = withContext(ioDispatcher) {
val lockPath = systemLockPath()
Files.createDirectories(lockPath.parent)
FileChannel.open(
lockPath,
StandardOpenOption.CREATE,
StandardOpenOption.READ,
StandardOpenOption.WRITE,
).use { channel ->
val fileLock = channel.lock()
try {
val current = readSyncLockFromChannel(channel) ?: return@withContext
if (current.holderId != holderId) {
return@withContext
}
channel.truncate(0)
channel.force(true)
} finally {
fileLock.release()
}
}
}
private fun systemLockPath(): Path =
_filesystemBasePath.resolve(SYSTEM_HIDDEN_DIRNAME).resolve(SYNC_LOCK_FILENAME)
private fun readSyncLockFromChannel(channel: FileChannel): StorageSyncLock? {
channel.position(0)
val size = channel.size().toInt()
if (size <= 0) {
return null
}
val buffer = ByteBuffer.allocate(size)
while (buffer.hasRemaining()) {
val read = channel.read(buffer)
if (read <= 0) break
}
val bytes = buffer.array().copyOf(buffer.position())
if (bytes.isEmpty()) {
return null
}
return runCatching { jackson.readValue(bytes, StorageSyncLock::class.java) }.getOrNull()
}
private fun writeSyncLockToChannel(channel: FileChannel, lock: StorageSyncLock) {
val bytes = jackson.writeValueAsBytes(lock)
channel.truncate(0)
channel.position(0)
channel.write(ByteBuffer.wrap(bytes))
channel.force(true)
}
private suspend fun appendSyncEntry(path: String, operation: StorageSyncOperation) {
val cleanedPath = if (path.startsWith("/")) path else "/$path"
val entries = readSyncJournal()
val nextSequence = (entries.maxOfOrNull { it.revision.sequence } ?: 0L) + 1L
val entry = StorageSyncJournalEntry(
path = cleanedPath,
operation = operation,
revision = StorageSyncRevision(
sequence = nextSequence,
actorId = syncActorId,
createdAt = Instant.now(),
),
)
appendSyncJournal(listOf(entry))
}
companion object {
// Файлы, которые можно использовать для чтения и записи, но не отображаются в хранилище
private const val SYSTEM_HIDDEN_DIRNAME = "wallenc-local-storage-meta-dir"
private const val META_INFO_POSTFIX = ".wallenc-meta"
private const val DATA_PAGE_LENGTH = 10
private const val SYNC_JOURNAL_FILENAME = "sync-journal.json"
private const val SYNC_LOCK_FILENAME = "sync-lock.json"
private val jackson = jacksonObjectMapper().apply { findAndRegisterModules() }
}
}

106
domain-vault/src/main/java/com/github/nullptroma/wallenc/infrastructure/storages/yandex/YandexStorageAccessor.kt
View File

@@ -13,6 +13,10 @@ import com.github.nullptroma.wallenc.domain.datatypes.DataPage
import com.github.nullptroma.wallenc.domain.interfaces.IDirectory
import com.github.nullptroma.wallenc.domain.interfaces.IFile
import com.github.nullptroma.wallenc.domain.interfaces.IStorageAccessor
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncLock
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.CoroutineScope
@@ -30,6 +34,8 @@ import kotlinx.coroutines.flow.flowOn
import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withContext
import java.io.ByteArrayOutputStream
import java.io.File
@@ -53,6 +59,8 @@ class YandexStorageAccessor(
private val accessorScope: CoroutineScope,
private val reportAuthFailure: () -> Unit,
) : IStorageAccessor {
private val syncActorId = UUID.randomUUID().toString()
private val syncLockMutex = Mutex()
private val diskRoot = "app:/$storageUuid"
@@ -440,6 +448,7 @@ class YandexStorageAccessor(
_numberOfFiles.value = (_numberOfFiles.value ?: 0) + 1
persistStatsImmediate()
}
appendSyncEntry(path = path, operation = StorageSyncOperation.UPSERT)
}
override suspend fun touchDir(path: String): Unit = withContext(ioDispatcher) {
@@ -478,6 +487,7 @@ class YandexStorageAccessor(
}
guard { repo.delete(diskPath, permanently = true) }
scheduleStatsPersist()
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
}
override suspend fun openWrite(path: String): OutputStream = withContext(ioDispatcher) {
@@ -509,6 +519,7 @@ class YandexStorageAccessor(
info?.let {
_filesUpdates.emit(DataPage(listOf(it), pageLength = 1, pageIndex = 0))
}
appendSyncEntry(path = path, operation = StorageSyncOperation.UPSERT)
} finally {
tmp.delete()
}
@@ -522,6 +533,7 @@ class YandexStorageAccessor(
override suspend fun moveToTrash(path: String) = withContext(ioDispatcher) {
patchCustomProps(path, mapOf(PROP_DELETED to "true"))
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
}
override suspend fun openReadSystemFile(name: String): InputStream = withContext(ioDispatcher) {
@@ -549,6 +561,98 @@ class YandexStorageAccessor(
}
}
override suspend fun readSyncJournal(): List<StorageSyncJournalEntry> = withContext(ioDispatcher) {
val bytes = openReadSystemFile(SYNC_JOURNAL_FILENAME).use { it.readBytes() }
if (bytes.isEmpty()) {
return@withContext emptyList()
}
return@withContext runCatching {
val javaType = statsMapper.typeFactory.constructCollectionType(
List::class.java,
StorageSyncJournalEntry::class.java,
)
@Suppress("UNCHECKED_CAST")
(statsMapper.readValue(bytes, javaType) as List<StorageSyncJournalEntry>)
}.getOrElse {
emptyList()
}
}
override suspend fun appendSyncJournal(entries: List<StorageSyncJournalEntry>) = withContext(ioDispatcher) {
if (entries.isEmpty()) {
return@withContext
}
val next = readSyncJournal().toMutableList().apply {
addAll(entries)
}
openWriteSystemFile(SYNC_JOURNAL_FILENAME).use { out ->
statsMapper.writeValue(out, next)
}
}
override suspend fun rewriteSyncJournal(entries: List<StorageSyncJournalEntry>) = withContext(ioDispatcher) {
openWriteSystemFile(SYNC_JOURNAL_FILENAME).use { out ->
statsMapper.writeValue(out, entries)
}
}
override suspend fun readSyncLock(): StorageSyncLock? = withContext(ioDispatcher) {
val bytes = openReadSystemFile(SYNC_LOCK_FILENAME).use { it.readBytes() }
if (bytes.isEmpty()) {
return@withContext null
}
return@withContext runCatching {
statsMapper.readValue(bytes, StorageSyncLock::class.java)
}.getOrNull()
}
override suspend fun tryAcquireSyncLock(holderId: String, leaseUntil: Instant): Boolean = withContext(ioDispatcher) {
return@withContext syncLockMutex.withLock {
val current = readSyncLock()
if (current != null && current.holderId != holderId) {
return@withLock false
}
val next = StorageSyncLock(
holderId = holderId,
leaseUntil = leaseUntil,
updatedAt = Instant.now(),
)
openWriteSystemFile(SYNC_LOCK_FILENAME).use { out ->
statsMapper.writeValue(out, next)
}
readSyncLock()?.holderId == holderId
}
}
override suspend fun releaseSyncLock(holderId: String) = withContext(ioDispatcher) {
syncLockMutex.withLock {
val current = readSyncLock() ?: return@withLock
if (current.holderId != holderId) {
return@withLock
}
openWriteSystemFile(SYNC_LOCK_FILENAME).use { out ->
out.write(ByteArray(0))
}
}
}
private suspend fun appendSyncEntry(path: String, operation: StorageSyncOperation) {
val cleanedPath = if (path.startsWith("/")) path else "/$path"
val entries = readSyncJournal()
val nextSequence = (entries.maxOfOrNull { it.revision.sequence } ?: 0L) + 1L
val entry = StorageSyncJournalEntry(
path = cleanedPath,
operation = operation,
revision = StorageSyncRevision(
sequence = nextSequence,
actorId = syncActorId,
createdAt = Instant.now(),
),
originStorageUuid = storageUuid,
)
appendSyncJournal(listOf(entry))
}
private suspend fun touchParentDirs(path: String) {
val normalized = path.removeSuffix("/")
if (normalized == "/" || normalized.isBlank()) return
@@ -571,6 +675,8 @@ class YandexStorageAccessor(
private const val SYSTEM_HIDDEN_DIRNAME = "wallenc-yandex-system"
private const val STATS_FILENAME = "yandex-vault-stats.json"
private const val SYNC_JOURNAL_FILENAME = "sync-journal.json"
private const val SYNC_LOCK_FILENAME = "sync-lock.json"
private const val STATS_DEBOUNCE_MS = 450L
private const val DATA_PAGE_LENGTH = 10
private const val API_LIST_LIMIT = 200