fix(sync): исправил журнал при DELETE/TRASH и безопасный flush
Добавил recordSyncJournal для delete/moveToTrash, StorageSyncJournalBuffer с восстановлением pending при ошибке записи и немедленным flush без debounce.
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
package com.github.nullptroma.wallenc.domain.vault.storages.common
|
||||
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournal
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalMerge
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import java.util.UUID
|
||||
|
||||
/**
|
||||
* Буфер журнала sync: накопление записей и безопасный flush (без потери pending при ошибке записи).
|
||||
*/
|
||||
class StorageSyncJournalBuffer(
|
||||
private val syncActorId: String,
|
||||
private val originStorageUuid: UUID?,
|
||||
private val readJournal: suspend () -> StorageSyncJournal,
|
||||
private val writeJournal: suspend (StorageSyncJournal) -> Unit,
|
||||
) {
|
||||
private val pendingMutex = Mutex()
|
||||
private val flushMutex = Mutex()
|
||||
private var pendingJournalEntries: StorageSyncJournal = emptyMap()
|
||||
|
||||
@Volatile
|
||||
private var sequenceHighWatermark: Long? = null
|
||||
|
||||
suspend fun flushPending() {
|
||||
flushMutex.withLock {
|
||||
flushPendingUnderLock()
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun putEntries(entries: StorageSyncJournal) {
|
||||
flushPending()
|
||||
if (entries.isEmpty()) {
|
||||
return
|
||||
}
|
||||
val current = readJournal()
|
||||
val merged = StorageSyncJournalMerge.merge(current, entries)
|
||||
if (merged == current) {
|
||||
return
|
||||
}
|
||||
writeJournal(merged)
|
||||
refreshSequenceHighWatermark(merged)
|
||||
}
|
||||
|
||||
suspend fun appendEntry(path: String, entry: StorageSyncJournalEntry) {
|
||||
pendingMutex.withLock {
|
||||
pendingJournalEntries = pendingJournalEntries + (path to entry)
|
||||
}
|
||||
flushPending()
|
||||
}
|
||||
|
||||
suspend fun nextSequence(): Long {
|
||||
flushPending()
|
||||
val diskMax = readJournal().values.maxOfOrNull { it.revision.sequence } ?: 0L
|
||||
val pendingMax = pendingMutex.withLock {
|
||||
pendingJournalEntries.values.maxOfOrNull { it.revision.sequence } ?: 0L
|
||||
}
|
||||
val base = maxOf(diskMax, pendingMax, sequenceHighWatermark ?: 0L)
|
||||
val next = base + 1L
|
||||
sequenceHighWatermark = next
|
||||
return next
|
||||
}
|
||||
|
||||
fun buildEntry(
|
||||
path: String,
|
||||
operation: com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation,
|
||||
sequence: Long,
|
||||
): StorageSyncJournalEntry {
|
||||
return StorageSyncJournalEntry(
|
||||
path = path,
|
||||
operation = operation,
|
||||
revision = StorageSyncRevision(
|
||||
sequence = sequence,
|
||||
actorId = syncActorId,
|
||||
createdAt = java.time.Instant.now(),
|
||||
),
|
||||
originStorageUuid = originStorageUuid,
|
||||
)
|
||||
}
|
||||
|
||||
private suspend fun flushPendingUnderLock() {
|
||||
val pending = pendingMutex.withLock {
|
||||
if (pendingJournalEntries.isEmpty()) {
|
||||
return
|
||||
}
|
||||
val snapshot = pendingJournalEntries
|
||||
pendingJournalEntries = emptyMap()
|
||||
snapshot
|
||||
}
|
||||
try {
|
||||
val current = readJournal()
|
||||
val merged = StorageSyncJournalMerge.merge(current, pending)
|
||||
if (merged != current) {
|
||||
writeJournal(merged)
|
||||
}
|
||||
refreshSequenceHighWatermark(merged)
|
||||
} catch (e: Exception) {
|
||||
pendingMutex.withLock {
|
||||
pendingJournalEntries = StorageSyncJournalMerge.merge(pending, pendingJournalEntries)
|
||||
}
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
private fun refreshSequenceHighWatermark(journal: StorageSyncJournal) {
|
||||
val maxSeq = journal.values.maxOfOrNull { it.revision.sequence } ?: return
|
||||
val current = sequenceHighWatermark
|
||||
sequenceHighWatermark = if (current == null) maxSeq else maxOf(current, maxSeq)
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package com.github.nullptroma.wallenc.domain.vault.storages.encrypt
|
||||
|
||||
import com.fasterxml.jackson.module.kotlin.readValue
|
||||
import com.github.nullptroma.wallenc.domain.errors.WallencException
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.StorageSyncJournalBuffer
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.StorageSyncJournalCodec
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.readSystemFileBytesOrEmpty
|
||||
import com.github.nullptroma.wallenc.domain.vault.utils.CloseHandledStreamExtension.Companion.onClosed
|
||||
@@ -18,14 +19,12 @@ import com.github.nullptroma.wallenc.domain.interfaces.IFile
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IMetaInfo
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IStorageAccessor
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournal
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalMerge
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncPaths
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncLock
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.DisposableHandle
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
@@ -52,6 +51,12 @@ class EncryptedStorageAccessor(
|
||||
) : IStorageAccessor, DisposableHandle {
|
||||
private val syncActorId = UUID.randomUUID().toString()
|
||||
private val syncLockMutex = Mutex()
|
||||
private val journalBuffer = StorageSyncJournalBuffer(
|
||||
syncActorId = syncActorId,
|
||||
originStorageUuid = null,
|
||||
readJournal = { readSyncJournalUnchecked() },
|
||||
writeJournal = { writeSyncJournal(it) },
|
||||
)
|
||||
private val _size = MutableStateFlow<Long?>(null)
|
||||
override val size: StateFlow<Long?> = _size
|
||||
|
||||
@@ -257,10 +262,12 @@ class EncryptedStorageAccessor(
|
||||
source.touchDir(encryptPath(path))
|
||||
}
|
||||
|
||||
override suspend fun delete(path: String) {
|
||||
source.delete(encryptPath(path))
|
||||
override suspend fun delete(path: String, recordSyncJournal: Boolean) {
|
||||
source.delete(encryptPath(path), recordSyncJournal = false)
|
||||
if (recordSyncJournal) {
|
||||
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun openWrite(path: String, recordSyncJournal: Boolean): OutputStream =
|
||||
openWriteInternal(path, recordJournal = recordSyncJournal)
|
||||
@@ -270,12 +277,17 @@ class EncryptedStorageAccessor(
|
||||
return dataEncryptor.decryptStream(stream)
|
||||
}
|
||||
|
||||
override suspend fun moveToTrash(path: String) {
|
||||
source.moveToTrash(encryptPath(path))
|
||||
override suspend fun moveToTrash(path: String, recordSyncJournal: Boolean) {
|
||||
source.moveToTrash(encryptPath(path), recordSyncJournal = false)
|
||||
if (recordSyncJournal) {
|
||||
appendSyncEntry(path = path, operation = StorageSyncOperation.TRASH)
|
||||
}
|
||||
}
|
||||
|
||||
override fun dispose() {
|
||||
runBlocking {
|
||||
runCatching { journalBuffer.flushPending() }
|
||||
}
|
||||
dataEncryptor.dispose()
|
||||
}
|
||||
|
||||
@@ -297,16 +309,21 @@ class EncryptedStorageAccessor(
|
||||
}
|
||||
|
||||
override suspend fun readSyncJournal(): StorageSyncJournal {
|
||||
val bytes = readSystemFileBytesOrEmpty { openReadSystemFile(SYNC_JOURNAL_FILENAME) }
|
||||
return StorageSyncJournalCodec.read(jackson, bytes)
|
||||
journalBuffer.flushPending()
|
||||
return readSyncJournalUnchecked()
|
||||
}
|
||||
|
||||
override suspend fun flushPendingSyncJournal() {
|
||||
journalBuffer.flushPending()
|
||||
}
|
||||
|
||||
override suspend fun putSyncJournalEntries(entries: StorageSyncJournal) {
|
||||
if (entries.isEmpty()) {
|
||||
return
|
||||
journalBuffer.putEntries(entries)
|
||||
}
|
||||
val merged = StorageSyncJournalMerge.merge(readSyncJournal(), entries)
|
||||
writeSyncJournal(merged)
|
||||
|
||||
private suspend fun readSyncJournalUnchecked(): StorageSyncJournal {
|
||||
val bytes = readSystemFileBytesOrEmpty { openReadSystemFile(SYNC_JOURNAL_FILENAME) }
|
||||
return StorageSyncJournalCodec.read(jackson, bytes)
|
||||
}
|
||||
|
||||
private suspend fun writeSyncJournal(journal: StorageSyncJournal) {
|
||||
@@ -373,21 +390,9 @@ class EncryptedStorageAccessor(
|
||||
if (!StorageSyncPaths.isSyncableUserPath(cleanedPath)) {
|
||||
return
|
||||
}
|
||||
val journal = readSyncJournal()
|
||||
val nextSequence = (journal.values.maxOfOrNull { it.revision.sequence } ?: 0L) + 1L
|
||||
putSyncJournalEntries(
|
||||
mapOf(
|
||||
cleanedPath to StorageSyncJournalEntry(
|
||||
path = cleanedPath,
|
||||
operation = operation,
|
||||
revision = StorageSyncRevision(
|
||||
sequence = nextSequence,
|
||||
actorId = syncActorId,
|
||||
createdAt = Instant.now(),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
val sequence = journalBuffer.nextSequence()
|
||||
val entry = journalBuffer.buildEntry(cleanedPath, operation, sequence)
|
||||
journalBuffer.appendEntry(cleanedPath, entry)
|
||||
}
|
||||
|
||||
private suspend fun openWriteInternal(path: String, recordJournal: Boolean): OutputStream {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.github.nullptroma.wallenc.domain.vault.storages.local
|
||||
|
||||
import com.github.nullptroma.wallenc.domain.errors.WallencException
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.StorageSyncJournalBuffer
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.StorageSyncJournalCodec
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.readSystemFileBytesOrEmpty
|
||||
|
||||
@@ -26,6 +27,8 @@ import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
|
||||
import kotlinx.coroutines.CoroutineDispatcher
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.SharedFlow
|
||||
@@ -73,6 +76,13 @@ class LocalStorageAccessor(
|
||||
private val _dirsUpdates = MutableSharedFlow<DataPage<IDirectory>>()
|
||||
override val dirsUpdates: SharedFlow<DataPage<IDirectory>> = _dirsUpdates
|
||||
|
||||
private val journalBuffer = StorageSyncJournalBuffer(
|
||||
syncActorId = syncActorId,
|
||||
originStorageUuid = null,
|
||||
readJournal = { readSyncJournalUnchecked() },
|
||||
writeJournal = { writeSyncJournal(it) },
|
||||
)
|
||||
|
||||
suspend fun init() = withContext(ioDispatcher) {
|
||||
// запускам сканирование хранилища
|
||||
scanSizeAndNumOfFiles()
|
||||
@@ -513,7 +523,7 @@ class LocalStorageAccessor(
|
||||
createDir(path)
|
||||
}
|
||||
|
||||
override suspend fun delete(path: String) = withContext(ioDispatcher) {
|
||||
override suspend fun delete(path: String, recordSyncJournal: Boolean) = withContext(ioDispatcher) {
|
||||
if (path == "/" || path.isBlank()) {
|
||||
throw WallencException.Storage.DeleteRootForbidden()
|
||||
}
|
||||
@@ -523,9 +533,11 @@ class LocalStorageAccessor(
|
||||
else pair.file.delete()
|
||||
pair.metaFile.delete()
|
||||
scanSizeAndNumOfFiles()
|
||||
if (recordSyncJournal) {
|
||||
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun openWrite(path: String, recordSyncJournal: Boolean): OutputStream = withContext(ioDispatcher) {
|
||||
touchFileInternal(path, recordJournal = false)
|
||||
@@ -548,13 +560,15 @@ class LocalStorageAccessor(
|
||||
return@withContext pair.file.inputStream()
|
||||
}
|
||||
|
||||
override suspend fun moveToTrash(path: String) = withContext(ioDispatcher) {
|
||||
override suspend fun moveToTrash(path: String, recordSyncJournal: Boolean) = withContext(ioDispatcher) {
|
||||
val pair = LocalStorageFilePair.from(_filesystemBasePath, path)
|
||||
?: throw WallencException.Storage.FileNotFound()
|
||||
val newMeta = pair.meta.copy(isDeleted = true)
|
||||
writeMeta(pair.metaFile, newMeta)
|
||||
if (recordSyncJournal) {
|
||||
appendSyncEntry(path = path, operation = StorageSyncOperation.TRASH)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun openReadSystemFile(name: String): InputStream = withContext(ioDispatcher) {
|
||||
val dirPath = _filesystemBasePath.resolve(SYSTEM_HIDDEN_DIRNAME)
|
||||
@@ -579,16 +593,21 @@ class LocalStorageAccessor(
|
||||
}
|
||||
|
||||
override suspend fun readSyncJournal(): StorageSyncJournal = withContext(ioDispatcher) {
|
||||
val bytes = readSystemFileBytesOrEmpty { openReadSystemFile(SYNC_JOURNAL_FILENAME) }
|
||||
StorageSyncJournalCodec.read(jackson, bytes)
|
||||
journalBuffer.flushPending()
|
||||
readSyncJournalUnchecked()
|
||||
}
|
||||
|
||||
override suspend fun flushPendingSyncJournal() = withContext(ioDispatcher) {
|
||||
journalBuffer.flushPending()
|
||||
}
|
||||
|
||||
override suspend fun putSyncJournalEntries(entries: StorageSyncJournal) = withContext(ioDispatcher) {
|
||||
if (entries.isEmpty()) {
|
||||
return@withContext
|
||||
journalBuffer.putEntries(entries)
|
||||
}
|
||||
val merged = StorageSyncJournalMerge.merge(readSyncJournal(), entries)
|
||||
writeSyncJournal(merged)
|
||||
|
||||
private suspend fun readSyncJournalUnchecked(): StorageSyncJournal {
|
||||
val bytes = readSystemFileBytesOrEmpty { openReadSystemFile(SYNC_JOURNAL_FILENAME) }
|
||||
return StorageSyncJournalCodec.read(jackson, bytes)
|
||||
}
|
||||
|
||||
private suspend fun writeSyncJournal(journal: StorageSyncJournal) {
|
||||
@@ -721,21 +740,9 @@ class LocalStorageAccessor(
|
||||
if (!StorageSyncPaths.isSyncableUserPath(cleanedPath)) {
|
||||
return
|
||||
}
|
||||
val journal = readSyncJournal()
|
||||
val nextSequence = (journal.values.maxOfOrNull { it.revision.sequence } ?: 0L) + 1L
|
||||
putSyncJournalEntries(
|
||||
mapOf(
|
||||
cleanedPath to StorageSyncJournalEntry(
|
||||
path = cleanedPath,
|
||||
operation = operation,
|
||||
revision = StorageSyncRevision(
|
||||
sequence = nextSequence,
|
||||
actorId = syncActorId,
|
||||
createdAt = Instant.now(),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
val sequence = journalBuffer.nextSequence()
|
||||
val entry = journalBuffer.buildEntry(cleanedPath, operation, sequence)
|
||||
journalBuffer.appendEntry(cleanedPath, entry)
|
||||
}
|
||||
|
||||
companion object {
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.github.nullptroma.wallenc.domain.vault.storages.yandex
|
||||
|
||||
import com.github.nullptroma.wallenc.domain.errors.WallencException
|
||||
import com.github.nullptroma.wallenc.domain.vault.errors.toVaultWallencException
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.StorageSyncJournalBuffer
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.StorageSyncJournalCodec
|
||||
import com.github.nullptroma.wallenc.domain.vault.storages.common.readSystemFileBytesOrEmpty
|
||||
|
||||
@@ -19,17 +20,16 @@ import com.github.nullptroma.wallenc.domain.interfaces.IDirectory
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IFile
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IStorageAccessor
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournal
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalMerge
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncPaths
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncLock
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.CoroutineDispatcher
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlin.coroutines.coroutineContext
|
||||
import kotlinx.coroutines.ensureActive
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
@@ -42,6 +42,7 @@ import kotlinx.coroutines.flow.flowOn
|
||||
import kotlinx.coroutines.flow.stateIn
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
@@ -95,6 +96,13 @@ class YandexStorageAccessor(
|
||||
|
||||
private var statsPersistJob: Job? = null
|
||||
|
||||
private val journalBuffer = StorageSyncJournalBuffer(
|
||||
syncActorId = syncActorId,
|
||||
originStorageUuid = storageUuid,
|
||||
readJournal = { readSyncJournalUnchecked() },
|
||||
writeJournal = { writeSyncJournal(it) },
|
||||
)
|
||||
|
||||
@Volatile
|
||||
private var systemDirEnsured: Boolean = false
|
||||
|
||||
@@ -240,6 +248,7 @@ class YandexStorageAccessor(
|
||||
val queue = ArrayDeque<String>()
|
||||
queue.add(relDir)
|
||||
while (queue.isNotEmpty()) {
|
||||
coroutineContext.ensureActive()
|
||||
val rel = queue.removeFirst()
|
||||
if (isSystemRel(rel)) continue
|
||||
val (files, dirs) = listImmediateChildren(rel)
|
||||
@@ -285,6 +294,7 @@ class YandexStorageAccessor(
|
||||
val dirs = mutableListOf<IDirectory>()
|
||||
var offset = 0
|
||||
while (true) {
|
||||
coroutineContext.ensureActive()
|
||||
val res = guard { repo.list(diskPath, API_LIST_LIMIT, offset) }
|
||||
val items = res.embedded?.items.orEmpty()
|
||||
for (it in items) {
|
||||
@@ -303,7 +313,7 @@ class YandexStorageAccessor(
|
||||
}
|
||||
|
||||
private suspend fun getMetadataAfterWrite(diskPath: String): ResourceDto {
|
||||
val maxAttempts = 6
|
||||
val maxAttempts = 3
|
||||
repeat(maxAttempts) { attempt ->
|
||||
try {
|
||||
return guard { repo.get(diskPath) }
|
||||
@@ -488,9 +498,9 @@ class YandexStorageAccessor(
|
||||
if (created) {
|
||||
_numberOfFiles.value = (_numberOfFiles.value ?: 0) + 1
|
||||
persistStatsImmediate()
|
||||
}
|
||||
appendSyncEntry(path = path, operation = StorageSyncOperation.UPSERT)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun touchDir(path: String): Unit = withContext(ioDispatcher) {
|
||||
val segments = pathSegments(path)
|
||||
@@ -506,7 +516,7 @@ class YandexStorageAccessor(
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun delete(path: String) = withContext(ioDispatcher) {
|
||||
override suspend fun delete(path: String, recordSyncJournal: Boolean) = withContext(ioDispatcher) {
|
||||
if (path == "/" || path.isBlank()) {
|
||||
throw WallencException.Storage.DeleteRootForbidden()
|
||||
}
|
||||
@@ -528,8 +538,10 @@ class YandexStorageAccessor(
|
||||
}
|
||||
guard { repo.delete(diskPath, permanently = true) }
|
||||
scheduleStatsPersist()
|
||||
if (recordSyncJournal) {
|
||||
appendSyncEntry(path = path, operation = StorageSyncOperation.DELETE)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun openWrite(path: String, recordSyncJournal: Boolean): OutputStream = withContext(ioDispatcher) {
|
||||
touchParentDirs(path)
|
||||
@@ -546,10 +558,12 @@ class YandexStorageAccessor(
|
||||
guard { repo.openDownloadStream(toDiskPath(path)) }
|
||||
}
|
||||
|
||||
override suspend fun moveToTrash(path: String) = withContext(ioDispatcher) {
|
||||
override suspend fun moveToTrash(path: String, recordSyncJournal: Boolean) = withContext(ioDispatcher) {
|
||||
patchCustomProps(path, mapOf(PROP_DELETED to "true"))
|
||||
if (recordSyncJournal) {
|
||||
appendSyncEntry(path = path, operation = StorageSyncOperation.TRASH)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun openReadSystemFile(name: String): InputStream = withContext(ioDispatcher) {
|
||||
ensureSystemDirExists()
|
||||
@@ -576,16 +590,21 @@ class YandexStorageAccessor(
|
||||
}
|
||||
|
||||
override suspend fun readSyncJournal(): StorageSyncJournal = withContext(ioDispatcher) {
|
||||
val bytes = readSystemFileBytesOrEmpty { openReadSystemFile(SYNC_JOURNAL_FILENAME) }
|
||||
StorageSyncJournalCodec.read(statsMapper, bytes)
|
||||
journalBuffer.flushPending()
|
||||
readSyncJournalUnchecked()
|
||||
}
|
||||
|
||||
override suspend fun flushPendingSyncJournal() = withContext(ioDispatcher) {
|
||||
journalBuffer.flushPending()
|
||||
}
|
||||
|
||||
override suspend fun putSyncJournalEntries(entries: StorageSyncJournal) = withContext(ioDispatcher) {
|
||||
if (entries.isEmpty()) {
|
||||
return@withContext
|
||||
journalBuffer.putEntries(entries)
|
||||
}
|
||||
val merged = StorageSyncJournalMerge.merge(readSyncJournal(), entries)
|
||||
writeSyncJournal(merged)
|
||||
|
||||
private suspend fun readSyncJournalUnchecked(): StorageSyncJournal {
|
||||
val bytes = readSystemFileBytesOrEmpty { openReadSystemFile(SYNC_JOURNAL_FILENAME) }
|
||||
return StorageSyncJournalCodec.read(statsMapper, bytes)
|
||||
}
|
||||
|
||||
private suspend fun writeSyncJournal(journal: StorageSyncJournal) {
|
||||
@@ -668,10 +687,12 @@ class YandexStorageAccessor(
|
||||
* Выполняется из [OutputStream.close]; ошибки upload пробрасываются вызывающему коду.
|
||||
*/
|
||||
private fun runCommitAfterStreamClosed(block: suspend () -> Unit) {
|
||||
runBlocking(ioDispatcher) {
|
||||
runBlocking {
|
||||
withContext(ioDispatcher) {
|
||||
block()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun commitUploadedFile(path: String, tmp: File, recordSyncJournal: Boolean) {
|
||||
try {
|
||||
@@ -710,22 +731,9 @@ class YandexStorageAccessor(
|
||||
if (!StorageSyncPaths.isSyncableUserPath(cleanedPath)) {
|
||||
return
|
||||
}
|
||||
val journal = readSyncJournal()
|
||||
val nextSequence = (journal.values.maxOfOrNull { it.revision.sequence } ?: 0L) + 1L
|
||||
putSyncJournalEntries(
|
||||
mapOf(
|
||||
cleanedPath to StorageSyncJournalEntry(
|
||||
path = cleanedPath,
|
||||
operation = operation,
|
||||
revision = StorageSyncRevision(
|
||||
sequence = nextSequence,
|
||||
actorId = syncActorId,
|
||||
createdAt = Instant.now(),
|
||||
),
|
||||
originStorageUuid = storageUuid,
|
||||
),
|
||||
),
|
||||
)
|
||||
val sequence = journalBuffer.nextSequence()
|
||||
val entry = journalBuffer.buildEntry(cleanedPath, operation, sequence)
|
||||
journalBuffer.appendEntry(cleanedPath, entry)
|
||||
}
|
||||
|
||||
private suspend fun touchParentDirs(path: String) {
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.github.nullptroma.wallenc.domain.vault.storages.common
|
||||
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import java.time.Instant
|
||||
import java.util.concurrent.atomic.AtomicReference
|
||||
|
||||
class StorageSyncJournalBufferTest {
|
||||
|
||||
@Test
|
||||
fun flushRestoresPendingOnWriteFailure() = runBlocking {
|
||||
val disk = AtomicReference(emptyMap<String, StorageSyncJournalEntry>())
|
||||
var shouldFail = true
|
||||
val buffer = StorageSyncJournalBuffer(
|
||||
syncActorId = "actor",
|
||||
originStorageUuid = null,
|
||||
readJournal = { disk.get() },
|
||||
writeJournal = {
|
||||
if (shouldFail) {
|
||||
error("disk unavailable")
|
||||
}
|
||||
disk.set(it)
|
||||
},
|
||||
)
|
||||
val entry = buffer.buildEntry("/a.txt", StorageSyncOperation.UPSERT, 1L)
|
||||
try {
|
||||
buffer.appendEntry("/a.txt", entry)
|
||||
} catch (_: IllegalStateException) {
|
||||
// expected
|
||||
}
|
||||
shouldFail = false
|
||||
buffer.flushPending()
|
||||
assertTrue(disk.get().containsKey("/a.txt"))
|
||||
assertEquals(1L, disk.get()["/a.txt"]?.revision?.sequence)
|
||||
}
|
||||
}
|
||||
@@ -39,10 +39,10 @@ interface IStorageAccessor {
|
||||
suspend fun setHidden(path: String, hidden: Boolean)
|
||||
suspend fun touchFile(path: String)
|
||||
suspend fun touchDir(path: String)
|
||||
suspend fun delete(path: String)
|
||||
suspend fun delete(path: String, recordSyncJournal: Boolean = true)
|
||||
suspend fun openWrite(path: String, recordSyncJournal: Boolean = true): OutputStream
|
||||
suspend fun openRead(path: String): InputStream
|
||||
suspend fun moveToTrash(path: String)
|
||||
suspend fun moveToTrash(path: String, recordSyncJournal: Boolean = true)
|
||||
|
||||
/**
|
||||
* Системный sidecar-файл для логических нужд хранилища (мета, ключи и т.п.).
|
||||
@@ -53,6 +53,10 @@ interface IStorageAccessor {
|
||||
suspend fun openWriteSystemFile(name: String): OutputStream
|
||||
|
||||
suspend fun readSyncJournal(): StorageSyncJournal
|
||||
|
||||
/** Сбрасывает отложенные записи журнала на носитель (перед sync и при закрытии storage). */
|
||||
suspend fun flushPendingSyncJournal() = Unit
|
||||
|
||||
suspend fun putSyncJournalEntries(entries: StorageSyncJournal)
|
||||
|
||||
suspend fun readSyncLock(): StorageSyncLock?
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.github.nullptroma.wallenc.usecases
|
||||
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncPaths
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.StorageSyncGroup
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.StorageSyncGroupEncryptionKind
|
||||
@@ -10,8 +11,11 @@ import com.github.nullptroma.wallenc.usecases.fakes.FakeStorage
|
||||
import com.github.nullptroma.wallenc.usecases.fakes.FakeStorageAccessor
|
||||
import com.github.nullptroma.wallenc.usecases.fakes.FakeStorageSyncGroupStore
|
||||
import com.github.nullptroma.wallenc.usecases.fakes.FakeVaultsManager
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.Assert.assertArrayEquals
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
@@ -19,6 +23,8 @@ import java.time.Instant
|
||||
|
||||
class StorageSyncEngineTest {
|
||||
|
||||
private fun norm(path: String): String = StorageSyncPaths.normalize(path)
|
||||
|
||||
@Test
|
||||
fun syncAllGroupsReportsNoGroupsWhenEmpty() = runBlocking {
|
||||
val labels = mutableListOf<TaskProgressLabel?>()
|
||||
@@ -63,7 +69,7 @@ class StorageSyncEngineTest {
|
||||
|
||||
assertArrayEquals(payload, target.fileBytes(path))
|
||||
assertTrue(labels.any { it is TaskProgressLabel.SyncGroupCompleted })
|
||||
assertTrue(target.syncJournalEntries().any { it.path == path })
|
||||
assertTrue(target.syncJournalEntries().any { it.path == norm(path) })
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -111,6 +117,81 @@ class StorageSyncEngineTest {
|
||||
engine.syncGroup(group.id) { _, _ -> }
|
||||
|
||||
assertNull(target.fileBytes(path))
|
||||
val targetEntry = target.syncJournalEntries().single { it.path == norm(path) }
|
||||
assertEquals(2L, targetEntry.revision.sequence)
|
||||
assertEquals("actor-b", targetEntry.revision.actorId)
|
||||
assertEquals(StorageSyncOperation.DELETE, targetEntry.operation)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun syncSkipsWhenTargetRevisionAlreadyWinner() = runBlocking {
|
||||
val source = FakeStorage()
|
||||
val target = FakeStorage()
|
||||
val path = "already-synced.txt"
|
||||
val payload = "same".encodeToByteArray()
|
||||
source.putFile(path, payload)
|
||||
target.putFile(path, payload)
|
||||
|
||||
val winner = StorageSyncJournalEntry(
|
||||
path = path,
|
||||
operation = StorageSyncOperation.UPSERT,
|
||||
revision = StorageSyncRevision(
|
||||
sequence = 5L,
|
||||
actorId = "winner-actor",
|
||||
createdAt = Instant.parse("2024-08-01T00:00:00Z"),
|
||||
),
|
||||
size = payload.size.toLong(),
|
||||
)
|
||||
source.addSyncJournalEntry(winner)
|
||||
target.addSyncJournalEntry(winner)
|
||||
|
||||
val group = StorageSyncGroup(
|
||||
id = "skip-group",
|
||||
storageUuids = setOf(source.uuid, target.uuid),
|
||||
encryptionKind = StorageSyncGroupEncryptionKind.NONE,
|
||||
)
|
||||
val engine = createEngine(
|
||||
storages = listOf(source, target),
|
||||
groups = listOf(group),
|
||||
)
|
||||
engine.syncGroup(group.id) { _, _ -> }
|
||||
|
||||
val targetJournal = target.syncJournalEntries().single { it.path == norm(path) }
|
||||
assertEquals(winner.revision, targetJournal.revision)
|
||||
assertEquals(winner.operation, targetJournal.operation)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun openReadDoesNotChangeJournal() = runBlocking {
|
||||
val storage = FakeStorage()
|
||||
val path = "read-only.txt"
|
||||
storage.putFile(path, "data".encodeToByteArray())
|
||||
val before = storage.syncJournalEntries().size
|
||||
|
||||
storage.accessor.openRead(path).use { it.readBytes() }
|
||||
|
||||
assertEquals(before, storage.syncJournalEntries().size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun deleteWithRecordSyncJournalFalseDoesNotBumpSequence() = runBlocking {
|
||||
val storage = FakeStorage()
|
||||
val path = "to-delete.txt"
|
||||
storage.putFile(path, "x".encodeToByteArray())
|
||||
storage.addSyncJournalEntry(
|
||||
StorageSyncJournalEntry(
|
||||
path = path,
|
||||
operation = StorageSyncOperation.UPSERT,
|
||||
revision = StorageSyncRevision(10L, "prior", Instant.EPOCH),
|
||||
),
|
||||
)
|
||||
|
||||
storage.accessor.delete(path, recordSyncJournal = false)
|
||||
|
||||
assertNull(storage.fileBytes(path))
|
||||
val entry = storage.syncJournalEntries().single { it.path == norm(path) }
|
||||
assertEquals(10L, entry.revision.sequence)
|
||||
assertEquals(StorageSyncOperation.UPSERT, entry.operation)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -145,7 +226,11 @@ class StorageSyncEngineTest {
|
||||
engine.syncGroup(group.id) { _, _ -> }
|
||||
|
||||
assertArrayEquals(payload, target.fileBytes(path))
|
||||
assertTrue(path in (target.accessor as FakeStorageAccessor).trashedPaths)
|
||||
assertTrue(norm(path) in (target.accessor as FakeStorageAccessor).trashedPaths)
|
||||
val targetEntry = target.syncJournalEntries().single { it.path == norm(path) }
|
||||
assertEquals(3L, targetEntry.revision.sequence)
|
||||
assertEquals("actor-trash", targetEntry.revision.actorId)
|
||||
assertEquals(StorageSyncOperation.TRASH, targetEntry.operation)
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -221,6 +306,45 @@ class StorageSyncEngineTest {
|
||||
assertNull((second.accessor as FakeStorageAccessor).syncLock)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun syncGroupCooperativeCancellationReleasesLocks() = runBlocking {
|
||||
val source = FakeStorage()
|
||||
val target = FakeStorage()
|
||||
val path = "slow.txt"
|
||||
val payload = "payload".encodeToByteArray()
|
||||
source.putFile(path, payload)
|
||||
(source.accessor as FakeStorageAccessor).openReadDelayMs = 5_000
|
||||
source.addSyncJournalEntry(
|
||||
StorageSyncJournalEntry(
|
||||
path = path,
|
||||
operation = StorageSyncOperation.UPSERT,
|
||||
revision = StorageSyncRevision(1L, "actor", Instant.EPOCH),
|
||||
size = payload.size.toLong(),
|
||||
),
|
||||
)
|
||||
val group = StorageSyncGroup(
|
||||
id = "cancel-group",
|
||||
storageUuids = setOf(source.uuid, target.uuid),
|
||||
encryptionKind = StorageSyncGroupEncryptionKind.NONE,
|
||||
)
|
||||
val engine = createEngine(
|
||||
storages = listOf(source, target),
|
||||
groups = listOf(group),
|
||||
)
|
||||
val job = async {
|
||||
engine.syncGroup(group.id) { _, _ -> }
|
||||
}
|
||||
kotlinx.coroutines.delay(50)
|
||||
job.cancel()
|
||||
try {
|
||||
job.await()
|
||||
} catch (_: CancellationException) {
|
||||
// expected
|
||||
}
|
||||
assertNull((source.accessor as FakeStorageAccessor).syncLock)
|
||||
assertNull((target.accessor as FakeStorageAccessor).syncLock)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun syncGroupReleasesLocksWhenJournalEmpty() = runBlocking {
|
||||
val first = FakeStorage()
|
||||
|
||||
@@ -41,10 +41,12 @@ class FakeStorage(
|
||||
}
|
||||
|
||||
fun putFile(path: String, bytes: ByteArray) {
|
||||
accessorImpl.dataFiles[path] = bytes
|
||||
accessorImpl.dataFiles[com.github.nullptroma.wallenc.domain.datatypes.StorageSyncPaths.normalize(path)] =
|
||||
bytes
|
||||
}
|
||||
|
||||
fun fileBytes(path: String): ByteArray? = accessorImpl.dataFiles[path]
|
||||
fun fileBytes(path: String): ByteArray? =
|
||||
accessorImpl.dataFiles[com.github.nullptroma.wallenc.domain.datatypes.StorageSyncPaths.normalize(path)]
|
||||
|
||||
fun addSyncJournalEntry(entry: StorageSyncJournalEntry) {
|
||||
accessorImpl.syncJournal = StorageSyncJournalMerge.merge(
|
||||
|
||||
@@ -5,10 +5,14 @@ import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournal
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalEntry
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncJournalMerge
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncLock
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncOperation
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncPaths
|
||||
import com.github.nullptroma.wallenc.domain.datatypes.StorageSyncRevision
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IDirectory
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IFile
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IMetaInfo
|
||||
import com.github.nullptroma.wallenc.domain.interfaces.IStorageAccessor
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
@@ -23,6 +27,8 @@ import java.time.Instant
|
||||
|
||||
class FakeStorageAccessor : IStorageAccessor {
|
||||
val dataFiles: MutableMap<String, ByteArray> = mutableMapOf()
|
||||
|
||||
private fun norm(path: String): String = StorageSyncPaths.normalize(path)
|
||||
val trashedPaths: MutableSet<String> = mutableSetOf()
|
||||
private val systemFiles: MutableMap<String, ByteArray> = mutableMapOf()
|
||||
private val _filesUpdates = MutableSharedFlow<DataPage<IFile>>(extraBufferCapacity = 16)
|
||||
@@ -31,6 +37,7 @@ class FakeStorageAccessor : IStorageAccessor {
|
||||
var syncLock: StorageSyncLock? = null
|
||||
var acquireLockResult: Boolean = true
|
||||
var readSyncJournalThrows: Throwable? = null
|
||||
var openReadDelayMs: Long = 0
|
||||
|
||||
override val size: StateFlow<Long?> = MutableStateFlow(0L)
|
||||
override val numberOfFiles: StateFlow<Int?> = MutableStateFlow(0)
|
||||
@@ -51,7 +58,9 @@ class FakeStorageAccessor : IStorageAccessor {
|
||||
override fun getDirsFlow(path: String): Flow<DataPage<IDirectory>> = emptyFlow()
|
||||
|
||||
override suspend fun getFileInfo(path: String): IFile {
|
||||
error("Not implemented in tests")
|
||||
val key = norm(path)
|
||||
val bytes = dataFiles[key] ?: throw IllegalStateException("File not found: $path")
|
||||
return FakeFile(key, bytes.size.toLong())
|
||||
}
|
||||
|
||||
override suspend fun getDirInfo(path: String): IDirectory {
|
||||
@@ -64,14 +73,17 @@ class FakeStorageAccessor : IStorageAccessor {
|
||||
|
||||
override suspend fun touchDir(path: String) = Unit
|
||||
|
||||
override suspend fun delete(path: String) {
|
||||
dataFiles.remove(path)
|
||||
override suspend fun delete(path: String, recordSyncJournal: Boolean) {
|
||||
dataFiles.remove(norm(path))
|
||||
if (recordSyncJournal) {
|
||||
recordDeleteJournal(path)
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun openWrite(path: String, recordSyncJournal: Boolean): OutputStream {
|
||||
return object : ByteArrayOutputStream() {
|
||||
override fun close() {
|
||||
dataFiles[path] = toByteArray()
|
||||
dataFiles[norm(path)] = toByteArray()
|
||||
_filesUpdates.tryEmit(
|
||||
DataPage(
|
||||
listOf(FakeFile(path)),
|
||||
@@ -84,13 +96,20 @@ class FakeStorageAccessor : IStorageAccessor {
|
||||
}
|
||||
|
||||
override suspend fun openRead(path: String): InputStream {
|
||||
val bytes = dataFiles[path] ?: throw IllegalStateException("File not found: $path")
|
||||
if (openReadDelayMs > 0) {
|
||||
delay(openReadDelayMs)
|
||||
}
|
||||
val bytes = dataFiles[norm(path)] ?: throw IllegalStateException("File not found: $path")
|
||||
return ByteArrayInputStream(bytes)
|
||||
}
|
||||
|
||||
override suspend fun moveToTrash(path: String) {
|
||||
if (path in dataFiles) {
|
||||
trashedPaths.add(path)
|
||||
override suspend fun moveToTrash(path: String, recordSyncJournal: Boolean) {
|
||||
val key = norm(path)
|
||||
if (key in dataFiles) {
|
||||
trashedPaths.add(key)
|
||||
if (recordSyncJournal) {
|
||||
recordTrashJournal(path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,10 +131,38 @@ class FakeStorageAccessor : IStorageAccessor {
|
||||
return syncJournal
|
||||
}
|
||||
|
||||
override suspend fun flushPendingSyncJournal() = Unit
|
||||
|
||||
override suspend fun putSyncJournalEntries(entries: StorageSyncJournal) {
|
||||
syncJournal = StorageSyncJournalMerge.merge(syncJournal, entries)
|
||||
}
|
||||
|
||||
private suspend fun recordDeleteJournal(path: String) {
|
||||
appendJournalEntry(path, StorageSyncOperation.DELETE)
|
||||
}
|
||||
|
||||
private suspend fun recordTrashJournal(path: String) {
|
||||
appendJournalEntry(path, StorageSyncOperation.TRASH)
|
||||
}
|
||||
|
||||
private suspend fun appendJournalEntry(path: String, operation: StorageSyncOperation) {
|
||||
val cleaned = norm(path)
|
||||
val nextSequence = (syncJournal.values.maxOfOrNull { it.revision.sequence } ?: 0L) + 1L
|
||||
putSyncJournalEntries(
|
||||
mapOf(
|
||||
cleaned to StorageSyncJournalEntry(
|
||||
path = cleaned,
|
||||
operation = operation,
|
||||
revision = StorageSyncRevision(
|
||||
sequence = nextSequence,
|
||||
actorId = "fake-actor",
|
||||
createdAt = Instant.now(),
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
override suspend fun readSyncLock(): StorageSyncLock? = syncLock
|
||||
|
||||
override suspend fun tryAcquireSyncLock(holderId: String, leaseUntil: Instant): Boolean {
|
||||
@@ -135,9 +182,12 @@ class FakeStorageAccessor : IStorageAccessor {
|
||||
}
|
||||
}
|
||||
|
||||
class FakeFile(path: String) : IFile {
|
||||
class FakeFile(
|
||||
path: String,
|
||||
size: Long = 0L,
|
||||
) : IFile {
|
||||
override val metaInfo: IMetaInfo = object : IMetaInfo {
|
||||
override val size: Long = 0L
|
||||
override val size: Long = size
|
||||
override val isDeleted: Boolean = false
|
||||
override val isHidden: Boolean = false
|
||||
override val lastModified: Instant = Instant.now()
|
||||
|
||||
Reference in New Issue
Block a user