Локальное хранилище теперь читает файлы и создаёт .wallenc-meta

2024-12-21 22:45:02 +03:00
parent cf443487ee
commit 577939e953
18 changed files with 324 additions and 102 deletions
--- a/data/build.gradle.kts
+++ b/data/build.gradle.kts
@@ -9,7 +9,7 @@ android {
    compileSdk = 34

    defaultConfig {
-        minSdk = 24
+        minSdk = 26

        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
        consumerProguardFiles("consumer-rules.pro")
@@ -34,6 +34,10 @@ android {
 }

 dependencies {
+    // jackson
+    implementation(libs.jackson.module.kotlin)
+    implementation(libs.jackson.datatype.jsr310)
+
    // Timber
    implementation(libs.timber)

@@ -45,9 +49,8 @@ dependencies {

    // Retrofit
    implementation(libs.retrofit)
-    implementation(libs.retrofit.converter.gson)
    implementation(libs.retrofit.converter.scalars)
-    implementation(libs.google.gson)
+    implementation(libs.retrofit.converter.jackson)

    implementation(libs.androidx.core.ktx)
    implementation(libs.androidx.appcompat)
--- a/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/LocalStorageAccessor.kt
+++ b/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/LocalStorageAccessor.kt
@@ -1,6 +1,14 @@
 package com.github.nullptroma.wallenc.data.vaults.local

+import com.fasterxml.jackson.core.JacksonException
+import com.fasterxml.jackson.core.JsonParseException
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import com.github.nullptroma.wallenc.data.vaults.local.entity.LocalDirectory
+import com.github.nullptroma.wallenc.data.vaults.local.entity.LocalFile
+import com.github.nullptroma.wallenc.data.vaults.local.entity.LocalMetaInfo
 import com.github.nullptroma.wallenc.domain.datatypes.DataPackage
+import com.github.nullptroma.wallenc.domain.datatypes.DataPage
 import com.github.nullptroma.wallenc.domain.models.IDirectory
 import com.github.nullptroma.wallenc.domain.models.IFile
 import com.github.nullptroma.wallenc.domain.models.IStorageAccessor
@@ -11,90 +19,256 @@ import kotlinx.coroutines.flow.MutableSharedFlow
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.SharedFlow
 import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.flow.flowOn
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import timber.log.Timber
 import java.io.File
 import java.io.InputStream
 import java.io.OutputStream
+import java.nio.file.Path
+import java.time.LocalDateTime
 import kotlin.io.path.Path
+import kotlin.io.path.absolute
 import kotlin.io.path.fileSize
+import kotlin.io.path.pathString
+import kotlin.io.path.relativeTo

 class LocalStorageAccessor(
-    private val absolutePath: String,
+    absolutePath: String,
    private val ioDispatcher: CoroutineDispatcher
 ) : IStorageAccessor {
-    private val _size = MutableStateFlow<Long?>(null)
-    private val _numberOfFiles = MutableStateFlow<Int?>(null)
-    private val _isAvailable = MutableStateFlow(false)
-    private val _filesUpdates = MutableSharedFlow<DataPackage<IFile>>()
-    private val _dirsUpdates = MutableSharedFlow<DataPackage<IDirectory>>()
+    private val _absolutePath: Path = Path(absolutePath).normalize().absolute()
+    private val _jackson = jacksonObjectMapper().apply { findAndRegisterModules() }

-    override val size: StateFlow<Long?>
-        get() = _size
-    override val numberOfFiles: StateFlow<Int?>
-        get() = _numberOfFiles
-    override val isAvailable: StateFlow<Boolean>
-        get() = _isAvailable
-    override val filesUpdates: SharedFlow<DataPackage<IFile>>
-        get() = _filesUpdates
-    override val dirsUpdates: SharedFlow<DataPackage<IDirectory>>
-        get() = _dirsUpdates
+    private val _size = MutableStateFlow<Long?>(null)
+    override val size: StateFlow<Long?> = _size
+
+    private val _numberOfFiles = MutableStateFlow<Int?>(null)
+    override val numberOfFiles: StateFlow<Int?> = _numberOfFiles
+
+    private val _isAvailable = MutableStateFlow(false)
+    override val isAvailable: StateFlow<Boolean> = _isAvailable
+
+    private val _filesUpdates = MutableSharedFlow<DataPackage<IFile>>()
+    override val filesUpdates: SharedFlow<DataPackage<IFile>> = _filesUpdates
+
+    private val _dirsUpdates = MutableSharedFlow<DataPackage<IDirectory>>()
+    override val dirsUpdates: SharedFlow<DataPackage<IDirectory>> = _dirsUpdates

    init {
+        // запускам сканирование хранилища
        CoroutineScope(ioDispatcher).launch {
-            scanStorage()
+            Timber.d("Local storage path: $_absolutePath")
+            updateSizeAndNumOfFiles()
        }
    }

+    /**
+     * Проверяет существование корневого пути Storage в файловой системе, изменяет _isAvailable
+     */
    private fun checkAvailable(): Boolean {
-        _isAvailable.value = File(absolutePath).exists()
+        _isAvailable.value = _absolutePath.toFile().exists()
        return _isAvailable.value
    }

-    private fun forAllFiles(dir: File, callback: (File) -> Unit) {
-        if (dir.exists() == false)
+    /**
+     * Перебирает все файлы в файловой системе
+     * @param dir стартовый каталог
+     * @param maxDepth максимальная глубина (отрицательное для бесконечной)
+     * @param callback метод обратного вызова для каждого файла и директории
+     */
+    private suspend fun scanFileSystem(
+        dir: File,
+        maxDepth: Int,
+        callback: suspend (File) -> Unit,
+        useCallbackForSelf: Boolean = true
+    ) {
+        if (!dir.exists())
            return
-        callback(dir)

-        val nextDirs = dir.listFiles()
-        if (nextDirs != null) {
-            for (nextDir in nextDirs) {
-                forAllFiles(nextDir, callback)
+
+        val children = dir.listFiles()
+        if (children != null) {
+            // вызвать коллбек для каждого элемента директории
+            for (child in children) {
+                callback(child)
+            }
+
+            if (maxDepth != 0) {
+                val nextMaxDepth = if (maxDepth > 0) maxDepth - 1 else maxDepth
+                for (child in children) {
+                    if (child.isDirectory) {
+                        scanFileSystem(child, nextMaxDepth, callback, false)
+                    }
+                }
            }
        }
+
+        if (useCallbackForSelf)
+            callback(dir)
    }

-    private suspend fun scanStorage() = withContext(ioDispatcher) {
-        _isAvailable.value = File(absolutePath).exists()
+    /**
+     * Перебирает все файлы и каталоги в relativePath и возвращает с мета-информацией
+     *
+     */
+    private suspend fun scanStorage(
+        baseStoragePath: String,
+        maxDepth: Int,
+        fileCallback: suspend (LocalFile) -> Unit = {},
+        dirCallback: suspend (LocalDirectory) -> Unit = {}
+    ) {
+        if (!checkAvailable())
+            throw Exception("Not available")
+        val basePath = Path(_absolutePath.pathString, baseStoragePath)
+        scanFileSystem(basePath.toFile(), maxDepth, { file ->
+            val filePath = Path(file.absolutePath)
+
+            // если это файл с мета-информацией - пропустить
+            if (filePath.pathString.endsWith(META_INFO_POSTFIX)) {
+                // Если не удаётся прочитать метаданные или они указывают на несуществующий файл - удалить
+                try {
+                    val reader = file.bufferedReader()
+                    val meta : LocalMetaInfo = _jackson.readValue(reader)
+                    val fileInMeta = File(Path(_absolutePath.pathString, meta.path).pathString)
+                    if (!fileInMeta.exists())
+                        file.delete()
+                } catch (e: JacksonException) {
+                    file.delete()
+                }
+                return@scanFileSystem
+            }
+
+            val metaFilePath = Path(
+                if (file.isFile) {
+                    file.absolutePath + META_INFO_POSTFIX
+                } else {
+                    Path(file.absolutePath, META_INFO_POSTFIX).pathString
+                }
+            )
+            val metaFile = metaFilePath.toFile()
+            val metaInfo: LocalMetaInfo
+            val storageFilePath = "/" + filePath.relativeTo(_absolutePath)
+
+            if (!metaFile.exists()) {
+                metaInfo = createNewLocalMetaInfo(storageFilePath, filePath.fileSize())
+                _jackson.writeValue(metaFile, metaInfo)
+            } else {
+                var readMeta: LocalMetaInfo
+                try {
+                    val reader = metaFile.bufferedReader()
+                    readMeta = _jackson.readValue(reader)
+                } catch (e: JacksonException) {
+                    // если файл повреждён - пересоздать
+                    readMeta = createNewLocalMetaInfo(storageFilePath, filePath.fileSize())
+                    _jackson.writeValue(metaFile, readMeta)
+                }
+                metaInfo = readMeta
+            }
+
+            if (file.isFile) {
+                fileCallback(LocalFile(metaInfo))
+            } else {
+                dirCallback(LocalDirectory(metaInfo, null))
+            }
+        })
+    }
+
+    /**
+     * Создаёт LocalMetaInfo, не требуя наличие файла в файловой системе
+     * @param storagePath полный путь в Storage
+     * @param size размер файла
+     */
+    private fun createNewLocalMetaInfo(storagePath: String, size: Long): LocalMetaInfo {
+        return LocalMetaInfo(
+            size = size,
+            isDeleted = false,
+            isHidden = false,
+            lastModified = LocalDateTime.now(),
+            path = storagePath
+        )
+    }
+
+
+    /**
+     * Считает файлы и их размер. Не бросает исключения, если файлы недоступны
+     * @throws none Если возникла ошибка, оставляет размер и количества файлов равными null
+     */
+    private suspend fun updateSizeAndNumOfFiles() {
+        if (!checkAvailable()) {
+            _size.value = null
+            _numberOfFiles.value = null
+            return
+        }

        var size = 0L
        var numOfFiles = 0

-        forAllFiles(File(absolutePath)) {
-            if (it.isFile) {
-                numOfFiles++
-                size += Path(it.path).fileSize()
-            }
-        }
+        scanStorage(baseStoragePath = "/", maxDepth = -1, fileCallback = {
+            size += it.metaInfo.size
+            numOfFiles++
+        })
+
        _size.value = size
        _numberOfFiles.value = numOfFiles
    }

    override suspend fun getAllFiles(): List<IFile> = withContext(ioDispatcher) {
-        if(checkAvailable() == false)
+        if (!checkAvailable())
            return@withContext listOf()

        val list = mutableListOf<IFile>()
-        return@withContext listOf()
+        scanStorage(baseStoragePath = "/", maxDepth = -1, fileCallback = {
+            list.add(it)
+        })
+        return@withContext list
    }

    override suspend fun getFiles(path: String): List<IFile> = withContext(ioDispatcher) {
-        TODO("Not yet implemented")
+        if (!checkAvailable())
+            return@withContext listOf()
+
+        val list = mutableListOf<IFile>()
+        scanStorage(baseStoragePath = path, maxDepth = 0, fileCallback = {
+            list.add(it)
+        })
+        return@withContext list
    }

-    override fun getFilesFlow(path: String): Flow<DataPackage<IFile>> {
-        TODO("Not yet implemented")
-    }
+    override fun getFilesFlow(path: String): Flow<DataPackage<List<IFile>>> = flow {
+        if (!checkAvailable())
+            return@flow
+
+        val buf = mutableListOf<IFile>()
+        var pageNumber = 0
+        scanStorage(baseStoragePath = path, maxDepth = 0, fileCallback = {
+            if(buf.size == DATA_PAGE_LENGTH) {
+                val page = DataPage(
+                    list = buf.toList(),
+                    isLoading = false,
+                    isError = false,
+                    hasNext = true,
+                    pageLength = DATA_PAGE_LENGTH,
+                    pageIndex = pageNumber++
+                )
+                emit(page)
+                buf.clear()
+            }
+            buf.add(it)
+        })
+        // отправка последней страницы
+        val page = DataPage(
+            list = buf.toList(),
+            isLoading = false,
+            isError = false,
+            hasNext = false,
+            pageLength = DATA_PAGE_LENGTH,
+            pageIndex = pageNumber++
+        )
+        emit(page)
+    }.flowOn(ioDispatcher)

    override suspend fun getAllDirs(): List<IDirectory> = withContext(ioDispatcher) {
        TODO("Not yet implemented")
@@ -104,7 +278,7 @@ class LocalStorageAccessor(
        TODO("Not yet implemented")
    }

-    override fun getDirsFlow(path: String): Flow<DataPackage<IDirectory>> {
+    override fun getDirsFlow(path: String): Flow<DataPackage<List<IDirectory>>> {
        TODO("Not yet implemented")
    }

@@ -139,4 +313,9 @@ class LocalStorageAccessor(
    override suspend fun moveToTrash(path: String) = withContext(ioDispatcher) {
        TODO("Not yet implemented")
    }
+
+    companion object {
+        private const val META_INFO_POSTFIX = ".wallenc-meta"
+        private const val DATA_PAGE_LENGTH = 10
+    }
 }
--- a/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/LocalVault.kt
+++ b/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/LocalVault.kt
@@ -18,12 +18,23 @@ import kotlin.io.path.createDirectory
 import kotlin.io.path.pathString

 class LocalVault(private val ioDispatcher: CoroutineDispatcher, context: Context) : IVault {
-    private val _path = MutableStateFlow<File?>(null)
-    private val _storages = MutableStateFlow(listOf<IStorage>())
-    private val _totalSpace = MutableStateFlow(null)
-    private val _availableSpace = MutableStateFlow(null)
-    private val _isAvailable = MutableStateFlow(false)
+    override val type: VaultType = VaultType.LOCAL
+    override val uuid: UUID
+        get() = TODO("Not yet implemented")

+    private val _storages = MutableStateFlow(listOf<IStorage>())
+    override val storages: StateFlow<List<IStorage>> = _storages
+
+    private val _isAvailable = MutableStateFlow(false)
+    override val isAvailable: StateFlow<Boolean> = _isAvailable
+
+    private val _totalSpace = MutableStateFlow(null)
+    override val totalSpace: StateFlow<Int?> = _totalSpace
+
+    private val _availableSpace = MutableStateFlow(null)
+    override val availableSpace: StateFlow<Int?> = _availableSpace
+
+    private val _path = MutableStateFlow<File?>(null)

    init {
        CoroutineScope(ioDispatcher).launch {
@@ -35,7 +46,7 @@ class LocalVault(private val ioDispatcher: CoroutineDispatcher, context: Context

    private fun readStorages() {
        val path = _path.value
-        if (path == null || _isAvailable.value == false)
+        if (path == null || !_isAvailable.value)
            return

        val dirs = path.listFiles()?.filter { it.isDirectory }
@@ -49,7 +60,7 @@ class LocalVault(private val ioDispatcher: CoroutineDispatcher, context: Context

    override suspend fun createStorage(): IStorage = withContext(ioDispatcher) {
        val path = _path.value
-        if (path == null || _isAvailable.value == false)
+        if (path == null || !_isAvailable.value)
            throw Exception("Not available")

        val uuid = UUID.randomUUID()
@@ -78,16 +89,4 @@ class LocalVault(private val ioDispatcher: CoroutineDispatcher, context: Context
    override suspend fun remove(storage: IStorage) = withContext(ioDispatcher) {
        TODO("Not yet implemented")
    }
-
-    override val type: VaultType = VaultType.LOCAL
-    override val uuid: UUID
-        get() = TODO("Not yet implemented")
-    override val storages: StateFlow<List<IStorage>>
-        get() = _storages
-    override val isAvailable: StateFlow<Boolean>
-        get() = _isAvailable
-    override val totalSpace: StateFlow<Int?>
-        get() = _totalSpace
-    override val availableSpace: StateFlow<Int?>
-        get() = _availableSpace
 }
--- a/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/entity/LocalDirectory.kt
+++ b/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/entity/LocalDirectory.kt
@@ -2,7 +2,7 @@ package com.github.nullptroma.wallenc.data.vaults.local.entity

 import com.github.nullptroma.wallenc.domain.models.IDirectory

-class LocalDirectory(
+data class LocalDirectory(
    override val metaInfo: LocalMetaInfo,
-    override val elementsCount: Int
+    override val elementsCount: Int?
 ) : IDirectory
--- a/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/entity/LocalFile.kt
+++ b/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/entity/LocalFile.kt
@@ -2,4 +2,4 @@ package com.github.nullptroma.wallenc.data.vaults.local.entity

 import com.github.nullptroma.wallenc.domain.models.IFile

-class LocalFile(override val metaInfo: LocalMetaInfo) : IFile
+data class LocalFile(override val metaInfo: LocalMetaInfo) : IFile
--- a/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/entity/LocalMetaInfo.kt
+++ b/data/src/main/java/com/github/nullptroma/wallenc/data/vaults/local/entity/LocalMetaInfo.kt
@@ -3,20 +3,10 @@ package com.github.nullptroma.wallenc.data.vaults.local.entity
 import com.github.nullptroma.wallenc.domain.models.IMetaInfo
 import java.time.LocalDateTime

-class LocalMetaInfo : IMetaInfo {
-    override val name: String
-    override val size: Int
-        get() = TODO("Not yet implemented")
-    override val isDeleted: Boolean
-        get() = TODO("Not yet implemented")
-    override val isHidden: Boolean
-        get() = TODO("Not yet implemented")
-    override val lastModified: LocalDateTime
-        get() = TODO("Not yet implemented")
+data class LocalMetaInfo(
+    override val size: Long,
+    override val isDeleted: Boolean,
+    override val isHidden: Boolean,
+    override val lastModified: LocalDateTime,
    override val path: String
-        get() = TODO("Not yet implemented")
-
-    init {
-        name = ""
-    }
-}
+) : IMetaInfo