From a094a3d574d12a1c55b3d00d301c494cf4bbdeb8 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 15 Mar 2025 19:54:45 +0100 Subject: [PATCH] move to read only memory mapped files --- CMakeLists.txt | 2 +- .../backends/sha1_mapped_filesystem.cpp | 26 ++++++--- .../{file_rw_mapped.hpp => file2_mapped.hpp} | 55 ++++++++++++++++++- solanaceae/ngc_ft1_sha1/file_constructor.cpp | 6 +- solanaceae/ngc_ft1_sha1/file_constructor.hpp | 1 + 5 files changed, 78 insertions(+), 12 deletions(-) rename solanaceae/ngc_ft1_sha1/{file_rw_mapped.hpp => file2_mapped.hpp} (61%) diff --git a/CMakeLists.txt b/CMakeLists.txt index d41dfdf..27d0e45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,7 @@ target_link_libraries(solanaceae_ngcft1 PUBLIC add_library(solanaceae_sha1_ngcft1 # hacky deps ./solanaceae/ngc_ft1_sha1/mio.hpp - ./solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp + ./solanaceae/ngc_ft1_sha1/file2_mapped.hpp ./solanaceae/ngc_ft1_sha1/file_constructor.hpp ./solanaceae/ngc_ft1_sha1/file_constructor.cpp diff --git a/solanaceae/ngc_ft1_sha1/backends/sha1_mapped_filesystem.cpp b/solanaceae/ngc_ft1_sha1/backends/sha1_mapped_filesystem.cpp index 9967f68..ec76b43 100644 --- a/solanaceae/ngc_ft1_sha1/backends/sha1_mapped_filesystem.cpp +++ b/solanaceae/ngc_ft1_sha1/backends/sha1_mapped_filesystem.cpp @@ -68,7 +68,7 @@ void SHA1MappedFilesystem::newFromFile(std::string_view file_name, std::string_v file_path_ = std::string(file_path) ]() mutable { // 0. open and fail - std::unique_ptr file_impl = construct_file2_rw_mapped(file_path_, -1); + std::unique_ptr file_impl = construct_file2_r_mapped(file_path_); if (!file_impl->isGood()) { { std::lock_guard l{ibs->info_builder_queue_mutex}; @@ -124,7 +124,8 @@ void SHA1MappedFilesystem::newFromFile(std::string_view file_name, std::string_v // executed on iterate thread // reopen, cant move, since std::function needs to be copy consturctable (meh) - std::unique_ptr file_impl = construct_file2_rw_mapped(file_path_, sha1_info.file_size); + std::unique_ptr file_impl = construct_file2_r_mapped(file_path_); + //std::unique_ptr file_impl = construct_file2_rw_mapped(file_path_, sha1_info.file_size); if (!file_impl->isGood()) { std::cerr << "SHA1MF error: failed opening file '" << file_path_ << "'!\n"; return; @@ -234,15 +235,22 @@ std::unique_ptr SHA1MappedFilesystem::file2(Object ov, FILE2_FLAGS flags return nullptr; } - // TODO: read-only one too // since they are mapped, is this efficent to have multiple? - auto res = construct_file2_rw_mapped(file_path, -1); - if (!res || !res->isGood()) { - std::cerr << "SHA1MF error: failed constructing mapped file '" << file_path << "'\n"; - return nullptr; + if (flags & FILE2_WRITE) { + auto res = construct_file2_rw_mapped(file_path, -1); + if (!res || !res->isGood()) { + std::cerr << "SHA1MF error: failed constructing mapped RW file '" << file_path << "'\n"; + return nullptr; + } + return res; + } else { // read + auto res = construct_file2_r_mapped(file_path); + if (!res || !res->isGood()) { + std::cerr << "SHA1MF error: failed constructing mapped R file '" << file_path << "'\n"; + return nullptr; + } + return res; } - - return res; } } // Backends diff --git a/solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp b/solanaceae/ngc_ft1_sha1/file2_mapped.hpp similarity index 61% rename from solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp rename to solanaceae/ngc_ft1_sha1/file2_mapped.hpp index 05d13ca..fe811b1 100644 --- a/solanaceae/ngc_ft1_sha1/file_rw_mapped.hpp +++ b/solanaceae/ngc_ft1_sha1/file2_mapped.hpp @@ -48,7 +48,8 @@ struct File2RWMapped : public File2I { } } - virtual ~File2RWMapped(void) override {} + virtual ~File2RWMapped(void) { + } bool isGood(void) override { return _file_map.is_mapped(); @@ -91,3 +92,55 @@ struct File2RWMapped : public File2I { } }; +struct File2RMapped : public File2I { + mio::ummap_source _file_map; + + File2RMapped(std::string_view file_path) : File2I(false, true) { + std::filesystem::path native_file_path{file_path}; + + if (!std::filesystem::exists(native_file_path)) { + std::cerr << "FileRMapped error: file does not exist\n"; + return; + } + + _file_size = std::filesystem::file_size(native_file_path); + + std::error_code err; + _file_map.map(native_file_path.u8string(), err); + + if (err) { + std::cerr << "FileRMapped error: mapping file failed: " << err.message() << " (" << err << ")\n"; + return; + } + + if (_file_size != _file_map.length()) { + std::cerr << "FileRMapped warning: file size and mapped file size mismatch.\n"; + _file_size = _file_map.length(); + } + } + + virtual ~File2RMapped(void) { + } + + bool isGood(void) override { + return _file_map.is_mapped(); + } + + bool write(const ByteSpan, int64_t = -1) override { return false; } + + ByteSpanWithOwnership read(uint64_t size, int64_t pos = -1) override { + // TODO: support streaming read + if (pos < 0) { + assert(false && "streaming not implemented"); + return ByteSpan{}; + } + + if (pos+size > _file_size) { + assert(false && "read past end"); + return ByteSpan{}; + } + + // return non-owning + return ByteSpan{_file_map.data()+pos, size}; + } +}; diff --git a/solanaceae/ngc_ft1_sha1/file_constructor.cpp b/solanaceae/ngc_ft1_sha1/file_constructor.cpp index d6bec6f..8b1228f 100644 --- a/solanaceae/ngc_ft1_sha1/file_constructor.cpp +++ b/solanaceae/ngc_ft1_sha1/file_constructor.cpp @@ -1,8 +1,12 @@ #include "./file_constructor.hpp" -#include "./file_rw_mapped.hpp" +#include "./file2_mapped.hpp" std::unique_ptr construct_file2_rw_mapped(std::string_view file_path, int64_t file_size) { return std::make_unique(file_path, file_size); } +std::unique_ptr construct_file2_r_mapped(std::string_view file_path) { + return std::make_unique(file_path); +} + diff --git a/solanaceae/ngc_ft1_sha1/file_constructor.hpp b/solanaceae/ngc_ft1_sha1/file_constructor.hpp index 9d18a0f..551cbdc 100644 --- a/solanaceae/ngc_ft1_sha1/file_constructor.hpp +++ b/solanaceae/ngc_ft1_sha1/file_constructor.hpp @@ -6,4 +6,5 @@ #include std::unique_ptr construct_file2_rw_mapped(std::string_view file_path, int64_t file_size = -1); +std::unique_ptr construct_file2_r_mapped(std::string_view file_path);