From c737715c6603e4d1f05184dd284642dd7d29169a Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 1 Apr 2024 13:28:33 +0200 Subject: [PATCH] more refactor and transition meta write --- src/fragment_store/fragment_store.cpp | 125 +++++++++++++++++++------- 1 file changed, 95 insertions(+), 30 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 3af7f694..1b00d644 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -50,6 +50,63 @@ static ByteSpan spanFromRead(const std::variant>& } } +// TODO: these stacks are file specific +static std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for reading '" << file_path << "'\n"; + return {}; + } + + // TODO: decrypt here + assert(encryption == Encryption::NONE); + + // add layer based on enum + if (compression == Compression::ZSTD) { + file_stack.push(std::make_unique(*file_stack.top().get())); + } else { + // TODO: make error instead + assert(compression == Compression::NONE); + } + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer '" << file_path << "'\n"; + return {}; + } + + return file_stack; +} + +static std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for writing '" << file_path << "'\n"; + return {}; + } + + // TODO: decrypt here + assert(encryption == Encryption::NONE); + + // add layer based on enum + if (compression == Compression::ZSTD) { + file_stack.push(std::make_unique(*file_stack.top().get())); + } else { + // TODO: make error instead + assert(compression == Compression::NONE); + } + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: file failed to add " << (int)compression << " compression layer '" << file_path << "'\n"; + return {}; + } + + return file_stack; +} + FragmentStore::FragmentStore(void) { registerSerializers(); } @@ -227,13 +284,12 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); - std::ofstream meta_file{ - meta_tmp_path, - std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text - }; + auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); + //meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()}, true)); - if (!meta_file.is_open()) { - std::cerr << "FS error: failed to create temporary meta file\n"; + if (meta_file_stack.empty()) { + std::cerr << "FS error: failed to create temporary meta file stack\n"; + std::filesystem::remove(meta_tmp_path); // might have created an empty file return false; } @@ -250,7 +306,8 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(meta_header_data.data()), meta_header_data.size()); + //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); + meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); } else if (meta_type == MetaFileType::TEXT_JSON) { // cant be compressed or encrypted - meta_file << meta_data_j.dump(2, ' ', true); + const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); + meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); } // now data @@ -385,8 +444,9 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function> data_file_stack; - data_file_stack.push(std::make_unique(std::string_view{frag_path})); - - if (!data_file_stack.top()->isGood()) { - std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; - return false; - } - - // TODO: decrypt here - Compression data_comp = Compression::NONE; if (_reg.all_of(fid)) { data_comp = _reg.get(fid).comp; } - // add layer based on enum - if (data_comp == Compression::ZSTD) { - data_file_stack.push(std::make_unique(*data_file_stack.top().get())); - } else { - // TODO: make error instead - assert(data_comp == Compression::NONE); - } + //std::stack> data_file_stack; + //data_file_stack.push(std::make_unique(std::string_view{frag_path})); - if (!data_file_stack.top()->isGood()) { - std::cerr << "FS error: fragment data file failed to add " << (int)data_comp << " decompression layer '" << frag_path << "'\n"; + //if (!data_file_stack.top()->isGood()) { + //std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; + //return false; + //} + + //// TODO: decrypt here + + + //// add layer based on enum + //if (data_comp == Compression::ZSTD) { + //data_file_stack.push(std::make_unique(*data_file_stack.top().get())); + //} else { + //// TODO: make error instead + //assert(data_comp == Compression::NONE); + //} + + //if (!data_file_stack.top()->isGood()) { + //std::cerr << "FS error: fragment data file failed to add " << (int)data_comp << " decompression layer '" << frag_path << "'\n"; + //return false; + //} + auto data_file_stack = buildFileStackRead(std::string_view{frag_path}, Encryption::NONE, data_comp); + if (data_file_stack.empty()) { return false; }