From fd0b210bbb6af9a0e923d7d28e119f7fea28bfd3 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 6 Apr 2024 14:23:43 +0200 Subject: [PATCH] fragment store fully file2 zstd. no more zstd.h in fragment store --- src/fragment_store/fragment_store.cpp | 57 +++++++++++++-------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index d8fe80d..fc2b0e3 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -13,8 +13,6 @@ #include "./file2_stack.hpp" -#include - #include #include #include @@ -281,44 +279,44 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function meta_data = nlohmann::json::to_msgpack(meta_data_j); - std::vector meta_data_compressed; // empty if none - //std::vector meta_data_encrypted; // empty if none + std::vector binary_meta_data; + { + std::stack> binary_writer_stack; + binary_writer_stack.push(std::make_unique(binary_meta_data)); - if (meta_comp == Compression::ZSTD) { - meta_data_compressed.resize(ZSTD_compressBound(meta_data.size())); - - size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3 - if (ZSTD_isError(cSize)) { - std::cerr << "FS error: compressing meta failed\n"; - meta_data_compressed.clear(); - meta_comp = Compression::NONE; - } else { - meta_data_compressed.resize(cSize); + if (!buildStackWrite(binary_writer_stack, meta_enc, meta_comp)) { + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: binary writer creation failed '" << _reg.get(fid).path << "'\n"; + return false; + } + + { + const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); + if (!binary_writer_stack.top()->write(ByteSpan{meta_data})) { + // i feel like exceptions or refactoring would be nice here + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: binary writer failed '" << _reg.get(fid).path << "'\n"; + return false; + } } - } else if (meta_comp == Compression::NONE) { - // do nothing - } else { - assert(false && "implement me"); } - // TODO: encryption - - // the meta file is itself msgpack data + //// the meta file is itself msgpack data nlohmann::json meta_header_j = nlohmann::json::array(); meta_header_j.emplace_back() = "SOLMET"; meta_header_j.push_back(meta_enc); meta_header_j.push_back(meta_comp); - if (false) { // TODO: encryption - } else if (!meta_data_compressed.empty()) { - meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed)); - } else { - meta_header_j.push_back(nlohmann::json::binary(meta_data)); - } + // with a custom msgpack impl like cmp, we can be smarter here and dont need an extra buffer + meta_header_j.push_back(nlohmann::json::binary(binary_meta_data)); const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); - //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); } else if (meta_type == MetaFileType::TEXT_JSON) { // cant be compressed or encrypted @@ -337,6 +335,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function buffer(chunk_size); uint64_t buffer_actual_size {0};