From 3b010bd16fbcfb524b31aa76c1b819500d9e39ca Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 5 Apr 2024 18:18:21 +0200 Subject: [PATCH] catch write errors (eg unsanitzed strings) --- src/fragment_store/fragment_store.cpp | 157 ++++++++++++-------------- 1 file changed, 72 insertions(+), 85 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 31a1876..bdb92cd 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -308,76 +308,86 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::functionsecond(storage.value(fid), meta_data[storage.type().hash()]); - //} else if (meta_type == MetaFileType::TEXT_JSON) { - s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]); - //} - } - - if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file - const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); - std::vector meta_data_compressed; // empty if none - //std::vector meta_data_encrypted; // empty if none - - if (meta_comp == Compression::ZSTD) { - meta_data_compressed.resize(ZSTD_compressBound(meta_data.size())); - - size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3 - if (ZSTD_isError(cSize)) { - std::cerr << "FS error: compressing meta failed\n"; - meta_data_compressed.clear(); - meta_comp = Compression::NONE; - } else { - meta_data_compressed.resize(cSize); + for (const auto& [type_id, storage] : _reg.storage()) { + if (!storage.contains(fid)) { + continue; } - } else if (meta_comp == Compression::NONE) { - // do nothing - } else { - assert(false && "implement me"); + + //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; + + // use type_id to find serializer + auto s_cb_it = _sc._serl_json.find(type_id); + if (s_cb_it == _sc._serl_json.end()) { + // could not find serializer, not saving + continue; + } + + // noooo, why cant numbers be keys + //if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead + //s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]); + //} else if (meta_type == MetaFileType::TEXT_JSON) { + s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]); + //} } - // TODO: encryption + if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file + const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); + std::vector meta_data_compressed; // empty if none + //std::vector meta_data_encrypted; // empty if none - // the meta file is itself msgpack data - nlohmann::json meta_header_j = nlohmann::json::array(); - meta_header_j.emplace_back() = "SOLMET"; - meta_header_j.push_back(meta_enc); - meta_header_j.push_back(meta_comp); + if (meta_comp == Compression::ZSTD) { + meta_data_compressed.resize(ZSTD_compressBound(meta_data.size())); - if (false) { // TODO: encryption - } else if (!meta_data_compressed.empty()) { - meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed)); - } else { - meta_header_j.push_back(nlohmann::json::binary(meta_data)); + size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3 + if (ZSTD_isError(cSize)) { + std::cerr << "FS error: compressing meta failed\n"; + meta_data_compressed.clear(); + meta_comp = Compression::NONE; + } else { + meta_data_compressed.resize(cSize); + } + } else if (meta_comp == Compression::NONE) { + // do nothing + } else { + assert(false && "implement me"); + } + + // TODO: encryption + + // the meta file is itself msgpack data + nlohmann::json meta_header_j = nlohmann::json::array(); + meta_header_j.emplace_back() = "SOLMET"; + meta_header_j.push_back(meta_enc); + meta_header_j.push_back(meta_comp); + + if (false) { // TODO: encryption + } else if (!meta_data_compressed.empty()) { + meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed)); + } else { + meta_header_j.push_back(nlohmann::json::binary(meta_data)); + } + + const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); + //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); + meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); + } else if (meta_type == MetaFileType::TEXT_JSON) { + // cant be compressed or encrypted + const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); + meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); } - const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); - //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); - meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); - } else if (meta_type == MetaFileType::TEXT_JSON) { - // cant be compressed or encrypted - const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); - meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); + } catch (...) { + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: failed to serialize json data\n"; + return false; } // now data @@ -460,29 +470,6 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function(fid).comp; } - //std::stack> data_file_stack; - //data_file_stack.push(std::make_unique(std::string_view{frag_path})); - - //if (!data_file_stack.top()->isGood()) { - //std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; - //return false; - //} - - //// TODO: decrypt here - - - //// add layer based on enum - //if (data_comp == Compression::ZSTD) { - //data_file_stack.push(std::make_unique(*data_file_stack.top().get())); - //} else { - //// TODO: make error instead - //assert(data_comp == Compression::NONE); - //} - - //if (!data_file_stack.top()->isGood()) { - //std::cerr << "FS error: fragment data file failed to add " << (int)data_comp << " decompression layer '" << frag_path << "'\n"; - //return false; - //} auto data_file_stack = buildFileStackRead(std::string_view{frag_path}, Encryption::NONE, data_comp); if (data_file_stack.empty()) { return false;