From 267f8dffc18b269f01662b0172b3f07aa17cf31c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 13 Feb 2024 00:15:18 +0100 Subject: [PATCH 01/98] working prototpying code --- external/CMakeLists.txt | 11 +- flake.nix | 2 + src/CMakeLists.txt | 24 ++ src/fragment_store/fragment_store.cpp | 318 ++++++++++++++++++++++++ src/fragment_store/fragment_store.hpp | 171 +++++++++++++ src/fragment_store/fragment_store_i.hpp | 11 + src/fragment_store/test_fragstore.cpp | 82 ++++++ 7 files changed, 618 insertions(+), 1 deletion(-) create mode 100644 src/fragment_store/fragment_store.cpp create mode 100644 src/fragment_store/fragment_store.hpp create mode 100644 src/fragment_store/fragment_store_i.hpp create mode 100644 src/fragment_store/test_fragstore.cpp diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 59914ab..0748b3f 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.9 FATAL_ERROR) +cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR) add_subdirectory(./entt) @@ -19,3 +19,12 @@ add_subdirectory(./stb) add_subdirectory(./libwebp) add_subdirectory(./qoi) +if (NOT TARGET nlohmann_json::nlohmann_json) + FetchContent_Declare(json + URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz + URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(json) +endif() + diff --git a/flake.nix b/flake.nix index f957dd0..9f580c7 100644 --- a/flake.nix +++ b/flake.nix @@ -58,6 +58,8 @@ cmakeFlags = [ "-DTOMATO_ASAN=OFF" "-DCMAKE_BUILD_TYPE=RelWithDebInfo" + "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here + # do we really care less about the version? do we need a stable abi? ]; # TODO: replace with install command diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6328a18..375f95b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,29 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) +add_library(fragment_store + ./fragment_store/fragment_store_i.hpp + ./fragment_store/fragment_store.hpp + ./fragment_store/fragment_store.cpp +) + +target_link_libraries(fragment_store PUBLIC + nlohmann_json::nlohmann_json + EnTT::EnTT + solanaceae_util +) + +######################################## + +add_executable(fragment_store_test + fragment_store/test_fragstore.cpp +) + +target_link_libraries(fragment_store_test PUBLIC + fragment_store +) + +######################################## + add_executable(tomato ./main.cpp ./icon.rc diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp new file mode 100644 index 0000000..fad1246 --- /dev/null +++ b/src/fragment_store/fragment_store.cpp @@ -0,0 +1,318 @@ +#include "./fragment_store.hpp" + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static const char* metaFileTypeSuffix(MetaFileType mft) { + switch (mft) { + case MetaFileType::TEXT_JSON: return ".json"; + //case MetaFileType::BINARY_ARB: return ".bin"; + case MetaFileType::BINARY_MSGPACK: return ".msgpack"; + } + return ""; // .unk? +} + +FragmentStore::FragmentStore(void) { + registerSerializers(); +} + +FragmentStore::FragmentStore( + std::array session_uuid_namespace +) : _session_uuid_namespace(std::move(session_uuid_namespace)) { + registerSerializers(); +} + +entt::basic_handle> FragmentStore::fragmentHandle(FragmentID fid) { + return {_reg, fid}; +} + +FragmentID FragmentStore::newFragmentMemoryOwned( + const std::vector& id, + size_t initial_size +) { + { // first check if id is already used + auto exising_id = getFragmentByID(id); + if (_reg.valid(exising_id)) { + return entt::null; + } + } + + { // next check if space in memory budget + const auto free_memory = _memory_budget - _memory_usage; + if (initial_size > free_memory) { + return entt::null; + } + } + + // actually allocate and create + auto new_data = std::make_unique>(initial_size); + if (!static_cast(new_data)) { + // allocation failure + return entt::null; + } + _memory_usage += initial_size; + + const auto new_frag = _reg.create(); + + _reg.emplace(new_frag, id); + // TODO: memory comp + _reg.emplace>>(new_frag) = std::move(new_data); + + return new_frag; +} + +FragmentID FragmentStore::newFragmentFile( + std::string_view store_path, + MetaFileType mft, + const std::vector& id +) { + { // first check if id is already used + const auto exising_id = getFragmentByID(id); + if (_reg.valid(exising_id)) { + return entt::null; + } + } + + if (store_path.empty()) { + store_path = _default_store_path; + } + + std::filesystem::create_directories(store_path); + + const auto id_hex = bin2hex(id); + std::filesystem::path fragment_file_path; + + if (id_hex.size() < 6) { + fragment_file_path = std::filesystem::path{store_path}/id_hex; + } else { + // use the first 2hex (1byte) as a subfolder + std::filesystem::create_directories(std::string{store_path} + id_hex.substr(0, 2)); + fragment_file_path = std::filesystem::path{std::string{store_path} + id_hex.substr(0, 2)} / id_hex.substr(2); + } + + if (std::filesystem::exists(fragment_file_path)) { + return entt::null; + } + + const auto new_frag = _reg.create(); + + _reg.emplace(new_frag, id); + + // file (info) comp + _reg.emplace(new_frag, fragment_file_path.generic_u8string()); + + _reg.emplace(new_frag, mft); + + // meta needs to be synced to file + std::function empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; }; + if (!syncToStorage(new_frag, empty_data_cb)) { + _reg.destroy(new_frag); + return entt::null; + } + + return new_frag; +} + +FragmentID FragmentStore::getFragmentByID( + const std::vector& id +) { + // TODO: accelerate + // maybe keep it sorted and binary search? hash table lookup? + for (const auto& [frag, id_comp] : _reg.view().each()) { + if (id == id_comp.v) { + return frag; + } + } + + return entt::null; +} + +FragmentID FragmentStore::getFragmentCustomMatcher( + std::function& fn +) { + return entt::null; +} + +template +static void writeBinaryMetafileHeader(F& file, const Encryption enc, const Compression comp) { + file.write("SOLMET", 6); + file.put(static_cast>(enc)); + + // TODO: is compressiontype encrypted? + file.put(static_cast>(comp)); +} + +bool FragmentStore::syncToStorage(FragmentID fid, std::function& data_cb) { + if (!_reg.valid(fid)) { + return false; + } + + if (!_reg.all_of(fid)) { + // not a file fragment? + return false; + } + + // split object storage + + MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults + if (_reg.all_of(fid)) { + meta_type = _reg.get(fid).type; + } + + Encryption meta_enc = Encryption::NONE; // TODO: better defaults + Compression meta_comp = Compression::NONE; // TODO: better defaults + + if (meta_type != MetaFileType::TEXT_JSON) { + if (_reg.all_of(fid)) { + meta_enc = _reg.get(fid).enc; + } + + if (_reg.all_of(fid)) { + meta_comp = _reg.get(fid).comp; + } + } else { + // we cant have encryption or compression + + // TODO: warning/error? + + // TODO: forcing for testing + //if (_reg.all_of(fid)) { + _reg.emplace_or_replace(fid, Encryption::NONE); + //} + //if (_reg.all_of(fid)) { + _reg.emplace_or_replace(fid, Compression::NONE); + //} + } + + std::ofstream meta_file{ + _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type), + std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text + }; + + if (!meta_file.is_open()) { + return false; + } + + std::ofstream data_file{ + _reg.get(fid).path, + std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text + }; + + if (!data_file.is_open()) { + return false; + } + + // metadata type + if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file + writeBinaryMetafileHeader(meta_file, meta_enc, meta_comp); + } + + // sharing code between binary msgpack and text json for now + nlohmann::json meta_data = nlohmann::json::object(); // metadata needs to be an object, null not allowed + // metadata file + + for (const auto& [type_id, storage] : _reg.storage()) { + if (!storage.contains(fid)) { + continue; + } + + std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; + + // use type_id to find serializer + auto s_cb_it = _sc._serl_json.find(type_id); + if (s_cb_it == _sc._serl_json.end()) { + // could not find serializer, not saving + continue; + } + + // noooo, why cant numbers be keys + //if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead + //s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]); + //} else if (meta_type == MetaFileType::TEXT_JSON) { + s_cb_it->second(storage.value(fid), meta_data[storage.type().name()]); + //} + } + + if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file + const auto res = nlohmann::json::to_msgpack(meta_data); + meta_file.write(reinterpret_cast(res.data()), res.size()); + } else if (meta_type == MetaFileType::TEXT_JSON) { + meta_file << meta_data.dump(2, ' ', true); + } + + // now data + std::array buffer; + uint64_t buffer_actual_size {0}; + do { + buffer_actual_size = data_cb(buffer.data(), buffer.size()); + if (buffer_actual_size == 0) { + break; + } + if (buffer_actual_size > buffer.size()) { + // wtf + break; + } + + data_file.write(reinterpret_cast(buffer.data()), buffer_actual_size); + } while (buffer_actual_size == buffer.size()); + + meta_file.flush(); + data_file.flush(); + + // TODO: use temp files and move to old location + + if (_reg.all_of(fid)) { + _reg.remove(fid); + } + + return true; +} + +static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) { + if (comp == nullptr) { + return false; + } + + auto& r_comp = *reinterpret_cast(comp); + + out = static_cast>(r_comp.enc); + + return true; +} + +static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { + if (comp == nullptr) { + return false; + } + + auto& r_comp = *reinterpret_cast(comp); + + out = static_cast>(r_comp.comp); + + return true; +} + +void FragmentStore::registerSerializers(void) { + _sc.registerSerializerJson(serl_json_data_enc_type); + _sc.registerSerializerJson(serl_json_data_comp_type); + + std::cout << "registered serl text json cbs:\n"; + for (const auto& [type_id, _] : _sc._serl_json) { + std::cout << " " << type_id << "\n"; + } +} + diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp new file mode 100644 index 0000000..58b3aef --- /dev/null +++ b/src/fragment_store/fragment_store.hpp @@ -0,0 +1,171 @@ +#pragma once + +#include "./fragment_store_i.hpp" +#include "entt/entity/fwd.hpp" + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +enum class Encryption : uint8_t { + NONE = 0x00, +}; +enum class Compression : uint8_t { + NONE = 0x00, +}; +enum class MetaFileType : uint8_t { + TEXT_JSON, + //BINARY_ARB, + BINARY_MSGPACK, +}; + +namespace Components { + + // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) + struct ID { + std::vector v; + }; + + struct DataEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct DataCompressionType { + Compression comp {Compression::NONE}; + }; + + + // meta that is not written to (meta-)file + namespace Ephemeral { + + // excluded from file meta + struct FilePath { + // contains store path, if any + std::string path; + }; + + // TODO: seperate into remote and local? + // (remote meaning eg. the file on disk was changed by another program) + struct DirtyTag {}; + + + // type as comp + struct MetaFileType { + ::MetaFileType type {::MetaFileType::TEXT_JSON}; + }; + + struct MetaEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct MetaCompressionType { + Compression comp {Compression::NONE}; + }; + + } // Ephemeral + +} // Components + +struct SerializerCallbacks { + // nlohmann + // json/msgpack + using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); + entt::dense_map _serl_json; + + using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in); + entt::dense_map _deserl_json; + + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { + _serl_json[type_info.hash()] = fn; + } + template + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerSerializerJson(fn, type_info); } + + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { + _deserl_json[type_info.hash()] = fn; + } + template + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerDeSerializerJson(fn, type_info); } +}; + +struct FragmentStore : public FragmentStoreI { + entt::basic_registry _reg; + + std::array _session_uuid_namespace; + + std::string _default_store_path; + + uint64_t _memory_budget {10u*1024u*1024u}; + uint64_t _memory_usage {0u}; + + SerializerCallbacks _sc; + + FragmentStore(void); + FragmentStore(std::array session_uuid_namespace); + + // HACK: get access to the reg + entt::basic_handle> fragmentHandle(FragmentID fid); + + // TODO: make the frags ref counted + + // ========== new fragment ========== + + // memory backed owned + FragmentID newFragmentMemoryOwned( + const std::vector& id, + size_t initial_size + ); + + // memory backed view (can only be added? not new?) + + // file backed (rw...) + // needs to know which store path to put into + FragmentID newFragmentFile( + std::string_view store_path, + MetaFileType mft, + const std::vector& id + ); + // this variant generate a new, mostly unique, id for us + FragmentID newFragmentFile( + std::string_view store_path, + MetaFileType mft + ); + + // ========== add fragment ========== + + // ========== get fragment ========== + FragmentID getFragmentByID( + const std::vector& id + ); + FragmentID getFragmentCustomMatcher( + std::function& fn + ); + + // remove fragment? + + // syncs fragment to file + + using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); + // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. + bool syncToStorage(FragmentID fid, std::function& data_cb); + + // unload frags? + // if frags are file backed, we can close the file if not needed + + // fragment discovery? + + private: + void registerSerializers(void); // internal comps + // internal actual backends + bool syncToMemory(FragmentID fid, std::function& data_cb); + bool syncToFile(FragmentID fid, std::function& data_cb); +}; + diff --git a/src/fragment_store/fragment_store_i.hpp b/src/fragment_store/fragment_store_i.hpp new file mode 100644 index 0000000..3a77f4c --- /dev/null +++ b/src/fragment_store/fragment_store_i.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include + +// internal id +enum class FragmentID : uint32_t {}; + +struct FragmentStoreI { + virtual ~FragmentStoreI(void) {} +}; + diff --git a/src/fragment_store/test_fragstore.cpp b/src/fragment_store/test_fragstore.cpp new file mode 100644 index 0000000..898b397 --- /dev/null +++ b/src/fragment_store/test_fragstore.cpp @@ -0,0 +1,82 @@ +#include +#include + +#include "./fragment_store.hpp" + +#include +#include + +namespace Components { + struct MessagesTimestampRange { + uint64_t begin {0}; + uint64_t end {1000}; + }; +} // Components + + +static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { + if (comp == nullptr) { + return false; + } + + out = nlohmann::json::object(); + + auto& r_comp = *reinterpret_cast(comp); + + out["begin"] = r_comp.begin; + out["end"] = r_comp.end; + + return true; +} + +int main(void) { + FragmentStore fs; + fs._default_store_path = "test_store/"; + fs._sc.registerSerializerJson(serl_json_msg_ts_range); + + const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9}); + + const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK, {0xff, 0xff, 0xf0, 0xf0, 0xf0, 0xf0, 0xff, 0xf9}); + + { + auto frag0h = fs.fragmentHandle(frag0); + + frag0h.emplace_or_replace(); + frag0h.emplace_or_replace(); + frag0h.emplace_or_replace(); + + std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { + uint64_t i = 0; + for (; i+read < 3000 && i < buffer_size; i++) { + request_buffer[i] = uint8_t((i+read) & 0xff); + } + read += i; + + return i; + }; + fs.syncToStorage(frag0, fn_cb); + } + + { + auto frag1h = fs.fragmentHandle(frag1); + + frag1h.emplace_or_replace(); + frag1h.emplace_or_replace(); + //frag1h.emplace_or_replace(MetaFileType::BINARY_MSGPACK); + + std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { + static constexpr std::string_view text = "This is some random data"; + uint64_t i = 0; + for (; i+read < text.size() && i < buffer_size; i++) { + request_buffer[i] = text[i+read]; + } + read += i; + + return i; + }; + fs.syncToStorage(frag1, fn_cb); + } + + return 0; +} + From 98ab9745155815a5211af36c8c0e286cdf48a2bc Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 13 Feb 2024 01:19:12 +0100 Subject: [PATCH 02/98] random ids --- src/fragment_store/fragment_store.cpp | 45 +++++++++++++++++++++++++++ src/fragment_store/fragment_store.hpp | 6 +++- src/fragment_store/test_fragstore.cpp | 5 +-- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index fad1246..a13431b 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -27,6 +27,21 @@ static const char* metaFileTypeSuffix(MetaFileType mft) { } FragmentStore::FragmentStore(void) { + { // random namespace + const auto num0 = _rng(); + const auto num1 = _rng(); + + _session_uuid_namespace[0] = (num0 >> 0) & 0xff; + _session_uuid_namespace[1] = (num0 >> 8) & 0xff; + _session_uuid_namespace[2] = (num0 >> 16) & 0xff; + _session_uuid_namespace[3] = (num0 >> 24) & 0xff; + + _session_uuid_namespace[4] = (num1 >> 0) & 0xff; + _session_uuid_namespace[5] = (num1 >> 8) & 0xff; + _session_uuid_namespace[6] = (num1 >> 16) & 0xff; + _session_uuid_namespace[7] = (num1 >> 24) & 0xff; + + } registerSerializers(); } @@ -40,6 +55,30 @@ entt::basic_handle> FragmentStore::fragmentHand return {_reg, fid}; } +std::vector FragmentStore::generateNewUID(std::array& uuid_namespace) { + std::vector new_uid(uuid_namespace.cbegin(), uuid_namespace.cend()); + new_uid.resize(new_uid.size() + 8); + + const auto num0 = _rng(); + const auto num1 = _rng(); + + new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff; + new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff; + new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff; + new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff; + + new_uid[uuid_namespace.size()+4] = (num1 >> 0) & 0xff; + new_uid[uuid_namespace.size()+5] = (num1 >> 8) & 0xff; + new_uid[uuid_namespace.size()+6] = (num1 >> 16) & 0xff; + new_uid[uuid_namespace.size()+7] = (num1 >> 24) & 0xff; + + return new_uid; +} + +std::vector FragmentStore::generateNewUID(void) { + return generateNewUID(_session_uuid_namespace); +} + FragmentID FragmentStore::newFragmentMemoryOwned( const std::vector& id, size_t initial_size @@ -126,6 +165,12 @@ FragmentID FragmentStore::newFragmentFile( return new_frag; } +FragmentID FragmentStore::newFragmentFile( + std::string_view store_path, + MetaFileType mft +) { + return newFragmentFile(store_path, mft, generateNewUID()); +} FragmentID FragmentStore::getFragmentByID( const std::vector& id diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 58b3aef..90cff43 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -1,7 +1,6 @@ #pragma once #include "./fragment_store_i.hpp" -#include "entt/entity/fwd.hpp" #include #include @@ -14,6 +13,7 @@ #include #include #include +#include enum class Encryption : uint8_t { NONE = 0x00, @@ -99,6 +99,7 @@ struct SerializerCallbacks { struct FragmentStore : public FragmentStoreI { entt::basic_registry _reg; + std::minstd_rand _rng{std::random_device{}()}; std::array _session_uuid_namespace; std::string _default_store_path; @@ -116,6 +117,9 @@ struct FragmentStore : public FragmentStoreI { // TODO: make the frags ref counted + std::vector generateNewUID(std::array& uuid_namespace); + std::vector generateNewUID(void); + // ========== new fragment ========== // memory backed owned diff --git a/src/fragment_store/test_fragstore.cpp b/src/fragment_store/test_fragstore.cpp index 898b397..9c02165 100644 --- a/src/fragment_store/test_fragstore.cpp +++ b/src/fragment_store/test_fragstore.cpp @@ -36,7 +36,9 @@ int main(void) { const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9}); - const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK, {0xff, 0xff, 0xf0, 0xf0, 0xf0, 0xf0, 0xff, 0xf9}); + const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK); + + const auto frag2 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK); { auto frag0h = fs.fragmentHandle(frag0); @@ -62,7 +64,6 @@ int main(void) { frag1h.emplace_or_replace(); frag1h.emplace_or_replace(); - //frag1h.emplace_or_replace(MetaFileType::BINARY_MSGPACK); std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { static constexpr std::string_view text = "This is some random data"; From e67d7d37b5588eb82d60de80c5dc99d93da3ac50 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 13 Feb 2024 12:30:29 +0100 Subject: [PATCH 03/98] refactoring, add to mainscreen --- src/CMakeLists.txt | 7 ++ src/fragment_store/fragment_store.cpp | 59 ++++++++----- src/fragment_store/fragment_store.hpp | 103 +++------------------- src/fragment_store/meta_components.hpp | 60 +++++++++++++ src/fragment_store/meta_components_id.inl | 26 ++++++ src/fragment_store/serializer.cpp | 10 +++ src/fragment_store/serializer.hpp | 25 ++++++ src/fragment_store/test_fragstore.cpp | 17 +++- src/fragment_store/types.hpp | 16 ++++ src/main_screen.cpp | 5 ++ src/main_screen.hpp | 3 + 11 files changed, 214 insertions(+), 117 deletions(-) create mode 100644 src/fragment_store/meta_components.hpp create mode 100644 src/fragment_store/meta_components_id.inl create mode 100644 src/fragment_store/serializer.cpp create mode 100644 src/fragment_store/serializer.hpp create mode 100644 src/fragment_store/types.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 375f95b..4bc6c63 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) add_library(fragment_store ./fragment_store/fragment_store_i.hpp + ./fragment_store/types.hpp + ./fragment_store/meta_components.hpp + ./fragment_store/meta_components_id.inl + ./fragment_store/serializer.hpp + ./fragment_store/serializer.cpp ./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.cpp ) @@ -106,6 +111,8 @@ target_link_libraries(tomato PUBLIC solanaceae_tox_contacts solanaceae_tox_messages + fragment_store + SDL3::SDL3 imgui diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index a13431b..be4c393 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -51,7 +51,7 @@ FragmentStore::FragmentStore( registerSerializers(); } -entt::basic_handle> FragmentStore::fragmentHandle(FragmentID fid) { +FragmentStore::FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) { return {_reg, fid}; } @@ -107,7 +107,7 @@ FragmentID FragmentStore::newFragmentMemoryOwned( const auto new_frag = _reg.create(); - _reg.emplace(new_frag, id); + _reg.emplace(new_frag, id); // TODO: memory comp _reg.emplace>>(new_frag) = std::move(new_data); @@ -149,12 +149,12 @@ FragmentID FragmentStore::newFragmentFile( const auto new_frag = _reg.create(); - _reg.emplace(new_frag, id); + _reg.emplace(new_frag, id); // file (info) comp - _reg.emplace(new_frag, fragment_file_path.generic_u8string()); + _reg.emplace(new_frag, fragment_file_path.generic_u8string()); - _reg.emplace(new_frag, mft); + _reg.emplace(new_frag, mft); // meta needs to be synced to file std::function empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; }; @@ -177,7 +177,7 @@ FragmentID FragmentStore::getFragmentByID( ) { // TODO: accelerate // maybe keep it sorted and binary search? hash table lookup? - for (const auto& [frag, id_comp] : _reg.view().each()) { + for (const auto& [frag, id_comp] : _reg.view().each()) { if (id == id_comp.v) { return frag; } @@ -206,7 +206,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { + if (!_reg.all_of(fid)) { // not a file fragment? return false; } @@ -214,20 +214,20 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - meta_type = _reg.get(fid).type; + if (_reg.all_of(fid)) { + meta_type = _reg.get(fid).type; } Encryption meta_enc = Encryption::NONE; // TODO: better defaults Compression meta_comp = Compression::NONE; // TODO: better defaults if (meta_type != MetaFileType::TEXT_JSON) { - if (_reg.all_of(fid)) { - meta_enc = _reg.get(fid).enc; + if (_reg.all_of(fid)) { + meta_enc = _reg.get(fid).enc; } - if (_reg.all_of(fid)) { - meta_comp = _reg.get(fid).comp; + if (_reg.all_of(fid)) { + meta_comp = _reg.get(fid).comp; } } else { // we cant have encryption or compression @@ -236,15 +236,15 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - _reg.emplace_or_replace(fid, Encryption::NONE); + _reg.emplace_or_replace(fid, Encryption::NONE); //} //if (_reg.all_of(fid)) { - _reg.emplace_or_replace(fid, Compression::NONE); + _reg.emplace_or_replace(fid, Compression::NONE); //} } std::ofstream meta_file{ - _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type), + _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type), std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text }; @@ -253,7 +253,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path, + _reg.get(fid).path, std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text }; @@ -320,19 +320,32 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - _reg.remove(fid); + if (_reg.all_of(fid)) { + _reg.remove(fid); } return true; } +bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) { + std::function fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { + uint64_t i = 0; + for (; i+read < data_size && i < buffer_size; i++) { + request_buffer[i] = data[i+read]; + } + read += i; + + return i; + }; + return syncToStorage(fid, fn_cb); +} + static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) { if (comp == nullptr) { return false; } - auto& r_comp = *reinterpret_cast(comp); + auto& r_comp = *reinterpret_cast(comp); out = static_cast>(r_comp.enc); @@ -344,7 +357,7 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { return false; } - auto& r_comp = *reinterpret_cast(comp); + auto& r_comp = *reinterpret_cast(comp); out = static_cast>(r_comp.comp); @@ -352,8 +365,8 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { } void FragmentStore::registerSerializers(void) { - _sc.registerSerializerJson(serl_json_data_enc_type); - _sc.registerSerializerJson(serl_json_data_comp_type); + _sc.registerSerializerJson(serl_json_data_enc_type); + _sc.registerSerializerJson(serl_json_data_comp_type); std::cout << "registered serl text json cbs:\n"; for (const auto& [type_id, _] : _sc._serl_json) { diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 90cff43..6dda33d 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -2,101 +2,24 @@ #include "./fragment_store_i.hpp" -#include +#include "./types.hpp" +#include "./meta_components.hpp" + +#include "./serializer.hpp" + #include #include -#include #include -#include #include #include #include #include -enum class Encryption : uint8_t { - NONE = 0x00, -}; -enum class Compression : uint8_t { - NONE = 0x00, -}; -enum class MetaFileType : uint8_t { - TEXT_JSON, - //BINARY_ARB, - BINARY_MSGPACK, -}; - -namespace Components { - - // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) - struct ID { - std::vector v; - }; - - struct DataEncryptionType { - Encryption enc {Encryption::NONE}; - }; - - struct DataCompressionType { - Compression comp {Compression::NONE}; - }; - - - // meta that is not written to (meta-)file - namespace Ephemeral { - - // excluded from file meta - struct FilePath { - // contains store path, if any - std::string path; - }; - - // TODO: seperate into remote and local? - // (remote meaning eg. the file on disk was changed by another program) - struct DirtyTag {}; - - - // type as comp - struct MetaFileType { - ::MetaFileType type {::MetaFileType::TEXT_JSON}; - }; - - struct MetaEncryptionType { - Encryption enc {Encryption::NONE}; - }; - - struct MetaCompressionType { - Compression comp {Compression::NONE}; - }; - - } // Ephemeral - -} // Components - -struct SerializerCallbacks { - // nlohmann - // json/msgpack - using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); - entt::dense_map _serl_json; - - using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in); - entt::dense_map _deserl_json; - - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { - _serl_json[type_info.hash()] = fn; - } - template - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerSerializerJson(fn, type_info); } - - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { - _deserl_json[type_info.hash()] = fn; - } - template - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerDeSerializerJson(fn, type_info); } -}; - struct FragmentStore : public FragmentStoreI { + using FragmentHandle = entt::basic_handle>; + entt::basic_registry _reg; std::minstd_rand _rng{std::random_device{}()}; @@ -113,10 +36,11 @@ struct FragmentStore : public FragmentStoreI { FragmentStore(std::array session_uuid_namespace); // HACK: get access to the reg - entt::basic_handle> fragmentHandle(FragmentID fid); + FragmentHandle fragmentHandle(FragmentID fid); // TODO: make the frags ref counted + // TODO: check for exising std::vector generateNewUID(std::array& uuid_namespace); std::vector generateNewUID(void); @@ -154,21 +78,20 @@ struct FragmentStore : public FragmentStoreI { ); // remove fragment? + // unload? - // syncs fragment to file - + // ========== sync fragment to storage ========== using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. bool syncToStorage(FragmentID fid, std::function& data_cb); - - // unload frags? - // if frags are file backed, we can close the file if not needed + bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size); // fragment discovery? private: void registerSerializers(void); // internal comps // internal actual backends + // TODO: seperate out bool syncToMemory(FragmentID fid, std::function& data_cb); bool syncToFile(FragmentID fid, std::function& data_cb); }; diff --git a/src/fragment_store/meta_components.hpp b/src/fragment_store/meta_components.hpp new file mode 100644 index 0000000..a8ec20e --- /dev/null +++ b/src/fragment_store/meta_components.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include "./types.hpp" + +#include +#include +#include + +namespace Fragment::Components { + + // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) + struct ID { + std::vector v; + }; + + struct DataEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct DataCompressionType { + Compression comp {Compression::NONE}; + }; + + + // meta that is not written to (meta-)file + namespace Ephemeral { + + // excluded from file meta + struct FilePath { + // contains store path, if any + std::string path; + }; + + // TODO: seperate into remote and local? + // (remote meaning eg. the file on disk was changed by another program) + struct DirtyTag {}; + + + // type as comp + struct MetaFileType { + ::MetaFileType type {::MetaFileType::TEXT_JSON}; + }; + + struct MetaEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct MetaCompressionType { + Compression comp {Compression::NONE}; + }; + + } // Ephemeral + +} // Components + +// shortened to save bytes (until I find a way to save by ID in msgpack) +namespace FragComp = Fragment::Components; + +#include "./meta_components_id.inl" + diff --git a/src/fragment_store/meta_components_id.inl b/src/fragment_store/meta_components_id.inl new file mode 100644 index 0000000..c912cce --- /dev/null +++ b/src/fragment_store/meta_components_id.inl @@ -0,0 +1,26 @@ +#pragma once + +#include "./meta_components.hpp" + +#include + +// TODO: move more central +#define DEFINE_COMP_ID(x) \ +template<> \ +constexpr entt::id_type entt::type_hash::value() noexcept { \ + using namespace entt::literals; \ + return #x##_hs; \ +} \ +template<> \ +constexpr std::string_view entt::type_name::value() noexcept { \ + return #x; \ +} + +// cross compiler stable ids + +DEFINE_COMP_ID(FragComp::DataEncryptionType) +DEFINE_COMP_ID(FragComp::DataCompressionType) + +#undef DEFINE_COMP_ID + + diff --git a/src/fragment_store/serializer.cpp b/src/fragment_store/serializer.cpp new file mode 100644 index 0000000..1f3bfe4 --- /dev/null +++ b/src/fragment_store/serializer.cpp @@ -0,0 +1,10 @@ +#include "./serializer.hpp" + +void SerializerCallbacks::registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { + _serl_json[type_info.hash()] = fn; +} + +void SerializerCallbacks::registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { + _deserl_json[type_info.hash()] = fn; +} + diff --git a/src/fragment_store/serializer.hpp b/src/fragment_store/serializer.hpp new file mode 100644 index 0000000..a09feec --- /dev/null +++ b/src/fragment_store/serializer.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include + +struct SerializerCallbacks { + // nlohmann + // json/msgpack + using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); + entt::dense_map _serl_json; + + using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in); + entt::dense_map _deserl_json; + + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info); + template + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerSerializerJson(fn, type_info); } + + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info); + template + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerDeSerializerJson(fn, type_info); } +}; + diff --git a/src/fragment_store/test_fragstore.cpp b/src/fragment_store/test_fragstore.cpp index 9c02165..581f64a 100644 --- a/src/fragment_store/test_fragstore.cpp +++ b/src/fragment_store/test_fragstore.cpp @@ -43,8 +43,8 @@ int main(void) { { auto frag0h = fs.fragmentHandle(frag0); - frag0h.emplace_or_replace(); - frag0h.emplace_or_replace(); + frag0h.emplace_or_replace(); + frag0h.emplace_or_replace(); frag0h.emplace_or_replace(); std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { @@ -62,8 +62,8 @@ int main(void) { { auto frag1h = fs.fragmentHandle(frag1); - frag1h.emplace_or_replace(); - frag1h.emplace_or_replace(); + frag1h.emplace_or_replace(); + frag1h.emplace_or_replace(); std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { static constexpr std::string_view text = "This is some random data"; @@ -78,6 +78,15 @@ int main(void) { fs.syncToStorage(frag1, fn_cb); } + { + auto frag2h = fs.fragmentHandle(frag2); + + frag2h.emplace_or_replace(); + frag2h.emplace_or_replace(); + + static constexpr std::string_view text = "This is more random data"; + fs.syncToStorage(frag2, reinterpret_cast(text.data()), text.size()); + } return 0; } diff --git a/src/fragment_store/types.hpp b/src/fragment_store/types.hpp new file mode 100644 index 0000000..cbb64c5 --- /dev/null +++ b/src/fragment_store/types.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +enum class Encryption : uint8_t { + NONE = 0x00, +}; +enum class Compression : uint8_t { + NONE = 0x00, +}; +enum class MetaFileType : uint8_t { + TEXT_JSON, + //BINARY_ARB, + BINARY_MSGPACK, +}; + diff --git a/src/main_screen.cpp b/src/main_screen.cpp index e84a2c9..9db04a3 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -1,4 +1,5 @@ #include "./main_screen.hpp" +#include "fragment_store/fragment_store.hpp" #include @@ -49,6 +50,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n"; { // setup plugin instances + // TODO: make interface useful + g_provideInstance("FragmentStoreI", "host", &fs); + g_provideInstance("FragmentStore", "host", &fs); + g_provideInstance("ConfigModelI", "host", &conf); g_provideInstance("Contact3Registry", "1", "host", &cr); g_provideInstance("RegistryMessageModel", "host", &rmm); diff --git a/src/main_screen.hpp b/src/main_screen.hpp index 27f7d7b..aabcb51 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -2,6 +2,7 @@ #include "./screen.hpp" +#include "./fragment_store/fragment_store.hpp" #include #include #include @@ -43,6 +44,8 @@ extern "C" { struct MainScreen final : public Screen { SDL_Renderer* renderer; + FragmentStore fs; + SimpleConfigModel conf; Contact3Registry cr; RegistryMessageModel rmm; From 2bc30ffcdcc7673c9d866286c6187a3e13b87afa Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 13 Feb 2024 14:48:38 +0100 Subject: [PATCH 04/98] start with messages (no fragments get created yet) --- src/CMakeLists.txt | 13 ++ src/fragment_store/message_fragment_store.cpp | 131 ++++++++++++++++++ src/fragment_store/message_fragment_store.hpp | 66 +++++++++ src/main_screen.cpp | 1 + src/main_screen.hpp | 2 + 5 files changed, 213 insertions(+) create mode 100644 src/fragment_store/message_fragment_store.cpp create mode 100644 src/fragment_store/message_fragment_store.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4bc6c63..9acd1e3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,6 +19,18 @@ target_link_libraries(fragment_store PUBLIC ######################################## +add_library(message_fragment_store + ./fragment_store/message_fragment_store.hpp + ./fragment_store/message_fragment_store.cpp +) +target_compile_features(message_fragment_store PRIVATE cxx_std_20) +target_link_libraries(message_fragment_store PUBLIC + fragment_store + solanaceae_message3 +) + +######################################## + add_executable(fragment_store_test fragment_store/test_fragstore.cpp ) @@ -112,6 +124,7 @@ target_link_libraries(tomato PUBLIC solanaceae_tox_messages fragment_store + message_fragment_store SDL3::SDL3 diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp new file mode 100644 index 0000000..2f1324c --- /dev/null +++ b/src/fragment_store/message_fragment_store.cpp @@ -0,0 +1,131 @@ +#include "./message_fragment_store.hpp" + +#include + +#include + +#include +#include + +static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { + if (comp == nullptr) { + return false; + } + + out = nlohmann::json::object(); + + auto& r_comp = *reinterpret_cast(comp); + + out["begin"] = r_comp.begin; + out["end"] = r_comp.end; + + return true; +} + +void MessageFragmentStore::handleMessage(const Message3Handle& m) { + if (!static_cast(m)) { + return; // huh? + } + + if (!m.all_of()) { + return; // we only handle msg with ts + } + + const auto msg_ts = m.get().ts; + + if (!m.all_of()) { + // missing fuid + // find closesed non-sealed off fragment + + std::vector fragment_uid; + + // first search for fragment where the ts falls into the range + for (const auto& [tsrage, fid] : _fuid_open) { + if (tsrage.ts_begin <= msg_ts && tsrage.ts_end >= msg_ts) { + fragment_uid = fid; + // TODO: check conditions for open here + // TODO: mark msg (and frag?) dirty + } + } + + // if it did not fit into an existing fragment, we next look for fragments that could be extended + if (fragment_uid.empty()) { + for (const auto& [tsrage, fid] : _fuid_open) { + const int64_t frag_range = int64_t(tsrage.ts_end) - int64_t(tsrage.ts_begin); + //constexpr static int64_t max_frag_ts_extent {1000*60*60}; + constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing + const int64_t possible_extention = max_frag_ts_extent - frag_range; + + // which direction + if ((tsrage.ts_begin - possible_extention) <= msg_ts) { + fragment_uid = fid; + auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fid)); + assert(static_cast(fh)); + + // assuming ts range exists + auto& fts_comp = fh.get(); + fts_comp.begin = msg_ts; // extend into the past + + // TODO: check conditions for open here + // TODO: mark msg (and frag?) dirty + } else if ((tsrage.ts_end + possible_extention) >= msg_ts) { + fragment_uid = fid; + auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fid)); + assert(static_cast(fh)); + + // assuming ts range exists + auto& fts_comp = fh.get(); + fts_comp.end = msg_ts; // extend into the future + + // TODO: check conditions for open here + // TODO: mark msg (and frag?) dirty + } + } + } + + // if its still not found, we need a new fragment + if (fragment_uid.empty()) { + } + + // if this is still empty, something is very wrong and we exit here + if (fragment_uid.empty()) { + return; + } + + m.emplace(fragment_uid); + } + + // TODO: do we use fid? + + // on new message: assign fuid + // on new and update: mark as fragment dirty +} + +MessageFragmentStore::MessageFragmentStore( + RegistryMessageModel& rmm, + FragmentStore& fs +) : _rmm(rmm), _fs(fs) { + _rmm.subscribe(this, RegistryMessageModel_Event::message_construct); + _rmm.subscribe(this, RegistryMessageModel_Event::message_updated); + _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); + + _fs._sc.registerSerializerJson(serl_json_msg_ts_range); +} + +MessageFragmentStore::~MessageFragmentStore(void) { +} + +float MessageFragmentStore::tick(float time_delta) { + return 1000.f*60.f*60.f; +} + +bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) { + handleMessage(e.e); + return false; +} + +bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) { + handleMessage(e.e); + return false; +} + diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp new file mode 100644 index 0000000..aca4d50 --- /dev/null +++ b/src/fragment_store/message_fragment_store.hpp @@ -0,0 +1,66 @@ +#pragma once + +#include "./meta_components.hpp" +#include "./fragment_store_i.hpp" +#include "./fragment_store.hpp" + +#include +#include +#include + +#include + +#include + +namespace Message::Components { + + using FUID = FragComp::ID; + + struct FID { + FragmentID fid {entt::null}; + }; + +} // Message::Components + +namespace Fragment::Components { + struct MessagesTSRange { + // timestamp range within the fragment + uint64_t begin {0}; + uint64_t end {0}; + }; +} // Fragment::Components + +// handles fragments for messages +// on new message: assign fuid +// on new and update: mark as fragment dirty +// on delete: mark as fragment dirty? +class MessageFragmentStore : public RegistryMessageModelEventI { + protected: + RegistryMessageModel& _rmm; + FragmentStore& _fs; + + void handleMessage(const Message3Handle& m); + + struct TSRange final { + uint64_t ts_begin {0}; + uint64_t ts_end {0}; + }; + // only contains fragments with <1024 messages and <28h tsrage + std::map> _fuid_open; + + std::map> _fuid_save_queue; + + public: + MessageFragmentStore( + RegistryMessageModel& rmm, + FragmentStore& fs + ); + virtual ~MessageFragmentStore(void); + + float tick(float time_delta); + + protected: // rmm + bool onEvent(const Message::Events::MessageConstruct& e) override; + bool onEvent(const Message::Events::MessageUpdated& e) override; +}; + diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 9db04a3..9e00875 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -14,6 +14,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri renderer(renderer_), rmm(cr), mts(rmm), + mfs(rmm, fs), tc(save_path, save_password), tpi(tc.getTox()), ad(tc), diff --git a/src/main_screen.hpp b/src/main_screen.hpp index aabcb51..39baf7e 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -7,6 +7,7 @@ #include #include #include +#include "./fragment_store/message_fragment_store.hpp" #include #include #include "./tox_private_impl.hpp" @@ -50,6 +51,7 @@ struct MainScreen final : public Screen { Contact3Registry cr; RegistryMessageModel rmm; MessageTimeSort mts; + MessageFragmentStore mfs; ToxEventLogger tel{std::cout}; ToxClient tc; From 3d41eedf48e54e8a980774e282d0202c204656b7 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 13 Feb 2024 20:10:33 +0100 Subject: [PATCH 05/98] message fragment meta is saved, but still empty data --- src/fragment_store/message_fragment_store.cpp | 59 +++++++++++++++++-- src/fragment_store/message_fragment_store.hpp | 17 ++++-- src/main_screen.cpp | 1 + 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 2f1324c..9009cfb 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -1,11 +1,14 @@ #include "./message_fragment_store.hpp" +#include + #include #include #include #include +#include static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { if (comp == nullptr) { @@ -40,8 +43,8 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { std::vector fragment_uid; // first search for fragment where the ts falls into the range - for (const auto& [tsrage, fid] : _fuid_open) { - if (tsrage.ts_begin <= msg_ts && tsrage.ts_end >= msg_ts) { + for (const auto& [ts_begin, ts_end, fid] : _fuid_open) { + if (ts_begin <= msg_ts && ts_end >= msg_ts) { fragment_uid = fid; // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty @@ -50,32 +53,39 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // if it did not fit into an existing fragment, we next look for fragments that could be extended if (fragment_uid.empty()) { - for (const auto& [tsrage, fid] : _fuid_open) { - const int64_t frag_range = int64_t(tsrage.ts_end) - int64_t(tsrage.ts_begin); + for (auto& [ts_begin, ts_end, fid] : _fuid_open) { + const int64_t frag_range = int64_t(ts_end) - int64_t(ts_begin); //constexpr static int64_t max_frag_ts_extent {1000*60*60}; constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing const int64_t possible_extention = max_frag_ts_extent - frag_range; // which direction - if ((tsrage.ts_begin - possible_extention) <= msg_ts) { + if ((ts_begin - possible_extention) <= msg_ts && ts_begin > msg_ts) { fragment_uid = fid; auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fid)); assert(static_cast(fh)); + std::cout << "MFS: extended begin from " << ts_begin << " to " << msg_ts << "\n"; + // assuming ts range exists auto& fts_comp = fh.get(); fts_comp.begin = msg_ts; // extend into the past + ts_begin = msg_ts; + // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty - } else if ((tsrage.ts_end + possible_extention) >= msg_ts) { + } else if ((ts_end + possible_extention) >= msg_ts && ts_end < msg_ts) { fragment_uid = fid; auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fid)); assert(static_cast(fh)); + std::cout << "MFS: extended end from " << ts_end << " to " << msg_ts << "\n"; + // assuming ts range exists auto& fts_comp = fh.get(); fts_comp.end = msg_ts; // extend into the future + ts_end = msg_ts; // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty @@ -85,14 +95,34 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // if its still not found, we need a new fragment if (fragment_uid.empty()) { + const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::TEXT_JSON); + auto fh = _fs.fragmentHandle(new_fid); + if (!static_cast(fh)) { + std::cout << "MFS error: failed to create new fragment for message\n"; + return; + } + + fragment_uid = fh.get().v; + + auto& new_ts_range = fh.emplace(); + new_ts_range.begin = msg_ts; + new_ts_range.end = msg_ts; + + _fuid_open.emplace_back(OpenFrag{msg_ts, msg_ts, fragment_uid}); + + std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n"; } // if this is still empty, something is very wrong and we exit here if (fragment_uid.empty()) { + std::cout << "MFS error: failed to find/create fragment for message\n"; return; } m.emplace(fragment_uid); + + // in this case we know the fragment needs an update + _fuid_save_queue.push({Message::getTimeMS(), fragment_uid}); } // TODO: do we use fid? @@ -116,6 +146,21 @@ MessageFragmentStore::~MessageFragmentStore(void) { } float MessageFragmentStore::tick(float time_delta) { + // sync dirty fragments here + + if (!_fuid_save_queue.empty()) { + const auto fid = _fs.getFragmentByID(_fuid_save_queue.front().id); + auto j = nlohmann::json::object(); + // if save as binary + //nlohmann::json::to_msgpack(j); + auto j_dump = j.dump(2, ' ', true); + if (_fs.syncToStorage(fid, reinterpret_cast(j_dump.data()), j_dump.size())) { + std::cout << "MFS: dumped " << j_dump << "\n"; + // succ + _fuid_save_queue.pop(); + } + } + return 1000.f*60.f*60.f; } @@ -129,3 +174,5 @@ bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) { return false; } +// TODO: handle deletes? diff between unload? + diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index aca4d50..ea76c25 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -4,13 +4,14 @@ #include "./fragment_store_i.hpp" #include "./fragment_store.hpp" -#include #include #include #include -#include +#include +#include +#include namespace Message::Components { @@ -41,14 +42,20 @@ class MessageFragmentStore : public RegistryMessageModelEventI { void handleMessage(const Message3Handle& m); - struct TSRange final { + struct OpenFrag final { uint64_t ts_begin {0}; uint64_t ts_end {0}; + std::vector uid; }; // only contains fragments with <1024 messages and <28h tsrage - std::map> _fuid_open; + // TODO: this needs to move into the message reg + std::vector _fuid_open; - std::map> _fuid_save_queue; + struct QueueEntry final { + uint64_t ts_since_dirty{0}; + std::vector id; + }; + std::queue _fuid_save_queue; public: MessageFragmentStore( diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 9e00875..07c4d06 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -422,6 +422,7 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { tdch.tick(time_delta); // compute mts.iterate(); // compute + mfs.tick(time_delta); // TODO: use delta _min_tick_interval = std::min( // HACK: pow by 1.6 to increase 50 -> ~500 (~522) From 58e9fd551436df4f3bc77f65aae2b02071e928be Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 14 Feb 2024 00:58:21 +0100 Subject: [PATCH 06/98] dump messages to data (some comps) --- CMakeLists.txt | 4 +- src/fragment_store/message_fragment_store.cpp | 92 ++++++++++++++++++- src/fragment_store/message_fragment_store.hpp | 13 +-- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d9000e4..856947f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF) if (TOMATO_ASAN) if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") if (NOT WIN32) # exclude mingw - link_libraries(-fsanitize=address) - #link_libraries(-fsanitize=address,undefined) + #link_libraries(-fsanitize=address) + link_libraries(-fsanitize=address,undefined) #link_libraries(-fsanitize=undefined) message("II enabled ASAN") else() diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 9009cfb..f67a300 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -10,6 +10,34 @@ #include #include +namespace Message::Components { + +// ctx +struct OpenFragments { + struct OpenFrag final { + uint64_t ts_begin {0}; + uint64_t ts_end {0}; + std::vector uid; + }; + // only contains fragments with <1024 messages and <28h tsrage + // TODO: this needs to move into the message reg + std::vector fuid_open; +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text) +//NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TagMessageIsAction, void) + +} // Message::Components + +template +static bool serl_json_default(void* comp, nlohmann::json& out) { + out = *reinterpret_cast(comp); + return true; +} + static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { if (comp == nullptr) { return false; @@ -34,6 +62,13 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; // we only handle msg with ts } + if (!m.registry()->ctx().contains()) { + // first message in this reg + m.registry()->ctx().emplace(); + } + + auto& fuid_open = m.registry()->ctx().get().fuid_open; + const auto msg_ts = m.get().ts; if (!m.all_of()) { @@ -43,7 +78,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { std::vector fragment_uid; // first search for fragment where the ts falls into the range - for (const auto& [ts_begin, ts_end, fid] : _fuid_open) { + for (const auto& [ts_begin, ts_end, fid] : fuid_open) { if (ts_begin <= msg_ts && ts_end >= msg_ts) { fragment_uid = fid; // TODO: check conditions for open here @@ -53,7 +88,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // if it did not fit into an existing fragment, we next look for fragments that could be extended if (fragment_uid.empty()) { - for (auto& [ts_begin, ts_end, fid] : _fuid_open) { + for (auto& [ts_begin, ts_end, fid] : fuid_open) { const int64_t frag_range = int64_t(ts_end) - int64_t(ts_begin); //constexpr static int64_t max_frag_ts_extent {1000*60*60}; constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing @@ -108,7 +143,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { new_ts_range.begin = msg_ts; new_ts_range.end = msg_ts; - _fuid_open.emplace_back(OpenFrag{msg_ts, msg_ts, fragment_uid}); + fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{msg_ts, msg_ts, fragment_uid}); std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n"; } @@ -122,7 +157,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { m.emplace(fragment_uid); // in this case we know the fragment needs an update - _fuid_save_queue.push({Message::getTimeMS(), fragment_uid}); + _fuid_save_queue.push({Message::getTimeMS(), fragment_uid, m.registry()}); } // TODO: do we use fid? @@ -140,6 +175,18 @@ MessageFragmentStore::MessageFragmentStore( _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); _fs._sc.registerSerializerJson(serl_json_msg_ts_range); + + _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); + //_sc.registerSerializerJson(serl_json_default<); + + // files + //_sc.registerSerializerJson() + //_sc.registerSerializerJson(); + //_sc.registerSerializerJson(); + //_sc.registerSerializerJson(); } MessageFragmentStore::~MessageFragmentStore(void) { @@ -150,7 +197,42 @@ float MessageFragmentStore::tick(float time_delta) { if (!_fuid_save_queue.empty()) { const auto fid = _fs.getFragmentByID(_fuid_save_queue.front().id); - auto j = nlohmann::json::object(); + auto* reg = _fuid_save_queue.front().reg; + assert(reg != nullptr); + auto j = nlohmann::json::array(); + + // TODO: does every message have ts? + auto msg_view = reg->view(); + // we also assume all messages have fuid (hack: call handle when not?) + for (const Message3 m : msg_view) { + if (!reg->all_of(m)) { + continue; + } + + if (!reg->any_of(m)) { + continue; + } + + auto& j_entry = j.emplace_back(nlohmann::json::object()); + + for (const auto& [type_id, storage] : reg->storage()) { + if (!storage.contains(m)) { + continue; + } + + std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; + + // use type_id to find serializer + auto s_cb_it = _sc._serl_json.find(type_id); + if (s_cb_it == _sc._serl_json.end()) { + // could not find serializer, not saving + continue; + } + + s_cb_it->second(storage.value(m), j_entry[storage.type().name()]); + } + } + // if save as binary //nlohmann::json::to_msgpack(j); auto j_dump = j.dump(2, ' ', true); diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index ea76c25..8c3f35d 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -40,20 +40,15 @@ class MessageFragmentStore : public RegistryMessageModelEventI { RegistryMessageModel& _rmm; FragmentStore& _fs; - void handleMessage(const Message3Handle& m); + // for message components only + SerializerCallbacks _sc; - struct OpenFrag final { - uint64_t ts_begin {0}; - uint64_t ts_end {0}; - std::vector uid; - }; - // only contains fragments with <1024 messages and <28h tsrage - // TODO: this needs to move into the message reg - std::vector _fuid_open; + void handleMessage(const Message3Handle& m); struct QueueEntry final { uint64_t ts_since_dirty{0}; std::vector id; + Message3Registry* reg{nullptr}; }; std::queue _fuid_save_queue; From 84987216cb1184b271f8cd3e5e79f990ae2c3178 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 14 Feb 2024 12:43:02 +0100 Subject: [PATCH 07/98] handle empty type --- src/fragment_store/message_fragment_store.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index f67a300..08fc47d 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -34,7 +34,9 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text) template static bool serl_json_default(void* comp, nlohmann::json& out) { - out = *reinterpret_cast(comp); + if constexpr (!std::is_empty_v) { + out = *reinterpret_cast(comp); + } // do nothing if empty type return true; } @@ -180,7 +182,7 @@ MessageFragmentStore::MessageFragmentStore( _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); - //_sc.registerSerializerJson(serl_json_default<); + _sc.registerSerializerJson(serl_json_default); // files //_sc.registerSerializerJson() From aa7a5d601380fca7d45fcca03154bcb8da2e1a52 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 14 Feb 2024 17:38:51 +0100 Subject: [PATCH 08/98] more comps --- src/fragment_store/message_fragment_store.cpp | 49 ++++++++++++------- src/fragment_store/message_fragment_store.hpp | 6 ++- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 08fc47d..8d899a8 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -12,23 +12,31 @@ namespace Message::Components { -// ctx -struct OpenFragments { - struct OpenFrag final { - uint64_t ts_begin {0}; - uint64_t ts_end {0}; - std::vector uid; + // ctx + struct OpenFragments { + struct OpenFrag final { + uint64_t ts_begin {0}; + uint64_t ts_end {0}; + std::vector uid; + }; + // only contains fragments with <1024 messages and <28h tsrage + // TODO: this needs to move into the message reg + std::vector fuid_open; }; - // only contains fragments with <1024 messages and <28h tsrage - // TODO: this needs to move into the message reg - std::vector fuid_open; -}; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text) -//NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TagMessageIsAction, void) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text) + + namespace Transfer { + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list) + } // Transfer } // Message::Components @@ -179,16 +187,20 @@ MessageFragmentStore::MessageFragmentStore( _fs._sc.registerSerializerJson(serl_json_msg_ts_range); _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); // files //_sc.registerSerializerJson() - //_sc.registerSerializerJson(); - //_sc.registerSerializerJson(); - //_sc.registerSerializerJson(); + _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(serl_json_default); } MessageFragmentStore::~MessageFragmentStore(void) { @@ -228,6 +240,7 @@ float MessageFragmentStore::tick(float time_delta) { auto s_cb_it = _sc._serl_json.find(type_id); if (s_cb_it == _sc._serl_json.end()) { // could not find serializer, not saving + std::cout << "missing " << storage.type().name() << "\n"; continue; } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 8c3f35d..d7112a5 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -31,6 +31,10 @@ namespace Fragment::Components { }; } // Fragment::Components +struct MessageSerializerCallbacks : public SerializerCallbacks { + // TODO: add contact and message reg, so entities can be looked up and be converted to fragment uids OR persistent ids +}; + // handles fragments for messages // on new message: assign fuid // on new and update: mark as fragment dirty @@ -41,7 +45,7 @@ class MessageFragmentStore : public RegistryMessageModelEventI { FragmentStore& _fs; // for message components only - SerializerCallbacks _sc; + MessageSerializerCallbacks _sc; void handleMessage(const Message3Handle& m); From d2783915283b8a774d8e70336ec433af68b2246c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 15 Feb 2024 00:35:39 +0100 Subject: [PATCH 09/98] add contact id to meta --- src/fragment_store/message_fragment_store.cpp | 28 ++++++++++++++++++- src/fragment_store/message_fragment_store.hpp | 7 +++++ src/main_screen.cpp | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 8d899a8..cba9b6b 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -2,6 +2,7 @@ #include +#include #include #include @@ -63,6 +64,20 @@ static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { return true; } +static bool serl_json_msg_c_id(void* comp, nlohmann::json& out) { + if (comp == nullptr) { + return false; + } + + out = nlohmann::json::object(); + + auto& r_comp = *reinterpret_cast(comp); + + out["id"] = r_comp.id; + + return true; +} + void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (!static_cast(m)) { return; // huh? @@ -153,6 +168,15 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { new_ts_range.begin = msg_ts; new_ts_range.end = msg_ts; + { + const auto msg_reg_contact = m.registry()->ctx().get(); + if (_cr.all_of(msg_reg_contact)) { + fh.emplace(_cr.get(msg_reg_contact).data); + } else { + // ? rage quit? + } + } + fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{msg_ts, msg_ts, fragment_uid}); std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n"; @@ -177,14 +201,16 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } MessageFragmentStore::MessageFragmentStore( + Contact3Registry& cr, RegistryMessageModel& rmm, FragmentStore& fs -) : _rmm(rmm), _fs(fs) { +) : _cr(cr), _rmm(rmm), _fs(fs) { _rmm.subscribe(this, RegistryMessageModel_Event::message_construct); _rmm.subscribe(this, RegistryMessageModel_Event::message_updated); _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); _fs._sc.registerSerializerJson(serl_json_msg_ts_range); + _fs._sc.registerSerializerJson(serl_json_msg_c_id); _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index d7112a5..58ded02 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -29,6 +30,10 @@ namespace Fragment::Components { uint64_t begin {0}; uint64_t end {0}; }; + + struct MessagesContact { + std::vector id; + }; } // Fragment::Components struct MessageSerializerCallbacks : public SerializerCallbacks { @@ -41,6 +46,7 @@ struct MessageSerializerCallbacks : public SerializerCallbacks { // on delete: mark as fragment dirty? class MessageFragmentStore : public RegistryMessageModelEventI { protected: + Contact3Registry& _cr; RegistryMessageModel& _rmm; FragmentStore& _fs; @@ -58,6 +64,7 @@ class MessageFragmentStore : public RegistryMessageModelEventI { public: MessageFragmentStore( + Contact3Registry& cr, RegistryMessageModel& rmm, FragmentStore& fs ); diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 07c4d06..a17fd2d 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -14,7 +14,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri renderer(renderer_), rmm(cr), mts(rmm), - mfs(rmm, fs), + mfs(cr, rmm, fs), tc(save_path, save_password), tpi(tc.getTox()), ad(tc), From 0b0245d844cd53467821eca126377da445b83030 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 15 Feb 2024 15:06:51 +0100 Subject: [PATCH 10/98] loading fragments mostly working (not notifying anyone yet) --- src/fragment_store/fragment_store.cpp | 198 +++++++++++++++++- src/fragment_store/fragment_store.hpp | 10 +- src/fragment_store/fragment_store_i.hpp | 4 + src/fragment_store/message_fragment_store.cpp | 30 +++ src/fragment_store/serializer.hpp | 4 +- 5 files changed, 241 insertions(+), 5 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index be4c393..97dd67a 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include @@ -13,6 +15,7 @@ #include #include #include +#include #include #include @@ -51,7 +54,7 @@ FragmentStore::FragmentStore( registerSerializers(); } -FragmentStore::FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) { +FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) { return {_reg, fid}; } @@ -340,6 +343,184 @@ bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uin return syncToStorage(fid, fn_cb); } +size_t FragmentStore::scanStoragePath(std::string_view path) { + if (path.empty()) { + path = _default_store_path; + } + // TODO: extract so async can work (or/and make iteratable generator) + + if (!std::filesystem::is_directory(path)) { + std::cerr << "FS error: scan path not a directory '" << path << "'\n"; + return 0; + } + + // step 1: make snapshot of files, validate metafiles and save id/path+meta.ext + // can be extra thread (if non vfs) + struct FragFileEntry { + std::string id_str; + std::filesystem::path frag_path; + std::string meta_ext; + + bool operator==(const FragFileEntry& other) const { + // only compare by id + return id_str == other.id_str; + } + }; + struct FragFileEntryHash { + size_t operator()(const FragFileEntry& it) const { + return entt::hashed_string(it.id_str.data(), it.id_str.size()); + } + }; + entt::dense_set file_frag_list; + + std::filesystem::path storage_path{path}; + + auto handle_file = [&](const std::filesystem::path& file_path) { + if (!std::filesystem::is_regular_file(file_path)) { + return; + } + // handle file + + if (file_path.has_extension()) { + // skip over metadata, assuming only metafiles have extentions (might be wrong?) + // also skips temps + return; + } + + auto relative_path = std::filesystem::proximate(file_path, storage_path); + std::string id_str = relative_path.generic_u8string(); + // delete all '/' + id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end()); + if (id_str.size() % 2 != 0) { + std::cerr << "FS error: non hex fragment uid detected: '" << id_str << "'\n"; + } + + if (file_frag_list.contains(FragFileEntry{id_str, {}, ""})) { + std::cerr << "FS error: fragment duplicate detected: '" << id_str << "'\n"; + return; // skip + } + + const char* meta_ext = ".meta.msgpack"; + { // find meta + // TODO: this as to know all possible extentions + bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack"); + bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json"); + const size_t meta_sum = + (has_meta_msgpack?1:0) + + (has_meta_json?1:0) + ; + + if (meta_sum > 1) { // has multiple + std::cerr << "FS error: fragment with multiple meta files detected: " << id_str << "\n"; + return; // skip + } + + if (meta_sum == 0) { + std::cerr << "FS error: fragment missing meta file detected: " << id_str << "\n"; + return; // skip + } + + if (has_meta_json) { + meta_ext = ".meta.json"; + } + } + + file_frag_list.emplace(FragFileEntry{ + std::move(id_str), + file_path, + meta_ext + }); + }; + + for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) { + if (std::filesystem::is_regular_file(outer_path)) { + handle_file(outer_path); + } else if (std::filesystem::is_directory(outer_path)) { + // subdir, part of id + for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) { + //if (std::filesystem::is_regular_file(inner_path)) { + + //// handle file + //} // TODO: support deeper recursion? + handle_file(inner_path); + } + } + } + + std::cout << "FS: scan found:\n"; + for (const auto& it : file_frag_list) { + std::cout << " " << it.id_str << "\n"; + } + + // step 2: check if files preexist in reg + // main thread + // (merge into step 3 and do more error checking?) + for (auto it = file_frag_list.begin(); it != file_frag_list.end();) { + auto id = hex2bin(it->id_str); + auto fid = getFragmentByID(id); + if (_reg.valid(fid)) { + // pre exising (handle differently??) + // check if store differs? + it = file_frag_list.erase(it); + } else { + it++; + } + } + + size_t count {0}; + // step 3: parse meta and insert into reg of non preexising + // main thread + // TODO: check timestamps of preexisting and reload? mark external/remote dirty? + for (const auto& it : file_frag_list) { + nlohmann::json j; + if (it.meta_ext == ".meta.msgpack") { + // uh + // read binary header + } else if (it.meta_ext == ".meta.json") { + std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext); + if (!file.is_open()) { + std::cout << "FS error: failed opening meta " << it.frag_path << "\n"; + continue; + } + + file >> j; + + if (!j.is_object()) { + std::cerr << "FS error: json in meta is broken " << it.id_str << "\n"; + continue; + } + + // TODO: existing fragment file + //newFragmentFile(); + FragmentHandle fh{_reg, _reg.create()}; + fh.emplace(hex2bin(it.id_str)); + + for (const auto& [k, v] : j.items()) { + // type id from string hash + const auto type_id = entt::hashed_string(k.data(), k.size()); + const auto deserl_fn_it = _sc._deserl_json.find(type_id); + if (deserl_fn_it != _sc._deserl_json.cend()) { + // TODO: check return value + deserl_fn_it->second(fh, v); + } else { + std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; + } + } + count++; + } else { + assert(false); + } + } + + return count; +} + +void FragmentStore::scanStoragePathAsync(std::string path) { + // add path to queue + // HACK: if path is known/any fragment is in the path, this operation blocks (non async) + scanStoragePath(path); // TODO: make async and post result +} + static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) { if (comp == nullptr) { return false; @@ -352,6 +533,20 @@ static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) { return true; } +static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) { + // TODO: this is ugly in multiple places + try { + fh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + } catch(...) { + return false; + } + return true; +} + static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { if (comp == nullptr) { return false; @@ -366,6 +561,7 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { void FragmentStore::registerSerializers(void) { _sc.registerSerializerJson(serl_json_data_enc_type); + _sc.registerDeSerializerJson(deserl_json_data_enc_type); _sc.registerSerializerJson(serl_json_data_comp_type); std::cout << "registered serl text json cbs:\n"; diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 6dda33d..1025f9e 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -17,10 +17,11 @@ #include #include -struct FragmentStore : public FragmentStoreI { - using FragmentHandle = entt::basic_handle>; +// fwd +struct SerializerCallbacks; - entt::basic_registry _reg; +struct FragmentStore : public FragmentStoreI { + FragmentRegistry _reg; std::minstd_rand _rng{std::random_device{}()}; std::array _session_uuid_namespace; @@ -87,6 +88,9 @@ struct FragmentStore : public FragmentStoreI { bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size); // fragment discovery? + // returns number of new fragments + size_t scanStoragePath(std::string_view path); + void scanStoragePathAsync(std::string path); private: void registerSerializers(void); // internal comps diff --git a/src/fragment_store/fragment_store_i.hpp b/src/fragment_store/fragment_store_i.hpp index 3a77f4c..be3f9c1 100644 --- a/src/fragment_store/fragment_store_i.hpp +++ b/src/fragment_store/fragment_store_i.hpp @@ -1,9 +1,13 @@ #pragma once +#include + #include // internal id enum class FragmentID : uint32_t {}; +using FragmentRegistry = entt::basic_registry; +using FragmentHandle = entt::basic_handle; struct FragmentStoreI { virtual ~FragmentStoreI(void) {} diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index cba9b6b..24c4b31 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -64,6 +64,19 @@ static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { return true; } +static bool deserl_json_msg_ts_range(FragmentHandle fh, const nlohmann::json& in) { + // TODO: this is ugly in multiple places + try { + fh.emplace_or_replace(FragComp::MessagesTSRange{ + in["begin"], + in["end"] + }); + } catch(...) { + return false; + } + return true; +} + static bool serl_json_msg_c_id(void* comp, nlohmann::json& out) { if (comp == nullptr) { return false; @@ -78,6 +91,18 @@ static bool serl_json_msg_c_id(void* comp, nlohmann::json& out) { return true; } +static bool deserl_json_msg_c_id(FragmentHandle fh, const nlohmann::json& in) { + // TODO: this is ugly in multiple places + try { + fh.emplace_or_replace(FragComp::MessagesContact{ + in["id"] + }); + } catch(...) { + return false; + } + return true; +} + void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (!static_cast(m)) { return; // huh? @@ -210,7 +235,9 @@ MessageFragmentStore::MessageFragmentStore( _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); _fs._sc.registerSerializerJson(serl_json_msg_ts_range); + _fs._sc.registerDeSerializerJson(deserl_json_msg_ts_range); _fs._sc.registerSerializerJson(serl_json_msg_c_id); + _fs._sc.registerDeSerializerJson(deserl_json_msg_c_id); _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); @@ -227,9 +254,12 @@ MessageFragmentStore::MessageFragmentStore( _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); _sc.registerSerializerJson(serl_json_default); + + _fs.scanStoragePath("test_message_store/"); } MessageFragmentStore::~MessageFragmentStore(void) { + // TODO: sync all dirty fragments } float MessageFragmentStore::tick(float time_delta) { diff --git a/src/fragment_store/serializer.hpp b/src/fragment_store/serializer.hpp index a09feec..bf12c4e 100644 --- a/src/fragment_store/serializer.hpp +++ b/src/fragment_store/serializer.hpp @@ -5,13 +5,15 @@ #include +#include "./fragment_store_i.hpp" + struct SerializerCallbacks { // nlohmann // json/msgpack using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); entt::dense_map _serl_json; - using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in); + using deserialize_json_fn = bool(*)(FragmentHandle fh, const nlohmann::json& in); entt::dense_map _deserl_json; void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info); From f6e55851ccb5e810c1b6d77e2ddcc72c01a9f094 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 15 Feb 2024 18:34:52 +0100 Subject: [PATCH 11/98] improve deserialization and provide message comp deserl --- src/fragment_store/message_fragment_store.cpp | 46 ++++++++----------- src/fragment_store/serializer.hpp | 32 ++++++++++++- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 24c4b31..07383a7 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -41,6 +41,11 @@ namespace Message::Components { } // Message::Components +namespace Fragment::Components { + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id) +} // Fragment::Components + template static bool serl_json_default(void* comp, nlohmann::json& out) { if constexpr (!std::is_empty_v) { @@ -64,19 +69,6 @@ static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { return true; } -static bool deserl_json_msg_ts_range(FragmentHandle fh, const nlohmann::json& in) { - // TODO: this is ugly in multiple places - try { - fh.emplace_or_replace(FragComp::MessagesTSRange{ - in["begin"], - in["end"] - }); - } catch(...) { - return false; - } - return true; -} - static bool serl_json_msg_c_id(void* comp, nlohmann::json& out) { if (comp == nullptr) { return false; @@ -91,18 +83,6 @@ static bool serl_json_msg_c_id(void* comp, nlohmann::json& out) { return true; } -static bool deserl_json_msg_c_id(FragmentHandle fh, const nlohmann::json& in) { - // TODO: this is ugly in multiple places - try { - fh.emplace_or_replace(FragComp::MessagesContact{ - in["id"] - }); - } catch(...) { - return false; - } - return true; -} - void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (!static_cast(m)) { return; // huh? @@ -235,25 +215,37 @@ MessageFragmentStore::MessageFragmentStore( _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); _fs._sc.registerSerializerJson(serl_json_msg_ts_range); - _fs._sc.registerDeSerializerJson(deserl_json_msg_ts_range); + _fs._sc.registerDeSerializerJson(); _fs._sc.registerSerializerJson(serl_json_msg_c_id); - _fs._sc.registerDeSerializerJson(deserl_json_msg_c_id); + _fs._sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); // files //_sc.registerSerializerJson() _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _sc.registerSerializerJson(serl_json_default); + _sc.registerDeSerializerJson(); _fs.scanStoragePath("test_message_store/"); } diff --git a/src/fragment_store/serializer.hpp b/src/fragment_store/serializer.hpp index bf12c4e..1f94f0d 100644 --- a/src/fragment_store/serializer.hpp +++ b/src/fragment_store/serializer.hpp @@ -2,6 +2,7 @@ #include #include +#include #include @@ -16,12 +17,41 @@ struct SerializerCallbacks { using deserialize_json_fn = bool(*)(FragmentHandle fh, const nlohmann::json& in); entt::dense_map _deserl_json; + template + static bool component_emplace_or_replace_json(FragmentHandle fh, const nlohmann::json& j) { + if constexpr (std::is_empty_v) { + fh.emplace_or_replace(); // assert empty json? + } else { + fh.emplace_or_replace(static_cast(j)); + } + return true; + } + + template + static bool component_get_json(const FragmentHandle fh, nlohmann::json& j) { + if (fh.all_of()) { + if constexpr (!std::is_empty_v) { + j = fh.get(); + } + return true; + } + + return false; + } + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info); + template void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerSerializerJson(fn, type_info); } void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info); + template - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerDeSerializerJson(fn, type_info); } + void registerDeSerializerJson( + deserialize_json_fn fn = component_emplace_or_replace_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerDeSerializerJson(fn, type_info); + } }; From 73d1d6514255103d019c795dabc8f0895ffd3833 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 15 Feb 2024 19:06:56 +0100 Subject: [PATCH 12/98] further serializer refactoring --- src/CMakeLists.txt | 1 - src/fragment_store/fragment_store.cpp | 60 ++++++++----------- src/fragment_store/fragment_store.hpp | 5 +- src/fragment_store/message_fragment_store.cpp | 30 +++++----- src/fragment_store/message_fragment_store.hpp | 2 +- src/fragment_store/serializer.cpp | 10 ---- src/fragment_store/serializer.hpp | 51 +++++++++------- src/fragment_store/test_fragstore.cpp | 20 ++----- 8 files changed, 78 insertions(+), 101 deletions(-) delete mode 100644 src/fragment_store/serializer.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9acd1e3..6e3e87e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,7 +6,6 @@ add_library(fragment_store ./fragment_store/meta_components.hpp ./fragment_store/meta_components_id.inl ./fragment_store/serializer.hpp - ./fragment_store/serializer.cpp ./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.cpp ) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 97dd67a..c0aa02c 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -291,7 +291,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::functionsecond(storage.value(fid), meta_data[storage.type().hash()]); //} else if (meta_type == MetaFileType::TEXT_JSON) { - s_cb_it->second(storage.value(fid), meta_data[storage.type().name()]); + s_cb_it->second({_reg, fid}, meta_data[storage.type().name()]); //} } @@ -521,52 +521,44 @@ void FragmentStore::scanStoragePathAsync(std::string path) { scanStoragePath(path); // TODO: make async and post result } -static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) { - if (comp == nullptr) { - return false; - } - - auto& r_comp = *reinterpret_cast(comp); - - out = static_cast>(r_comp.enc); - +static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) { + out = static_cast>( + fh.get().enc + ); return true; } static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) { - // TODO: this is ugly in multiple places - try { - fh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - } catch(...) { - return false; - } + fh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); return true; } -static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { - if (comp == nullptr) { - return false; - } - - auto& r_comp = *reinterpret_cast(comp); - - out = static_cast>(r_comp.comp); - +static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) { + out = static_cast>( + fh.get().comp + ); return true; } +static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) { + fh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + +// TODO: dserl comp type + void FragmentStore::registerSerializers(void) { _sc.registerSerializerJson(serl_json_data_enc_type); _sc.registerDeSerializerJson(deserl_json_data_enc_type); _sc.registerSerializerJson(serl_json_data_comp_type); - - std::cout << "registered serl text json cbs:\n"; - for (const auto& [type_id, _] : _sc._serl_json) { - std::cout << " " << type_id << "\n"; - } + _sc.registerDeSerializerJson(deserl_json_data_comp_type); } diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 1025f9e..101a933 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -17,9 +17,6 @@ #include #include -// fwd -struct SerializerCallbacks; - struct FragmentStore : public FragmentStoreI { FragmentRegistry _reg; @@ -31,7 +28,7 @@ struct FragmentStore : public FragmentStoreI { uint64_t _memory_budget {10u*1024u*1024u}; uint64_t _memory_usage {0u}; - SerializerCallbacks _sc; + SerializerCallbacks _sc; FragmentStore(void); FragmentStore(std::array session_uuid_namespace); diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 07383a7..bc7804b 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -214,37 +214,37 @@ MessageFragmentStore::MessageFragmentStore( _rmm.subscribe(this, RegistryMessageModel_Event::message_updated); _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); - _fs._sc.registerSerializerJson(serl_json_msg_ts_range); + _fs._sc.registerSerializerJson(); _fs._sc.registerDeSerializerJson(); - _fs._sc.registerSerializerJson(serl_json_msg_c_id); + _fs._sc.registerSerializerJson(); _fs._sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); // files //_sc.registerSerializerJson() - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(serl_json_default); + _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); _fs.scanStoragePath("test_message_store/"); @@ -292,7 +292,7 @@ float MessageFragmentStore::tick(float time_delta) { continue; } - s_cb_it->second(storage.value(m), j_entry[storage.type().name()]); + s_cb_it->second({*reg, m}, j_entry[storage.type().name()]); } } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 58ded02..8039470 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -36,7 +36,7 @@ namespace Fragment::Components { }; } // Fragment::Components -struct MessageSerializerCallbacks : public SerializerCallbacks { +struct MessageSerializerCallbacks : public SerializerCallbacks { // TODO: add contact and message reg, so entities can be looked up and be converted to fragment uids OR persistent ids }; diff --git a/src/fragment_store/serializer.cpp b/src/fragment_store/serializer.cpp deleted file mode 100644 index 1f3bfe4..0000000 --- a/src/fragment_store/serializer.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "./serializer.hpp" - -void SerializerCallbacks::registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { - _serl_json[type_info.hash()] = fn; -} - -void SerializerCallbacks::registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { - _deserl_json[type_info.hash()] = fn; -} - diff --git a/src/fragment_store/serializer.hpp b/src/fragment_store/serializer.hpp index 1f94f0d..a1daa98 100644 --- a/src/fragment_store/serializer.hpp +++ b/src/fragment_store/serializer.hpp @@ -6,32 +6,24 @@ #include -#include "./fragment_store_i.hpp" - +template struct SerializerCallbacks { + using Registry = entt::basic_registry; + using Handle = entt::basic_handle; + // nlohmann // json/msgpack - using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); + using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out); entt::dense_map _serl_json; - using deserialize_json_fn = bool(*)(FragmentHandle fh, const nlohmann::json& in); + using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in); entt::dense_map _deserl_json; template - static bool component_emplace_or_replace_json(FragmentHandle fh, const nlohmann::json& j) { - if constexpr (std::is_empty_v) { - fh.emplace_or_replace(); // assert empty json? - } else { - fh.emplace_or_replace(static_cast(j)); - } - return true; - } - - template - static bool component_get_json(const FragmentHandle fh, nlohmann::json& j) { - if (fh.all_of()) { + static bool component_get_json(const Handle h, nlohmann::json& j) { + if (h.template all_of()) { if constexpr (!std::is_empty_v) { - j = fh.get(); + j = h.template get(); } return true; } @@ -39,12 +31,31 @@ struct SerializerCallbacks { return false; } - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info); + template + static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) { + if constexpr (std::is_empty_v) { + h.template emplace_or_replace(); // assert empty json? + } else { + h.template emplace_or_replace(static_cast(j)); + } + return true; + } + + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { + _serl_json[type_info.hash()] = fn; + } template - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerSerializerJson(fn, type_info); } + void registerSerializerJson( + serialize_json_fn fn = component_get_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerSerializerJson(fn, type_info); + } - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info); + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { + _deserl_json[type_info.hash()] = fn; + } template void registerDeSerializerJson( diff --git a/src/fragment_store/test_fragstore.cpp b/src/fragment_store/test_fragstore.cpp index 581f64a..7136643 100644 --- a/src/fragment_store/test_fragstore.cpp +++ b/src/fragment_store/test_fragstore.cpp @@ -11,28 +11,16 @@ namespace Components { uint64_t begin {0}; uint64_t end {1000}; }; + + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end) } // Components -static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { - if (comp == nullptr) { - return false; - } - - out = nlohmann::json::object(); - - auto& r_comp = *reinterpret_cast(comp); - - out["begin"] = r_comp.begin; - out["end"] = r_comp.end; - - return true; -} - int main(void) { FragmentStore fs; fs._default_store_path = "test_store/"; - fs._sc.registerSerializerJson(serl_json_msg_ts_range); + fs._sc.registerSerializerJson(); + fs._sc.registerDeSerializerJson(); const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9}); From 1bfd04680e57d20b7b4e6edbeaaa85642c73d9f6 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 16 Feb 2024 16:29:37 +0100 Subject: [PATCH 13/98] refactor message serializer to allow access to eg contacts --- src/CMakeLists.txt | 2 + src/fragment_store/message_fragment_store.cpp | 4 +- src/fragment_store/message_fragment_store.hpp | 6 +- src/fragment_store/message_serializer.cpp | 94 +++++++++++++++++++ src/fragment_store/message_serializer.hpp | 85 +++++++++++++++++ 5 files changed, 185 insertions(+), 6 deletions(-) create mode 100644 src/fragment_store/message_serializer.cpp create mode 100644 src/fragment_store/message_serializer.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6e3e87e..6540420 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,6 +19,8 @@ target_link_libraries(fragment_store PUBLIC ######################################## add_library(message_fragment_store + ./fragment_store/message_serializer.hpp + ./fragment_store/message_serializer.cpp ./fragment_store/message_fragment_store.hpp ./fragment_store/message_fragment_store.cpp ) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index bc7804b..cecc02e 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -209,7 +209,7 @@ MessageFragmentStore::MessageFragmentStore( Contact3Registry& cr, RegistryMessageModel& rmm, FragmentStore& fs -) : _cr(cr), _rmm(rmm), _fs(fs) { +) : _cr(cr), _rmm(rmm), _fs(fs), _sc{_cr, {}, {}} { _rmm.subscribe(this, RegistryMessageModel_Event::message_construct); _rmm.subscribe(this, RegistryMessageModel_Event::message_updated); _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); @@ -292,7 +292,7 @@ float MessageFragmentStore::tick(float time_delta) { continue; } - s_cb_it->second({*reg, m}, j_entry[storage.type().name()]); + s_cb_it->second(_sc, {*reg, m}, j_entry[storage.type().name()]); } } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 8039470..559f477 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -4,6 +4,8 @@ #include "./fragment_store_i.hpp" #include "./fragment_store.hpp" +#include "./message_serializer.hpp" + #include #include @@ -36,10 +38,6 @@ namespace Fragment::Components { }; } // Fragment::Components -struct MessageSerializerCallbacks : public SerializerCallbacks { - // TODO: add contact and message reg, so entities can be looked up and be converted to fragment uids OR persistent ids -}; - // handles fragments for messages // on new message: assign fuid // on new and update: mark as fragment dirty diff --git a/src/fragment_store/message_serializer.cpp b/src/fragment_store/message_serializer.cpp new file mode 100644 index 0000000..98c6778 --- /dev/null +++ b/src/fragment_store/message_serializer.cpp @@ -0,0 +1,94 @@ +#include "./message_serializer.hpp" + +#include +#include + +#include + +#include + +template<> +bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) { + const Contact3 c = h.get().c; + if (!msc.cr.valid(c)) { + // while this is invalid registry state, it is valid serialization + j = nullptr; + std::cerr << "MSC warning: encountered invalid contact\n"; + return true; + } + + if (!msc.cr.all_of(c)) { + // unlucky, this contact is purely ephemeral + j = nullptr; + std::cerr << "MSC warning: encountered contact without ID\n"; + return true; + } + + j = nlohmann::json::binary(msc.cr.get(c).data); + + return true; +} +template<> +bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) { + if (j.is_null()) { + std::cerr << "MSC warning: encountered null contact\n"; + h.emplace_or_replace(); + return true; + } + + const auto id = static_cast>(j); + + // TODO: id lookup table, this is very inefficent + for (const auto& [c_it, id_it] : msc.cr.view().each()) { + if (id == id_it.data) { + h.emplace_or_replace(c_it); + return true; + } + } + + // TODO: should we really return false if the contact is unknown?? + return false; +} + +template<> +bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) { + const Contact3 c = h.get().c; + if (!msc.cr.valid(c)) { + // while this is invalid registry state, it is valid serialization + j = nullptr; + std::cerr << "MSC warning: encountered invalid contact\n"; + return true; + } + + if (!msc.cr.all_of(c)) { + // unlucky, this contact is purely ephemeral + j = nullptr; + std::cerr << "MSC warning: encountered contact without ID\n"; + return true; + } + + j = nlohmann::json::binary(msc.cr.get(c).data); + + return true; +} +template<> +bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) { + if (j.is_null()) { + std::cerr << "MSC warning: encountered null contact\n"; + h.emplace_or_replace(); + return true; + } + + const auto id = static_cast>(j); + + // TODO: id lookup table, this is very inefficent + for (const auto& [c_it, id_it] : msc.cr.view().each()) { + if (id == id_it.data) { + h.emplace_or_replace(c_it); + return true; + } + } + + // TODO: should we really return false if the contact is unknown?? + return false; +} diff --git a/src/fragment_store/message_serializer.hpp b/src/fragment_store/message_serializer.hpp new file mode 100644 index 0000000..70d6fcd --- /dev/null +++ b/src/fragment_store/message_serializer.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include + +#include + +#include + +struct MessageSerializerCallbacks { + using Registry = Message3Registry; + using Handle = Message3Handle; + + Contact3Registry& cr; + + // nlohmann + // json/msgpack + using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out); + entt::dense_map _serl_json; + + using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in); + entt::dense_map _deserl_json; + + template + static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) { + if (h.template all_of()) { + if constexpr (!std::is_empty_v) { + j = h.template get(); + } + return true; + } + + return false; + } + + template + static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) { + if constexpr (std::is_empty_v) { + h.template emplace_or_replace(); // assert empty json? + } else { + h.template emplace_or_replace(static_cast(j)); + } + return true; + } + + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { + _serl_json[type_info.hash()] = fn; + } + + template + void registerSerializerJson( + serialize_json_fn fn = component_get_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerSerializerJson(fn, type_info); + } + + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { + _deserl_json[type_info.hash()] = fn; + } + + template + void registerDeSerializerJson( + deserialize_json_fn fn = component_emplace_or_replace_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerDeSerializerJson(fn, type_info); + } +}; + +// fwd +namespace Message::Components { +struct ContactFrom; +struct ContactTo; +} + +// make specializations known +template<> +bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j); +template<> +bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j); +template<> +bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j); +template<> +bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j); From 2e7d5538d12e4171469b123073c123e113f40cb5 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 16 Feb 2024 18:56:25 +0100 Subject: [PATCH 14/98] fragment events + 256bit uuids --- src/CMakeLists.txt | 1 + src/fragment_store/fragment_store.cpp | 62 ++++++++++++++----- src/fragment_store/fragment_store.hpp | 8 +-- src/fragment_store/fragment_store_i.cpp | 23 +++++++ src/fragment_store/fragment_store_i.hpp | 52 +++++++++++++++- src/fragment_store/message_fragment_store.cpp | 4 ++ src/fragment_store/message_fragment_store.hpp | 1 + 7 files changed, 127 insertions(+), 24 deletions(-) create mode 100644 src/fragment_store/fragment_store_i.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6540420..f793b77 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) add_library(fragment_store ./fragment_store/fragment_store_i.hpp + ./fragment_store/fragment_store_i.cpp ./fragment_store/types.hpp ./fragment_store/meta_components.hpp ./fragment_store/meta_components_id.inl diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index c0aa02c..3c0a4cc 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -33,23 +33,34 @@ FragmentStore::FragmentStore(void) { { // random namespace const auto num0 = _rng(); const auto num1 = _rng(); + const auto num2 = _rng(); + const auto num3 = _rng(); - _session_uuid_namespace[0] = (num0 >> 0) & 0xff; - _session_uuid_namespace[1] = (num0 >> 8) & 0xff; - _session_uuid_namespace[2] = (num0 >> 16) & 0xff; - _session_uuid_namespace[3] = (num0 >> 24) & 0xff; + _session_uuid_namespace[0+0] = (num0 >> 0) & 0xff; + _session_uuid_namespace[0+1] = (num0 >> 8) & 0xff; + _session_uuid_namespace[0+2] = (num0 >> 16) & 0xff; + _session_uuid_namespace[0+3] = (num0 >> 24) & 0xff; - _session_uuid_namespace[4] = (num1 >> 0) & 0xff; - _session_uuid_namespace[5] = (num1 >> 8) & 0xff; - _session_uuid_namespace[6] = (num1 >> 16) & 0xff; - _session_uuid_namespace[7] = (num1 >> 24) & 0xff; + _session_uuid_namespace[4+0] = (num1 >> 0) & 0xff; + _session_uuid_namespace[4+1] = (num1 >> 8) & 0xff; + _session_uuid_namespace[4+2] = (num1 >> 16) & 0xff; + _session_uuid_namespace[4+3] = (num1 >> 24) & 0xff; + _session_uuid_namespace[8+0] = (num2 >> 0) & 0xff; + _session_uuid_namespace[8+1] = (num2 >> 8) & 0xff; + _session_uuid_namespace[8+2] = (num2 >> 16) & 0xff; + _session_uuid_namespace[8+3] = (num2 >> 24) & 0xff; + + _session_uuid_namespace[12+0] = (num3 >> 0) & 0xff; + _session_uuid_namespace[12+1] = (num3 >> 8) & 0xff; + _session_uuid_namespace[12+2] = (num3 >> 16) & 0xff; + _session_uuid_namespace[12+3] = (num3 >> 24) & 0xff; } registerSerializers(); } FragmentStore::FragmentStore( - std::array session_uuid_namespace + std::array session_uuid_namespace ) : _session_uuid_namespace(std::move(session_uuid_namespace)) { registerSerializers(); } @@ -58,22 +69,34 @@ FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) { return {_reg, fid}; } -std::vector FragmentStore::generateNewUID(std::array& uuid_namespace) { +std::vector FragmentStore::generateNewUID(std::array& uuid_namespace) { std::vector new_uid(uuid_namespace.cbegin(), uuid_namespace.cend()); - new_uid.resize(new_uid.size() + 8); + new_uid.resize(new_uid.size() + 16); const auto num0 = _rng(); const auto num1 = _rng(); + const auto num2 = _rng(); + const auto num3 = _rng(); new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff; new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff; new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff; new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff; - new_uid[uuid_namespace.size()+4] = (num1 >> 0) & 0xff; - new_uid[uuid_namespace.size()+5] = (num1 >> 8) & 0xff; - new_uid[uuid_namespace.size()+6] = (num1 >> 16) & 0xff; - new_uid[uuid_namespace.size()+7] = (num1 >> 24) & 0xff; + new_uid[uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff; + new_uid[uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff; + new_uid[uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff; + new_uid[uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff; + + new_uid[uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff; + new_uid[uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff; + new_uid[uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff; + new_uid[uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff; + + new_uid[uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff; + new_uid[uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff; + new_uid[uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff; + new_uid[uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff; return new_uid; } @@ -114,6 +137,8 @@ FragmentID FragmentStore::newFragmentMemoryOwned( // TODO: memory comp _reg.emplace>>(new_frag) = std::move(new_data); + throwEventConstruct(new_frag); + return new_frag; } @@ -159,6 +184,8 @@ FragmentID FragmentStore::newFragmentFile( _reg.emplace(new_frag, mft); + throwEventConstruct(new_frag); + // meta needs to be synced to file std::function empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; }; if (!syncToStorage(new_frag, empty_data_cb)) { @@ -476,6 +503,7 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { if (it.meta_ext == ".meta.msgpack") { // uh // read binary header + assert(false); } else if (it.meta_ext == ".meta.json") { std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext); if (!file.is_open()) { @@ -506,6 +534,8 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; } } + // throw new frag event here + throwEventConstruct(fh); count++; } else { assert(false); @@ -553,8 +583,6 @@ static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& return true; } -// TODO: dserl comp type - void FragmentStore::registerSerializers(void) { _sc.registerSerializerJson(serl_json_data_enc_type); _sc.registerDeSerializerJson(deserl_json_data_enc_type); diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 101a933..0aec782 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -18,10 +18,8 @@ #include struct FragmentStore : public FragmentStoreI { - FragmentRegistry _reg; - std::minstd_rand _rng{std::random_device{}()}; - std::array _session_uuid_namespace; + std::array _session_uuid_namespace; std::string _default_store_path; @@ -31,7 +29,7 @@ struct FragmentStore : public FragmentStoreI { SerializerCallbacks _sc; FragmentStore(void); - FragmentStore(std::array session_uuid_namespace); + FragmentStore(std::array session_uuid_namespace); // HACK: get access to the reg FragmentHandle fragmentHandle(FragmentID fid); @@ -39,7 +37,7 @@ struct FragmentStore : public FragmentStoreI { // TODO: make the frags ref counted // TODO: check for exising - std::vector generateNewUID(std::array& uuid_namespace); + std::vector generateNewUID(std::array& uuid_namespace); std::vector generateNewUID(void); // ========== new fragment ========== diff --git a/src/fragment_store/fragment_store_i.cpp b/src/fragment_store/fragment_store_i.cpp new file mode 100644 index 0000000..75a9e44 --- /dev/null +++ b/src/fragment_store/fragment_store_i.cpp @@ -0,0 +1,23 @@ +#include "./fragment_store_i.hpp" + +#include + +void FragmentStoreI::throwEventConstruct(const FragmentID fid) { + std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n"; + dispatch( + FragmentStore_Event::fragment_construct, + Fragment::Events::FragmentConstruct{ + FragmentHandle{_reg, fid} + } + ); +} + +void FragmentStoreI::throwEventUpdate(const FragmentID fid) { + std::cout << "FSI debug: event updated " << entt::to_integral(fid) << "\n"; + dispatch( + FragmentStore_Event::fragment_updated, + Fragment::Events::FragmentUpdated{ + FragmentHandle{_reg, fid} + } + ); +} diff --git a/src/fragment_store/fragment_store_i.hpp b/src/fragment_store/fragment_store_i.hpp index be3f9c1..8dffdf3 100644 --- a/src/fragment_store/fragment_store_i.hpp +++ b/src/fragment_store/fragment_store_i.hpp @@ -1,6 +1,9 @@ #pragma once +#include + #include +#include #include @@ -9,7 +12,52 @@ enum class FragmentID : uint32_t {}; using FragmentRegistry = entt::basic_registry; using FragmentHandle = entt::basic_handle; -struct FragmentStoreI { - virtual ~FragmentStoreI(void) {} +namespace Fragment::Events { + + struct FragmentConstruct { + const FragmentHandle e; + }; + struct FragmentUpdated { + const FragmentHandle e; + }; + //struct MessageDestory { + //const Message3Handle e; + //}; + +} // Fragment::Events + +enum class FragmentStore_Event : uint32_t { + fragment_construct, + fragment_updated, + //message_destroy, + + MAX +}; + +struct FragmentStoreEventI { + using enumType = FragmentStore_Event; + + virtual ~FragmentStoreEventI(void) {} + + virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; } + virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; } + //virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; } + + // mm3 + // send text + // send file path +}; +using FragmentStoreEventProviderI = EventProviderI; + +struct FragmentStoreI : public FragmentStoreEventProviderI { + static constexpr const char* version {"1"}; + + FragmentRegistry _reg; + + virtual ~FragmentStoreI(void) {} + + void throwEventConstruct(const FragmentID fid); + void throwEventUpdate(const FragmentID fid); + //void throwEventDestroy(); }; diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index cecc02e..fffa084 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -185,6 +185,10 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{msg_ts, msg_ts, fragment_uid}); std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n"; + + _fs_ignore_event = true; + _fs.throwEventConstruct(fh); + _fs_ignore_event = false; } // if this is still empty, something is very wrong and we exit here diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 559f477..8093948 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -47,6 +47,7 @@ class MessageFragmentStore : public RegistryMessageModelEventI { Contact3Registry& _cr; RegistryMessageModel& _rmm; FragmentStore& _fs; + bool _fs_ignore_event {false}; // for message components only MessageSerializerCallbacks _sc; From 97aedca844b90bd2e0b5237d711345c1ae96acae Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 16 Feb 2024 19:55:07 +0100 Subject: [PATCH 15/98] scan laters --- src/fragment_store/message_fragment_store.cpp | 6 ++++-- src/fragment_store/message_fragment_store.hpp | 2 ++ src/main_screen.cpp | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index fffa084..e61112d 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -250,8 +250,6 @@ MessageFragmentStore::MessageFragmentStore( _sc.registerDeSerializerJson(); _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); - - _fs.scanStoragePath("test_message_store/"); } MessageFragmentStore::~MessageFragmentStore(void) { @@ -313,6 +311,10 @@ float MessageFragmentStore::tick(float time_delta) { return 1000.f*60.f*60.f; } +void MessageFragmentStore::triggerScan(void) { + _fs.scanStoragePath("test_message_store/"); +} + bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) { handleMessage(e.e); return false; diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 8093948..39357a5 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -71,6 +71,8 @@ class MessageFragmentStore : public RegistryMessageModelEventI { float tick(float time_delta); + void triggerScan(void); + protected: // rmm bool onEvent(const Message::Events::MessageConstruct& e) override; bool onEvent(const Message::Events::MessageUpdated& e) override; diff --git a/src/main_screen.cpp b/src/main_screen.cpp index a17fd2d..ba61d9c 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -80,6 +80,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri } conf.dump(); + + mfs.triggerScan(); // HACK: after plugins and tox contacts got loaded } MainScreen::~MainScreen(void) { From 3d0863ff9a50082cba303f450c4560a6557f9c1a Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 11:49:03 +0100 Subject: [PATCH 16/98] basically working, but some dup glitch is still there --- src/fragment_store/fragment_store.cpp | 62 ++++++- src/fragment_store/fragment_store.hpp | 6 + src/fragment_store/message_fragment_store.cpp | 166 ++++++++++++++---- src/fragment_store/message_fragment_store.hpp | 7 +- src/fragment_store/message_serializer.cpp | 4 +- 5 files changed, 203 insertions(+), 42 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 3c0a4cc..98a08e0 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -370,6 +370,64 @@ bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uin return syncToStorage(fid, fn_cb); } +bool FragmentStore::loadFromStorage(FragmentID fid, std::function& data_cb) { + if (!_reg.valid(fid)) { + return false; + } + + if (!_reg.all_of(fid)) { + // not a file fragment? + // TODO: memory fragments + return false; + } + + const auto& frag_path = _reg.get(fid).path; + + // TODO: check if metadata dirty? + // TODO: what if file changed on disk? + + std::cout << "FS: loading fragment '" << frag_path << "'\n"; + + std::ifstream data_file{ + frag_path, + std::ios::in | std::ios::binary // always binary, also for text + }; + + if (!data_file.is_open()) { + std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; + // error + return false; + } + + std::array buffer; + uint64_t buffer_actual_size {0}; + do { + data_file.read(reinterpret_cast(buffer.data()), buffer.size()); + buffer_actual_size = data_file.gcount(); + + if (buffer_actual_size == 0) { + break; + } + + data_cb(buffer.data(), buffer_actual_size); + } while (buffer_actual_size == buffer.size() && !data_file.eof()); + + return true; +} + +nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) { + std::vector tmp_buffer; + std::function cb = [&tmp_buffer](const uint8_t* buffer, const uint64_t buffer_size) { + tmp_buffer.insert(tmp_buffer.end(), buffer, buffer+buffer_size); + }; + + if (!loadFromStorage(fid, cb)) { + return nullptr; + } + + return nlohmann::json::parse(tmp_buffer); +} + size_t FragmentStore::scanStoragePath(std::string_view path) { if (path.empty()) { path = _default_store_path; @@ -505,7 +563,7 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { // read binary header assert(false); } else if (it.meta_ext == ".meta.json") { - std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext); + std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); if (!file.is_open()) { std::cout << "FS error: failed opening meta " << it.frag_path << "\n"; continue; @@ -523,6 +581,8 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { FragmentHandle fh{_reg, _reg.create()}; fh.emplace(hex2bin(it.id_str)); + fh.emplace(it.frag_path.generic_u8string()); + for (const auto& [k, v] : j.items()) { // type id from string hash const auto type_id = entt::hashed_string(k.data(), k.size()); diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 0aec782..811b4be 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -82,6 +82,12 @@ struct FragmentStore : public FragmentStoreI { bool syncToStorage(FragmentID fid, std::function& data_cb); bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size); + // ========== load fragment data from storage ========== + using read_from_storage_put_data_cb = void(const uint8_t* buffer, const uint64_t buffer_size); + bool loadFromStorage(FragmentID fid, std::function& data_cb); + // convenience function + nlohmann::json loadFromStorageNJ(FragmentID fid); + // fragment discovery? // returns number of new fragments size_t scanStoragePath(std::string_view path); diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index e61112d..b81ca07 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -7,10 +7,13 @@ #include +#include #include #include #include +// https://youtu.be/CU2exyhYPfA + namespace Message::Components { // ctx @@ -46,43 +49,6 @@ namespace Fragment::Components { NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id) } // Fragment::Components -template -static bool serl_json_default(void* comp, nlohmann::json& out) { - if constexpr (!std::is_empty_v) { - out = *reinterpret_cast(comp); - } // do nothing if empty type - return true; -} - -static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) { - if (comp == nullptr) { - return false; - } - - out = nlohmann::json::object(); - - auto& r_comp = *reinterpret_cast(comp); - - out["begin"] = r_comp.begin; - out["end"] = r_comp.end; - - return true; -} - -static bool serl_json_msg_c_id(void* comp, nlohmann::json& out) { - if (comp == nullptr) { - return false; - } - - out = nlohmann::json::object(); - - auto& r_comp = *reinterpret_cast(comp); - - out["id"] = r_comp.id; - - return true; -} - void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (!static_cast(m)) { return; // huh? @@ -95,6 +61,30 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (!m.registry()->ctx().contains()) { // first message in this reg m.registry()->ctx().emplace(); + + // TODO: move this to async + // new reg -> load all fragments for this contact (for now, ranges later) + for (const auto& [fid, tsrange, fmc] : _fs._reg.view().each()) { + Contact3 frag_contact = entt::null; + // TODO: id lookup table, this is very inefficent + for (const auto& [c_it, id_it] : _cr.view().each()) { + if (fmc.id == id_it.data) { + //h.emplace_or_replace(c_it); + //return true; + frag_contact = c_it; + break; + } + } + if (!_cr.valid(frag_contact)) { + // unkown contact + continue; + } + + // registry is the same as the one the message event is for + if (static_cast(_rmm).get(frag_contact) == m.registry()) { + loadFragment(*m.registry(), FragmentHandle{_fs._reg, fid}); + } + } } auto& fuid_open = m.registry()->ctx().get().fuid_open; @@ -209,6 +199,60 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // on new and update: mark as fragment dirty } +void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) { + std::cout << "MFS: loadFragment\n"; + const auto j = _fs.loadFromStorageNJ(fh); + + if (!j.is_array()) { + // wrong data + return; + } + + for (const auto& j_entry : j) { + auto new_real_msg = Message3Handle{reg, reg.create()}; + // load into staging reg + for (const auto& [k, v] : j_entry.items()) { + std::cout << "K:" << k << " V:" << v.dump() << "\n"; + const auto type_id = entt::hashed_string(k.data(), k.size()); + const auto deserl_fn_it = _sc._deserl_json.find(type_id); + if (deserl_fn_it != _sc._deserl_json.cend()) { + try { + if (!deserl_fn_it->second(_sc, new_real_msg, v)) { + std::cerr << "MFS error: failed deserializing '" << k << "'\n"; + } + } catch(...) { + std::cerr << "MFS error: failed deserializing (threw) '" << k << "'\n"; + } + } else { + std::cerr << "MFS warning: missing deserializer for meta key '" << k << "'\n"; + } + } + + new_real_msg.emplace_or_replace(fh.get()); + + // TODO: dup checking + const bool is_dup {false}; + + // dup check (hacky, specific to protocols) + if (is_dup) { + // -> merge with preexisting + // -> throw update + reg.destroy(new_real_msg); + //_rmm.throwEventUpdate(reg, new_real_msg); + } else { + if (!new_real_msg.all_of()) { + // does not have needed components to be stand alone + reg.destroy(new_real_msg); + std::cerr << "MFS warning: message with missing basic compoments\n"; + continue; + } + + // -> throw create + _rmm.throwEventConstruct(reg, new_real_msg); + } + } +} + MessageFragmentStore::MessageFragmentStore( Contact3Registry& cr, RegistryMessageModel& rmm, @@ -250,6 +294,8 @@ MessageFragmentStore::MessageFragmentStore( _sc.registerDeSerializerJson(); _sc.registerSerializerJson(); _sc.registerDeSerializerJson(); + + _fs.subscribe(this, FragmentStore_Event::fragment_construct); } MessageFragmentStore::~MessageFragmentStore(void) { @@ -302,7 +348,7 @@ float MessageFragmentStore::tick(float time_delta) { //nlohmann::json::to_msgpack(j); auto j_dump = j.dump(2, ' ', true); if (_fs.syncToStorage(fid, reinterpret_cast(j_dump.data()), j_dump.size())) { - std::cout << "MFS: dumped " << j_dump << "\n"; + //std::cout << "MFS: dumped " << j_dump << "\n"; // succ _fuid_save_queue.pop(); } @@ -327,3 +373,47 @@ bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) { // TODO: handle deletes? diff between unload? +bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) { + if (_fs_ignore_event) { + return false; // skip self + } + + if (!e.e.all_of()) { + return false; // not for us + } + + // TODO: are we sure it is a *new* fragment? + + //std::cout << "MFS: got frag for us!\n"; + + Contact3 frag_contact = entt::null; + { // get contact + const auto& frag_contact_id = e.e.get().id; + // TODO: id lookup table, this is very inefficent + for (const auto& [c_it, id_it] : _cr.view().each()) { + if (frag_contact_id == id_it.data) { + //h.emplace_or_replace(c_it); + //return true; + frag_contact = c_it; + break; + } + } + if (!_cr.valid(frag_contact)) { + // unkown contact + return false; + } + } + + // only load if msg reg open + auto* msg_reg = static_cast(_rmm).get(frag_contact); + if (msg_reg == nullptr) { + // msg reg not created yet + return false; + } + + // TODO: should this be done async / on tick() instead of on event? + loadFragment(*msg_reg, e.e); + + return false; +} + diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 39357a5..cc60870 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -42,7 +42,7 @@ namespace Fragment::Components { // on new message: assign fuid // on new and update: mark as fragment dirty // on delete: mark as fragment dirty? -class MessageFragmentStore : public RegistryMessageModelEventI { +class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentStoreEventI { protected: Contact3Registry& _cr; RegistryMessageModel& _rmm; @@ -54,6 +54,8 @@ class MessageFragmentStore : public RegistryMessageModelEventI { void handleMessage(const Message3Handle& m); + void loadFragment(Message3Registry& reg, FragmentHandle fh); + struct QueueEntry final { uint64_t ts_since_dirty{0}; std::vector id; @@ -76,5 +78,8 @@ class MessageFragmentStore : public RegistryMessageModelEventI { protected: // rmm bool onEvent(const Message::Events::MessageConstruct& e) override; bool onEvent(const Message::Events::MessageUpdated& e) override; + + protected: // fs + bool onEvent(const Fragment::Events::FragmentConstruct& e) override; }; diff --git a/src/fragment_store/message_serializer.cpp b/src/fragment_store/message_serializer.cpp index 98c6778..78362e9 100644 --- a/src/fragment_store/message_serializer.cpp +++ b/src/fragment_store/message_serializer.cpp @@ -36,7 +36,7 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json>(j); + const auto id = static_cast>(j.is_binary()?j:j["bytes"]); // TODO: id lookup table, this is very inefficent for (const auto& [c_it, id_it] : msc.cr.view().each()) { @@ -79,7 +79,7 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json>(j); + const auto id = static_cast>(j.is_binary()?j:j["bytes"]); // TODO: id lookup table, this is very inefficent for (const auto& [c_it, id_it] : msc.cr.view().each()) { From 24dc5a03f3790741aeba5154912d16083a0f5925 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 12:21:26 +0100 Subject: [PATCH 17/98] fix dup on write --- src/fragment_store/message_fragment_store.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index b81ca07..c1050a9 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -50,6 +50,12 @@ namespace Fragment::Components { } // Fragment::Components void MessageFragmentStore::handleMessage(const Message3Handle& m) { + if (_fs_ignore_event) { + // message event because of us loading a fragment, ignore + // TODO: this barely makes a difference + return; + } + if (!static_cast(m)) { return; // huh? } @@ -319,10 +325,15 @@ float MessageFragmentStore::tick(float time_delta) { continue; } + // require msg and file for now if (!reg->any_of(m)) { continue; } + if (_fuid_save_queue.front().id != reg->get(m).v) { + continue; // not ours + } + auto& j_entry = j.emplace_back(nlohmann::json::object()); for (const auto& [type_id, storage] : reg->storage()) { From 20f7c6d0116b0b98b5c0f13ebef4b394b7f6106c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 14:44:50 +0100 Subject: [PATCH 18/98] add dup check, would work for ngc if we saved tox group msg id yet --- src/fragment_store/message_fragment_store.cpp | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index c1050a9..c886c21 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -205,6 +206,8 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // on new and update: mark as fragment dirty } +// assumes new frag +// need update from frag void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) { std::cout << "MFS: loadFragment\n"; const auto j = _fs.loadFromStorageNJ(fh); @@ -236,11 +239,28 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh new_real_msg.emplace_or_replace(fh.get()); - // TODO: dup checking - const bool is_dup {false}; - // dup check (hacky, specific to protocols) - if (is_dup) { + Message3 dup_msg {entt::null}; + { + // get comparator from contact + if (reg.ctx().contains()) { + const auto c = reg.ctx().get(); + if (_cr.all_of(c)) { + auto& comp = _cr.get(c).comp; + // walking EVERY existing message OOF + // this needs optimizing + for (const Message3 other_msg : reg.view()) { + if (comp({reg, other_msg}, new_real_msg)) { + // dup + dup_msg = other_msg; + break; + } + } + } + } + } + + if (reg.valid(dup_msg)) { // -> merge with preexisting // -> throw update reg.destroy(new_real_msg); From 4ec87337c83ba5f663c9d5eb0e0a27410914884f Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 16:38:46 +0100 Subject: [PATCH 19/98] reverse message write order --- src/fragment_store/message_fragment_store.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index c886c21..7fe8d6e 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -340,7 +340,9 @@ float MessageFragmentStore::tick(float time_delta) { // TODO: does every message have ts? auto msg_view = reg->view(); // we also assume all messages have fuid (hack: call handle when not?) - for (const Message3 m : msg_view) { + for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) { + const Message3 m = *it; + if (!reg->all_of(m)) { continue; } From 7ac62274f426bfb0d680020ebea6464872bc466f Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 16:50:56 +0100 Subject: [PATCH 20/98] move json around and disable files for now --- src/CMakeLists.txt | 2 ++ src/fragment_store/message_fragment_store.cpp | 32 ++++++------------- src/json/message_components.hpp | 27 ++++++++++++++++ 3 files changed, 39 insertions(+), 22 deletions(-) create mode 100644 src/json/message_components.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f793b77..24bd59a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,6 +9,8 @@ add_library(fragment_store ./fragment_store/serializer.hpp ./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.cpp + + ./json/message_components.hpp # TODO: move ) target_link_libraries(fragment_store PUBLIC diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 7fe8d6e..e8af12a 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -1,5 +1,7 @@ #include "./message_fragment_store.hpp" +#include "../json/message_components.hpp" + #include #include @@ -29,20 +31,6 @@ namespace Message::Components { std::vector fuid_open; }; - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text) - - namespace Transfer { - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list) - } // Transfer - } // Message::Components namespace Fragment::Components { @@ -314,12 +302,12 @@ MessageFragmentStore::MessageFragmentStore( // files //_sc.registerSerializerJson() - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); + //_sc.registerSerializerJson(); + //_sc.registerDeSerializerJson(); + //_sc.registerSerializerJson(); + //_sc.registerDeSerializerJson(); + //_sc.registerSerializerJson(); + //_sc.registerDeSerializerJson(); _fs.subscribe(this, FragmentStore_Event::fragment_construct); } @@ -347,8 +335,8 @@ float MessageFragmentStore::tick(float time_delta) { continue; } - // require msg and file for now - if (!reg->any_of(m)) { + // require msg for now + if (!reg->any_of(m)) { continue; } diff --git a/src/json/message_components.hpp b/src/json/message_components.hpp new file mode 100644 index 0000000..c272e6c --- /dev/null +++ b/src/json/message_components.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include + +#include + +namespace Message::Components { + + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts) + // TODO: SyncedBy + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text) + + namespace Transfer { + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size) + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list) + } // Transfer + +} // Message::Components + From d21dbb43e295384333a65e3c9d65bd1833bca4cd Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 19:30:52 +0100 Subject: [PATCH 21/98] comp refactor and make groups work --- src/CMakeLists.txt | 8 ++++ src/fragment_store/fragment_store.cpp | 2 +- src/fragment_store/message_fragment_store.cpp | 42 +++++-------------- src/fragment_store/message_fragment_store.hpp | 2 + .../register_mfs_json_message_components.cpp | 35 ++++++++++++++++ .../register_mfs_json_message_components.hpp | 6 +++ ...gister_mfs_json_tox_message_components.cpp | 10 +++++ ...gister_mfs_json_tox_message_components.hpp | 6 +++ src/json/tox_message_components.hpp | 16 +++++++ src/main_screen.cpp | 6 ++- 10 files changed, 100 insertions(+), 33 deletions(-) create mode 100644 src/fragment_store/register_mfs_json_message_components.cpp create mode 100644 src/fragment_store/register_mfs_json_message_components.hpp create mode 100644 src/fragment_store/register_mfs_json_tox_message_components.cpp create mode 100644 src/fragment_store/register_mfs_json_tox_message_components.hpp create mode 100644 src/json/tox_message_components.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 24bd59a..b70c3e9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,12 +11,15 @@ add_library(fragment_store ./fragment_store/fragment_store.cpp ./json/message_components.hpp # TODO: move + ./json/tox_message_components.hpp # TODO: move ) target_link_libraries(fragment_store PUBLIC nlohmann_json::nlohmann_json EnTT::EnTT solanaceae_util + + solanaceae_tox_messages # TODO: move ) ######################################## @@ -26,6 +29,11 @@ add_library(message_fragment_store ./fragment_store/message_serializer.cpp ./fragment_store/message_fragment_store.hpp ./fragment_store/message_fragment_store.cpp + + ./fragment_store/register_mfs_json_message_components.hpp + ./fragment_store/register_mfs_json_message_components.cpp + ./fragment_store/register_mfs_json_tox_message_components.hpp + ./fragment_store/register_mfs_json_tox_message_components.cpp ) target_compile_features(message_fragment_store PRIVATE cxx_std_20) target_link_libraries(message_fragment_store PUBLIC diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 98a08e0..1776f27 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -305,7 +305,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function()) { + if (other_msg == new_real_msg) { + continue; // skip self + } + if (comp({reg, other_msg}, new_real_msg)) { // dup dup_msg = other_msg; @@ -281,34 +285,6 @@ MessageFragmentStore::MessageFragmentStore( _fs._sc.registerSerializerJson(); _fs._sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - _sc.registerSerializerJson(); - _sc.registerDeSerializerJson(); - - // files - //_sc.registerSerializerJson() - //_sc.registerSerializerJson(); - //_sc.registerDeSerializerJson(); - //_sc.registerSerializerJson(); - //_sc.registerDeSerializerJson(); - //_sc.registerSerializerJson(); - //_sc.registerDeSerializerJson(); - _fs.subscribe(this, FragmentStore_Event::fragment_construct); } @@ -316,6 +292,10 @@ MessageFragmentStore::~MessageFragmentStore(void) { // TODO: sync all dirty fragments } +MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) { + return _sc; +} + float MessageFragmentStore::tick(float time_delta) { // sync dirty fragments here @@ -351,13 +331,13 @@ float MessageFragmentStore::tick(float time_delta) { continue; } - std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; + //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; // use type_id to find serializer auto s_cb_it = _sc._serl_json.find(type_id); if (s_cb_it == _sc._serl_json.end()) { // could not find serializer, not saving - std::cout << "missing " << storage.type().name() << "\n"; + std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n"; continue; } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index cc60870..b61fe9a 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -71,6 +71,8 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS ); virtual ~MessageFragmentStore(void); + MessageSerializerCallbacks& getMSC(void); + float tick(float time_delta); void triggerScan(void); diff --git a/src/fragment_store/register_mfs_json_message_components.cpp b/src/fragment_store/register_mfs_json_message_components.cpp new file mode 100644 index 0000000..ed6bda6 --- /dev/null +++ b/src/fragment_store/register_mfs_json_message_components.cpp @@ -0,0 +1,35 @@ +#include "./register_mfs_json_message_components.hpp" + +#include "./message_serializer.hpp" +#include "../json/message_components.hpp" + +void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) { + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); + + // files + //_sc.registerSerializerJson() + //_sc.registerSerializerJson(); + //_sc.registerDeSerializerJson(); + //_sc.registerSerializerJson(); + //_sc.registerDeSerializerJson(); + //_sc.registerSerializerJson(); + //_sc.registerDeSerializerJson(); +} + diff --git a/src/fragment_store/register_mfs_json_message_components.hpp b/src/fragment_store/register_mfs_json_message_components.hpp new file mode 100644 index 0000000..5efef28 --- /dev/null +++ b/src/fragment_store/register_mfs_json_message_components.hpp @@ -0,0 +1,6 @@ +#pragma once + +#include "./message_serializer.hpp" + +void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc); + diff --git a/src/fragment_store/register_mfs_json_tox_message_components.cpp b/src/fragment_store/register_mfs_json_tox_message_components.cpp new file mode 100644 index 0000000..83b7dad --- /dev/null +++ b/src/fragment_store/register_mfs_json_tox_message_components.cpp @@ -0,0 +1,10 @@ +#include "./register_mfs_json_message_components.hpp" + +#include "./message_serializer.hpp" +#include "../json/tox_message_components.hpp" + +void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) { + msc.registerSerializerJson(); + msc.registerDeSerializerJson(); +} + diff --git a/src/fragment_store/register_mfs_json_tox_message_components.hpp b/src/fragment_store/register_mfs_json_tox_message_components.hpp new file mode 100644 index 0000000..fdf86bc --- /dev/null +++ b/src/fragment_store/register_mfs_json_tox_message_components.hpp @@ -0,0 +1,6 @@ +#pragma once + +#include "./message_serializer.hpp" + +void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc); + diff --git a/src/json/tox_message_components.hpp b/src/json/tox_message_components.hpp new file mode 100644 index 0000000..09ebbdc --- /dev/null +++ b/src/json/tox_message_components.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +#include + +#include + +namespace Message::Components { + + // TODO: friend msg id, does not have the same qualities + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id) + // TODO: transfer stuff, needs content rewrite + +} // Message::Components + diff --git a/src/main_screen.cpp b/src/main_screen.cpp index ba61d9c..29ff026 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -1,5 +1,7 @@ #include "./main_screen.hpp" -#include "fragment_store/fragment_store.hpp" + +#include "./fragment_store/register_mfs_json_message_components.hpp" +#include "./fragment_store/register_mfs_json_tox_message_components.hpp" #include @@ -35,6 +37,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri tdch(tpi) { tel.subscribeAll(tc); + registerMFSJsonMessageComponents(mfs.getMSC()); + registerMFSJsonToxMessageComponents(mfs.getMSC()); conf.set("tox", "save_file_path", save_path); From 527a7c63f61376a12f51c9190ea63c67c0c6d02b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 22:52:55 +0100 Subject: [PATCH 22/98] add zstd dep --- external/CMakeLists.txt | 19 +++++++++++++++++++ flake.nix | 2 ++ 2 files changed, 21 insertions(+) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 0748b3f..cbcdf2c 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -28,3 +28,22 @@ if (NOT TARGET nlohmann_json::nlohmann_json) FetchContent_MakeAvailable(json) endif() +if (NOT TARGET zstd::zstd) + # TODO: try find_package() first + # TODO: try pkg-config next (will work on most distros) + + set(ZSTD_BUILD_STATIC ON) + set(ZSTD_BUILD_SHARED OFF) + set(ZSTD_BUILD_PROGRAMS OFF) + set(ZSTD_BUILD_CONTRIB OFF) + set(ZSTD_BUILD_TESTS OFF) + FetchContent_Declare(zstd + URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz" + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + SOURCE_SUBDIR build/cmake + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(zstd) + + add_library(zstd::zstd ALIAS libzstd_static) +endif() diff --git a/flake.nix b/flake.nix index 9f580c7..8210243 100644 --- a/flake.nix +++ b/flake.nix @@ -58,8 +58,10 @@ cmakeFlags = [ "-DTOMATO_ASAN=OFF" "-DCMAKE_BUILD_TYPE=RelWithDebInfo" + "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here # do we really care less about the version? do we need a stable abi? + "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" ]; # TODO: replace with install command From fb885b5c21348c8b389b898b66c5eb19eee6c9f8 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 17 Feb 2024 23:04:34 +0100 Subject: [PATCH 23/98] simplify array cast a little --- src/fragment_store/message_serializer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/message_serializer.cpp b/src/fragment_store/message_serializer.cpp index 78362e9..6bb7df0 100644 --- a/src/fragment_store/message_serializer.cpp +++ b/src/fragment_store/message_serializer.cpp @@ -36,7 +36,7 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json>(j.is_binary()?j:j["bytes"]); + const std::vector id = j.is_binary()?j:j["bytes"]; // TODO: id lookup table, this is very inefficent for (const auto& [c_it, id_it] : msc.cr.view().each()) { @@ -79,7 +79,7 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json>(j.is_binary()?j:j["bytes"]); + const std::vector id = j.is_binary()?j:j["bytes"]; // TODO: id lookup table, this is very inefficent for (const auto& [c_it, id_it] : msc.cr.view().each()) { From 6f511016bc23d140bdc0dc2c9f14260383964748 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 18 Feb 2024 02:40:17 +0100 Subject: [PATCH 24/98] save msg json zstd compressed (3x compression) --- external/CMakeLists.txt | 5 +- src/CMakeLists.txt | 2 + src/fragment_store/fragment_store.cpp | 89 ++++++++++++++++--- src/fragment_store/message_fragment_store.cpp | 2 + src/fragment_store/types.hpp | 1 + 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index cbcdf2c..a7066be 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -45,5 +45,8 @@ if (NOT TARGET zstd::zstd) ) FetchContent_MakeAvailable(zstd) - add_library(zstd::zstd ALIAS libzstd_static) + add_library(zstd INTERFACE) # somehow zstd fkd this up + target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/) + target_link_libraries(zstd INTERFACE libzstd_static) + add_library(zstd::zstd ALIAS zstd) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b70c3e9..4c85e70 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,6 +19,8 @@ target_link_libraries(fragment_store PUBLIC EnTT::EnTT solanaceae_util + zstd::zstd + solanaceae_tox_messages # TODO: move ) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 1776f27..4f705a2 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -8,6 +8,8 @@ #include +#include + #include #include #include @@ -282,6 +284,11 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { + data_comp = _reg.get(fid).comp; + } + std::ofstream data_file{ _reg.get(fid).path, std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text @@ -324,8 +331,22 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(res.data()), res.size()); + + // TODO: refactor + if (meta_comp == Compression::NONE) { + meta_file.write(reinterpret_cast(res.data()), res.size()); + } else if (meta_comp == Compression::ZSTD) { + std::vector compressed_buffer; + compressed_buffer.resize(ZSTD_compressBound(res.size())); + + size_t const cSize = ZSTD_compress(compressed_buffer.data(), compressed_buffer.size(), res.data(), res.size(), 0); // 0 is default is probably 3 + + compressed_buffer.resize(cSize); // maybe skip this resize + + meta_file.write(reinterpret_cast(compressed_buffer.data()), compressed_buffer.size()); + } } else if (meta_type == MetaFileType::TEXT_JSON) { + // cant be compressed or encrypted meta_file << meta_data.dump(2, ' ', true); } @@ -342,7 +363,16 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(buffer.data()), buffer_actual_size); + if (data_comp == Compression::NONE) { + data_file.write(reinterpret_cast(buffer.data()), buffer_actual_size); + } else if (data_comp == Compression::ZSTD) { + std::vector compressed_buffer; + compressed_buffer.resize(ZSTD_compressBound(buffer_actual_size)); + + size_t const cSize = ZSTD_compress(compressed_buffer.data(), compressed_buffer.size(), buffer.data(), buffer_actual_size, 0); // 0 is default is probably 3 + + data_file.write(reinterpret_cast(compressed_buffer.data()), cSize); + } } while (buffer_actual_size == buffer.size()); meta_file.flush(); @@ -399,18 +429,53 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function buffer; - uint64_t buffer_actual_size {0}; - do { - data_file.read(reinterpret_cast(buffer.data()), buffer.size()); - buffer_actual_size = data_file.gcount(); + Compression data_comp = Compression::NONE; + if (_reg.all_of(fid)) { + data_comp = _reg.get(fid).comp; + } - if (buffer_actual_size == 0) { - break; - } + if (data_comp == Compression::NONE) { + std::array buffer; + uint64_t buffer_actual_size {0}; + do { + data_file.read(reinterpret_cast(buffer.data()), buffer.size()); + buffer_actual_size = data_file.gcount(); - data_cb(buffer.data(), buffer_actual_size); - } while (buffer_actual_size == buffer.size() && !data_file.eof()); + if (buffer_actual_size == 0) { + break; + } + + data_cb(buffer.data(), buffer_actual_size); + } while (buffer_actual_size == buffer.size() && !data_file.eof()); + } else if (data_comp == Compression::ZSTD) { + std::vector in_buffer(ZSTD_DStreamInSize()); + std::vector out_buffer(ZSTD_DStreamOutSize()); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + + uint64_t buffer_actual_size {0}; + do { + data_file.read(reinterpret_cast(in_buffer.data()), in_buffer.size()); + buffer_actual_size = data_file.gcount(); + if (buffer_actual_size == 0) { + break; + } + + ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 }; + do { + ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 }; + size_t const ret = ZSTD_decompressStream(dctx, &output , &input); + if (ZSTD_isError(ret)) { + // error <.< + std::cerr << "FS error: decompression error\n"; + break; + } + + data_cb(out_buffer.data(), output.pos); + } while (input.pos < input.size); + } while (buffer_actual_size == in_buffer.size() && !data_file.eof()); + + ZSTD_freeDCtx(dctx); + } return true; } diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 761a256..b0b4213 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -154,6 +154,8 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { fragment_uid = fh.get().v; + fh.emplace_or_replace().comp = Compression::ZSTD; + auto& new_ts_range = fh.emplace(); new_ts_range.begin = msg_ts; new_ts_range.end = msg_ts; diff --git a/src/fragment_store/types.hpp b/src/fragment_store/types.hpp index cbb64c5..c259bd9 100644 --- a/src/fragment_store/types.hpp +++ b/src/fragment_store/types.hpp @@ -7,6 +7,7 @@ enum class Encryption : uint8_t { }; enum class Compression : uint8_t { NONE = 0x00, + ZSTD = 0x01, }; enum class MetaFileType : uint8_t { TEXT_JSON, From 182d844e32a24a7071ac544390adc29e2d4c5352 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 18 Feb 2024 13:25:15 +0100 Subject: [PATCH 25/98] update fs readme a little --- src/fragment_store/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/fragment_store/README.md b/src/fragment_store/README.md index eda2607..a1d18a8 100644 --- a/src/fragment_store/README.md +++ b/src/fragment_store/README.md @@ -25,17 +25,16 @@ Just keeps the Fragments in memory. # File formats -Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted. +Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted. ### Text Json Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes). Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised. -A Metadata json object has the following keys: -- `enc` (uint) Encryption type of the data, if any -- `comp` (uint) Compression type of the data, if any -- `metadata` (obj) the +A Metadata json object can have arbitrary keys, some are predefined: +- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any +- `FragComp::DataCompressionType` (uint) Compression type of the data, if any ## Binary file headers @@ -70,3 +69,7 @@ file magic bytes `SOLFIL` (6 bytes) ...data here... +## Compression types + +- `0x00` none +- `0x01` zstd (without dict) From 4fb2b51b7d93bc7bc95bf8edd4d710ce63231e24 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 18 Feb 2024 18:19:49 +0100 Subject: [PATCH 26/98] switch to streaming compressor for data to drastically improve ratio. would still benefit from a abstract file refactor --- src/fragment_store/fragment_store.cpp | 74 ++++++++++++++++++++------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 4f705a2..6b4b40e 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -340,6 +340,10 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function buffer; - uint64_t buffer_actual_size {0}; - do { - buffer_actual_size = data_cb(buffer.data(), buffer.size()); - if (buffer_actual_size == 0) { - break; - } - if (buffer_actual_size > buffer.size()) { - // wtf - break; - } + if (data_comp == Compression::NONE) { + std::array buffer; + uint64_t buffer_actual_size {0}; + do { + buffer_actual_size = data_cb(buffer.data(), buffer.size()); + if (buffer_actual_size == 0) { + break; + } + if (buffer_actual_size > buffer.size()) { + // wtf + break; + } - if (data_comp == Compression::NONE) { data_file.write(reinterpret_cast(buffer.data()), buffer_actual_size); - } else if (data_comp == Compression::ZSTD) { - std::vector compressed_buffer; - compressed_buffer.resize(ZSTD_compressBound(buffer_actual_size)); + } while (buffer_actual_size == buffer.size()); + } else if (data_comp == Compression::ZSTD) { + std::vector buffer(ZSTD_CStreamInSize()); + std::vector compressed_buffer(ZSTD_CStreamOutSize()); + uint64_t buffer_actual_size {0}; - size_t const cSize = ZSTD_compress(compressed_buffer.data(), compressed_buffer.size(), buffer.data(), buffer_actual_size, 0); // 0 is default is probably 3 + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 0); // default (3) + ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) + do { + buffer_actual_size = data_cb(buffer.data(), buffer.size()); + //if (buffer_actual_size == 0) { + //break; + //} + if (buffer_actual_size > buffer.size()) { + // wtf + break; + } + bool const lastChunk = (buffer_actual_size < buffer.size()); - data_file.write(reinterpret_cast(compressed_buffer.data()), cSize); - } - } while (buffer_actual_size == buffer.size()); + ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; + ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 }; + + while (input.pos < input.size) { + ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 }; + + size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode); + if (ZSTD_isError(remaining)) { + std::cerr << "FS error: compressing data failed\n"; + break; + } + + data_file.write(reinterpret_cast(compressed_buffer.data()), output.pos); + + if (remaining == 0) { + break; + } + } + // same as if lastChunk break; + } while (buffer_actual_size == buffer.size()); + } meta_file.flush(); data_file.flush(); From 6aac44cda9d72d5a97324b3c701044be5f7b9fb3 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 19 Feb 2024 12:32:09 +0100 Subject: [PATCH 27/98] change binary meta format and add zstd to metadata --- src/fragment_store/README.md | 15 +- src/fragment_store/fragment_store.cpp | 203 +++++++++++++----- src/fragment_store/fs_binary_msgpack1.png | Bin 0 -> 37255 bytes src/fragment_store/fs_binary_msgpack2.png | Bin 0 -> 46969 bytes src/fragment_store/message_fragment_store.cpp | 3 +- src/fragment_store/types.hpp | 6 +- 6 files changed, 160 insertions(+), 67 deletions(-) create mode 100644 src/fragment_store/fs_binary_msgpack1.png create mode 100644 src/fragment_store/fs_binary_msgpack2.png diff --git a/src/fragment_store/README.md b/src/fragment_store/README.md index a1d18a8..0eba57b 100644 --- a/src/fragment_store/README.md +++ b/src/fragment_store/README.md @@ -40,25 +40,26 @@ A Metadata json object can have arbitrary keys, some are predefined: ### Split Metadata -file magic bytes `SOLMET` (6 bytes) +msgpack array: -1 byte encryption type (`0x00` is none) - -1 byte compression type (`0x00` is none) - -...metadata here... +- `[0]`: file magic string `SOLMET` (6 bytes) +- `[1]`: uint8 encryption type (`0x00` is none) +- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd) +- `[3]`: binary metadata (optionally compressed and encrypted) note that the encryption and compression are for the metadata only. The metadata itself contains encryption and compression info about the data. ### Split Data -(none) all the data is in the metadata file. +All the metadata is in the metadata file. (like encryption and compression) This is mostly to allow direct storage for files in the Fragment store without excessive duplication. Keep in mind to not use the actual file name as the data/meta file name. ### Single fragment +Note: this format is unused for now + file magic bytes `SOLFIL` (6 bytes) 1 byte encryption type (`0x00` is none) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 6b4b40e..a32573a 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -224,15 +224,6 @@ FragmentID FragmentStore::getFragmentCustomMatcher( return entt::null; } -template -static void writeBinaryMetafileHeader(F& file, const Encryption enc, const Compression comp) { - file.write("SOLMET", 6); - file.put(static_cast>(enc)); - - // TODO: is compressiontype encrypted? - file.put(static_cast>(comp)); -} - bool FragmentStore::syncToStorage(FragmentID fid, std::function& data_cb) { if (!_reg.valid(fid)) { return false; @@ -298,13 +289,8 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::functionsecond(storage.value(fid), meta_data[storage.type().hash()]); //} else if (meta_type == MetaFileType::TEXT_JSON) { - s_cb_it->second({_reg, fid}, meta_data[storage.type().name()]); + s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]); //} } if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file - const auto res = nlohmann::json::to_msgpack(meta_data); + const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); + std::vector meta_data_compressed; // empty if none + //std::vector meta_data_encrypted; // empty if none - // TODO: refactor - if (meta_comp == Compression::NONE) { - meta_file.write(reinterpret_cast(res.data()), res.size()); - } else if (meta_comp == Compression::ZSTD) { - std::vector compressed_buffer; - compressed_buffer.resize(ZSTD_compressBound(res.size())); + if (meta_comp == Compression::ZSTD) { + meta_data_compressed.resize(ZSTD_compressBound(meta_data.size())); - size_t const cSize = ZSTD_compress(compressed_buffer.data(), compressed_buffer.size(), res.data(), res.size(), 0); // 0 is default is probably 3 + size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3 if (ZSTD_isError(cSize)) { std::cerr << "FS error: compressing meta failed\n"; - return false; // HACK + meta_data_compressed.clear(); + meta_comp = Compression::NONE; + } else { + meta_data_compressed.resize(cSize); } - - compressed_buffer.resize(cSize); // maybe skip this resize - - meta_file.write(reinterpret_cast(compressed_buffer.data()), compressed_buffer.size()); + } else if (meta_comp == Compression::NONE) { + // do nothing + } else { + assert(false && "implement me"); } + + // TODO: encryption + + // the meta file is itself msgpack data + nlohmann::json meta_header_j = nlohmann::json::array(); + meta_header_j.emplace_back() = "SOLMET"; + meta_header_j.push_back(meta_enc); + meta_header_j.push_back(meta_comp); + + if (false) { // TODO: encryption + } else if (!meta_data_compressed.empty()) { + meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed)); + } else { + meta_header_j.push_back(nlohmann::json::binary(meta_data)); + } + + const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); + meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); } else if (meta_type == MetaFileType::TEXT_JSON) { // cant be compressed or encrypted - meta_file << meta_data.dump(2, ' ', true); + meta_file << meta_data_j.dump(2, ' ', true); } // now data @@ -409,6 +414,8 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function> j; + // file is a msgpack within a msgpack - if (!j.is_object()) { - std::cerr << "FS error: json in meta is broken " << it.id_str << "\n"; + std::vector full_meta_data; + { // read meta file + // figure out size + file.seekg(0, file.end); + uint64_t file_size = file.tellg(); + file.seekg(0, file.beg); + + full_meta_data.resize(file_size); + + file.read(reinterpret_cast(full_meta_data.data()), full_meta_data.size()); + } + + const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data); + + if (!meta_header_j.is_array() || meta_header_j.size() < 4) { + std::cerr << "FS error: broken binary meta " << it.frag_path << "\n"; continue; } - // TODO: existing fragment file - //newFragmentFile(); - FragmentHandle fh{_reg, _reg.create()}; - fh.emplace(hex2bin(it.id_str)); - - fh.emplace(it.frag_path.generic_u8string()); - - for (const auto& [k, v] : j.items()) { - // type id from string hash - const auto type_id = entt::hashed_string(k.data(), k.size()); - const auto deserl_fn_it = _sc._deserl_json.find(type_id); - if (deserl_fn_it != _sc._deserl_json.cend()) { - // TODO: check return value - deserl_fn_it->second(fh, v); - } else { - std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; - } + if (meta_header_j.at(0) != "SOLMET") { + std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.frag_path << "\n"; + continue; } - // throw new frag event here - throwEventConstruct(fh); - count++; + + Encryption meta_enc = meta_header_j.at(1); + if (meta_enc != Encryption::NONE) { + std::cerr << "FS error: unknown encryption " << it.frag_path << "\n"; + continue; + } + + Compression meta_comp = meta_header_j.at(2); + if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) { + std::cerr << "FS error: unknown compression " << it.frag_path << "\n"; + continue; + } + + //const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data"); + if (!meta_header_j.at(3).is_binary()) { + std::cerr << "FS error: meta data not binary " << it.frag_path << "\n"; + continue; + } + const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3); + + std::vector meta_data_decomp; + if (meta_comp == Compression::NONE) { + // do nothing + } else if (meta_comp == Compression::ZSTD) { + meta_data_decomp.resize(ZSTD_DStreamOutSize()); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + + ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0 }; + ZSTD_outBuffer output = { meta_data_decomp.data(), meta_data_decomp.size(), 0 }; + do { + size_t const ret = ZSTD_decompressStream(dctx, &output , &input); + if (ZSTD_isError(ret)) { + // error <.< + std::cerr << "FS error: decompression error\n"; + meta_data_decomp.clear(); + break; + } + } while (input.pos < input.size); + meta_data_decomp.resize(output.pos); + + ZSTD_freeDCtx(dctx); + } else { + assert(false && "implement me"); + } + + // TODO: enc + + if (!meta_data_decomp.empty()) { + j = nlohmann::json::from_msgpack(meta_data_decomp); + } else { + j = nlohmann::json::from_msgpack(meta_data_ref); + } + } else if (it.meta_ext == ".meta.json") { + std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); + if (!file.is_open()) { + std::cerr << "FS error: failed opening meta " << it.frag_path << "\n"; + continue; + } + + file >> j; } else { assert(false); } + + if (!j.is_object()) { + std::cerr << "FS error: json in meta is broken " << it.id_str << "\n"; + continue; + } + + // TODO: existing fragment file + //newFragmentFile(); + FragmentHandle fh{_reg, _reg.create()}; + fh.emplace(hex2bin(it.id_str)); + + fh.emplace(it.frag_path.generic_u8string()); + + for (const auto& [k, v] : j.items()) { + // type id from string hash + const auto type_id = entt::hashed_string(k.data(), k.size()); + const auto deserl_fn_it = _sc._deserl_json.find(type_id); + if (deserl_fn_it != _sc._deserl_json.cend()) { + // TODO: check return value + deserl_fn_it->second(fh, v); + } else { + std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; + } + } + // throw new frag event here + throwEventConstruct(fh); + count++; } return count; diff --git a/src/fragment_store/fs_binary_msgpack1.png b/src/fragment_store/fs_binary_msgpack1.png new file mode 100644 index 0000000000000000000000000000000000000000..b536b203ca5e1f586564c93e732fb464f798f3f7 GIT binary patch literal 37255 zcmdSBWk6K#w>CTy(jn4~lt@W;OG}r4faFMbgMgGEIS5D%QqnneBT6%%bT>nH2n_IU ze*bfx^PJ~B=XuZ9_tPGB-h1EoTI*WZx)xEI>WX;Sl-M8;2v1q*l{N^3stN+3e8)rq z_B{Oz=?8&=LCUXWbbV&`7yJWgegz8MiPSI+);M6zL1MUjMf(I@6C7929?5uFX`9Jr zehuR<;2++YAfgWw{4&a-F_yx=|CBOaIE2jh&$_K_`X+*fd6^ael>iw9g#$`oO+dm4 zaT>j9Jx@wGqwzGvQ=~(RG{xH6%gYAr4fYyA+@1_p=85=U!S|rBB|iT&-M~PJN5Ig6 z(b%+UfGr4whD$*UY^49tccTJZB}yxwhuzAI$t&j<7ow_};ZLf{`B%xm=u!}C*JT6e zxSYU`{FdO?lYa_S9iNJ+qI7)<8K8(*KmM9N7-3=$ed%GEI;yV&eTj9e^vU5faK&Gp zA-+WXUOpNNkk+&vaCa3KYad+&R*Vsb`8|9rq>sk_dsm#-EHMSDsTUU)>3p-MVyU*q zsh)laA`}OV2297_C$HINEMhc848X0s4yJaxE1tz}f*&E0Ccxc<-crz>(}QUPB@mus zxj4tK)x=S8wi|cf(uhT2G2+5WNf$^@azq=_)7ZsOzp-qtVC!eS1HL0Ruc>QyyvI^G4CV`OO>Gr?t9f7mSk=X?s zh2kQgY3l}%Sh<7^?L)6DJmw)Tq9fQYhv>l9{dDH*<2oW0unx9ag47+Le;^(43o)sZ zH7cX z!)k$jAmz}1+ddt{b)KowE0L2viZUVt=KII8sQB~u^44UpN6NUNu-c*k=*}C^GXYO8 zmsQ=v=|gBHC5<)>4hW5af|j@YVZM#`We? zdFH2M-1K3@@mVTrYUHJutr6MR0>_?0Yn~DbJJ74eK5lmR}k;UretHRq9w2R%72M6FtCl7s=7;($n z>+@{nI_oenwB-;)cF^5H%i*2b{cccfY-|+Qesly*$4MwSW{QlntobnRGK(sMe<|>I zqwD6w;uGEV;^L^8N=XTxKzLd@OPZ^h76nWcmcl=lY~Cehp)9C5?!74*@k8*bwGni z)+W>UYN%CUtjCO7GW)c8GHP27F^CxX;7Hf6@kDk&y+fSWc^LTS?{|<#-2@tHY73rI zCT7=vQePn?q$DRUcWl=@mr4t>16CXje|z8CS@P>WPZw5Hl1z{<_zoWtJw{%g;N(TU zbw~Ev?tFpE!otASnS-^%t4*)rdF@tlKSWxLLyx8;Vn}@ThxybIf9Canew{Xpwfc@0 zjJ7s>^gTAaUxQst$KBVrqQ!?p@kEC&!oT+gDM|Lg)tlYvps*0Xlx2qQ@X=&Nnz4$u6i+5F=8|B_JICfe&P?+x= z*RPS!qKL&%h!Eg9dJf8vNo;EDU(v z_-G9e$-z8{stmj`cNTUYC?ZhE>fv;V0lKa&psZ)VW(thPzC&3}Evz~vw@514^Y#5ae#2oXLI+It6X?S zz!6zH6kV7lW#U@1d+wX6-hNfp@a8#ZTK(yEYO%9p`!g7#Q(BU4%aJsPBBSYjiBHep znDkRVGW|>+)4S~m-D2C~8LARR=y28t2bYVRO_JfS-l^?X)xDXCaCqEfN#)zB#u$Nz_QV7czD$&C4fp^o1}vln(sr#7+;%~Accwm$ zN+3?y7pj$Ouptz3s>|zFb25F903S+^2=WZr*_(4>r@akgL}0+fn*rPjqR$1}$~8~% z=Kn&Go8G2ju-Z9ShRXs6FjJ1QNFTt!H<4|IDv$-v6fJ-nw4>&2ACO>FmCK5@Y+oW{ zTM25&G6mJtpGX67MyVF)&{w%=a7mmR>eI?3_d@MLf<&x&3g7~C^A{QltlDcj#l>$- zXw7tAsYIVx|GIjHIBm2@pF>LwYfW1r8R?s9{rqJ}+l1ATc!UXNvpHCzUD+1QL;wO2 zIRn?j$<%sYH25=~u<;5er+-z^`=Z{)+5g$u>FFz?lb$TntC{m^hmTdsLjxl{h7JHM zZT}smOFbqwwoXiup??7;!g)(!8@K&MACMQAba^->36YT_|Q?2X^1}t2KSLQ zJK&^;%9%g@TH&$FTC9-d)q2t$sj&Bo+nU*Q)6{(U;PAbi-hKOAOpy5{1x1@Ky#2*e zfWzBrR@w|si^qXm$2UclB4Ac7+ZwIDtsYp#7fQ$+5il0Wzz`~cBiiSjTGUwDV7w4< zMpQnSgt6=gr31g zTFOT!&&Jh&z}m0@;BltKBn?ShE70YFw;BC*R;)k>Y||7jKIEvZ3>;3m|*s$fhN_7#b$LLYZRxlY7?e8n|d{ zUY>E@EJOzbMiwkSH*r~Se`*~T2<7*yZuvCW+e5O|4g2mIgL4vQL#Be4rTUHQ>BD4G zC{tMYv0xDp{iJ-scLej>t{3VIZP~owb&mvb%bF)F!5; zp30u6E^^Pl4riRWEZQLiM)$Gy-`BrQ|7a7Fxb*%<3TKIPVTz{Mc&4Azjmp!2#|44j zh2sKi7XaA_D-Tw~q_&e=Dpe>7%s%MTcy3fGCIjlPL0t@#E1d&YlZ&jd*ktn17`*2%S2!R+XVh!CBT;ffq2%|)-rWc zzI-XdCmolj`&ZWXcj(Bii5krCZ~@kgEdLv@ZLFiFo?d}yU)5K!w7R%)*}IuoH>d)R z<^D;RyMC8Q%l=b#(U!V-#UojJ1cCC^CT6}2)6_vzSmzX16SwjKE?m&N6yI{<&I=V3 zM9THH@m&{+CC1=}n_@hzgxk6jz3S#YhfX}DQzTIUX|7h|)QaJy_^Ci$LXg*w+cBn? zxEC40STA4y17K>?j3qwrMOX^qML0quwnzXqL|mp2kX1n}~6 zm`drJOaoK$P3owO&y4Yhgt8C(@+c)L_{W_?gQW&T5SvU_ z?g8?5_Ihpv`?s0n1XK6K2Q^txRKi<;$Q8HY`pY}>69b7mbJPp zWbt=@U&B{5_jZE99kk&f)-f9ol8i!*ixb5EPD2=eeRopWY&B_WErVPPDN3lP^KnGA ztO`7{5_F<2b^g3$NdE2y^nTF2cQS{MfE=x&JL&b0A_D=o);I_y!F_?j-j0tKV@vrb z*nv4(W&QRu9La$M;SrXwYLQR#?LX;~5wq6m@XJ|LtD9)cLt2F>75$w)`aB>ng3A>s@FVsx5o zd{u|fob&_TuYTz9kybE;Ws3dryil4-lhb%As|^g~GzA)1xBP+M4ldB07x%Z+JST>+ zTrq|WM~E}OU6s?Dt4olgZgg{GfXyYxvA9y0nRZQJ25E zZRwN3awrKKe#+j>3ZWRxbL=p;AAukM{DC^5RkAvxYqV56% z6AJBE;iftfCheJdztbNrR|ZRx{=T2D^ldjj4tP63oAUQ3SnHq2epk!Z6Kwc!wx09y zN8g5<@b)39L^rJd`;}3*EX5=mJZ&hgKYPexX1W*)oW9Ck>Yic34^n_q+8rUU=pNoA zO(gRw-AKuEzuA6>8Xlm=>0vp%;&7FjcE$7Nso1H2SSg?3=?lR3jpBK#+|3O4&*!ch z)_G#dcHOKYO|0%TtJ?@IGQzJzR4_~_T>Eygf7Wi3Fp~)M*c%!jHSmNEEpiA?re6G2lf=0et8ZESn2fZ5<5les&K8&LePx%PZj zuw`EZi0zJ)4F{2+(0GTmCshR=^=EOv{mGypf)&-u#>rC%2lz1SJ!pKLmls|Zcl+e! zQtUSEu9d?ouw2(pTA0%GuN4j#%C1X`wzJGHPEridhwBZ?N;q_<`f0V7uR`_TOs#0? z7~tWAGN%<(k06bjvIy$3ewq)d@1y5)CAoWjFdyO~Zgp3=3T3X#i(Bvq%kLMR?zLGi zYaA9`%BF|!!Nu(#+YH9~z>|o!?RSc=-qIHu%I$PArVw*ZBZqqBDODP?CcV8ta%E`< z@E}=*g@$3f%S4)-|Kit;m`hs>1@81yH;%lYUi~R)naZXE)*QX!de4a=qtRcKeq@o} zrBBaZ+7PK7W~#o+ysA6d6SqXLI~`uPT&U}k!%h))doar=hH-!J@XFBBI$p3XUM7Y^ z+JR8RcDOpMdE-;gCdK5g;%!!f=RvLOOyo=k?zV}=>kL!#)~_qcD-@TeZ@|^Ln{rO6 zCL#n|<4{2a$=j1uH>_|bSZ-exulU{AiXBcM%*N6tXTQ!k#l~N?1~G!MD?(3qrB((e zuuU6p7lq@CueynaAL5IjT(BSMW>v%n*WtV@s^zG0%Fn;+1Yv5DizGC?wxfeeAz{56 z^IR3({>E-LZwQC^!rbh2Xu%N>pv}2m92?|qmHcngT8K( zX>H@JgL4C|-g9;{YC>oV|HyiY%tct(3DkQ9PmB#YCvMhN}5nKEwt8 z+n($_Uy7tBmr1RA31QHN+^}lbB$q;TAfBMvyE#||tP89q`Y7NdUZ0e0FZ}JZhr?>2 zspE4j%S~uYoW-lJ>tBJv^R_unKL(C$aLx3qP%Fqz_9!Fr11@wJE!)BImGYm-%HK9Y zu2T*pGg+nW6;~!{rWG`3U%W^&7Z=@aM_Q};4a!gbxdUW!OHTDrf zvmYJ3f-K$UXhjC4D%WQzV388@{+glB|7I+>uiVjQ@A{ja%)0>GljAkY42ktC7^zqJ ztpvVN17*nOcNPUjjW%AvfP9Y&6!&j9|BU{$Fn*J@R}TZz;|w?6FI&|rJnZ6=*j2Bd zSaqc836Gd3wZ}-fCfU|i+v_apt8x5&$YqO}2?2Fj7|UuIILThylQ z3)Ri&TXJLGPp%psgAB^+6~1*K*Z)pyW+UC`c|}bl!8C4hL4LpMUN+kW(CLUQc^YS} z%V^ac=qu=Uc@exJgPT&`%PjQnK_{vm1ED6eZe|u7tTlCHn-xkXaWM4EK%4QEg-n?7 zeeHz~ujtCFmW1;C?t?j&AxZd=9U7VdEHW14CTRUOO?o;1?Ss zNh&9SALn|bKjN9XIq38UMoHzroHSi-$xY^4%Yimsyv@6sttT{YIusTOZAkW4FE)+^ zNk_y5`s4;60w89}PB2w3kx%%SPFau+(i5{@$kN)l-N~hhG90c$mV^Cx5t>*|(~|3` z6~fUgWGDaXY2a-s7U;=4Yaej}THMM6sr;|lmYtp(R@dtR2%!_$)70S{FDm(Zuf1q^ zb1xUh$--2Ts0#>FmFW7YmeL^Z6r`-CcqY<#-O?yI&4O*~^t}9%6>G#w zga13+q$f_GAy+&wpfbVps_;BYB3xuy6htV?Wj5wAh?mjeFtZxKo;oP++w1h>5B-kg z@^z+>a{lqLkRM#oVOl!t-Bg81&u0jPXa}ul`3wVp&$6<3^ZCZMtp%{Q&zN&hP*j~zw`0-2b+<2s$!MgxxGpzObJ&?>^>|EzX?V7QB zcR26didnK2TV3&0#Wc24Q?OytZ-u5Cz?=h{lkre-CYb1Mf#IV+I~KRkN%SMUXKH-? z*}s>@p@7E8x9Eg@j_A+J)$B=|hdp<}&Uk_`on%FCPwddb5^TrVRbJTrE#xjPqM#Fz z6cMywZ-_JNgx7{W7ISeMT*u~XV|;+Wgo5H<&{p3Yu^AlF_>(g3^7b>i&=1rR#QUw4 zM1;4a^V&2WKAK{lqE$FboRG->4pXN;eJP&XyGdxL(c_|rY>+1E5JO#4H@({t%iCze z5l%>n4_?QEoqRrtyhaiSQIV~FIz^TRrttNWKNY4+U-6YD8^D-{!n;H$z)G$%BS&Q> zS)2BFIM$j3Hi>wRTum1*KTd(Gd~+teAl%|*Y$z+`ZiiXOyu_(j(W zTM#xLY9bBdtmLb!dl~t{bcdW$xLpk!z;%@g*}wK0U>o2}43FOdolVcdy*YDxU7O^h zqsOA+zf+Ru$?x!0&BcXePS}u%=#?vspUYKKL>Ry4+nMu!>iF(LJLqeJC2rJTJ;jv$ zYj~zXZU4rKSTsoUcxczq&~C~|Q-k!2vnc3flV>3XahzRYt&tX^%fVEJ8=o#Budl=z z&jBM9Z6n#cEM9NOA9{bBYkzEVnP4+@H2Q-g8QGrSn)=Hz=3Nj$8V(b8Vn6ZBLYge2 zhOmy~NZiObg@Kf9xW{fMqbZ%VgjCdW_xiMwYxQ(n6nu{3b^SrZ59)G$3(<=eioZ!s z-$ak5!`0!H>c7K6U7fi0W{SH8Gp88a)nV6PC0awH`IL!vP?Mr$KA2&rSS?(Ig019{ zo~c*FE#*_?E$~UnaT-INkE0ySumc`$Dpf}_o{M4@SI(9hK0dBrdPvx> zL~gJqv#JJ_VwC#4&nx`fT^o&S-DL0x-kZf=IE%K<25PN2Q zx(^7V$g%kNRs{)F`#_^;*r-VL+#WPMCQ5?Ua;f5p4gg8+%0yyhWm}v@YVtEq{SXDU zJWZ()eAxcWFm7x7%Y&w!gw-#$R|$QY9!c}m5Sa0y(b3zFLvZQ$ROB6jl>>(FIAs(-mMlHrD@(mr0OV7}-STtg*RM>-debHu@IF#(@(&9zI zt)ik*^prewj83>JzAg7(30zI&_hAjSObIU+I}2UU(oU+F*wQsaqX(bUHU4vJ#uB4< zA=^^?!zTG3F^O$|Zbx#czwOa@-`AbzAnDkD#|--Ry-y~7V>RB3T~cumeQ5FdMR3e3 zz$5CnlU%*RSO6n?Gcv6nmn9N({r-d7ThCx=gEth5)`Oe-<%Ist1R$m!>HE&EPz<1O zU;3N5-F|Qk0syd3gzhzu*!N5Vo2PNPFv79x%?Mn48IU~(6lkZ)R#cFo&E$GD+D_>1z@ z8`dTwoUO3QjazpEVezl=4C%q?Nun$tYxJZ$Is0`&P`O8ankuT0=@{bV+8q9qu2oau zku>0zd!-(~RSN%^xQ{M|LUaBohZm+=Ve-K%XJyga#@qP2rvAFW{0R(zBJQhi+}y(q ziZMUN5AF%V<9K=u{pa|dZta}199JCe=eBy_%|7}`Pg>s#Fz{%Pskm@Ewr)*Ff?c4Q z)*9QP>jsdyy-Zb{fPGn>s*XJ0zom7YAcSmgT9ygf2uYKg0a*}EFJ&+o#Gyh;@RPQ%cBZG-UPacN%sCHGJbA3t43q+nWKgXA9U8mLyP%r+a>+R?t!k!XV=lgbi zZcEJKZQ`C{FYu_i&$Bxb?K)N+XQI{CbCp_VRA;Xo^oZY^2};)PyX9nyJg+k(OC6kU zxBBJG!=CIh{|#sSpIN7pH~TJ2y3yHzKY8T3W^87At< zs2Tdl=>CkF?UTFLl{v1}W3A=v1w*ypa|D3|ms~R*C!8aOjK()%Q}hrR#x#A*G1aP` zw_WTjV`f}GjaJO~H$B`gq_Ra@6to%yc0S>gFQ*y$ z-^GV7Uj?A;>>;~DRPm{4#~asT);oJqr-i!{97OwP!tH+EH*)WwqFklsRY?@fU=MRW z4IuXfe*wv9F1<(~|6g%xpL5NM9Z^+4h#5HYhtfE!kkP>H-sMB;jKViW_Cl|cnv?3< zh+DwWf#Gn$P|iWOPYKLQorQwHve6s55G<8(DI|l2XHi;a^tEQx}-uT9AEm3ppEmHnsSk6FK7pNMDs4U+ynbxn#+ zUO6tD3n6P}c6`Ega}r5?=gtEGJ6hSKO1-4w{^%!LiH~Q2V|um+eZ4SIUpNP+ckc>b z-@rcEXOmK?MCSVZu)KfaZY<{SGs}#{+op_u$1wc9BLOMF%*yAGN zx$HfL*Ag1rHL3|06drrsPu12?_BwIaOK9#YwRXNB+wj}LsZ#GnI#%T~v}(4Qz=&ck zdYC(xnz=vk;)a8U7twO%wv}W=*QS+CFBPKLYR77vO94vKRBp0${pgPJ(R2({TvwMj z6j99vd}vw9QIr5nYo9%0v2oc3_M>M?LnLw1bPmn^L4}wVO>8dxa}*!T6Nk7lE_i+Q zU+w3*x3J6G$b)qALfbaTDy#_9GP2XjY4(!M$lz^{OO-r9#CoJ*mZ~@H+v$FuNbO$C zGbC&fSw$NA*2ZhZ?z-hl?!Cp58mpAQ6-p}>OY7sD16;p)R`5c&c@MASVEk`YlwL)`$SIjFS>PDnU)w={oz=ltIcNDzp!>j6QalV6su>g$6N5VMhrZLR|8 zvJH`jk*?g1RiVRA59YKs6kmHAYoT>T@{55XD?NxWnW~`rMJz?^h@wtz59rWF&4zB_lf%*NF6Y zbvDMMuFb2+1DkkR27b-9(KU@g4!)yi80&KJN8^RSc-3QoZ?eG$w|^%+5z319@u6+# zaU%3mV)%6^LLUh(x1LnbB4WbGz9k3LVxgxv^Np#4oUdaqz<5fopA?Of&i zn>pjSl!lHkVIo}gjP%}*qWi^BZ782I$-ue%6zyNH!-dPL;}KYb7Q7=DMAql@Y*)#; zpXvu1sOXKik!H(4E}MrK1C{P!7DIm5rgcxiI$e)0uTMYuI!1&yyaM@7ZokO#a#3V= zgWIeSf2fmA1j=u})*=(;_te@bBMUFmj|M8DUU8y94GL4~^|y62*!KPI(e5s}#Ckpr zJB`2EYME2mjLG+8fHE9UnY`C!zHsILp1U33nR_6668e_SfzX!$kCbf8WZbdr^H+Tg zkZ|=s?13RekdiXdFcxvc5O0xg_oZ(s<_=tXzjSS{(;er{Rz>Q&U0StBO~|Xs&)@SE zn4Waxfr7K+rdFa%4G@Q-tuK}0Na_B3+HPNqaud1RIheF2IbJc83o#IWAZ`4m>2u!;SiuJRlQa=1F{Qthhs@KJ=HfHC=$DJjQkAoCXPy z-4)$Yo**TNuC;Z+4teg_^-n0k(cR9(mF-A_q(lBy*x2MWt`}Y!0I2#d6(f0NWh_q=OP6a@jEmw5Y_yT-Mp0IB+y03sfgsb)EaM#Yb(u*&oL)+dU?;YpTm=o}p#UVljLKjCF^09TJd5YO z?wT0zM^h51{S=|=?{{CVD=~`9x4dpqt}tmo020%owYaA+uOKnV`Yh^F5=E(tF$a_h z7ex8zn4|nX3gR%T*Qw2$cTIi3N?uYQk%4@7TE8+B{*l0?$Ul%0&7L)dwOPKN@71VB zz1b55^ofHCK=JS{Bp;M@+Jo|mY>%iBgqkA!?(r|ZUN~j=qi&W@su$i8x z0TmZ?? zH!TIM5BCYx#_#b{YHy!iyfuz;rvB6XYSR{~d~ZM}@E^;VhgoX6wWPan&Sg_LHxx?@ z6%=grfZPX`JbiSLsA;Nk^vX02k~LavsMHRNqHS9fMvp20f6dlechh$ckP|**X^c;F z49q779ZBFbIIV;r;@ry>pja$Y88%j>88O#)km*gJ$+||z2@Ohv=wNfoxK%q5z%jk{ zp;IeYIFjvn&otcjAsKSOa_7%F$0xFbL@ug;Mm7Jb*W>=S4YU|(lbXJBP}KLN2Yu_@ zu%~otGCxh>G|qJ1a1#k?5PGC{*dCeWp%okPe_51TOTV&6W5Qnw(`^iLcls;G#+GhB z=x-=x{V>o=?>qB)KL09RsuU$XFH4Oy9OK=+QrG|(TmOTCE`a_m$pe7|NI6Yw-8NKq zjLT^hbrmtbRoxSsZXLDW5bJg-J?xNkX*Ys>AZ zJC+msYHO^)2jVbkh0m8y#Osp7(z(VdwZNA=(WxEAp$iv9M;90hne7ARQ~LkHUkAi2Sex$mLLpE9 zRArhIYoAJa{n*u5?r!^x-w1z0f5WcrhURhAHOokTki`(gK+R;oE+PiT9j82Y)*ngu zQPeV5(P!rR3^LJFzEG}^c+jm9q3aqg_t_26WG8mR84s@+v`PG#)k<;Ks#SqpO%O8axhIdy|f z>NAu76t+uhh9Z{IopfG^@C*oxRSLG@1f-3tk@m@d_y)^o@w%(h5rsz0Bvn79g4Y(l z@pzI*GBB=E-p8BYX+S*Oz&A3n6z-LoAl^^1>uyzz4;`qNDlPxgVgDFJ|A@a0Ut`Om ztb=b-H5uSgdV0%=?k1nXUe~E1T28rJDU~M|8m~wHDhA+z_cBd*eS!o|s*SZn$wVbM zYehBamgb-9^ewUQ4xaIdjck-d7;;bvq=g>1nz0^p;*&@wwj-XwK-|fLn>2iQp{^4C zq`HZXm0B-%zMd-Aq)0)4(jD#B9~D~Up5E##P5z3czEuvp$pJ^P?rD$-tQSmIGmq#g z|7Vaap$x4Ez+=^oMJ()3-lkJI!%{cab^hZxN@z+`JG+udfqnn z0Mx!zeb-ONWa^DJ$YoI;+dYy!nO6-hROr!Fd<3%NFlv3jF_1joU?dh-6o(I@R|3M4 zNxj2jX7u(y*8TVz1D{r0DbRi_J^X)#B@oD{i?9MjNHn#3R3ubk_lEq_w+R1Bp?Vfi zevKR+M|tpNCL~DHq`_5bhTw03I?L16LoGFlUAVhY5pvVTIB|5X(wZ`irzL(F^{v>- zVrr2NwnRtwm9iC5IWT~+WdSL%ogiverFp7=8n2>EzU0j${bf3p=1@j154is{~^Pww}So`tlvOrZqpTNb5a2ubN-o$z{_<-C+83AD(deMr~M zU@1_`GTK#c{y$w@G+Ki&gw*t7t|(pR7?gpAV`p>l0b%LZlyK^VtvlCF$yDQ4m-7Own{| zfSorgO6G|LpK(0(tH1&2noDqo&+3uNFs!Zggj_D7e{g|NIGAoJmpkd4NLd1F=Y447 zx%P942DQ86AH2po9{o^uq6I>TCqaN$zl2CkIRyup*7Cy#5qCBCe7dN9m~o!S4W-1* zV0qHa_v1Zo#gS*haE}05=oXe?kcgR5AOY_!7ZPz!Zc8z;&9w6cvP~1Tu<-H8_4bHx zt?^sG;eE=+H$$|K?-G7sPVdX!yq~1)*&MZkkcD+{8hKAWEPQSVKp!HRLzO@CKEXN8xnVXFd~MO940=Hn4v3?fv3L5@1{ z-kCgA>^L;7m7au?_9Po48PI+D&Rnm)SS=K<`v;>m;d8vfJ{^C)d*GWIH3Uu)j&LWr zeT#{H3czvz{)74VSyZm>aGm04C5*JSA2eFqV}(y;dwP1}s72h)wzMW(tNW~MRfX=- zc*J}F^gqtN!q0&#_meEi@YpP}LD#9%GY98xx_sRkD@L{Q)uEZ72F~O5m1p+MJJDsA z>(^gR+SBd4<@;$k-Z+s>^X&Y9PWQ(1(9!t76Uf7%u+z8aIY)!A1hv<%sYN}uMswu& zY=!{QxtO=Q=c{j@|7U0t`-VmjiyHy}%E)USYQX$O3OU%=a#E=N#lK!V`HXb$-X8kL zE{g5qoSNA8LJggE_xZpccgiN;J+YhyJs$Vd5qVm`NtE6EX`v6*UO;d}ufVT-2s&GN z{d*o;o=l0OpHzVC3vPe?YdUsp4FG74wDJ4cS-tio27T+v{qdN-vOWPBEI7_KO4A~IW`#^;5wr`@88J)=+}~2%C5f zjongZX$Inc#k*iD%dqXz;7ru>4;NKskxd(07O!RwhT78s5LOSUx+vk}Fi4eziL}iz zZu+9Q_%D)xk>fKG*#@#Ei7Z|2kH2XsL3R8j^Fu}bt=96;?qmqt5(Q(jLim=e44x<@ zJc>|sw96_bIR167OT6<%A!^?XWA_U*&;ia?%znb5JDsLDa#WP3^8U^9&>&FMOPdC+ z-X&fMy7pgm9qwK^;72>8ty0)>7V*-{SMPB8hjUgQ`1WX~%q)h1XjEe29T6S-oH_d* zSN^%v3_6Yw>u8l6mj-ewc zr^B4wUoawGv76wd<5djvt9rgF3i4p|>O>6e1(h0lHmFv&^BX;zf-#86xcLbVMzwN*-SJAo0(C{&~=f=`a((DS+pN=3~{u1cybXIRTBsI#9{`l z?nqv9_@yK!WRM_F!D%WNy+(loVzMpX)VS5g5q`9xi8&w9{gFHe$m~z^ZBB>{8GK>8 zaHCbh0_Qs?S4=yve(MP00De3ouOmrPRg>=lu~xE>`~Y_)2$p1ig17-^?c&P|A$U~x zG-sZzJBSccQdXu*Bj;h1J?(x-P%kx?s7HeddhT+S9%1Y>-D*8ZiZE_^t8Sj^cmmL~ z!h4_Xu;>5&6-In{kZQR7r^Ju^pJ+^P`+tGY&u|zAJReDSy(q&ytg$}b)O~I6K&Xj`hjTdUXA z%HnX8HPl_kL;wtG2NM);3melXq!3u9@hqgDzy_C6X&jDn?WaQKJ~@VY?Uw%VuYE+X zutfnq$8z7?eQ@y*8421WsWlZhuhqLPHuztO>~AchVk?qHeN$OdA#0f?cPmzqLEHd) zqQuaKA6X%@D&`LOcgq64+F2dfHFTQ2Z^gJy%^Td1M%m? zGpEt0)cmh&(H(vW_u1A6FoO+Fj_iU^-=UqmSoZgLa7CJIU7`I1WhXzMP1~s_y%P@P zr{Prxsr^f=@IW_&e6}I>+F{g?pIQ}zC-m4~ns`6^Ae%I*9`$&R9vKo|EnrQX&2Dxn z4Q*%V+o=v%K&6~8`U$qZ zKE|6?C#Ds|RK?A8_Bw{#Bsf>f!GW!DPO-bASI1Sax z%G0~WaTKZe)A<)>-sQ7{Z4)aoxvEnAcCV8Qosm7+em`HXq}z%BB`Evt-%SMs+PExe z6t9h-vZQ3->OJIPYO&}46e*Y&s@t})M6`vyT-7!9mIy_(QcLESv%xOu!$Yua5LZJAXyy>Y0{_$G(W$cJZ`*zJ-=Ne!?)!g43Zt#4++40G!7+d^ ziOX2f+4+~pKjjP5TT~1J+Whu9+hN(Ojim=>(SSz_I1x7!AYP8-uG&sPAl!;TpK&UK zJhocN+n?@U%0_sX>S%|GI9B_cUu0n!@v49OE5&BQ&4=LRW3qQw^pZ|K>m3+LL0->l z`1){9x;`b{7q^P;^|ymB@d||>NrM6D$aRAG4qoTRgj>cdMS*E<_lzee^i^ zLFWeudDL>|UXsY1FH-PRA4t?JZ^-3d)K<4Fp=3%-GYNF|wp}ycqZx%WzNsSHJ#=Tf z;yG&&C^m_Ay8Y=ig6GBc!hBt<706c70UcGW>aIvNVcMgqs2id5A(umBuhB(nnN=up zI!MFg+V~?-mr20%-q|E=MJjoScSRoajC0PYk=GJQbNq*yDiihTS^xg#RquMy**{`h z)n7z)p~jjk^6Y*O!v06CEHob12#cQdVVyVVG)wj&^g^gten`Ha6~#FpVM!j_cjFmG z{{R>D(yQWiCenCqPF{`%3d#Eyp+cudWXO;s@$#3W%91Rli;2yC*v%j2t4CATtM|Yx zW@!lK0AF3q|GD=aMgV3V>;1&L@uGks+L;QaS6)==kgmb01+!T3&__f+?;eAGEB%{5 zuUuoV=@Mfud%v#)9}Ps@7@*e8Dde#Dk4En;S8viM#=@2Hmcx~wX1aNk|BcHBGKvR1 zH%LEyQI!gUFr^e1;QcO&{oF%AL)+aQwj;gF36Li9i_QPkzgH(hB9jI|=oSA7SA`LG z#;zRK0IRv7;i0@;l6Q)ALoUtG{XmtAc?3U_o(J@9)y}SXEeS@ndu_Fkd&*PA4fN&? z*q&0JWcuwsN8IM0!1#Jf5Ogn~jpCgny_%O0*waKR19_^S-b$Gdoh)c^&=teT#Fu7@ z`bvTO7Ztb{?r+0w$Y>JBM8#79RWW(bp3@q$%Y|_tfidylJqV*Tn(%jYrv@|-h-S+t zWiqAHm&N@K>N9lz^#jzF$VHRjr9fgp*jGsndcN78-r^9L+?n5{gac9sh@Qb5lUbzW z6*d2sLl_oA^2mf!wKHSek)*&WE`UI^KvC=wkum7@nEDt3H;hn)LTsm z!X|Aa7)iHGTd+wa(H>m*ex`Oez?sI3T%XE>>{2#pr z|Hpq;`2Xp}I%T2{hDWHQt(_~1HcE5-pBpOvo0qh3=XRW#|EwDcLU%k{A1sjrZm}FL z`L712C^b50&P9pw!%@WFv3uaYARkq(RG{jhO(WmR7$z${0<(aVN-+ zx$4E{<{|u;b#IGgGW&I`_WO$x%kxW0u$5__1Jy0??Vh<1Iw$Cqg{&2csX4vvJ! zIqfM934OCkS~yn%C}A>tqOJ28Md6cx8WD}|alzf@m_1M`!zw&Nb-KCJS|Hl7ou;qH z9a0n|Rf|Wxe?fjs-fushgYxo|wU3@NulHNad%>?shB(!qTdp!RDh2MYpu9&hYTXI+ zyyMw+xw(6X4ZmzZ%)NxK+M2$h$0LeJ&|jbN5bi=3b(c^bY{%|On)~usoc!Uj9oOAn z2>UxWd3Ux8qc34-%~3aX6j&{Pl_k?Y$%nq0d_q49y%wSYWdONc3mER+q(@q^OHMV3 z808-iHT>R=PcITm?hVWn$itx6ez#*o-5C`TKV8qcSD zTI3)hpq!=MEqmqlSw8fo?1lp#!WWT%FH}#*wAWXxC!LNt@S9J(;TE!BzmHTp;%<6* zT&Tv2m7@89R88qyVTn*S_6o;%EZ>Vct!$c(h;Z~j^w6Bg+TO~E^7w==VvN3%F;b1~ zq@d%;ebj8qn)>{WPb2#@>KkV{T&zU~gLaD^(15TteR{9nmonV;iOGJPmk;dL!ad;| zUovM>nt9n~N2I++o|h5}4KVbiU#X!z?{9yx_xJmq>-=@NbS~y~eCMraJ@>lrd#wkJEAaPD9Pz?} z+KV7yqSax0{pz*yUm^m!p=-G|Td3WaeOfI_BGGB<0HHB#E267wbD4K;Fb-v;m4&fv zD66P2z(Oj+EGS0qs2nGCb<@1jkxo`P!?qxt;m8*9vEVxt>Ke&P!XM4JRY(1qQ?6C$ z^~D|euYsvfT4cieew&x9Zv<3CViq~e?Lg;_<@0=N!vacK z8P|VrfE4F@Qo+wrbuD=$ODSOcp&inb$ee-h+lFi?W!AZS@V*?4255rdK_0fo7zd#c zHPMzAJm`H)BqfjM6L`(()nw9&Y0c2w$g7Zr54qBBLGA;)ZN6%4oTr%gsj6D*_E&u6 z)rwr0Qtd-NwNC}|)uVE9ovyCYx0GFuOB`nOg63Ap<3Br?bd?_pSrm1nsrTkLa2}|Q z4&@YLQVj>*T=^6_(rIsj-OT~rZM%5|vQ5s=MZoIBYL44L7;{xo{ctSaa+e?1=x8Yb zuj;X%FkFyUVr2-oq)3Hyke{?aLm2PODV)fp-J$8;-pk^3vz^I>uYhS&2p!!<%O+;pxrSA zny85E>hTrUW0>$88mv~8xf?nc^o^u1GkaGCZ>?2eo|5Ehjf-a+bICHk5x6fvwCEg@ zUm{GUxsEAVRDwC-LFVcTCxSW?p#=bLjbZ=VsN}fnQk7=?`U&<*IY}2VT(&>VPaqMR zohnyH$^Z@I?jbANwHea;K1xRu&6z?bDdT1dLqv)(($Ms|MA7q>-A_uD8-kVdW6`aA zRI9TgEu6l?>vJi3jyrl^zvV#+SrZh?qZFuQOk)$n*o!@yi`8uQSD3NrfCn2O8_oxG^A~EasZgPe*G{r#hoyJBq_gl^G7~5 zo&ooaT825^abhnoRGIMDjdpe*81N0~I>}D83nKSwGxJELT0Pe)6Z_6KOn7tfiCSXnw3U^W6%`+Hb31K}m0OL%A%Ge6o)$Uj4cFAv3>}(0ojByNnaG6~f&G4< ze?_5dXSeIQi6y|;b0M$;olV>LoOAz4nn&I;8nWmQ zf+1J4h6$HCe*btC9~1@Q;}k`|-8Q(DKIU$rB&>DFx!E+l~k?oQe9LOc^ABl3Lm!VC19 zF`zvi2Dj6DFO@bdG_-(>ilr32i<6V?&K z>x!)WeD+Lk46KK{iLOp|=zc56B)IOWM~o3^_4GG;b_J z*R-@WPLnP{_q`=zc|p4Y6}6JmQh7N!Z_r37HZBfP<8|5;M&*UsJ3c->JZ#F^y0}|i zU0p*~=CnE}N$a&&IeEU($vf#eVia?}mm#84Y}#8|vkx88^;q9q?(^~jrEZ-&fOhpd z7$E>x>SAB-f<#K3SnTBFq+@j*fk1Q&)gt%v?Wk!?zmw-pNbPiN@N0D~f?fqbn&N~J zx}1ZrYQ6(!@OnSVNCiDbJJ;Y6sX4h>VXsaz=z36N1066!d{(jG$$Rt7UGgl|4l4pc2V|LA{;lf~Sm`8h_j!t?X&&PPN7rE@x ziE=t{V6y@_tcyeuCM0oxxr4i57l13J?$#Ii9kZ>&8^_J;@sS#OxOT<{pd&*|UdKge zFz6bzE3(2dUL?U834?xrbjQgVesS8kJ5;k-h3 zOifAD)YLAMUZouvHnrym;BoWvINV_HW6>@vC@XUvDKct)PtAPqUM#Q8L`zFcv00z4 z&$(0g{2J;4Bnp9U91+*95h|USVITP+=yb=snz7=y7A(q0{mrzHl653^q#~;I{oTjfc!6ui??>)rllQb7R*-Bz=V}L1%~jf-uot&|gbeLu16BOxN_Q zi{FP!nE2XW`vXQsMzYZ?a1mIi9YUUM%p|Y7`l^g120c9^Hc#3wa!z z9W{#xyX}14ZiyPNbRodUpKx1*fC6*SHc9k;)JdPnxl$q?yE^Fago=ZqL$Wb!Ru&eV zx_XYiPZw6@y;hXKaFuC%jb4_Vcr_~F1(u18#w`n%JO&Br_zxefq4OOgnd43o?Q=}U z&N*F)YRK7`b)UU&B^TT62jK0SFq~U>;2d;x#^rrZly($c%RMyq|s>clt)Gk%rGHVr1tGPB_Q`F@5cW6=F|S`qj4O>VE#=87=^ zyt=0N>GOlq+VgcA9}8q`r!j1Vh0buY#_M&X-NBlMn3R^*Xeb4z@P3wjmUVRLF&NfX zg4?bG*(#5Pw#h}syQpvzVsam~hXs$P$vop|?m9Or;yu^D6;wMi0%AcZSyQXV zYBd?(Q`{6(@&BGn&5>p+FX@Ta(h;>}%Q*Mmp`(uF%K4;J$;7e^_VM$>Ioj&C$V6Fv zK`+-x>(q z`fN?vgR0YFgHjH9w}A*M?Zk(g_U^W9^%qRYy_UQQcp1@x#-Bi?G`yM85_eO6Af9E&I2J>jceDBlpVKhgOJE9*jwTZ_h#?I zx&EaOp#L8D!FtG9^`~}wv%yK=vuX3Y=_&(|@e6T!kXfL8dCQyE5TB?Kz9-iNdh+#g zuBrg*0qUSpy>~RS`6GtKsQ~csWt?c)vR#J=7NZ)3_!)v8b%_kBdl_j}TkNzCFWmAX z_^)+`?##^s)(Nd=3^aAA1)XoSl=w}8pBIi&p}NL3g`oP!f$FXA407N@Xg-OaJFQeU zRokbN7sxQ)iKDqldCFzTHo_|U)yMhTfosO?X_@m~V&<)t*2n&e8wnldx33yJYT7#k zo)~PfgD(V}t)QdLsdeQ#kF&ev6OakwbDF!)b>wBjwE8$;3PmRGMdTdO!QqRt2!cjB z67C>)9TxTp)VFL6kJ}>^Ok9kec8+EnzoRY%Aufpfygw>Iha6~5PP++hBIjh0Q`K=0 zOPm}Bl5gGub!vp@t0gWvIy%4zMiR^-_~7MNCB9Bn)6QVS%98pj#_sOwVz~v@>!J3m z_tdoLoH*RpRA94?@u(I&qO=k+ChLU`(Zi_5QP^}*R5f}0NA%dSm1Yu=35tnsA7ovjLG$qgluzBAtX`g-X7N#{|kkHvW`n|S$^rx)uZ zFqfe`CWwuW*M7xeO{s%I*6e67+hR~>EbDzM|6yo!~vatDc4xYej}cOq-t z^mVbknv<2B{2IjBZlcEqonO^fJua9f-YMFHAbLPV^mYgGy@ldb!NH-8dxDwZsGh9G79e19|2Ok^rmHamSg)DY(^S;esbP3a>lE30)>Cs=n$2qftzTo(*hXJW(;{CwIqg>;PMq%N`#^S^LXI6W zr1&AoDNKWnS&s|Q`&C4t-muI1J`YJB8-?=xGpg0Zm+ z&T#MGU~pjISp5r>z~*pcyr~*%wA4ngK>xMkQLDUngcmz>8qDFW`(nKE$OW7>0K#ex zO7gu<`jUhlmp*jZ|BM2n`ZPXgcC+7^gtxJ2VU_#SIzH$7A62GPY0oo9wV31*hO0eX z6qr*I@e>mg&RQgeIk>qQ%54krY$mG8!3`T)=+3{EZ4Hi)P>D%C(Q_p$BXo#_goK-$ z8$iT`-7Vmakx$^GtLOmF0xfbYu)h>Md;l~$BcE2!Qg3d#WcZJ0EYT;qZetm;zM`0D z7D2{h-A6USIyv166tS#H6!tn{EgjpLrM)-;Su01o=X{hJ=%+?2b7I}W5ajKfII^~=;FV%y!xf`R8dm0;TnL*bmX%Y@2O0uBC}qB0uvtx zQ1K2;ti*kv{a>yHD}mJT+3 zyf2W8Dw5YLii(QjZECV|bDufgx{^V7z(!|EDD@TN(Wge)CkC42RLCE)lU7Kxnt%_V zX?dKL^xT&#ai*Kt2d6Uv4CmH>M5?DBoqnsH_A6#)}COux*pr!`o zt2JBxwQ~u*&L5oN*gnh*PMFTCS@rpD%%j2$rkQ?Tvx+|E?i-widG@b{@kii83tXWL zK=;|XaX9^mxAvGs-|IEaJ-0KfmLqipT#lvcuzGc&Bet)%aPnyP4KADOrpM6w#Q6=U zR}!S{cc;zL$VoQj_Mwd>d8srK*qpSkk@4iDyJ6(jnU#7A)M}Io znxRdxey?~~oi}TgdnfdSl(Ce$we@qyVTOu>wZ|Knb8wdZ%&~y25^iB~Lo~3y6qA$` zYrnNKGq?<0r;2-z9@dbLHrgf`W#FEpI(@fpoNohg920{FSo=^^^SIwEE^b?m1^^o3 z%C(|rzCl@3>g#t+rO~iJLnEys!}1O>Fl_IaU!DpaJh=9}*|%w0&DToBS4B?HvTqJJ zcdUmt>?-CWBw88JME=7QFDKj1y@p-Y{X^G%*~5|{OZk3M_bATcrjLY^5eioN5_YX_ z6^$(B`tb1ZI6FIQ%7THZq;nlL2wmT9^}Udhm5q;!1FtHQY*RXV9?yRcmLeeCL_|r{ z>M}4tC!h8(>sxk|mX_v+fSf2w!|rVB!)(!q$fi&QwX#PY=N~%Ar(B5F3 zfQ5J`WXH8nRI*Ivp|x4|qLss{Gew$N14cUk&Gu?8y_`W>_Fy7jJ%uh~B*o2}9flkc z6uOGiBSj*|>=v!E7mKOcT9Cpxw2Y7#qs zBCEt=C{MGbjb#KN7?4@Vu=~uySMu`mw5^{xe$Vho;vdxF-ip4zFHwGjToktf{W{GBq_F zuW$mh>iPNkrAwEfQ0TJb$xH~lwT(@;F$}p;%y)1kWE|kKw(T)Y9B&rO{nUX}bJB^? zL2A=Ca93ev4HIZ(Rxix6mA|i z6eS;mxcHo8y4gg{SuJIQrie2dqC%|W(Oj{#1BWIjMNp)MnYlS7xTEv-9LtbzOHI3L zb*V*8LvC_6^>ZENJj{c+sEm1@$Fmk&6-P1 zG8YkP8aE0QARsaqdg1)Uu9`A#Zf?(?KX-6&07Mpe1;EA4t=#i+GtH^HloL#+`mjF6 z{KMjF5qOAeolI#{;S~eiXMqh-weVL%vb4q~x&?;Ot-~{+U4q?Wbm(2WpvmF@(|g6o zNHMIpqO{;K1Pt5C(o%~t?AoS4e*iRF#Px)+3x~wkghP3tom(XDCgg6jbq&0s zT0KnPWvbkFBtS`axCM6P$pQTwy~QNhKE&;|m7Vo*TMthTGTo9#s}kBB8-|5kagLQl zk?ZpY2kP3?=;mqnnVGGgf+Xywf$|BXO9U5((rpd0-Tc$E$5H#4QP+A@Indh1C@h6n3vP^>S^Z!H-DN7y zrl!Re{5m=ip7y{1iHEur0JFWwsu)7&Rd3nt+RBAEfFLmSbsgK$O~(9Fm2qg|)@M?g zDWILBE#XmMF9mp0PXrT7s$il}0-EE%0PY#;PG1ytXp*tlNZDi85&S6-W~1dKxDKZm zD(O`F3cWtmK%AUh)sns+)wZ|`uNMz2%4Wv(BYeT{6ShJKIz|oP5B$t)->>ou&YImz z2e#y{E;$SAjm&|zP%Q#L;43X|{ zkT6`qe>3fQMjvn(E<09~mhu+eC7=~fkjt2lxir;!ChERnk5ZDK%s%Pk%SODrG0aQ9 z|3QOqAUt%(8Gc1yhuep|Nlba2ri&nbZV1DmRrya*NzvF9t+&A2a+sEG1%$U0yA-pV zDppUGU5K8rx2>zUIGQf{S?RUrUsd#`lb}0o<0ADrDDtiA|$?aLC$-j9uXWk^D^=gsDkt1U|Rvil;rxz;~LQvZZzrC0}F09dS z#4nyg8c-a$<-=NbrMs4G0#-a?>Fnz2I@=TmhN#~MPvN9iL+6dLk;zF~{Z8>W-4CQV zg#Au#J}-IGG##@B$9cetO9ggoYhkh?=^IyJm)Ei$IgeMD5a9ljwuFG)vTG{KEG;}A z|MlwD$?0ieU?4UfhyZWjz6E+J=`wZivI$$`o1P&V797-Jzt4vM@C#SU$zB=@l};&u zSRH>bb-yu@mY48SVaVFUYrTZGfb8~G@u!`gow&HTg@px>QGi##-GMD=In?N9)|Y&W zD-hDWrZ>H>{7_x$xJ0ETfa!hxX|9My=oB>A81ZJy=pBZaYwR;)bMtH!qzikOcUKDs!1y{LRMT)a*+?iM5KAl`Gz$H{SMlx!4k2d+F3UB41j~QC=?U zU&p6{uKH-Z;JHCYni0AO1|UsibWdndvBMr;3GQ>j2|@=d0{Pj3)?b3G-}{G$het-t zSie!1pf;ZnlK0gsXlf$*`;|2{v-$0lCiY8)P=8THJ!qO*hf5+#?G83O+87s~5HGa+ zFLQm-tUvsb-nSeP5#h)`J~~RJt4<;tm*9WLtM;Sb4z6A{=--RCP%5pW5)}?KbISR| zk4!aone%7gEJjuN`$O8K0utRX$TEGh!`cWfAB3jUbX`+ZRncS$@pzMBTPq`R={|kJ zv=K+_w{PE`F%p{xs42gIugN3ZHFtho9GmVzFZnmc(G)5%tb8)Zm2T1Vw==uBVu740 zv0|oPArTn-%K0YLfsZ%0q6y7r`@aO1H`i7jHihW(R;x2j1rJ?BS6*2Di=ZT=cLwtmSD6Ya>uMdLI|aB8L@hH^U`+lxoV# zwfhHP#iysI!>D)<_V=%g?HqeDMvjy_nE3ekNPP^0tOx)wgX_K71+WdLb9VT{G&?iM zajgcMTQxP?&?Vwu&_-^rIzIFPh9&d!6{80NTe<#Srz9OT!mFS9#b&%K-xB|UYDC>J zA2Qfd>-zXyY;8TD`M+EVefPHxGr(;k{fyX|2dW8Y*T;PUI)-Tj`k|+&*-EkG~MGN47qD{@lsY(GWHg5D);O8;Ta%*ie`iD8E=l z8qfLM$8nOepyS`g3EaQ6QgwI2{aqM;8bS=#=z9HVTbM1Z!8*zG$Cn2PnLzj8qIAgw z7iD!^fLvDI35w4oK->=~M18W_WKQ$L0DYNjFoH@s{!47)z_TG^_*2@#{5(6Cew%2W zZQ!fN0I^g23Skxg!==_}s0PZjGriWpX&psPNeRu9Q&Y?6+(}MW3{?Z@(*dgI!$Uta@=oq913~Ge%B6zR#_OH;i3@GADArEDvSi9kYGgLzF9U&H2`Y?M-sj zlN<5DC!w*zF4OG=2IBsV`{P@-d#_Ew5YJLJUq_Sxjh;i+8Ne7IFVDBfQ297JInCP} zC|<1w!XhTE=T^mP9w*Xs>l^0b+6b``;2>vqJnDj2XBs@UDFZ1B>Qyjl0`S9p<|j+y z@}XXq#^s8pdvz543xPV~sf=}kjpenz9!Xu~Xcw$(0X+@e>BXgm_?%M4%IkWkZhBoz& zRLDH{t}7T@iD^KAYiMYo;^hITpm)>z(`E~zj3XO~a{6qewtyPAa%Zbr1bai!=l={ZT zwDt5R!K;>*mac9d&xwuqZoK&=UyS5AqSCS;+1VxAOst;~D{3T`j3w5nXw%V7u0!K4 zecedka&bTspRm^n)CI0Z7mP?a`2IG9!Lvo>e)6Vh!+|veo>x#Q#rEZUaxA}Z~ zn`KTO{g^j4e8j-f>2fBju;?ed($~8^2+i8xzt;2_aDc2JgKiF|7R>8ex~P z;H>IMa&qs=ilfuMmX#0}7iG+L8=nVvj|Wf<^LsOB$dg#3A{TcH_g zpDC%R1Ox=240h$Ia8c#+@4hR9Tv4YBw}0kOPaf+(!xc9(GXrK9GqbFf7L-6VHD_Pm zh)LLT-|wGOqczH_w`1CYGYGVTd)08J?Y6=5D-gK2a=j0*BC)IfluC1qlX3DQeSF4)n=!@?3#J?Dm6mGxEfy2kaOiC5W?Ay1}Kuh4KOb+likqrUO!kjlk471 zrN9Y1!{lq@yCXc2YWKvzw&uM@^|Sbz?Ix>dP#0~pf6bL@hA)vx2`C-UQ=Xmp1;0m=+fvE{kuqmtTXt5BYYu> zQ9*w@tB3C_Vq6tH-RNZl*zFi%1Qs9mFm@CJlEh+haoa<2uloyH8tQB4u@A&3HDDFznS#(f0mO2aeNCH%3D_cU4jLLC7c&V*k|pfUPYg5 zG^mBw{yIBi4owU!2@dldf%Za%{P-9B4O6Y{c4MMThCqIV5Ch$L=>3EMB>eg0Tphhe zoS<|SOK+{WrvM@(qJb_kFL>B?@$%48e$;RW0*o}eKvzQ9OwNw%#H4uQh3sY;-;pry zb#Gzqt_eF4=y&6`HVtc$=Yw0NulICzX;4+u>+vIE0gK>Ki`AqqGQ6W#;S%KQ9h?sZ znC(ydb|V>zZyz(>wvlJc>hmk*wKsBY`eG<9o;onFys}cPA^hizsD5gMl`Ke96>yO2 zlNZl22NZ69)7djQdF5~g4^7mU=ToSMlqbY(fyoz|lhSEUb zDEU_|fiIi?0V;re$;s30`GYmK_HsQxXK&AS+MWrak3l=@LDVEKlY=J#rvQpm&=wqM zMJWK7m|&0O+nPx7xxw)7zN@Rsi4BhSC9b?(3!5E$Sf29P}dL zX?m!bb0;-`&P?`57vadS^WsH>T!x2-hYBWbfg^tyk90*vwo=>I!GtSI_k%2&bOL11J^` zAK#cGb`C6~FqlujI25_~lY9vs>KD8x_^~8vap#Uf`Y}yOJ3G0-YM`;upG{1*xm9pv8j2*#pa+qVN0PW4+x0zGybec=TA3UHFJ~Qvky5rMDJ~_1^4~Febh+#JQZWB(ClN09sWIb+liU{sg7gpE z^a6i@6PL{iNcUo8WhIzrF|S_L)Y9^BcL$@#d2?cXVq&AJMAc!L_jtHE^!&%++82>h zrK{FV)Wa}IZ7DI~mF8-@lpD{rsJ$Z*ZIsef`(hPcB1ayxNgA){-Ha1WHTDrng!rJg71;gN>|hZEXQ7;OWT;7TvL@aAfP+SUIe=R?mq<-BN_C!@hMK z%j0G8%ZvL1GJAF@c1QlbswaENaXOBTIToFsWEz?`W6zx%)$}O$l?X%FKMY?l!_Tj+ z^>KCmO!xz&CjdLP&vVIdjwy7P;(V+5YiS-^wYOWCo5#h+udc4f$H#-&y1l(!rFoOJAOEW30twb+Oz0EZui5+N6MRp? z#TB&?ZJ2^fUq#Et=gp&cJ$pX+5aIg$H4U4cb1U?MDw;p7gs{@AUV%4n-}~99#V9~{ z!KX@iJ$%=dMpb3Xbd*h43BIaz!*2(DUN2Z0BRTiF<9{-)Qt-!!@zsT``_qR z@(DH>_#hjk_NswqS=LcbWo~NpOGzWq*)8Te_>Zq`d}KFoJ%``h()t??g<%K;5AytH z;^gswy7*EDroF+gM?Fa;;W%iw+};SS<9<-CH$Yh}?)o+2`(mE+R}nM4&&CHp1dy-U zIb=uW_?zF!1dImVTN%P4WYN4$ttNXM@lfQ{u<8}{`NjsGFi3Rtm4DKH+8#X{0q&8^ zJQmb$2$ST&fTvkYS;9Aa4wPL)Vjx1YK_NI+)} z?i0M(oD#M7Cw?ULLBkzCL9h@d(A8^CH!Ahq=1F{HMJX}m25pEBcz+hg{6_NKKDr;0 zeAngaUBd|q-TP6RmCozgDmfhy)B?_yKyf+V0S5)lLgi!Xd&zo0VF%`k^Kq01Y1}2G z%fd5ns}oD`wjBPJXm?Dco?5O$Z8@hfdgp z?dql=)APH>AO`_PYLFrA1M>dxkN^*_7!C)tQV4rZS#`BgeCPeDzL^lcvy_jER+BXl zA2!eXzTXb2%><+#q$_Kg3ivlPqJt}{Zi@twwi3+1&Q8y*^x!~!N@Kt}@03DB9$nWypz2t={=Gjpj_ z#?8Q!QfSKTlQ3?-7Hq|SWO98z`h^nH9UJ+$EdM8S9e)yX^x50YZ0A!(8PRAkhNkzy zc@Pi~ivgDODki2wc>-cjZU?Nm!+WO3ur_%p1L#&9s%96w?d%w;MC(V-A0VO;vMZ;z zrMklI7I(DBwQ;)h-ihqjd(unzSGrAhS$_2a0*LS6oSumPjw!ILfOYtatZ{7Ujn}C4 zk5I`7s3zkY3mLACLg%@%MZAQ9odDTjKKv*c z?dr;a=0dpC>N3rrX#2Eowb)TB(m}uQ-crd<-0_8Z?Q|+%~@VSha-Q$(H?p?4hNU*S0l8EX$fr=ov{C4@PEDs*7CIPnxaL^1!l#MB(s8|K8f01Og>!x|H@G$`DrSa3v#7{R# z?O39jJlU=IA#C^VNoVu{t9teu;J}~MQB{STbkigkSTQW*8F;n3hE->OP;|jV{zIf` zyBnVVLRcFpA|ACx(ybg}3mjTa2O0MVXG3=M`7WsvWw`OZf6BZp{f{(6;$?s&*6}j% z^iI7cupWhX>hTzln-yGo9qiNm1ymjWx?cmTmusaAuq0`N0sy@rlQsyJ5iuXQ4>N3j zPpwsItyAM!zHOqWrpCrg>a{ck9Am(NT?&W=@F%z{gn(qLVG|wYH+g_)Q-9AWJrU(I zWd2twF?pE5`J^SU1BQixrpesBxg?>wJnhxWqek?%|1)YMXG<<*6!>=l&vFWk)W;x< z)Hkmk`T0g# zFrs^rGqwX*_D)Yv2UP3yXcN(s2&bM1-CvbZ(Zi>d@VQ^e=$(y50SGdBP;Y^zGR6r#nclR(`pDX8b3dO(TJtExHyoL!HnVJ z;RX@BQ98CFD#;p(2+VKqaFYv$i2n{)4Q9CxRcRk*^WX7ve9{Yaq6*nObbXzD=n^1O4PKKwPho?mk@YOqro(tn{kadQ3VaI)n#uyI8i?6{gprNOi_n!e6yiXmOBkA_ke7YP{hI}4#$NL0Vv2Nx zHGL^S`p(MAs;oRcJKoKHySSg1Uu|6FIxWln809%8EYxOGQ&Ax~m%*mHp0Cy_LoC79 zAo~x!S52kocje|~$4tj?W^2MW!V?$g!*+=nUyVklQZ_3+oeyhM+CS7A+5U8!HLL1b z8HV;^FH){n@{aX1@n6tHJuAC}>8))5ctpVD#~B-1kC&gBT68Q9cw8~*Cy$lCT8<>0 z#gxpYUF<3*GgJYIB^ESyD^e6^T29B?_-C*&03_#fEDGvuygmD z$;#a&gTqkB%5Ev)ZngluTOD9nS6dMu3B!Ijr^J;;iTn%8`myvdS`jz6B_otS&4lb$ z2`QZ6NdejN-vFBIy)J#n+2f76OSTOcD)mS&TD#>qJX)6s9~6;OK!zbo}#er z*lzZwcd8#Y9hyVumEQvD9ItQInfpZ2?~^HVy*pxJWoOR^0ui7@^Yd-MD-;U0s8O8* zJ!iv4VvH46%#v)|H|jSr2S1S6$Lf1tjBrO+m3}KrWZ`!Th+85mKvidF%jC%4Onr0a zPk8mUFp9wE<{s$n{n%+tMM;TDt5#MjDl17OeC#i}&pVHR*JW!-z_!TShTo@1Oa4$H zLs=`RX&aBbXaVR@ne#FSm+${uhzcEloAE*9%G@Nsu(A9Wg%8#{(Ev3ao%0@X0%{TO z1j2|;V_-xBmROt;ibOnmXVIrWp^kVmls-){b9Py}nA94w^O z6qn{E_v{HAW4jHPAx4_D*RT)wd%o6iYZF+4t+WmQT9~R@29TAFk9&GdwVRk{tE@;( zYygINrt)sXsqPx-&Yu9!2EXGPI+`y*Tml=x+XzaAe2P@2Ke4V~f49vf@YyLl>7xP* zuqsu5`9sGZI{{V!y=aZqneSxMZ0JYHC0^ov55L$Qk?3myJ0LN^D3>!xtqPd7gk`n# zCiR)6(_S2VPN=#rDX2Oby(ScsS~sQSI)zQnVTFZ-Fj$`^ zQH+2&ZX>(=g$UqSR^GzcRsY_gVo*FBr1p~GpMj9#)s<0TUI9e=nf({h&q<=);{iWq zIKi)8G$;z}BgScTD$O0-UqtL6n&s*|-rEaKDO1|py(8m{aJEMfImu%)k}ecEKQ*P# z7#S=ZfLYglGGgF^p9Lb`li9nLD5uKpO%esU<0e)@{$kiOyokw5VlYld)zj_&=2f{d z_$`vP-1+o?D`WAv7tdcjF!V8(^0hwU`4pR*rDzzT-~PZqRl`7zf)i0MReni8YgmuNWdgH^3k>KMzO8$G+_@A$6J7aozbY= zMogU=)L&dr@5(uwZT)-vbIbDR>{9UA3^qmYW+6zQ9-1@LVmxcFPxq9+OK82m7M=D` z=7|U&9YlBP49jBYA=!W6O^A{vu0y~>+0&rI*vU)t*uuOyH~gjIa+Owt=fx}ls0s8sHkE#-x} z-m7xyH?B?#S3F`N*noKrDuNhUClhK`?v62ENUA&PC>|{SVO(=`Tzhztn{V@=sn`_M zFfK1Y@;)n5)Ya0eeD(buFyD(wi8mMl2g)n1D#*_0>fFd5&J-|a;O{j_Q>nS#e{mUc zYLC#t*XS~Nffzd))}_|hf8MK{H_RJWdKp|ry^WSGSbs?w~4uruxSARiz%{B z9fj)|^%E3{B@S#4iN>DVtfyk2-Mxv8d^IHVVpJoiG9QCZF#RbV;*M>&m&F0yXZ@^jM{zvs|pv3llCqC6Vz#``M-Z zb;&yzf3myTwQivuY1HBm>s|XP3zhoJUqMy}H1!rhgC?j+R%y;nI16Dl0cExuI${D$ z7n(;VCO#G4{(^NyHZeT^$;5nKc^5Q&5xl9fp^f2O;`i}vM~3J3Bnn@eCZc=}+iY`Y z?D*1g)eTqPM6d|+FAps!wE({WanT=`c2agmq8eeVL%q^Yy_YBiH+grr;VVmx!Yf@J!@->?MWyv>d-`mQ+jV zSm6_4?;1VHG4DB2E<~LIeiCL`ggduPo9R7m{rZ!!?l|}!P z$7~7RpZ=rz0=gP}n?QSZR*C5ZR8iBL{$aBM0j>kMI0>WNUm^sU4%X}bN^;S5IxFgl z#T48tM@Wl?;+wABw{uM@0CJf7btDh)X zg!nRQcF|y`eE&rkuZh7WR4gGSjJmYCy&Q&3O$=GoJr-eaoPWD^u*{Yjay#KNN7c&8 zN+778I@+G~`(VlAG1C}K!mQeOyTQR)-x+(KyHWsQy#mYDY4HSxglV6<>EVPuA-(8I;>xZ=gKMKv3vVL@b z3tzB45~ZnxuG~_2H$F1_yB1oPGM6Bpe4VB*rAMC(3=H34R0Ao*J59~JNB^Y5KGqt& z(_f0J%q!amOja!mZ18&sdxSQ|mbk@fK)S&%4)=cDeI^>X*_yH%yT1ICS2Vn)<{L_n zO=$Q;{ny+F7{Vy(;|syG0P8QM1Yol6B$0gnqfFM*7wmL%Y5A1%EmT3>#pV2bwqxf^aw!sl=;}X7eX6dU$SP9QsNeKW3?!i zDZTvpJ->s3ULHWJvYa(w%F%j)1#mr@9nP~2Uh_TjJSx~9Zap$j1%j=@UF2n6{C%fP zR^sPcA4{2>yTq<>BSBMgR8|ueTdE=ZYxU&IXc@P=@BK2k+IaQt%(o|bpDY7oNIV13 zTb`%A$Yy@U>oFg=YVBq(x0_VR1UqOQ(WSZ&gHXTvc7pq2Q7;ny?DkCJ{gmgkcg--n z>N4{qzpdrKsu%pUuidf*WbXgWyQ=xTL+8}QsZ{V$&a`GPh29C`#cWbIi`7VsyOA;G ze1b_$D3OT?-aa1`4j@8!-RR0zydW-jFOZ%lkCJ>W&J0|pg412P@I}xHrvGN(Yx;D^ z!6!Y;yQ@R7L+LxtCRt)SIz9|DgsyYZs((+h>sXTseBfCcwJs(!D-Y4qU1EXu&FsB0 zv`4GK{ts?={tOq6p=k}6kWV?~Mrjx~Qk?NuMCf+CExTWDnj>aVJ@;M{AI%z6a>V_G zoFeZscX2wm4RHbOF}m=p(2_aXDLQiYRWR#cg*+%Sb+WkWEReZw-4BJ%V5luGv!lnT z(1-t>tfKE?J$0p(VM|X$67oYcsH@7gkj4vg$f_w{s0dPumb>hkO)vc@C1AipGq)Zn zaiFA}Vxn!|e0Q$u`=0Zk^WX2@aqqaB0oi0Ed#%0pTF*1*eC7(#P*bENyG;fHfhd)p zKhpw%uGoV>m+xI82Htu9uJIgrAvRM{ddref#`{GYG`bdGUM6J^P6( z2*eIjdM2ymj$di;i#FBuIrl3YfnL75qDOsAGul+a@l~tCr&~eXsr<(>!~tmyMOH$W6nI~6 zNu4W80euGDFEns0rj`qq@?uRaKd%WSn(FxEJJkKnpSn40UhhXgodd1e#+C6R+ImA2 zo)s}V;rgrzJJDDP+xJa}($yZD+Hc^8+!qcmfk5fgg~oLd>n3ty^$+5c4NfL3S@vEy zN56LK9f{Lb!-CrV-jzeQl4)Z6*<{N0a@x+C%ta5_pV$r+)A^ka`AIpX+u=>;g7nY2 zHp`e%X|*K3w@gQ5-8)ZQvR7yGr;Ed1*GekbYBgzd3G@{ov8EpZt%BI^xs7||R39xB z%u~dM1#A!Kn?78t-|l%cBo1q}wK`v7txm3(s&qaZl6uR3w#52?AebZRP0V7zwav5mJ{~UDmfytga z?PP{KhAJ5$fCFx5|HYqYzIlToJZP68k1x3b`?};3t;g_9dt0ylNRN+^@;<%lyKBaZ z1aiiS_u+n>k*_;(VDpN-eTlN_bbN)efP7MsccX<{+!t8OAZoRduAS=0B|~~D{3M8) z7oT9a^xMT$4kM5ky#+ql9a*J)G5sVSw&IFmTO3EvL432W0@nBNL^kcClJABQ z)$3n@xVuAg+T0|6wE;XySJ%mtpX$C^s!*t|xPK(VeErBJaM*1f0wu}MSZ(#pnyGdW zYkL&lkMCa}mK1ZjBXgg>-16y#?1suc+Jt}FqC-~O z>pE}fwj8mO z6T7lV2P{9pwd1&QUBX4x#BA!-hqk`Lvem=E*NL0z#^$$x9xl3>1znyHJV3V+b74m8 zjvA`QniB-^jYNO1GLMGb)T>cr3imEL``>GfL4Z<2OfJ||Y^@`CHPgyOE&Hx#6{J#X zsk&)3*f*!i#}>OL_3*~U7l9-F3;XEbXCRR9&G$_Iocn)}3;*lfnxaH}hfVGphcCpP zHSSUcu&6m&-u?OW`tOfgU&+iYcuy^-sy6=A0tBZ5$vo1%2x}Kddg=eorQf|J`uh{0 zJ`HLh@cn(h!!#4*GC;kQe^4kLKg=D#r^A2 z)ZcJ8e%?zU-b|cM-s0k7uJFZo{V_kh_(33@pXtlwus3hsza%@@(p&C|r?)MCPSqf) zJ1?X^V!rh4jrRVO?x1C&)qO?j<328@K}4iC+S*END{o}+c+Ue<&l%d%&C`V*j8-q-m$?~qp!AoixO3BS$Y zNyKu{F010Qc3eQ0<*zPeuTC>kh=BFdSL9dFfozD*Ed%b)^t+1jpyX_JWf$T#)<7iInXmmMh(L!Q|;fRaM#Cb-f{Z(@u@@DFtT4byaq-)fT zZ9OISP!(Qoqxl79k^9?{QS+;cg~N|+YC#S2PyrbqoJdi}ZgYB<5SVKM!xkl1uZB8O zBL{&V(%5Tx2@Q|V!A*V90_G;a3mCuU)UJ!6t7hE$RS^+5ZLfUanl7*uPYzP2=Pchr zU;w`H&6zUcz#g=~fU#<#lzs1!a}@+)2$Uvj9u15yD_(=%(wRNT&g;CSlPF#&o>qEj z-xd2cpWqZ{aerHIbvZyDwO~)!ZJ2BREvWpl!?+SG&4-oW%5~Y`y59!9zS%;rl1V8$YIhhFK|5v0&0$|z zT;GDT8nAHalCsenLFs+CrN5C+cosR&fnnx2RP~9{nkz5<9_fJM-$O}ys=`2hs$ZTO zgZa^>YGWlM<1wVF6p(()88j-K5DbCIqgeEfY)6T$li!-Zygo4B)neuOEPp#`CtGh; zI8Si4__t+b8M4$AA&q)d-nBcmRFbqK*@CzCGL0@9i$C;T`r$?i$~qqpks6vmZpRo` z4O4c1p*DA!3RBA>6=R$Gtdd;n(bEB|<u}5B)eTPIW}`_tKWJ+$alJzyw;yq=?r$ zn)Nml&}sj~h&RNg#AaVfg_m~jwyx*p|A0D>1_BRXZ&ah|_)oi4#o<9HRsAIRy>KWzGsxZ3d(Lh&AOUnwGj`5Gm%lg>u{5i02FF)#H zm~JG${Ym;0o_f4;Z_03@MkKG>YJGy=WU~zsQN{C~nv;`LxRAJFLaJ*0rxSEdV8-&* z+YkenDVUk{h!gw>=u%4ttvZ9y@m?r0%}`S;#&nm@8EX8&pwEKMZJEI8ov8>V0=8R% zEklcgk0?;3WwT3wnw)sF1oN~jKI}WK36II>F=&bsr^eCd@;wl({3a@iI*NXPR#CdD z5W1Ue*2FXRp^IE;ujfATasDueL%}$6zD#=gkg!vHuigZh^>yp6%VF@1Two#oon6w$ z9*7-|ie+?{Ir~DHo2KL(M?=Yhq*Y&GrCeWarMyD$sRAJCRiZ$%pFihA8PX1<<47u} z_<9!5T8U^)1PkKm)4I!Df@3Kp=#hFi?_RrFT-goI;^Ajex4_?T`dI;sVz9y}i$9GV zbxRlmu}nVFF)~F`#dMdJf*alnNlEz>(rOr?>Z0RsPuN1$)k$WvzZOWe$Io?U@|W-| zsqm*Qdx9a_vjg$H;$5sxCf0_6n|bO!HhbHQti zcctQJhE9odjViFg*Z%7a6l5N^JL8&05d?-FS~p5UXt7^ zr4;TV8NMO%;AQ>IT2q^&1DaXq36zOI;~gj?e}lWZ#w+uf9O%fU8k@{KBJUWDX#z<{ zJGeU0>pbkdr80ompn4%82bh54Hgs?@HSbkI>&0ZWeY#)R!^o@80&K*81z9Jd}J4=Qr4xNEq5!Nk=>P}sGDeYI8Uy4TQ0}+ z5a-7qkj=Btp*T`N&uy0~z+nQefukF%Fvy{Vd>r&V{! zO|~M0&d58*NhZn5wqPBnovj1rcFot}m}zM%-(-A_OdOn{dB4Q`tspQ~2m3`8{&l&x zz=27wBH-jl_wSM3_N&)K<@?MAK9s!uvn_4urM}_eR(OX(CQ)$4yLV|Iq2*3>bxVSD ziTEsyp35S5KDJw(|5C?Y>4RVp$KKLSSpEZ3>u{#DqpgtFnPjRE!$5H08h_R-O#n{7 zYz)I^+K7FhB=}Y>rB|9zIPD2pM;i1dFhfvIY_lctj45@vUt`06nxbI#rdHA5{i6pIVg76-gruZZqLhT&$*M{uWdvrJ(>MAJCN z!CWXYlXpqD_$7B3WNaUU3zH1uWoS%&%=Ys1NX)Obu`Hn_5tk8BFPP|%v^>H~tM~xn z6~<$lSWz(HR0)Q!*74s2K^3p_p8QVbS>HCnZ~G4_V4JA4+X$bgdD>KA4$h-&){&0_ z?*A#yqz0m;<#{c2bd|$wIe5V{r1C1<&itjCW>%;`$#9uSUgOgJrjl2mosOP$dc_7V zmo$x6OIAHBvO(#Nx(!(V;$_C%5|!R`uzMFC4pZ6?#OA!wx48feb3S!YT)uv#KA zZhHH%@P3T;NRO2LjJsRR($yu1S$^7yh_h?ybqA{8 z4DZ!9-#0ckG?N~KWNvXAImtPpOpn6|trKUv`YosDd#7(8NGcIOi?r>D9m~?Sr}|-T zO~+&ZN^4-UfYQ}PAU=W(R@jAvgv!%~N=zN3T!HLgp7!y>hr;&A$YL6~YhW>tC?P6x z1_8SEG}i|v5w>D)Io`XUO*DzsF;@eL;-H1-dso15OERhP+i&;sTTcA{(0qFOyX)** zKe0Dqd|`nXEK(V!1_JS#NAq6b1AmCq!K5z`eR#k?6ykWFK=(#)(Zn_82;e*rV?Csu#x z{|<=X^h8hFyZI^4fVdvQz*?sdnzj)Yd$g$#5aY(6?Qed!-+m# ztzErOI3k(FWm;ud4?+dRzqE-&kjVowpc;xY*)w7MR>sx7yLiq+UKkaiHm=AYK7j1jvki%tlJ>``lyzn+mh#>s&>^Al9 z_@!Umy#_~C5A>fHyGeU5gLiz};`0`FFH_$cXLjFfpPF>`x&TRODdM_s4<@BIx3;6g zGCx}7KCq_aTEC5k262Q5yL}dH2dK0|I4rdruj>Biya+W(4 zpBw9aYRBNnWv(ew^#%tvxdUWIEet{}FfJ^c;27|cGMAzPL(9k0mPO{0_*>;mp*yv$ zMEj(GqR$fXV;+)aiy! zW4lGLS%!K;{_eW7lyNQGk7XAStb!p>>bE#|QK)%EGlxTs7 zJiKeVh3UcYOIaM6b*|MS^BTxddIBI2oWTm$yL2;G%sl;E6-YIl8`Y&(x%g?PE$!@7 znN$tN9BALXp<;@WlBr(2$8$A>Ss;%e#Un+5v;A;($DvliPHM(;u@~(Bb>PKHNw=7t z-aWjI&v{a;dTQwsdavTDp>Zr7Q{77|5U7DE#^4GA@>ZY$?FW@`9w9{M-Qrx?4?aDP z*|JX8V$l1)>7!sV=c;;mPPVb;Oy<&eL>z^;n}4y^7s|9EW1v{#FN=jd31SV zzBjyx0mMd3j0~3NPUZR#|8NK`U{tRPEH)Q#xkCxSMz~u(JtYD3l#bBeo4g=4dI~Bk zw(Q#{BQ>YjH`V>NuWkk&uVo8@Ks0avh4MGo+?ojPzjk(9wyI4mz6sqcG_fB)q{bp8 zI)42E_r~om4;e;{Vc>+7`7d8MmP`LFAx)z zv&Qf&Jd%Xh0>I6`!mOs7FDTUKn4TLXsUs=A)WxdPC2uAtrVb)}+PUz#DuG=3LRk|%@>=5mfq;+wOs#Z;%jnO}Zw0igi_$riK#XaP{J2a_X)httzd3zqK=Z*h=#%WXEdN(Y1d(W-Zt&fDH2<(ONo8qxUF z8;A9QLqeGO6lQA`cpG;oo44_XrK`0Qpg1zoFl2UVsYA&GJK^gOSVx^@;=cZnO#ygb zXS0!F_G<{_n)m>@L>?O2+E!Yc^05jU^`2=LdBvlS99CnxFp&LzK-RJYH0)yci`+NT zThDlpYru*hReD)n8*guzN}?Ov4{v8D7ZhDfP92MXvWzh&?k$yzP1t2~!h>K@9xiIw;~Mmh2(C!4>msMP zB4JIpUfbtultGzsJ(qeKT}RP&jR~q;o-MkF4D9=!vktmDOI{22*UIi@p6$|1LkzFT zB}XJU&3Ll9V86(r6?H>;mcI)}@ah<8Vx5Bdi1586H9upcQHck?@Qj&lvrQzLU#RHky=&RGz7bM|v{98Fi^IS1&uj!F z^^dpUPlF-RrqzET)Vzv_vh{@0^-K%j>i71qXw8i7P~K5XcBd_SqEpyU`UniexV*c; zIqFW%1XZ$aE(KEeh#ZsLGF} zS|_+KB{!LwzEGV~HP*K9u6L08jUPoCZLE<;w5OK+(XJt&9Mf0>68 zUaN#%>Tbk zS$~+KSjFxF`j!7yL3sFV+l z2X0ZZ5&Ox=(b~+8>~(6A?n>_sTW$rT(hyd#ncR)c=NA9ZvfXsuXKh2QlSX0EXD2bE z6et2gj}_?}rs%dcKHRBfbTa8^a4Kpvx24s^Q`BJc8VSF-%+^xYMj`PH1rU^KadwY$ z;R;ZySd|y0%vaIQo(X7N|0d1#A>zT;g_hX-=_z4SLvKccPneRe8Gv;13S+-mme&_po3{k%99M;(n8m^kvX0Z;U@b|Gp=l9-1H6rxT{Mrv z4x&J@?r$%EN(vlM4=|Aa!7{ggOSZgnHh_(~21ATFEqRp-8B`ooduY*}y17XT;n(_w zqfoe6+uKDcyt;b9O84ffpP}~Qk|GB6n?9QABI!8FGVjPmX9mz zE29=SkMC?r9rOS*#_VC=wcXb?uh$Ttm$D@J6*p~cBKw8axcKC~`~Koy&DJ7sS^-6= zg@85yt^KWIJ|`?bFNUespdVLWn#g+w->Pk`%koFHk=qyOI^L4T$;k^}ZStzC z{_EV^xDveO%C6J!FOh{`U1*CKRVD3ioBjpNuBeYkH%P79-s}s;lcw zmu#Z8Fh$;95OdmlWeDPE)K&722D)S~@Ac%S4e)3P%D+rAAI$|@cwePAd^Kf&s<~gU z=khM#_LJ;eC#b3Xj&}i`k=-WP1-ZjFr3&2R3GW`b-FaLJ^Nd~h;OW@>yi~^MoWi^P zf~%>K=-o}C&9dEB+x{UbT*zk2!bdlz|HMq6&ECF-X4L_}0uF*Y7{y=7;3Bxh)vcN- z6pItxZ~it(bD{XR^p(xxcY?x@Apr2^x7=H$6L@m|{vvF|5${G>Ja`wT624e28mVH? zpWU9U4S7;?g+1nAlJoNW;R5ZfyfW`E<4>h`XK9-i=o0KO#OCiKxUzNc^~kt6wz?^C z5DWJMr5`uar7wxS`FtDT{iJ!Kqn|+2jwf5>E$=SvJAD3cwOuCSJ@MT&CP)d>0$b5; zHID>kem+b`m%=2;x(Rk!=K0ro+-6YHODjxSmF~Bnbo+S%8r%aOE{SO4O7LUwDsj@c z3lvTRC!?pJ2))Oh#%JO^b*#fG6Y=9pR#7HI)r0rxP?weYxaep{N3bK$Dj=raIyt%m zqX?n<%&TD++?}vIV%HYWaW^e*iA&K3(rP*20aGEYnvH1h2tfVp;g?#>FMmfiIOvXV z3l8NU`4Z{yfIsIfbOgMKx~ouA|u4j*K|K&zbxatnP~Rx5dRY>RsYEpcJv!8 zHdMU_0^ObKStW_tYj6(`E@1s9yqB^!Ef1AcvDU!}n33M6PoEB&D=O-f0-{ZFeS6|Z z;1PvRtJkILhyk~B=|W8QIG$y@6!ePBmu&;?1(27Pi+T~oe`r5tZ~M#cLR@}CP&>EY zu7AVMjm~a@^C|8kbn}e!L=HPCI%K#ANkk8osTvx7HA(?5{>EB(H31 zJ~6%#fU$Vl*ln2e92+9Tg>pZ4VM6?L@Mx+Pb}xmi1il@#8%BBxU&DPKV{+Z*T_T(vYxiKoBt@`!3Ht&fb!xxR-xJ?k=sBi>$Nm zKhn#?YXbjUdI624i+?L1W+ygvB#uhj8m_l?HFAXS=o^R1Z%-E!J!@)ihc?v^8l<%=@^t2Na?^^T&B|j{)VFTu&un}@HGBKk5ysReTR9_du1wSsp;iI*SoCLn zk*pb%h=#^#NjbgQz{pmE$=D%7y|uqpr_?$;c-}trTE?^4Xp7$3VKHb;*01dD`;G{{ zZzc}m>J?klypFP1x@QTqKtU(0vWxb6^)P}EZDhmJk!nAhy~H}EoQW@v!=b=iB0XT}*&Ra68WO`XfR3j+D{)lbI>6rO6yZgFXF zHf|m}2lWZ(KgW34b%i;+EQC&Mt6K0I_7Yl^+6HnteO@i*U8Z*+vuCiWogX=Pb#~O0(H}5p)t$%^o^H0 zOnwH%U4k@7T#SUMGb54L%_w!a zMwv$iEcc$=`w*W>gE;$jmQx>pS(H>5aznwFZMT%Aj7>Aga4P(0WKJnl|KTKg|_NqNgeReuRTB8Wi*GWzPPw_aF=n%Omg zDpLL@NQeuCx;T;wUW2>_B!0nhQ7%V$O2HLp;BKyviZ(Xg!6nXx!m;m7IgLvBlVo%= zwC&znn(@mRrmqef3giVrCzgZeTIb}a$>>wW1wh7szG0(+r;(=Y^b`lDTps@jnPsyd zdDKDCL};a5}@&%)fe)Q9a>z+9`7#$C*p4!@VemBSuh;~THCi|Lv)F|3* zi6!2sAu<@!9l~G;AJgl_j=l??1{e3h_rKGr^I;z1r;f5h4{1>hW4om6l#0PaB|Zv_Oige(dS~q4To8@XTDB!}Lze8*k(#8-Zw|y6S%m2LHI@I%_e08kzF2Vb6d0c(^}!QOYziZ!>Ac)sp5`Z((k#%8ELru{sb5YmPd>Kuo@R>K-fNg<$;7*OG*JB6y zAD^~~a@p1ZMnDaD#|z7lO#rv!t@wjoU7h%RHd zib^2^r;UL(hCVle4|U}k$6S@qUayafnFZCZ(ipf9JwW(%2uwFNReXHK2iyv)7K@~g;H%wYD=X%?q@k1UR35UW6c*Vp-Q>n5{c5MkPYLI+RE3g zt-xI2WCgT#OV?Og)Ym`D{aX4-w&FT~-0txxomQ_in7w#W1C#fnbcnpN61E%cs4QP- zJ*|>krIfg+0frE6Vt>p*re_r#R&lWi)T0mjg7S$Zp+0mP=yY^1I&p4bz>0!5nXnpnH8faTwa@c;JRxF83K zxd&T+klui4k%@z~k)3#9N@y~iKXDX{`CZsUmQt?dgn`UXcf#;x zlDc7DIZMZn__ z{(dfnK24IWnw;l}m+79Kp2qUYIJ1GFoX>KBWxPQ3;OaR1CDSz8E<8yGY`=7?I_vh% z%?s7h`7G22%K6cnGe+4+3>c&N6T0KhSpA57?JUuV*i^_>`O(O!l3{&fQ!rPH%Ih(C zrum|D9XrDBPBI3IoX2F252C9dKE$lR7x>giHh9eO9&D%9TlFJ$odq?z-wET+U%>F4 z9}^MzLMSLO6FZ19pjCcR6Si@yC{|(vga6-H3;65%s35H__A% zZ@k|#BY@!)+n{!AwGfg&bom~v%}?SBtmp_8U&3$i7N$hG4l&sD$IlN)_&L3%a#c78 z8aYoF(Q1o(AIWZQ-Q+M#Dgk|3@^EX-34L*0660-gWH{klzSLO$>!vEu8NhD&FQo-Q z)1!nk0l%qNPN-ffPa`K3p{KfJ%k2{V8E!6h zRX-bnrQfR-j9JRS!v>Be^P~KNititc`4kg-Y>9%{KfO%8kY}7*KJx++ap~pHc_v!q zm!aP)JN5p|ks(dxl-U>sOt6sRl+<>>{iDIec`Kpdaj0b)#?C^CIx`-epg1gJySYg#~pd5mOSRe@{wo%cxH10 za@B4x&BPN}pIR#s7b@P631S%_>Qoqo<=CgYeWL`~Rt_ zm>Yh|Q-V2pE_meC_#HP)MJ$iosq#Hy;p8ld)lM&t-L4#qihoJ-!gy{N^CL3Cu+hto zE^+^SNnzF3xRyLkH{AsW38hQK4J33H;#{!BhL%rE5kHR`uyS2RhF{R3afNcr3jF0S zT>&c*T&^nCQ&ak{vxfD&)F9lx)X+9{ARJLT?0X$Zx1>{QsOgd6Xp*BwqT_^Hkg{(X zwA%TRVc5HNn-+Ha54nm$X2Wbo(;^n9v$Zzx7RL%#~7W_EK&=TqTUhP5Db( z)k!JBoX={-UrG`6HSV5k-YyO<_g62O_oWm^%%7n$)^%Fp#!nasiohh55jR6;w0M@@ zoL`iDoo^JB4{HgghHh6Y88@QAeS_ghI`cPC$z>T4-B<KYE`Naen+&)xP8`E&xV3`4bhQSzDM0BF4QlON!d#S0+K$O#!n-Nx$6aMg#V_~<^O%p!x(x)st4C8nC!QbegWK(_* zsnd}52#vMxjOFQYS`MT|CSKQ-A4M5xwU^)c;c%q=X-z`6Qy$xL+jxaWtZ40$CyREr zQ?z-$&UGi%I2*Y-;34?aM_4hNh4D1>eh&$HOU_rjli=Jr$BhKNp<(lW=1=W~&_o={ z0ZKm(sZPP9`>O13JqxH7FWi36gA3rpKp#N&(fuKUm;_c3{8ImgD;2M7Ei={ldb>8@ z3=H6moKQ#eX<$`I3cM|S80{!KGTAIXz8TpM*nOlq=E%hn8EOo?#5u(k24?c80)CJ4 zdfm^V0J;GTXQwxJ|0pkeqJpzSA+O2RjOKkxwmjU|3B`SDNOSznQ|mu@2a#^!HbKkp z$j5!YYmXEZjU@bM{^^iY z-Fy-0l^H79$fulWHkwOBDRud4{g(%*`bmig);5ME3;E2;`TiR#|CjwYtCV zRvm5up_diM>SpAyoZSo0cxZ8c&U%JiXdJ?Q?EC?>PP1G_*&%ESm813M6PGPOR4QR@NGALF{i4H&1~RcRr5-~#*$)VdA$VIl9(kuW9=rvtkU#D{DcbRT;w67{X=PuZ z;$D?UTTB#p)#2o_Y06oRkF}K(O8?Eqo0`+g#p6Oz{sXny-ms$e%H1!R#=gOv*#(qc zZ~f|Fg_IR$zIAX7zB)8`9ln|u>3S)B3;UBW7c8;_$;$pkR`QnOg}>&0fr>u^V8gs- z)jinl+;SVilnw|E7l{+(*}>`NZIWE~XP>vJ_#u6^Lvknem;1XFMOlc=w0B`@^`)fX zU}47&6A`)KH&fyD!iYaUnVwS}gyQT%#2%ReEdL+>w@l7AEc&(?S}yNh7!V}ON3PYF zU^i~OBGYVlc`y#~{U60AS$%HijCcBp=;Mq>+4}u2j*CyG%9q0I4~;!6kb-=t-t&+A4|6H+WU=#KkLWN$A4*(srbG4_ruE%I_uKdeUru2ClFUNz|rHX7R%(93a5_8|otx~qp)(>7LZr$C{1wqrv$01@l zkLWPVNxXC_c3-E9L)(KPj#E(D_lykg!~UHz+~P&oUUTWjdZkbUnRTWjzuyBNE2hTs zsO}kysa;^k@TC}gf4keNT`>Mo*&SVEz27(pA3u20MJ`;a;uBWx8574dt$wb-%(nMG zu?k}hocJsoD?ukO!tn!{R-(vqNDDa9aW0P>AtD zR645ObH^#3h$_9XD#BLjEld8!oiuY5?8ruq$-p1lcgq?y|Bx$!(`2nnLKu+85^sr` zo15cb{aErvv2%J2C#+oh0WAojvr6J$z=3tSTL^7dUo|Yr!T|b=CDwpABx$HKrD$lc zlxJ6xVsGXP0v&v{U}I}ccjiqpipJ0qdR*ip>ebIig{deAI(e87h_0CpL(ovf30wO0 zP2#O-zTfXtm%Sp_>%6fOhsHprWBJ6zjJ&07xfH8pq!~zYoII@@=qq>&`%{FcsG9}B zxD@)GMm$#ys!gm>3*&8-HoTq17>l-cv#~=-hll}xF=PRGJoOGU2an(|2L4Te%YyJ! zDgpaQz4mGzAv)fH(P3@=8mFFoO*v*m`Of%3VKYD>Y-KLZZly@S;MN{|2s#3$qLu{P z2rD%@`Vi-=7eO;M%&UbFX5xmQ^d&E90G@XOm`LkToi(d%s}13hx~;#$ceea2g{z|c z=H*l->hSkW zW?Mx+T@F{%_`W`xtuSSC#=N5ZLshvbzdKh=x5LD_x66d>@>96amm>A@N=5>(0r4rJ z!F?4gpm32kvX=yh@j-t}e2HRv%*FrMYTNO{|Fx;6l(lN**B+F7bbol1jY-XS+kw0M zbf>{jKXJ8qim<1ks2`G-XY6vB`WY`!dgA_PuU`eF?iCoP*ws&s!4HkUSJYRn=WAWz zQ+rxk7W+_RGpPura#e8e(kBufz!~eY?{b8EPr`428nra6H`kNg+n#}>c{lU?g!inR z`Z}RLwsulH#H~1PcNFY#H{m}sxB}1s_uF;n>O}n`#o(`ApACO~>dO1D+klZ49xMQ; z=*ZD6L(|t@b8H?1P4ZG!rM+Pdr{lB(spu;pP)j=h^22rHU*VP5@!*=^(&s6V?!ODr zL_w1ov<`N16nCJH^B-|K9EU;FdqFm{G%BH`w`d;5Jl0PV;gTzdrQe<RP(VFtf35%LbF5J2;npmA1ZQNhM+WAL;dex;x9k`&1isdApIbKfzy(`?%X-L_4Yjsi?A%kvYNeu2qI-2~QhB^Y zMf3}QL{V||&GOqZ)IX{(C)zU}mhr{rt*c>}IfsxTgT8fQfR=ECw-PS~;`zh zZ9TFS#`iSQd+xs1%H(5UFO7H@P|7~de&BSx6Ia`~c(%IVv+QQ)6IoIP-Bc|&C-CEd z<>kTx@ZA^S_SuM-BT4HgL_QCzS3bokS2%_UUW|F=RAevvT{?~oRYo~UyEopx-PcbC z>OvREk&UlsB?Xc-oXCW}1RmUMN+^Ih1yfi^!o8)Gw5~l6|I)13(($Jzn!xnq&68Hd zthCOm^T65G6eSyuYyy0$Po|E#x%=`&l(a}1md}DRj)A5$M4tfAeBQm4x?(tv7o>%O0Y=PIAVG^^N=WOQ82* zLN@icMXXCD={nV*}N9&lUg*?kXnM{3)&yvfgB5WVANL02fQu7L3<%_#ESvIoc z9sjdr-Ct53rTh?o(?s;kX)%9WL~+{ZDra|mb*jaLRoSoD#^__x$*!~0G-ljgh+^M=s0S2&Z1L z5lodxB5HKggX2B*y#0X`V4}}>#j|0LCCq-)hGuGAy^R8-@P*vVP_ll7eOu=j{cUsd zv`eo!sX|rdR97nUc%Lz8gUlwrwQRlO7k)U9{$SvhxK(%zZ_o!cxdF7GH~LjCkJ;G5 zQMbbi;))OfM?1?Xmi7@y65CC$m*%B-;kA}4|Gk&(#S>J?IYbEHoC$+H1&&=UkmA{ zv>NqV3SlUK9j_ki1v1y4ukg7R_~@281SzIzwGkA2i`8ppl)=KEi&qcPvz`i0yKH@S z*IC5xx~`8uy!efjSbGfgTA*Qvh7QZmWK|43zr6bc;7*dY2X8$C zCQH+R%-1(Pw4q_S#|r3eU$XfCMs@hdPXwI;?2(lG)@NWcyesz~DQ_1_(ktG7e{}eW zj3D$xc*`LU=WLuR1UPJ0-&k65E!BVFIIhD1J3oMa#Ki0yL`fWWHyNwcu+>a&m#0FO ztQZHF6xD`2mO3!1$96;7tF^^eyoPp+cJpmB_ed^O45fd2n$^7wz%2#M8*!Hk<0734d+AH3*ZdaYi6i*j|$J5gyJ-OLAEO629V%_!Qz{(?SXBszQ~Xi z-z9`ORNR9aSkmpaukfs zROF>PyiDQ7q4@=Bbydwe9m@aONCH^Eel<84GRpOAQ`RRHKr0#q?$>4L48TPZrel`* zWa4&p?W)a}AJNv0R;|{XHSuqhc$x%jn2GnsBcm_4iK8ODP3>L$|FF#xb0CO0rjG$c zx7OHudMDrz2tO)N94K;$&s=zFd|#+-ZY4LP;e9XXJTd0~i?O!=tEy|eMz^A*v~+`l zv>@H3DBU0_otp+}kdQ6`K^kc!q&p>~k(Tc6ZqD5J__gvNTb#Ts%>STEW=mvZ*=aK0 zY!v;zgFb9j(|S1fvFsrgzm2BKx&q~9nFK9*pU#Ftt&WM1UQfIW?{VY&H&E5stjX1b z?+T~6+E1BwpD2GJ<64>)>@m8kL6E*K=N>nq(mMKWrbnBAn9Zer5$^m%`wLt zlOs*aS$L2cX0vE_JwOZCc`dJ^2JXe{$Xom!Tj}sMj%nI4#FF>enTaJ^Mjo}F+UuaqFdlO#ZJ~|e*p8wp zN7(<8t|v|m&v{aHPJTPO{7Z0)`AZu7oPEWh=0@7WAO`%L)p0NfnWANJ&FvRLL2mJz z4|aG+NKc?Xww;HBMWaiH)4hy}^}_?=!|wWBa;mJUT%8L=bn-kV*RbYr^Cm*Y_tjl< zn{OiXq+4A|?C)B4P_6P>ohSfGRdHdBZmGY`A|%x-Yg8_&5P)jRqwt5H_iCDZ)9t)rp``bcUv)azp!iSN!HZ^JH29|L=*HlRL~1p_nzXIK4)? z(Vvru?CWZ!4u{0WucqO1?qm(Tg#H1-LNaMv@%c>hg#W+cSg%`7cBkn^waOxcAdtOQ z@kClNl4G?YHo5`*{|oD_UqVLub<~h(zocJ{oWLL>v{f#i*iLNtIs5mMuAa3 zh=pH)L{$uhSdM%fLDIAIo!uJokDJvt+ui$#VC(maqZ()9|A!8RK+2ls?(a`VOqm%98_(z!HOZpBohSUiahazI}F3$Ww3h=9}ESBxoAg1VzAD%?QaHNX%@J)#4Dp}wakn-p=+@(bQ3F~_99o|S@ojY?SL zts8q)afV)64BxXs*Qu#X zk&l))(T^Hg1=MwBT^B=j;;!pOXge-Ocww z)Rg$^t6`Ec*KB3zVS7@?#P%Ps)Qy_dSk!64!9t0uACL>8R~(}~B+xHSRpajh=(DvT zg_@Xp`)}xe4>o#rUpx)qfUz`e#E(mKCzQ(9vV@mG#+#_4-H~*Lg9O_$D?57?=}8-) zXxfW}?K*Rd5e<2fJEp!XSywmWY|p9@TvA;pHoV;^knO_+2qm4gS2*$p`(nV@DMJ-; z4DBarZM6D78>LHtgiGsZ{?Ei5P`xmQ(VR)94{a3>93F97z%=97l*>B(x`}9xod*== zvzRlm>};!ME+?d(erLW?kD%V#?^-$==5T|5HGAnoH`RiVb%L+*o#@ycZTkbVz#8; zc)mV4=8H_)WudQT3EYL*2_nnEy@zdM^jXLWHNx&_HljG*3R+bDRR$$*D;{xYCoB7z z5-RtkPR0N&6D?2u1GM4H&iq=F^k1SZql@2*azv79ppw9x8`e>_<;v^-&JhkCF;!@- zLi+L8tqS9g`ul9;b6-|b(+i-lauiUUxju-(Of=>^AU~gly0$QJhRja;kYq@*QRe3! z2}Tt>7K;yxhJa|6=K0EtjyZ(ggfhL?JL@93Tk#evbfWo)i*&B}->EuE_q&UHkezL z!DcSb$=X74(u$*2g)&~JqsOOj``cn+IIVDj2vU6gCs=(e?;JKKf zOGTO8Wt7b0m62_c$ho!7Hm-PmJMUHd^ zxFGhd&TkzKK-vMpBRy&x7ED=@Pn4znhb5!-b$(+#=1eP}O^~bTr?*honylKMMQd%0 zoRGFG*OpK+l%8C(=(2b)RZ*4)omeRUd8r>8jaHrP(mQTg0bPEr`+}ogT6y2y7FiLb zvJ8i9=jdmuTk!4Lfohr}=lAgAQN(pC3bNvD&_NK$g4%OFKDTJIy;W)Bf^;k_R=3gV zPAHS-0yz#!!BX?;V28EKlJcd7{0W|8eMH$V<0ik!W&i+;*VCC4p1c=%7| z7$W2z_xXmSfcWX|SZ3vi|26&v#qFd=B3tTjKm>f;o4$Rsu7W}@RE4xOuQ>Q^bM?l; z)HY#O$5~#|%pU3zmHkU*vS)roZJ^S&gGKPLFw%(#xP+{eAcO36&!j z=j2!#-q~abB;HrXm-ghFUTB?CRXD0w1vv^12!9n;Wf@CrLSa0mq<2Fdf--=NAoT4S z04S5PQClT3Ve?+UK5mWdXEayj&mJdx;n2P8mvPwh9ra2X9aGu2$DVu z*mtDWdz@?WyQD4tGf*3`)5X#O&Vx~rZp~*?ojoPm*_gx)nbJ(v^1Nt1FN~IknpbP& zqLq3fmPqerM+^j<=QoW6XP+O7?t-Z!aEB4StBBTEdWJ|lH?F%dxhjtz<;LQ>ZSWWp z=-t^T@|`t)^O`v*75b4o!U+qDg~5lHG+9a{c-e2p7rMuHG^EtAyYUop6VCLpp%%`M z;_2A=`T1obv=v}io^&LM6)gi;?hV^2fv=$qKnpy$Mc1`fq2G#> zAgvJ}N(Zg!!l0kXsCb*T1)G`!f_mQA9sh-yaZ3Fw@nfDXvECYoVM84gw@5#c@3e|+ zwf_nS>KlMp6Oo44Q0Y(^Ml1szg!DJ2qkT-J*`b!?X+d1vde7Igr-XsQ&qm;036dZ99P`}Lj-2-@~;NDoOI$mK0o zkrM+?rMYz}JEBvS9HlwzVw$se6vj|m0Qw2iQ57zBZftdGUn=Q*9KW%L3ei$w==w)J^UZ&N zT1G?ggaZtRp+poN?Il2oXaztfHDRT^LzVEs0_unBofJ>VWgtpF?y7In+RKH(rl5UL zeL&9(z#?^c+;Ufa!@GZQQsBn10t7d`1GdTsGuSk0HtV86h^`2{%?0V->++mZ*8~5E z(p0E19o$4b&D7f0)zj}dV?s)9F$%-yRBpQwF7wvBW`s=;Vk#qKv}JPyl#l-Y;e)N$ z-1DEXKTY!8MR!*>UBOjv2PDv}Zh2xL!Z2{&whJiS-*@<{LK(sCZf_q0WWzc;U2_a_ z9?w>A^3+I3NpF8`5kjIf&ky7s0j(uGCjqMKuz& ztN=d$OeID_aw|_E*Bb$D{0_(uexHK`gT8cohX70S|6*Ce-v4i}|I4Fl_J#4NGz7E- zI*f|EaK2t3{ded4L@1&q;PnDwpU`Bwr(&AVbrW**C46rFikJ?_*$<^lY4f~$_1|Ii z|J&-!`@y zuUj3*t$!}UZS3sNqbsFN3lEgver)*LQ3KC04D-^_tP+V= zL46FtX>#qaYOM)5s1qUb_O^9sJY9BX+H*4H%!g&}!8P#wm1T`-pH4A)DueV%Lu4$n zzy5zL7;)+S-{Oh#J~&isRsX4J`4U4To~eE>z$OXLXIw~naJ`Hlg3+~E?s z&Hyvss6n7k*C~^JClKDeRM4H{6(bV#&6Cb3Pnwc?<5hfGX5MQ5Z_b z0toU$akkPnW^fJ3ly_|GernQo#NfhQ1r`&2%V9j~yZdp(9bNy&{U*6LYTlgoXOjX^ z_Sx)De`akUe-X+bUbO>Vw43%nINat`j0a25X^*71)MR}i(iyKJz|LF~$HTMV9!;i| zYWiCS=6LFa+TV{@=X>zg<*CTp@jdSh2@Z%;!x0S7sSez$)tBk9<0hS z$)^L^rWUh^4BxI@4gGI|bSnJ(@qaom^vc!?bIVQQuU;A5F2x2ew56R5)w`|r`ItTY zw7|UVoOW`d4Zqg$PW{Hbh5WhkTWOy+lXJgSn6+M~7beIXryaj-V|U-h)O!T6Pukz# z&oO=G-i%8Rf6$6(V5NWoq$Gb+Wn`5UCD9*=f=B`=-OTos0x#cGu;qOS_Z!eTdiNe;~4{VZj^P zb#Vg9Nlj^#D0tM(KkR8A8dm@-A9eAsaO& zDP}X$yGYQ-Tzh5rAZyHE<&1jOD6haxNXr2!l3?A}gv`%-(=h}l7!2&w9t{*Um*w`G zPe6D_QCe1(6)nCEIa#baP!u5Rrkgu#E4IhEa7a^pTTE(QI$u$=)F2^8zGuHy->jc& zb&T(9Nc$N{_6I=mw1KUVJqw8)bPr2 zk(FA>9Nc0FdW10>kyN{5&YQj&ms@xu-(8eZ@TDg9pOfSj=XigljVR)o{^^i>!EF?i z&GGpjxzS!u@C*F}87y9|%-e?;NuH!mefM|{7{miOSUS8&VxLn5^U;bCJ4i&;KqZEf zvKO$MVriAPsdBA$q0bZztELlN?JP@Ra<9>~fzzTNW$j=>r;73VbD zYSB2&gRjzKIg^(k@i}|5he$bPXRcMJbRi~Vl7xC-hZeGm;5Q5EZ}1M;2=$53H-`C_ zs$mLnICveKeUdEF5?a(#W0&pabx!$OCIWZvvSWdqqzq32FUr4W(`zf_uaqKs%)L;H z{~^Yk(I`jE@9*0zOtB@(xJ*57x@$QE@*PM}p&`mV`m?bN+*gqG6Dc!A_}7PE^r~XN z_EpaO^Lq&7Ux9d`&>VWPbK)27Bg&=kN&5fVjslv?3c8xstdo@Xd`ji+?V<4tcDN9L^;e}+I*|Gv{mD+ zMAIsS;OQ1k9D8D#5ejlS0SxJz9t^V21vj#SBuO42`Q%T7vZIh)E z|D|q6ZK6TFNfZZ?Sn6#5(eicYpaR(_X)VEsTCM2Z{^;v>L3WB`G4z5*>BoGH6?O>+ z7TJBqG5wAt{q7qV&t|@1SV3xnCB1B%2OsO9iG-z?=wq1yw%sOxMuM zdXd<7e-PuOWktid;&9?q7PhTzMQp>EgW|@}lYwCh_@%sl*3+~uSA3p03$v7&W8>P5 z)!MDnX%ys(g8bf*MjaB4T@%w^Hhs>cIC%V;*b|s#J|y}6yYH}= zHv6(JuG*qKnqsM~Pbs@1R#QV@;*&}Tc}};3*6|!$nSlEPYDJw-rH@!n>o-`V*AGQ5 znKV)?Yr_x9rAT>?(x-Rmi8!4C-GHLB>Pqz({|ky;p@0{V=h?E=>sc@=MN%r<{C%0V z4S9z&ms#M#*ulB}Ox=E|eJW|O)ot>VJpDw5Rh0Cst;h4ZuvRSskun;x{$^>|h3xjj z2=gHM9S^n)*>GrZVckHu$|Of#VSk=qF6%kNZ^zBkv>Ry2KN$TvQk7(ZUOBGXMw!^X-gL#(NT7~A5%-z+kzJ-G6 z(zGd-H81-PHQimdnku0~`tdWW70bU1sf-l1C?85=ox2_Mhx04vJBJSS2C5F%6{8MA+B1`KDbkH& z4hS@TNmfI{>dD*ZaQ5uG4xihUQlz_#W0E->1f$%Y#eui&q=sDA_~@dJ>*ovyOEU{} z6%3E&3yu_8ouAV89x0C1qACg&xDFF2((p(HMQP2)PiR^oh6+>35v%8tk~S{x|-1s!EH5T7hrl;wN7E-2*%fxrdPJd#Y}b zWT**xz&H@$X~7NQ!U1W-QqP)A)kuG!+CmQp^ViqrA>n-Y_bj41R$~1cFj5827V z4os6CQ0Y%>30cQaIh>DO!ZNndrYW3j%{m$u4s|cetkr()62$u5_A}|*=*B z#9uEurfGeCD`9!E25p?JyMTkNVFLSHtGdj;*Aot;&S6BVsVgi7f#W)ZisXjVT^<|R z6aEt@A-oqt0LO^DKc`ICVP3@-xH{+&KELJ1o3S7Ryf5e1{MsE3>+vX-vYa#rYrLGC z6*qTJNRGbo_}Sa<#hJp_XHMTT`1&7`GH-v(1qT^H?j>G~&H{1@Y>!dmd9s?1?!mHl zCf<0!LWi2Fr-b=Rd)Rfw}-a z9J^#pYM@!3lH_4YkiYC?;7{BR;QZMbo_0&@FL|r z#OZodhdJ5(l`?%Z7~AlKsdkGMWOc#i6_1E%YSBS?a;A%Ipr=1E z-ijO>F^@gYsqEo)I$7cFgMA6F3-9Uem(biS%)-r0>#PEN7pJt}LQ%S@puOY5@Cf$L5qzu;*pk_~BBiJ*bU74+mQ` zg_tYcIqvz`wsCxP^1FGX-gx$fhRYpnB2LSEsjBlIf>)y3H}pky3U{p)yjc`1(L99> zTFO}~O{>4))%=qarnW<5#aDagZ19M z6C&oe6-#s&eS0@M&SG@VarWDoJ>;hx#JG+04&q>D8Dyz%6z;Hro%i zauWp^D1Z=c>kv3$u~{3S*k5M$d<&xN0CbN%pM|9;SzOL(`>63o&&JAaQGc|+m%+Ho z3tDz7UyOs8-Pa{cCyi6wE6y&)8U<>vYsTs=`%c6KFXVyrf&rH3#lp+`5D&L`GAq1Y z_l~xOeAOgY=Ivi8{7)P9!Yfrj$BM+UWPW)-0$ex*DLXTG;g;U7FN0f=XcEauJ+Ewv z-7g*YM=KkYx5kS_i3Qx$t~yh$0=aCL0;gI2bWQu&G_~+qeg4E$5DF5Sk3Lt zWg8Ii;{tsH{KqNx)9HoJ8kU%VUOuMvKJfbTR2+qL_Il@i+oz|Jt#@-~%BCYQ<>n~6 zOvo!4nVVz2>B*C&y#*F&EjoQcj+CsrabIFG%+4oVBW)=++hN=bAxzU^IeTAaib|v_ z>8Sd0E~OWwD13k38cMDHen2{TtSJ&{svfoDR!&Tv&~SBlvKw@IbJ4Z{#cRBYQ#(lV zZxsaOcthx~r#(Nt(G)Ww2MeW%m2M$*gd<)R(?6&S_}<(`zD;$t@1BGty~kF!0G_I$CTS)uime(FBeKvuJg~D z5Cq7zUDluO&zv@JI=6xIy8;`}f*UbXuD6VhPp5&sTVTFDV2;w+97?$gl=MJErH|EUY)-Opg z!1XZY_HuFskrVbpIO1ran_a*X0#V=BPQ}!93)6i{yN0!~p)8H}rz}^;&%!d#JjL*~ zM2tbF!A+$M6PDF~Q*1fSQ*6+QWdmE&rTqdLiEam-Z|PRP>0IZX9c2qL?$n>&TJ7HW zEavR;-8LFpv>{qO@pyd@f2`)y)ULLVj(8!$%0XFfz9R+wGN;)!@A|T5TkrMLMA3U1 z33<~(Kb3{&QN!LW+v=K^3QZTg{UGeX(N}S>{u%HP+lPVPr}cfL&oavyakq&W7ZK6G z6xo2N>^D!hnrzpVq3M2c9KS5Vp+A1<0 zx^A&>dt5o4%P-cx-~9{>eD!8wZANwVTT=HUc~cXUh{mh##v_ucVF__@e?Gkn*2cXS zl+2={qN@vU6s8=fdfgxJ+)jJsv-m>8G{#JoI4@ar*u9UG0&RJdDtz{RrbZUuC<-&} zO}mVU6*r{j{g(OG9O-i)=rPTFyPHxJeY?NuK|nnJ9{=^Ub8~jR$y5tb%xQ-7z+`wn zzfI4fO0Co;U;7bxrKR(+NXh1xd58I3TplqNh|T!dDZ$E%lS!%`DDQMd$7b=&}P>K{xce z-4L-y^ojeO;O24E)3La)8zw3$E#XWwT1u@bpCb9L+}!#&5^KMF&9ru1b-8sLpvOS zv$5aQT5pr8QTcux`Jqxbk7Y=g6Q|>0HKU?~yQQJ-wods>lABJ2K!qfDZCvTdb$&#& zXtj9qY%{fOiu&R3yk^VOvSJxk0xEIU6%_XoluL5`@i;i#+UJ?0KmfEA>PArMh@z0S zc=PZ#UJ)s)FdSuelc|v=T9!g?bPg#cunlC5uY^RL{Sbt^zac%HT0ynFjGSE_(brJg zpbttFf0|+ADmwsp3te9k&9|5q3O)pOxoH=rGaXLA$29un4{hL0CKq+Tj~Sb5VWK*kfECEjXSdiAccMtBbj5!W%Gu;#%Rx!$Q z)@-K?dHr00v@_MoLmTr*5zrt)6C~GTDe_1G^_3pZ^f3^VCfPYXcdK~UDW`|@j?2O; zxsI-1k-ZXI5H`u{=nkD?Mnr*9oz*Ud;O4;#uHs4ij=7Z|yfcN)a8FH>GR--%zux)d zP0WpOSlOW(MMN3l?$&mm8rzjmgf|)Z?ZkOD&>;#-K8uZ#%O1D7TZV)>r5EYakSF&( zh@Ck+_SZix}s#w^eivtf=b6&$yx!IstZOw?9ET(8c z7I9m}a0aY8R~b9(Xkg&m&8CyM%12ZWGlL=g#rx?}fUl!j-O!r9{uB zl5v%Tn*%ndFO7z>q=_HICpfn!Ru79jN@OuEbYpTYPb78@>VmZutBGgsdw@EZK1qI@ zP=^NSGGaJF^Dom-h0#-0ePBcieI9=6!yC0tYy2o_HLdcMa6xyYwd!hKhYY08LC6w8 zpg#r1vg6pj`?D5>sA!R#ZcgoYc!HH&g^L7XoYtJ;swCL#wstrkQN4^u%OFUT=96+I zJ~J1rZCE%qoK4^>E) zD$hg`oZ?J5e-X<(Vab1eaRZxCrbkzY{oTEsvznNL7!%}g@GGg4j4qOzr}W#H_N?jO z)7d3>{H{! zu8eTNBx^xhP&GqP@(bb9aBV*D=U$}Ao;T;VCWlceS7Mg~Syj3|t5In^yDWn8T}@G$ zd5b3?*`l3^L$WI(#6)6DEGBwBK7rkFHF4!}$Y!b3Sl0J;7WP1s zb8;-xH*TDJftnhG+~w74eN^Mbtj(U?Do$%-G~@i{5X(x5x2{WZzb2`ZvL143H@{(D zrh~hL*ZOQWhdpxh%N+K`9KmdB>M+@uk{QyMa*8BWA-~)o5xI;vnZYZA-vXHjQ|G0~ zpxY2rc#*`p{K{ABJrj3Rm+c*LZw4qgfok|k55HMRBa;5td5-5c?D|s{Q&qnlpX+Yg z(`VlcNc!w=Pj#nGo}trp8KdG zR|bqPon@o~s(rqGb(J5n<6=Y3BW>dLDsIO~Q;%(>m2%JWDgUxReFy2Sn!af*P?*mX zw@Q4r^6i*4UC4^a6{lHRwm*JAb#Fog8`PZulH%CEI997$HUCUuZF8z}_vtC;)$y*V z&T7!irrs~oWQTMOjW*x0agKpl39FdrtA`@tu}KcB?Vl&wAzOc=gSJsPKx4}TLk|MQVyKphv4ISU-p~ohpfUKWi}}Eb>5F+KDhswARSq^ zEK)|GQ6cdq*tqL8FT|b%PT89it6Rs3qwEw9b;q5lHxKV28UXW2i8cGdEp)pwp%1EX z!snAflj5cnM$ElYomy=-xxA@!OU(|Op^T_#XkD(m^{O83qi>BnccyDIv$Mr#Ja4ZL zj_msRRd2TXZ=`0NSCjHa$EKzjRuk>QrohjaO;xoVVl-_#pO)CI$~Z3s(kxva3KC(j z0*5WSa>hl{^mo6&W&Wz`$?h!k`zag$gOiifQ~Hs45Ja{&KNx;f;1l``>`W5~B1Fc? z&Aoejx6I59XC+ttx94?>t_=q<%C;lQ+-9)-Z^C7_{?uS3PdOzCu|JU(&@m&KJJ$7zoEJ!#drZLXnOsF^FRalPj+Shij8gwnR1!1!m<`x#nvfwNS%vw45waUtKT z-Z$u=i6AESa$Du)?4GLLKA63|yrs!h$o+ctk?6*!Cux!z zjcjr>yZPwPKx~?&rOv1l5I3Foy>FnROgW-M`VmPbzEoJ76`r8WFpFsIeYP&^Nk>Tu z6qlKAlb!&qUzj18=T$MWpob=@$C++5$Y3m|hYKt;g{4)^?!XOyGPXYaB~I2E?s@g% zZKFpLaOU4HrTO0^Bnn)o^(1mDTGnpkaywdRR#==D4c!Mh6NnHez{!_5AFi0B*A@IzfKh^>KHc;AR8SXV|^G9IHVH1nmY1 ztEFz8o06-mRl(R$BF-*w)GzVoqQbKL?C5Fhb*{pzcR#AAy*QVf15=#z5;Pr@WdQ%J zf&O;6(c*LM&PD&h#}kCh?{_p!q0pGl&d!0+m|ml0@N5nY zx9yw_6a@tZ2Lqz$o!~Sr43nk>5Jz*CX(Za$C`*VviR%2Ba(kY_`#A+P+ZLSh^>eD) z&V3d)3)>EoenDD`;h!yauXA}bSE2TO<1?M>jdDuLgAPx=*a4V(Ry_FoH!bM#*eDv= zE?8RZ;ArHVYIhr+jwUHD^&_kLW0kfz9>*DvOKAUTRrT3IFo*%ICXTizD=&B2-c=HR z0~yV8aH@|+RK-N?`=YJs+OZ0Y{e`xJ!<1W15U$sZ`*Sdc^DPk0TU)kV8RrcZbEOz#ZWk zbT@-SQK4@^?iT|g@Kt_2G|^?wninU+Z3>2e@lxV(*ofZMrWK0o;WrFvR2Ic=+(p39 zdGKfFLhum5+jK>;I4++s`HKbcIwJq06*&AD4h!y&jg5G%r4>8omP9tQ@5lhN_t@Fn z!bgY}5C_!W-rlEqFjr0}p7C#V&~fN~=S@P)jQ(*FAgMH#cOjM+Pl;ciNEg^ktWP^O+aIHN!a&9{BgydxQVE}Q2c(tx?PG(MX5PS&MaDsyjxqxEX zwZ^l!ka)is(a4Hh+$&*CHy9`uBHfpC^|`B-=L~pSesx)|(1o#Ro_# zK`o0FqN$htLau_)efMqiF-4d+fg~OQmjTy|T2^5-`>h20Q}RVGHSYhcVO9Uqs6ScS zFk`Fq{stjr75_C_d;Pt4IhzFv?cvKoI6C=pMjoLC+FLml`{28!jIL}A%iqsdlU)m^ zY`b6mt~pL?!b_L;WIE61^@j`xYeV&*=o1~Y`%?83U5mXtRSMVc^TcpG&;4Jt&jP0q zMA9oW?3RX*PyQ^HaFX-hLD z^4Ue|AjA6*In)BzDXwdic-ez|&!JK(Zkqj(WToFEG;bV7l)o*3l^prugZr{6EXS?s zFhcjc{_gwMy5!0uBM+!*kVB z04zlgmC>6hTbomuS**(I!?~J7H`2(?G_w;t8=b>SW%j3yINZeXJd&b;Y<};P36#Ur zN5d4u{S@ZMzZcaWK!)Qh_$~9!#|)Ufy{wtk>jfl_E+8onPx~J zNjSK(H$3;~`I#yv|Kt##wmg0~Qg+gh+q$<_b_%9~0^jBn&kpf5+yzmzq4+DOn(7;4 z>!qm&BoBrcG(XKUiS;tN*|^$1PD70#y<0nrFJ=1)35YH$2UG`#*~G!|K`9e5ToYe7 z?mNi3%K3X_*pNH#eR=85p!Qwv;=-}I!T5Ixy_lH6?G1nOE)k*2)97tF%W-mGU=?l} zwZ6=PHug*9!KORJte@VV==sv{k>Zs{ngayn7q#7l*5Q@Ou zUe_USa2N*e$A(Y(9W#Lj)dML1xuy^i_$?s#S*S8w>?W!hlAjr3$8^XtDzL*GvqFWY zhe(Ghn)cyIay3_n&8_w5?bH_8+C#_OAfMU}RdR9Y8w_~s)6#?`GCx6oH@Gv)m%>^% z*FtY$Lr0TwZ}6pQNP?TkZkcI!EOTsg^ZM< zd-+t$adwv*ENGz(oZy!QmC|F|L(Y?c@{4k$lq*F|Rch4x3B07Ir>9vBCqA4GGu#co z{-^>+DL`OlYKmmoP!+9QdOPgyaIDeqGM2`lk7sLX$TK3(UU>eid(gEbM!l#%$F=q( zK#{15MCZ*F>X7qo2z1Du_a4Lr`SFxN>05A|VXOrQ(HrX*`lA-rYs@#!6Gxo4^X6sv zaz5DFBiNFa=L-rdy*G-dBO_M)n_96a-%$_}yS&Wq!->E_v^k&?Pbeut&M%YItLh;q zV7HXmE=#Q7`qbq;cjvfKgVWHNOM~i_bZO5TdM*!tl^@=#NYF)*N6H6?S z;j4c9^d7$O^0AO<^K6IQXx@jUqY=NRd|sCO$7nfyRlv3XQodHcdQ5Sy?IlM1;}E1Q_b;>pyiG7=S9k;Os2( zxtl;)b6_}MWi@3L_ra9WvU6`*5W9g5U1>u&s0F{ReG}j_HHnqML5$d%kO4QMx zD@v}9>ayic2BAf^e01)nO-(~%c)T&f%E6(6=1}rGe`~5*5!p{9hsbebn#FnbcQ5O% z?Cz)c$BJ*b|Twy!_OiA2XF@mHt|?2>R;8lhe6lAscm$o(Nj6qPMs0 zx8lRiv-kRQp*|j@-XA^lHix*&0w>2swL3tf7f}=Btta(!(I`;)ZM-h_f$U3~Tm+~P zU}CQC?7-6Mw79`(V3$uH3r(5fu-tMPIs1b5TJ$x>i5u+kHx(PTefG05`9HN=6@AHY z*Qq&f*nh#H6y2f4zB#mCN*L>v;Q64rOn;opOTL;U2%Hi+)EYYP(UEeEKkKZ!cg(ey zl|>D6Ks1?Xl9G}>buVYs*K5Vc$5T!=QS}SR#Re4<(XW>8mSfJ5$85E~e?8b1>(FRF z(O)%IZ+6|j{WZAbB9G;?AvfmxST+w=`Nn>cg32}v;W4X*x;oZAx7>EFQZNs^MI#NV zbH;*Ttv0v3F9(>tG13VfF$nY=vf?FvBop}EWj6@7ZpT%fMy0Im>`&p~;8Mbjhn`kz z%JA^;7&tj`eM04H*nKm5$OX<@z^zm3PP7bUo#|jyd~NGAe^k^oItf3u5-j;eg4=;i z>w3enTc}|2$NE>bOes}uIBbm{#4)7YmiHpY4!rY=CKjK(c7;cE_V)1fE0(fMf4D)P zm&`GR_41u!_wg?(8t=KR@^WRcoimAtlj20mw$}rHERI3Dj;myj8nIID>FX~JJZY$( zfJZ8Idgz2P}bN)L%H>~6=#Nntz+p=K8C)cm%@7lDP53z&e)IG_QfBRVragjYm!+AoKq<_w6 zwjq|Q3piG6ReP%ovv08wQpJ(OG{V2P*Ggk^;wqBN^hiyh*${z1 z@%2<23EmBa(&*7Ei}$>bE8nz5&04q8<0HL*AY(1y?@H=>-6eEcwU3VSBY~-(M}#GL zBJc|e4l|0ZVmDnsGuDcTiGh%H@w_1|O*Kc!q@Nap#T$LPBNoX;QUyzT(?;>x71L2n zPl~7TlWsq46XOZQVg7AllKaF~1M=pZe02A67jP!jZbu`vnhYvaOcAa4tYS8{wn~y_ z1_sD86*?=nnm+KQJPrtWy*MSA>&wG)-H4C_1OAb}-0}|N@+Z>1Cqvjml*wXwa9%%@ z#i-Mu36Fq~Exo#~@SW#y5+TQzhwFSuh7R-&Is6vsF4WyD`t@>ZtEFtBh|4-FhRx%t z`-vIhp4cYV3+?T%VC!&xUYFbbf-MCNd`c>^KN<FNm7==DMaMw@nuvBO4FE_uQ{?ciHwSR5$fXQ+`lthH^43+y+jg@J&JDr?$ zRIibto0}#uQI5^1UK_G4O0vGN+5eXPQg*tdcE`aAy$D6oX=r`?{&vFBV$tYq(whaG zCZ}mM!)WX!a?{0$GbF3T6P?p+BR|3^Ha51OT`}~A>w}p%L+`i7DL`F2#t#MWb2ivp z-V{4Y1e2j}g4vDkGl~Aq@|N&X>%0L&;br#FYGRG-sd2NK2l0;t`rREykncb*EKagi zH9DQBVOT6bua_^QU4`$wxz4L_pBA_kZXil@>8{%LsqP4zKJ^>=7@TV$`4pxB;rnZ>7&N&JyOpl$UmMgq1iz^aT2xlexl_%S#+N$-zN_IYMT zn}C2IGB=kNL}Hakr>DF?*6cIu7pe0`@D_$SAfpq2yC_KuAIxG(fXZj zOmW?*_SWiqCbM%J?Ao{~D7QWO<7yq>iqX_9(9EW~AY8$lirGeI&UT85QDx=jZ8&%RvO`UZUmGxS>F zSBrZ@2ArwZ5$7AfW)g{>TvjrMZEv2`W4ASYV;bv!@BDZ+$*~(AQn^RHQz^yH#>p8g zj=i(~mI@AzS*0$mv{VwT{gltAKf#e;#QZL0nls=@gYlR@H#7)^C)iE42}Fh#hrOP)cyMZ$E`@ok|MiGNw$!+tR)G9ELk!tTWIY2 znx#?*6|xRV_GFu}4P|GtWEsoYC)+S~W9R#xI_KQyzR$Vu-{buC^N)W#X69o)pKE!& zp4atuD-R9T(D3ES&u6uP=kEq3z1vLP<@zG52x5%l?yT6Y%Jy*_P!0J2S_p(XHM-L` z2q?4c`ODF3h1S=dbS5#TEOO_}HqA^29gx}bIlD-3q=vKWAtEuV{*i2(bD9^Jj$%ajvIu0-QbWt&zr!QAm2E=Jv4;5y?~8Hi+14Km)IC! zH_F-;r5G5ZUuuKpvkT#HNd{+MtKkZ_$r=5Ro^od#8@wZgY%T zHqQk0>1c=cJ=!fHC7YGYACtMHfay(ME)>wVPM&01cc$DA_~UQ@o>eKU2z zhMvGM<{A9T+k48U-1MeaeCoJ*>1yIIkYFcM>vjWzw;b$FPhh)MN(E=c_6~{X%(<9+ zv_1W!cZV+=uJeG-CLAV(L&9~zc$FSgd)azpwtHnxno9wDhe?J(-sWrK+Oua={4uwY zmOCVLeZfy&uxyE-V@?PB_l%q=(nW>52T!+3O`^t6IcEuXv+*1z>f^2b+%apo;3jx{ zuYH)=O`eCjf49-;l zserf*1tnxDv%5I3S9aFIx$I7l>rWvyMC@0$W*hJZ^zNABtH@kilnQLuTLq#+Z=V){ zuzc5Bfr^l*`d-?nnfmnwJ3D(uWu*xiI-Ojb?atmgfu64(4GV3y8@0o;9ELo33yR6f z8c>*>MbNio-5un9P&~qQ%8i~QCPmzR{F$xR32^egtq51KEtciVeP&(sP2itYp<{Atq*r&8vvKtzfN$!UyP_{j79H%QxbnHbI2Wa%$Fp zBe~-4D0R%X7T)?RakG28f6cB}Jb*7ZW$S8(jZ&Q5en&@@&LbO}LiA>eAT;(sH<|i- zDK##X_ptb>J|%$poH}6beoyzPDu`;XOghK1_Zg3l$SSytBOdPrKUVMjVg*`Vdp|Pqv7`WaY+i$Ki~2o4Z59Ip2uXp>fDd3-9i~f;IU#H02_2 z9T+OtuLtMdE58#fY-%j(*Yt|B$en0xROa}=YY-4iC&Myrk;{t_)jUnq{`}LhZKJz| zc*>f$8#m3Ck~WHV{LQ|*G%l1Xl|I)zi8(aEMyQW5O;sFzj-tn24(d?87O7eA#1j0I zrHBVqtD{qw8wbU2N@VH8ci1Om;l=i0I6J zO|M9Vn8IofNtVXW&%QgD4$9Gh++-|?i_rc2P|C2d)wklAL!g!V24 zTzt&+WNuJK2~?}6+gB3P0xoc@89_}g`5W> zRVhPUz-;CMzTCmafqt&&b?3;`h@^p1Q__nG1OCj;EMuV1=igD3MFU4+1xWvQ81=n@ z(}4+a7%CHW#|3n2O0{4_YB=_TL~?vtTVx3z7IvJYtJev3;fcy=Ja2%9EO5ue0|mwd z)C&#sOdgqE_8t!D#WfO{Gi^KTsGQ0!9`^xCgU-f!w%J_1str7{#BolA67s}|qb`9L zuCO)%^o}Ra~ zw6wmmDVo2h@$rg<=4C&vXGp zImD%_M!sz3!;f%Wb=P1Pf!*5}`%j@xUEOp?0hUaQh zh|wh`xk<{5y@eKfQyZ6xeF_LyGMyF9&^^%H4cK_SIdBM4pLoa7{2IUQ!wE_oc-t*I zyN~aEA5H`(C};wuC>0j8#*AlXM-QgMa7Q3d%z#-%mYUQ-i=aPbU-0tbk}X)$;+?Ta z0a5ztODm#hCEq!*QQxIXwV&-ecz#A>#|3g5R~oVGe2JAP`5rx;eOL!+jyUGJq8i4F%6M@;GFyMYv?^11iG*x5HBL8aDyK zeIEP*KwPOql=#69v8JLo&&_m=pXn}5Ft?vBXj`OkRTy9Qc{nR+#?kt%d(Q2S=5SxZ zg4-kd^dUIcFd>WW?#Qr!$iWy}exJ2RG-u;e4+*%fM>voCy7^;v;t_~lyaxuuG;SQ5rYI2lMkUNta{CnMdls$9QZQ{FPqjH^mAgOyCuGB0k3e? z)8PY%IGK3Nq>B)1{6J4_pf=0K+k=EY7oUeDLOPR*jQ%y=MeacX?tmp2Z`0^fkPin& zn6*-MYJJ9pFXgLdw%+UgegF7(@1pc`4O&1+3|b2r-_6f!B+_lrq5{5LEg+!g$Ep{c zn$Lfu;W=mdZyH$AV#EaNl_V}%>=Xzh&P8h74^Q8608KAGT7dX~u z4_zxD2ti?KuMnRbBU9yCh6Z~CEOca$N35R=hDZkZ_IhL#ZtdP6_T?$q#u)|l2p5}( zygpYMa80O6#+)=gnIv+Luee<{E?AMN?$|=KdgFpx<8)58ij`#PE%Vi_l?8_Y38AWN z(&p+>xN8B^q)EogR+R=6+DhonwS4?IkAnbY)q!%COmJEG^daM67H+K!23@STJtLd4q4IiCY(vL*ckd=oTqhH@k8WC0hkILwb4IhMg? zmZr`|W4`J}@IW6Gli6z7yFWZUEG9263{2sj&c^#|6*R@Dh$C9z%0?k zo_APge}b$3UE9*?)L)2q4~y%Xc2Fxe)W4-X`Zzvu1NDYb*5}tHe5Y&8=tyw+hGu>U zT3)#5&7L2(J8Rb&I`hFQQt{j*ps~0K@>p+;#BsCXwy(^}ii{=l=n;k8+{$xy=PJ$H z7MG{>_~OF#xm|nl{cC@dCJZ@x0-6sll~|&^MxTC)q@ng@wb0Z69Ri{;h`YPn+sVXCUw8tba7fO87nT^IXL%f7Unzwhk`0QCo{_^xnk!`6s z8lHqN^^Sn)%E7tcBjso29JS9We}eu|!5vaG|2Y;cf0(@Mu`@`Wxq;I}kt(uP{b0_+ z^d(37KEmrhKFh&1(J?FHAaOeIk?Ya5nhSOv?CpoIe9oi0e!kr89BwF{~C5I!m zQyWhXFI&C5DZs^tnnrwDP?V8~G@|qvW!Sq^j)0dVK?Oy@xSk7-=}_fVx+M z(Hr2xIgWR)=k{Ru7UOKphnuml?a-UUHx+Q)7V-XZb7k_V5mgg)F^-AqTn-<9j6R`W zyiCTC2eBk-;61~y(Nh&ONWT`i7GF%;KTC7+EO-vye96MaG<+BBj` zSZg}i{65kL;6uo~R1B^7TSs*Ns0EFnq=c_CdrX_VY_GM%&UCLl46awId0edOOGb6$ z0;7|WM-(nCs3*e&l4;B*zD4$yZ%rMR4WwW&n{u+Poo8ht^-C!U*SDpGj)r+<-am7x zl%uI37p8}_^!zI4+~T%AtbhVF*GA;Zid^G{Xl<@E?bLf?KHxfSt%`(N1iv@_V|Ped zo7S~+VMZ*?|3N+_E=3UWYWU%CF+Fa^1uPS9j_k3)ots{lr}8tLMH0)xYd+i|zY_4I zX#K9o3L0^>IH$4v%R_@SvAgzki;HwkvZKx4vnBQ0Z#~GGhO{by^2$UADW#2mETrj^ z@Y>c>w?I~RxMM#|>u7gB_ye(kh+v4sG>V8l*V+t^rUI7;(I$&h?9>bRn_bcC0=Mz`piHfQApnm|p7KNl8Ex~wkDu9$G6;z{6P zZT6gt;looY@WtA%Hl%&KhKBL=+T{C~iafF%=7O@+Va>()S^WBTXDnyY_}k=~LZflH z*5vsj%6K8SeaHH7;sS^t1M|wSjj7MJ2GfS!!lbuN)=aP0S1sgLt^d2Q`uKeiJ7aX% z{sdx@Y@9W_Bu@5Jn)_BnAH_1M{llWO@^CW1ydjkI`3))@gHmX+376>LzC7uKK)UMk zrz37hop67L=gSkt9zaK9ElSm5@IO4M{>ra=I%xyIqC0{qFa0u7d}jN|UY+pbwMwH` z*HAK@VtFVi*|)?5m}FL>sV2P1^#{k#`eU&aZBL=fCV#9^Y^XL-y5jFFyPqY<8(VsT z%B33_KpU~^RTmVhp?Px4yRqa!&70|>Zk&Ai{^NTrHk~J?kS{}%wwgEs4E~q;0SIf? zL}SFsKADTiUIm92i9)7&u_Z^Dy_U6J^n+7{=d#NLSDY9@etSEj`!Fnj@cQ}wqs~gO z#m+_n4yQ{}9l73NZ|-J+{~L|BZx&S)3_+oQM(QCO)Jx?ikHDjtu}a>7Wj4`Mv!mlL zBL`l&R&q%=upo+ag4nPBW!D!~)=JZbZ{&X~x=}{)aKo}dN8}_yKBY?7%7%XN*j$+- zK_XqYV0ct1j?vQRDko|xMFu*K(8W>h36`}@XR<^h@YwPWsksf!$=k5-+iVMld>z{PXiy775XO3-t5p2C zq|*`J@j^mEFE%-{NP0Oc;JU^d0pkQZWiS9DcKPzstf!yb2C=pgI{<% zd3rqJJEoCs>YdMM)*(5?o;X&q z&#&^*$mfR*alDAp9UCvO{ye-H%p77Si9})%IA3CRZjOmD`qL+jB`~}$#e$hCXRp0= zC0i4vBx>Q0|Lnd(XEHkKe^%fOvZQu(tSZIs0%v70rK&v2A*9 zwKROAvNF}k7LmLw_gWmq3kpc@60Ln6+L}Q=6PE>&^*5INd%IWFCm?Hpwc6ysQ+?;j z1Al*S6XepXAf237k5&CIDk$VgN6FYn?M>4efp~?3pj&@ZNmJ9YSTDWbp7gI&>E{#| zV}4H@l+-Idg5Z6DIvn%1u}qYU+8*h<<;=>Fv~PIxpS>N!t3U7#AE>T7f&o*|#W9di z0V+|j9`}9eA_q}~A3ka?79ajCK|&oEChsrd!_0Qy|8Oo(D3AjCcQ;eQsbvaX6MyP> z>$2G&7uG9Mzh}$2)@ajBv{w3*njvt@Y<-LCAD&bFGQ-k|GBJhl5Glny_+SY?Glo=| zcX302ZceL0F_N+an7W`49(Zg3nnfo4GvFkUvd1{2CO8kFHd5eDo%A@+-7LWE!^+n~ zRt+Nq$nKaLEnFM%MKQDEG-Tu0NWKZ`nLh>5IvyN@?(AD*Vw6;GdGLCKPS)4p#Hqh( z=lc68myu0%r>e7IBsh7o?|%rRTQ`nFeRqf`OoiLrkFV6+Z!=T8FO`JF zU4}Zpc2uI?6VXT(hV3Sm&FM|)eX4sZ>egZX)!xs^;<*02bWGy3zR|1OKO-n!sw|-+ z6p>=|Ac_7Ek*2YnT6^>u_HwK>%Tm_ct1;!8QbGJEuOwu{qP(Z+5Du;?v15(mRsvj- z+)E;@x5lAij~INoqGqRqS5HaFSuAGI$w z|M>ACG}P6@0|TUjlVJ_@PbD8jIx+RWJG?t~?j=*}=jrYYLApQ);G%f>s~B|_CEk=Q zs-s4vbiy4@HcoeSY`bnhKNph%l|8emQmPb&2s6ptNclYIPlt48!t+>W z`wSc(Supl7J6mmVaF90+(*j18E`pKB!f)~7ssNn$0Z8c=SyAlvWlAvL<%TbO^y}M7 zQ#T-`($y2*Nex@8N`QEBvRbRow)A8oZ3odp^POP~y-s>zmW@J!G80eWP&I@A>nmkV4zl0@$74WZ2;(1?AkSsomNjUXmifU zzVqw(;i=+Sr{4lEN9tq-97e((m9=H+#usu!SDIS+Qm(Sf5YTLVQfC-SsV_s8ye<{6Wf(*T}f-}L}$MakdQ({oA>v8 zu8m4vSvwkYG}Y>w&yeADuy-BOGhK@a3$Rw>c)|ttIU`WFL4bX7EC66~SJe~Y# zEiHXclJ}Ecj?oij#sr{BN^bm=34@XgWJ6$iwDs-AHRCVuN2aSzt52047xd9%P!qqF>ZNmvgS#8bEj(c$Lk_2f6uQq6T zGG19bQlwG2^4S4bgv=e@uFa|Ip_T&qs1soFmXNAP@G=s&(4f6Ldg8?Y?`cR#(&P>(av7%en^o@ZKVz$|>vFh%kw z$Hx!%aNpZrFY@2>E(Z$(Ciiz1>4}FBOH?7U%V_N)9JeV+goYe$tK1s1_X+@vc&Z>@ z4)$&GX8d>Y#*5+|=0pkejbF)YlznQ{Yi<#3D{@Ie&bI289_dPxK~b-8|HwJV7Z2l| za=TN5o-rv=`OvWmt9(bgqg3PT-rFLj+QAqMpfL9K_QXU*&*U29zmAS(1d5mQ=F)>% zA~6P_lVQw4WVNwndbRL$(q_TIb^(T>ok{B&V@m^1RLC)1!0yFtxe9h~)t=9Gb3pg6 z`y4Z28$4%&Ps<)k_@QT?a(A7XuRyy=%aCb}<4ZmZpbIB@Mqp>cS_@UADR~vNvkguL zpQ)4U4|PsD%Nfh!s%VMgX@<+#TzeMd5YsxnDhBqW^wlYGH@f-6yL@X~^46c0EBSxo=@H6%$&74o0e< zZ1PNSK!@Q@YeUiLQT8|KgFlNb3=4-Q3nv7xB`j4~Hp1g!_~Z}?t*AZ6iu27Ko~D=MSC z7hhj7UCi(}9(o4KK^Op3pY*fCuwadJB%-&>S8|5&aOSZE+t%Q!QK9=se5!3p9gEv~ z)bgJi_VtT}l;*_0Rpd1x%g`c3x%$6p9|jj^z>X3=FlHo>c3ivRh1ER~=n(WRk))ov zr?Fya>@<}58Fy}$Wp`>ss`LbDyQ~>Xaxm$)1RB-XziAUnhg-=}sJr#?IB8z2 ziOD<+N6tROFOUc?pvadPfp*zx^1sP^)71XeVL@DpN+^J$@&09I!OE7sSUJRfYkPTK zLfVSMzE=nRC2gS`eLIu%Igu+9_hGl1Z{|s3Z##R5X6?T49hsCM4yK{5^J)IVG6$59 zp5Xs#1Q}k_%wbdV?r{tPC>ev?F!f%BwIn2!?UxheB*yTSCkBK{9reL{Pbay>i6jPy zc0{}$2>Oa*BAql~)m&%|2<<;f1s%OVfo%{-h_$if{}X&ueC#!Hsdr$&@^|1lkkZTc z=XfF{_4aM^p&zS0nVKfmzU8!ZbkM|GMsb&8sA9s5|#U`W9bg z-~e!%yP@U=G~4`pO;0MZ#0P!|+@j_Q;q6aVm3LwLUnKzhys*O)zIdc^m7;g@e8yh- z?hxUg!jy~hAR}b#?w_2TvuFRU5jbHS^>A-_vgx#%%(%W>EZM~|LMv1Pe-G6s>;D{E zAy*|#&~IfBN#XWiQHDgafL-su*w_IeTb z=_9mF5dwK&;>7DlEk5|fpPPHLm>X;6ysO<7D;<#zi zx2VRfPM8wHbF|v~FKgpGBD8v&kSnqCefki+T3MNUgX?g`FU4IKJZmwmm!uV=Qh z7)y-4@n=}LW6^2C-F{5+V=MxHrrGJn7+ACbrndoZ7S19kXJ>JLtl_1UtGheUSU0*? zxX3&raIa%AI2_Ir46$|f_3@oO3#Okifs|V>Edlcyc=BjX82r5cj>uP=*?6W6cPQmx zXazOkUkQHgQCN5`$rEqD?^*tLCFoo4KXAgvTfyR@%SwVN7xanr`Jpud{F7$o;30XR zxszH$dy5v5_3S4e_6GYV(6}8$0)}T-hCgPm?mIz`K(ZQtj(hOLv&u%Snq7@l}0 z{A0NTWaigk9f+8^zb44Qi56#nO^ZP~PyU*ChCKQBYnmCL%PRj^8=lJlkLha&*qi|_ m4E#KL`D;1E|A&`a51=sQk)+AmD{mox5;Ya*jZez=gZ>Xv91VN` literal 0 HcmV?d00001 diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index b0b4213..a39fc7d 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -145,7 +145,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // if its still not found, we need a new fragment if (fragment_uid.empty()) { - const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::TEXT_JSON); + const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::BINARY_MSGPACK); auto fh = _fs.fragmentHandle(new_fid); if (!static_cast(fh)) { std::cout << "MFS error: failed to create new fragment for message\n"; @@ -154,6 +154,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { fragment_uid = fh.get().v; + fh.emplace_or_replace().comp = Compression::ZSTD; fh.emplace_or_replace().comp = Compression::ZSTD; auto& new_ts_range = fh.emplace(); diff --git a/src/fragment_store/types.hpp b/src/fragment_store/types.hpp index c259bd9..97cfb14 100644 --- a/src/fragment_store/types.hpp +++ b/src/fragment_store/types.hpp @@ -8,10 +8,12 @@ enum class Encryption : uint8_t { enum class Compression : uint8_t { NONE = 0x00, ZSTD = 0x01, + // TODO: zstd without magic + // TODO: zstd meta dict + // TODO: zstd data(message) dict }; enum class MetaFileType : uint8_t { TEXT_JSON, - //BINARY_ARB, - BINARY_MSGPACK, + BINARY_MSGPACK, // msgpacked msgpack }; From 67c6f9adb0c8762267a309b0f0fedb406313f369 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 19 Feb 2024 15:11:32 +0100 Subject: [PATCH 28/98] make empty contacts from ids on message load --- src/fragment_store/message_serializer.cpp | 45 +++++++++++++++-------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/src/fragment_store/message_serializer.cpp b/src/fragment_store/message_serializer.cpp index 6bb7df0..35823e6 100644 --- a/src/fragment_store/message_serializer.cpp +++ b/src/fragment_store/message_serializer.cpp @@ -7,6 +7,17 @@ #include +static Contact3 findContactByID(Contact3Registry& cr, const std::vector& id) { + // TODO: id lookup table, this is very inefficent + for (const auto& [c_it, id_it] : cr.view().each()) { + if (id == id_it.data) { + return c_it; + } + } + + return entt::null; +} + template<> bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) { const Contact3 c = h.get().c; @@ -38,16 +49,17 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json id = j.is_binary()?j:j["bytes"]; - // TODO: id lookup table, this is very inefficent - for (const auto& [c_it, id_it] : msc.cr.view().each()) { - if (id == id_it.data) { - h.emplace_or_replace(c_it); - return true; - } + Contact3 other_c = findContactByID(msc.cr, id); + if (!msc.cr.valid(other_c)) { + // create sparse contact with id only + other_c = msc.cr.create(); + msc.cr.emplace_or_replace(other_c, id); } - // TODO: should we really return false if the contact is unknown?? - return false; + h.emplace_or_replace(other_c); + + // TODO: should we return false if the contact is unknown?? + return true; } template<> @@ -81,14 +93,15 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json id = j.is_binary()?j:j["bytes"]; - // TODO: id lookup table, this is very inefficent - for (const auto& [c_it, id_it] : msc.cr.view().each()) { - if (id == id_it.data) { - h.emplace_or_replace(c_it); - return true; - } + Contact3 other_c = findContactByID(msc.cr, id); + if (!msc.cr.valid(other_c)) { + // create sparse contact with id only + other_c = msc.cr.create(); + msc.cr.emplace_or_replace(other_c, id); } - // TODO: should we really return false if the contact is unknown?? - return false; + h.emplace_or_replace(other_c); + + // TODO: should we return false if the contact is unknown?? + return true; } From 0896038dd6be9faf94f1770d2f98e9bbac0479f3 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 19 Feb 2024 20:32:56 +0100 Subject: [PATCH 29/98] make writing safe (by using a tmp file and moving to actual location) --- src/fragment_store/fragment_store.cpp | 32 ++++++++++++++++--- src/fragment_store/message_fragment_store.cpp | 4 +-- src/fragment_store/message_fragment_store.hpp | 3 ++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index a32573a..932c079 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -186,15 +186,17 @@ FragmentID FragmentStore::newFragmentFile( _reg.emplace(new_frag, mft); - throwEventConstruct(new_frag); - // meta needs to be synced to file std::function empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; }; if (!syncToStorage(new_frag, empty_data_cb)) { + std::cerr << "FS error: syncToStorage failed while creating new fragment file\n"; _reg.destroy(new_frag); return entt::null; } + // while new metadata might be created here, making sure the file could be created is more important + throwEventConstruct(new_frag); + return new_frag; } FragmentID FragmentStore::newFragmentFile( @@ -266,12 +268,15 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; + meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); std::ofstream meta_file{ - _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type), + meta_tmp_path, std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text }; if (!meta_file.is_open()) { + std::cerr << "FS error: failed to create temporary meta file\n"; return false; } @@ -280,12 +285,17 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).comp; } + std::filesystem::path data_tmp_path = _reg.get(fid).path + ".tmp"; + data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); std::ofstream data_file{ - _reg.get(fid).path, + data_tmp_path, std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text }; if (!data_file.is_open()) { + meta_file.close(); + std::filesystem::remove(meta_tmp_path); + std::cerr << "FS error: failed to create temporary data file\n"; return false; } @@ -419,9 +429,21 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ); + + std::filesystem::rename( + data_tmp_path, + _reg.get(fid).path + ); + + // TODO: check return value of renames if (_reg.all_of(fid)) { _reg.remove(fid); diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index a39fc7d..b4b4186 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -105,8 +105,8 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (fragment_uid.empty()) { for (auto& [ts_begin, ts_end, fid] : fuid_open) { const int64_t frag_range = int64_t(ts_end) - int64_t(ts_begin); - //constexpr static int64_t max_frag_ts_extent {1000*60*60}; - constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing + constexpr static int64_t max_frag_ts_extent {1000*60*60}; + //constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing const int64_t possible_extention = max_frag_ts_extent - frag_range; // which direction diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index b61fe9a..363b32e 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -36,6 +36,9 @@ namespace Fragment::Components { struct MessagesContact { std::vector id; }; + + // TODO: add src contact (self id) + } // Fragment::Components // handles fragments for messages From 795ab2d4e1edca01b1728b7e5ae282a8ce240a83 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 20 Feb 2024 11:07:18 +0100 Subject: [PATCH 30/98] fix potential tsrange errors and deduplicate state --- src/fragment_store/message_fragment_store.cpp | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index b4b4186..d53f441 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -22,8 +22,6 @@ namespace Message::Components { // ctx struct OpenFragments { struct OpenFrag final { - uint64_t ts_begin {0}; - uint64_t ts_end {0}; std::vector uid; }; // only contains fragments with <1024 messages and <28h tsrage @@ -93,9 +91,15 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { std::vector fragment_uid; // first search for fragment where the ts falls into the range - for (const auto& [ts_begin, ts_end, fid] : fuid_open) { - if (ts_begin <= msg_ts && ts_end >= msg_ts) { - fragment_uid = fid; + for (const auto& fuid : fuid_open) { + auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fuid.uid)); + assert(static_cast(fh)); + + // assuming ts range exists + auto& fts_comp = fh.get(); + + if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) { + fragment_uid = fuid.uid; // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty } @@ -103,39 +107,37 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // if it did not fit into an existing fragment, we next look for fragments that could be extended if (fragment_uid.empty()) { - for (auto& [ts_begin, ts_end, fid] : fuid_open) { - const int64_t frag_range = int64_t(ts_end) - int64_t(ts_begin); + for (const auto& fuid : fuid_open) { + auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fuid.uid)); + assert(static_cast(fh)); + + // assuming ts range exists + auto& fts_comp = fh.get(); + + const int64_t frag_range = int64_t(fts_comp.end) - int64_t(fts_comp.begin); constexpr static int64_t max_frag_ts_extent {1000*60*60}; //constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing const int64_t possible_extention = max_frag_ts_extent - frag_range; // which direction - if ((ts_begin - possible_extention) <= msg_ts && ts_begin > msg_ts) { - fragment_uid = fid; - auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fid)); - assert(static_cast(fh)); + if ((fts_comp.begin - possible_extention) <= msg_ts && fts_comp.begin > msg_ts) { + fragment_uid = fuid.uid; - std::cout << "MFS: extended begin from " << ts_begin << " to " << msg_ts << "\n"; + std::cout << "MFS: extended begin from " << fts_comp.begin << " to " << msg_ts << "\n"; // assuming ts range exists - auto& fts_comp = fh.get(); fts_comp.begin = msg_ts; // extend into the past - ts_begin = msg_ts; // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty - } else if ((ts_end + possible_extention) >= msg_ts && ts_end < msg_ts) { - fragment_uid = fid; - auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fid)); - assert(static_cast(fh)); + } else if ((fts_comp.end + possible_extention) >= msg_ts && fts_comp.end < msg_ts) { + fragment_uid = fuid.uid; - std::cout << "MFS: extended end from " << ts_end << " to " << msg_ts << "\n"; + std::cout << "MFS: extended end from " << fts_comp.end << " to " << msg_ts << "\n"; // assuming ts range exists - auto& fts_comp = fh.get(); fts_comp.end = msg_ts; // extend into the future - ts_end = msg_ts; // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty @@ -157,7 +159,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { fh.emplace_or_replace().comp = Compression::ZSTD; fh.emplace_or_replace().comp = Compression::ZSTD; - auto& new_ts_range = fh.emplace(); + auto& new_ts_range = fh.emplace_or_replace(); new_ts_range.begin = msg_ts; new_ts_range.end = msg_ts; @@ -170,7 +172,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } } - fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{msg_ts, msg_ts, fragment_uid}); + fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{fragment_uid}); std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n"; @@ -306,6 +308,10 @@ float MessageFragmentStore::tick(float time_delta) { const auto fid = _fs.getFragmentByID(_fuid_save_queue.front().id); auto* reg = _fuid_save_queue.front().reg; assert(reg != nullptr); + + auto fh = _fs.fragmentHandle(fid); + auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); + auto j = nlohmann::json::array(); // TODO: does every message have ts? @@ -327,6 +333,15 @@ float MessageFragmentStore::tick(float time_delta) { continue; // not ours } + { // potentially adjust tsrange (some external processes can change timestamps) + const auto msg_ts = msg_view.get(m).ts; + if (ftsrange.begin > msg_ts) { + ftsrange.begin = msg_ts; + } else if (ftsrange.end < msg_ts) { + ftsrange.end = msg_ts; + } + } + auto& j_entry = j.emplace_back(nlohmann::json::object()); for (const auto& [type_id, storage] : reg->storage()) { From e442191aad3565fb78dfa6d194a5b9ab60dc5469 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 20 Feb 2024 12:44:28 +0100 Subject: [PATCH 31/98] msg frag before and after helper --- src/fragment_store/message_fragment_store.cpp | 130 +++++++++++++++++- src/fragment_store/message_fragment_store.hpp | 4 + 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index d53f441..c18aad9 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -355,7 +355,7 @@ float MessageFragmentStore::tick(float time_delta) { auto s_cb_it = _sc._serl_json.find(type_id); if (s_cb_it == _sc._serl_json.end()) { // could not find serializer, not saving - std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n"; + //std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n"; continue; } @@ -380,6 +380,134 @@ void MessageFragmentStore::triggerScan(void) { _fs.scanStoragePath("test_message_store/"); } +static bool isLess(const std::vector& lhs, const std::vector& rhs) { + size_t i = 0; + for (; i < lhs.size() && i < rhs.size(); i++) { + if (lhs[i] < rhs[i]) { + return true; + } else if (lhs[i] > rhs[i]) { + return false; + } + // else continue + } + + // here we have equality of common lenths + + // we define smaller arrays to be less + return lhs.size() < rhs.size(); +} + +FragmentID MessageFragmentStore::fragmentBefore(FragmentID fid) { + auto fh = _fs.fragmentHandle(fid); + if (!fh.all_of()) { + return entt::null; + } + + const auto& mtsrange = fh.get(); + const auto& fuid = fh.get(); + + FragmentHandle current; + + auto mts_view = fh.registry()->view(); + for (const auto& [it_f, it_mtsrange, it_fuid] : mts_view.each()) { + // before means we compare end, so we dont jump over any + + if (it_mtsrange.end > mtsrange.end) { + continue; + } + + if (it_mtsrange.end == mtsrange.end) { + // equal ts -> fallback to id + if (!isLess(it_fuid.v, fuid.v)) { + // we dont "care" about equality here, since that *should* never happen + continue; + } + } + + // here we know that "it" is before + // now we check for the largest, so the closest + + if (!static_cast(current)) { + current = {*fh.registry(), it_f}; + continue; + } + + const auto& curr_mtsrange = fh.get(); + if (it_mtsrange.end < curr_mtsrange.end) { + continue; // it is older than current selection + } + + if (it_mtsrange.end == curr_mtsrange.end) { + const auto& curr_fuid = fh.get(); + // it needs to be larger to be considered (ignoring equality again) + if (isLess(it_fuid.v, curr_fuid.v)) { + continue; + } + } + + // new best fit + current = {*fh.registry(), it_f}; + } + + return current; +} + +FragmentID MessageFragmentStore::fragmentAfter(FragmentID fid) { + auto fh = _fs.fragmentHandle(fid); + if (!fh.all_of()) { + return entt::null; + } + + const auto& mtsrange = fh.get(); + const auto& fuid = fh.get(); + + FragmentHandle current; + + auto mts_view = fh.registry()->view(); + for (const auto& [it_f, it_mtsrange, it_fuid] : mts_view.each()) { + // after means we compare begin + + if (it_mtsrange.begin < mtsrange.begin) { + continue; + } + + if (it_mtsrange.begin == mtsrange.begin) { + // equal ts -> fallback to id + // id needs to be larger + if (isLess(it_fuid.v, fuid.v)) { + // we dont "care" about equality here, since that *should* never happen + continue; + } + } + + // here we know that "it" is after + // now we check for the smallest, so the closest + + if (!static_cast(current)) { + current = {*fh.registry(), it_f}; + continue; + } + + const auto& curr_mtsrange = fh.get(); + if (it_mtsrange.begin > curr_mtsrange.begin) { + continue; // it is newer than current selection + } + + if (it_mtsrange.begin == curr_mtsrange.begin) { + const auto& curr_fuid = fh.get(); + // it needs to be smaller to be considered (ignoring equality again) + if (!isLess(it_fuid.v, curr_fuid.v)) { + continue; + } + } + + // new best fit + current = {*fh.registry(), it_f}; + } + + return current; +} + bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) { handleMessage(e.e); return false; diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 363b32e..a167005 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -80,6 +80,10 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS void triggerScan(void); + // uses ts ranges and id to create strict ordering + FragmentID fragmentBefore(FragmentID fid); + FragmentID fragmentAfter(FragmentID fid); + protected: // rmm bool onEvent(const Message::Events::MessageConstruct& e) override; bool onEvent(const Message::Events::MessageUpdated& e) override; From 22f2c8f5145da0edd682c954c87c4f043ea9402f Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 22 Feb 2024 23:07:59 +0100 Subject: [PATCH 32/98] load based on view cursers (untested and not used yet) --- src/fragment_store/fragment_store.cpp | 19 +- src/fragment_store/message_fragment_store.cpp | 182 +++++++++++++++++- src/fragment_store/message_fragment_store.hpp | 42 +++- 3 files changed, 225 insertions(+), 18 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 932c079..6397e73 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -684,7 +684,7 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { } } - size_t count {0}; + std::vector scanned_frags; // step 3: parse meta and insert into reg of non preexising // main thread // TODO: check timestamps of preexisting and reload? mark external/remote dirty? @@ -749,8 +749,8 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { meta_data_decomp.resize(ZSTD_DStreamOutSize()); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0 }; - ZSTD_outBuffer output = { meta_data_decomp.data(), meta_data_decomp.size(), 0 }; + ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0}; + ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0}; do { size_t const ret = ZSTD_decompressStream(dctx, &output , &input); if (ZSTD_isError(ret)) { @@ -809,12 +809,17 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; } } - // throw new frag event here - throwEventConstruct(fh); - count++; + scanned_frags.push_back(fh); } - return count; + // TODO: mutex and move code to async and return this list ? + + // throw new frag event here, after loading them all + for (const FragmentID fid : scanned_frags) { + throwEventConstruct(fid); + } + + return scanned_frags.size(); } void FragmentStore::scanStoragePathAsync(std::string path) { diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index c18aad9..e6a2117 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -29,6 +29,18 @@ namespace Message::Components { std::vector fuid_open; }; + // all message fragments of this contact + struct ContactFragments final { + // kept up-to-date by events + entt::dense_set frags; + }; + + // all LOADED message fragments + struct LoadedContactFragments final { + // kept up-to-date by events + entt::dense_set frags; + }; + } // Message::Components namespace Fragment::Components { @@ -75,6 +87,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // registry is the same as the one the message event is for if (static_cast(_rmm).get(frag_contact) == m.registry()) { + // TODO: make dirty instead (they already are) loadFragment(*m.registry(), FragmentHandle{_fs._reg, fid}); } } @@ -163,6 +176,18 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { new_ts_range.begin = msg_ts; new_ts_range.end = msg_ts; + // contact frag + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); + } + m.registry()->ctx().get().frags.emplace(fh); + + // loaded contact frag + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); + } + m.registry()->ctx().get().frags.emplace(fh); + { const auto msg_reg_contact = m.registry()->ctx().get(); if (_cr.all_of(msg_reg_contact)) { @@ -199,7 +224,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // on new and update: mark as fragment dirty } -// assumes new frag +// assumes not loaded frag // need update from frag void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) { std::cout << "MFS: loadFragment\n"; @@ -210,6 +235,18 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh return; } + // TODO: this should probably never be the case, since we already know here that it is a msg frag + if (!reg.ctx().contains()) { + reg.ctx().emplace(); + } + reg.ctx().get().frags.emplace(fh); + + // mark loaded + if (!reg.ctx().contains()) { + reg.ctx().emplace(); + } + reg.ctx().get().frags.emplace(fh); + for (const auto& j_entry : j) { auto new_real_msg = Message3Handle{reg, reg.create()}; // load into staging reg @@ -301,9 +338,50 @@ MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) { return _sc; } +// checks range against all cursers in msgreg +static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message3Registry& msg_reg) { + // 1D collision checks: + // - for range vs range: + // r1 rhs >= r0 lhs AND r1 lhs <= r0 rhs + // - for range vs point: + // p >= r0 lhs AND p <= r0 rhs + // NOTE: directions for us are reversed (begin has larger values as end) + + auto c_b_view = msg_reg.view(); + c_b_view.use(); + for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) { + // p and r1 rhs can be seen as the same + // but first we need to know if a curser begin is a point or a range + + // TODO: margin? + auto ts_begin = ts_begin_comp.ts; + auto ts_end = ts_begin_comp.ts; // simplyfy code by making a single begin curser act as an infinitly small range + if (msg_reg.valid(vcb.curser_end) && msg_reg.all_of(vcb.curser_end)) { + // TODO: respect curser end's begin? + // TODO: remember which ends we checked and check remaining + ts_end = msg_reg.get(vcb.curser_end).ts; + + // sanity check curser order + if (ts_end > ts_begin) { + std::cerr << "MFS warning: begin curser and end curser of view swapped!!\n"; + std::swap(ts_begin, ts_end); + } + } + + // perform both checks here + if (ts_begin < range_end || ts_end > range_begin) { + continue; + } + + // range hits a view + return true; + } + + return false; +} + float MessageFragmentStore::tick(float time_delta) { // sync dirty fragments here - if (!_fuid_save_queue.empty()) { const auto fid = _fs.getFragmentByID(_fuid_save_queue.front().id); auto* reg = _fuid_save_queue.front().reg; @@ -373,6 +451,92 @@ float MessageFragmentStore::tick(float time_delta) { } } + // load needed fragments here + + // last check event frags + // only checks if it collides with ranges, not adjacency ??? + // bc ~range~ msgreg will be marked dirty and checked next tick + if (!_event_check_queue.empty()) { + auto fh = _fs.fragmentHandle(_event_check_queue.front().fid); + auto c = _event_check_queue.front().c; + _event_check_queue.pop(); + + if (!static_cast(fh)) { + return 0.05f; + } + + if (!fh.all_of()) { + return 0.05f; + } + + // get ts range of frag and collide with all curser(s/ranges) + const auto& frag_range = fh.get(); + + auto* msg_reg = _rmm.get(c); + if (msg_reg == nullptr) { + return 0.05f; + } + + if (rangeVisible(frag_range.begin, frag_range.end, !msg_reg)) { + loadFragment(*msg_reg, fh); + _potentially_dirty_contacts.emplace(c); + } + + return 0.05f; // only one but soon again + } + + if (!_potentially_dirty_contacts.empty()) { + // here we check if any view of said contact needs frag loading + // only once per tick tho + + // TODO: this makes order depend on internal order and is not fair + auto it = _potentially_dirty_contacts.cbegin(); + + auto* msg_reg = _rmm.get(*it); + + // first do collision check agains every contact associated fragment + // that is not already loaded !! + if (msg_reg->ctx().contains()) { + for (const FragmentID fid : msg_reg->ctx().get().frags) { + // TODO: better ctx caching code? + if (msg_reg->ctx().contains()) { + if (msg_reg->ctx().get().frags.contains(fid)) { + continue; + } + } + + auto fh = _fs.fragmentHandle(fid); + + if (!static_cast(fh)) { + return 0.05f; + } + + if (!fh.all_of()) { + return 0.05f; + } + + // get ts range of frag and collide with all curser(s/ranges) + const auto& [range_begin, range_end] = fh.get(); + + if (rangeVisible(range_begin, range_end, *msg_reg)) { + loadFragment(*msg_reg, fh); + return 0.05f; + } + } + // no new visible fragment + + // now, finally, check for adjecent fragments that need to be loaded + // we do this by finding a fragment in a rage + } else { + // contact has no fragments, skip + } + + _potentially_dirty_contacts.erase(it); + + return 0.05f; + } + + return 1000.f*60.f*60.f; } @@ -531,8 +695,6 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) // TODO: are we sure it is a *new* fragment? - //std::cout << "MFS: got frag for us!\n"; - Contact3 frag_contact = entt::null; { // get contact const auto& frag_contact_id = e.e.get().id; @@ -551,15 +713,19 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) } } - // only load if msg reg open - auto* msg_reg = static_cast(_rmm).get(frag_contact); + // create if not exist + auto* msg_reg = _rmm.get(frag_contact); if (msg_reg == nullptr) { // msg reg not created yet return false; } - // TODO: should this be done async / on tick() instead of on event? - loadFragment(*msg_reg, e.e); + if (!msg_reg->ctx().contains()) { + msg_reg->ctx().emplace(); + } + msg_reg->ctx().get().frags.emplace(e.e); + + _event_check_queue.push(ECQueueEntry{e.e, frag_contact}); return false; } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index a167005..9e64e75 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -20,16 +21,40 @@ namespace Message::Components { using FUID = FragComp::ID; + // unused struct FID { FragmentID fid {entt::null}; }; + // points to the front/newer message + // together they define a range that is, + // eg the first(end) and last(begin) message being rendered + // MFS requires there to be atleast one other message after/before, + // if not loaded fragment with fitting tsrange(direction) available + // uses fragmentAfter/Before() + // they can exist standalone + // if they are a pair, the inside is filled first + // cursers require a timestamp ??? + struct ViewCurserBegin { + Message3 curser_end{entt::null}; + }; + struct ViewCurserEnd { + Message3 curser_begin{entt::null}; + }; + + // mfs will only load a limited number of fragments per tick (1), + // so this tag will be set if we loaded a fragment and + // every tick we check all cursers for this tag and continue + // and remove once no fragment could be loaded anymore + // (internal) + struct TagCurserUnsatisfied {}; + } // Message::Components namespace Fragment::Components { struct MessagesTSRange { // timestamp range within the fragment - uint64_t begin {0}; + uint64_t begin {0}; // newer msg -> higher number uint64_t end {0}; }; @@ -59,12 +84,23 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS void loadFragment(Message3Registry& reg, FragmentHandle fh); - struct QueueEntry final { + struct SaveQueueEntry final { uint64_t ts_since_dirty{0}; std::vector id; Message3Registry* reg{nullptr}; }; - std::queue _fuid_save_queue; + std::queue _fuid_save_queue; + + struct ECQueueEntry final { + FragmentID fid; + Contact3 c; + }; + std::queue _event_check_queue; + + // range changed or fragment loaded. + // we only load a limited number of fragments at once, + // so we need to keep them dirty until nothing was loaded. + entt::dense_set _potentially_dirty_contacts; public: MessageFragmentStore( From 78488daa9bbe7fef1837714a215523be90fc05d2 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 25 Feb 2024 18:45:56 +0100 Subject: [PATCH 33/98] loading logic implemented but broken (very funky and sometimes even out of contact) --- flake.nix | 10 +- src/chat_gui4.cpp | 127 +++++++--- src/chat_gui4.hpp | 8 + src/fragment_store/message_fragment_store.cpp | 232 +++++++++++++----- 4 files changed, 291 insertions(+), 86 deletions(-) diff --git a/flake.nix b/flake.nix index 8210243..934ef57 100644 --- a/flake.nix +++ b/flake.nix @@ -12,13 +12,15 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; }; + stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv); in { - packages.default = pkgs.stdenv.mkDerivation { + #packages.default = pkgs.stdenv.mkDerivation { + packages.default = stdenv.mkDerivation { pname = "tomato"; version = "0.0.0"; src = ./.; - submodules = 1; + submodules = 1; # does nothing nativeBuildInputs = with pkgs; [ cmake @@ -70,7 +72,7 @@ mv bin/tomato $out/bin ''; - dontStrip = true; + dontStrip = true; # does nothing # copied from nixpkgs's SDL2 default.nix # SDL is weird in that instead of just dynamically linking with @@ -97,6 +99,8 @@ ''; }; + #packages.debug = pkgs.enableDebugging self.packages.${system}.default; + devShells.${system}.default = pkgs.mkShell { #inputsFrom = with pkgs; [ SDL2 ]; buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this? diff --git a/src/chat_gui4.cpp b/src/chat_gui4.cpp index a55c7d7..7d91f0f 100644 --- a/src/chat_gui4.cpp +++ b/src/chat_gui4.cpp @@ -269,28 +269,6 @@ float ChatGui4::render(float time_delta) { auto* msg_reg_ptr = _rmm.get(*_selected_contact); - if (msg_reg_ptr != nullptr) { - const auto& mm = *msg_reg_ptr; - //const auto& unread_storage = mm.storage(); - if (const auto* unread_storage = mm.storage(); unread_storage != nullptr && !unread_storage->empty()) { - //assert(unread_storage->size() == 0); - //assert(unread_storage.cbegin() == unread_storage.cend()); - -#if 0 - std::cout << "UNREAD "; - Message3 prev_ent = entt::null; - for (const Message3 e : mm.view()) { - std::cout << entt::to_integral(e) << " "; - if (prev_ent == e) { - assert(false && "dup"); - } - prev_ent = e; - } - std::cout << "\n"; -#endif - } - } - constexpr ImGuiTableFlags table_flags = ImGuiTableFlags_BordersInnerV | ImGuiTableFlags_RowBg | @@ -303,6 +281,9 @@ float ChatGui4::render(float time_delta) { ImGui::TableSetupColumn("timestamp"); ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled); + Message3Handle message_view_oldest; // oldest visible message + Message3Handle message_view_newest; // last visible message + // very hacky, and we have variable hight entries //ImGuiListClipper clipper; @@ -389,12 +370,23 @@ float ChatGui4::render(float time_delta) { } // use username as visibility test - if (ImGui::IsItemVisible() && msg_reg.all_of(e)) { - // get time now - const uint64_t ts_now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - msg_reg.emplace_or_replace(e, ts_now); - msg_reg.remove(e); - msg_reg.emplace_or_replace(e, 1.f); + if (ImGui::IsItemVisible()) { + if (msg_reg.all_of(e)) { + // get time now + const uint64_t ts_now = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + msg_reg.emplace_or_replace(e, ts_now); + msg_reg.remove(e); + msg_reg.emplace_or_replace(e, 1.f); + } + + // track view + if (!static_cast(message_view_oldest)) { + message_view_oldest = {msg_reg, e}; + message_view_newest = {msg_reg, e}; + } else if (static_cast(message_view_newest)) { + // update to latest + message_view_newest = {msg_reg, e}; + } } // highlight self @@ -559,9 +551,88 @@ float ChatGui4::render(float time_delta) { //ImGui::TableNextRow(0, TEXT_BASE_HEIGHT); //ImGui::TableNextRow(0, TEXT_BASE_HEIGHT); + { // update view cursers + // any message in view + if (!static_cast(message_view_oldest)) { + // no message in view? should we setup a view at current time? + + if (static_cast(_view_end)) { + // TODO: throwEventDestroy + _view_end.destroy(); + } + //if (static_cast(_view_begin)) { + //// TODO: throwEventDestroy + //_view_begin.destroy(); + //} + + // HACK: create begin curser with current time until someone else manages that + if (!static_cast(_view_begin) || _view_begin.registry() != msg_reg_ptr) { + _view_begin = {msg_reg, msg_reg.create()}; + + _view_begin.emplace_or_replace(entt::null); + // TODO: this needs to be saved somewhere? + _view_begin.get_or_emplace().ts = Message::getTimeMS(); + + std::cout << "CG: created view FRONT begin ts\n"; + _rmm.throwEventConstruct(_view_begin); + } + + } else { + // clean up old view + // TODO: properly handle this on contact transition (will be removed once multi chat refactor lands) + if (static_cast(_view_end) && _view_end.registry() != msg_reg_ptr) { + // TODO: throwEventDestroy + _view_end.destroy(); + } + if (static_cast(_view_begin) && _view_begin.registry() != msg_reg_ptr) { + // TODO: throwEventDestroy + _view_begin.destroy(); + } + + bool end_created {false}; + if (!static_cast(_view_end)) { + _view_end = {msg_reg, msg_reg.create()}; + end_created = true; + } + bool begin_created {false}; + if (!static_cast(_view_begin)) { + _view_begin = {msg_reg, msg_reg.create()}; + begin_created = true; + } + _view_end.emplace_or_replace(_view_begin); + _view_begin.emplace_or_replace(_view_end); + + auto& old_end_ts = _view_end.get_or_emplace().ts; + auto& old_begin_ts = _view_begin.get_or_emplace().ts; + + if (old_end_ts != message_view_oldest.get().ts) { + old_end_ts = message_view_oldest.get().ts; + if (end_created) { + std::cout << "CG: created view end ts with " << old_end_ts << "\n"; + _rmm.throwEventConstruct(_view_end); + } else { + std::cout << "CG: updated view end ts to " << old_end_ts << "\n"; + _rmm.throwEventUpdate(_view_end); + } + } + + if (old_begin_ts != message_view_newest.get().ts) { + old_begin_ts = message_view_newest.get().ts; + if (begin_created) { + std::cout << "CG: created view begin ts with " << old_begin_ts << "\n"; + _rmm.throwEventConstruct(_view_begin); + } else { + std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n"; + _rmm.throwEventUpdate(_view_begin); + } + } + } + } + ImGui::EndTable(); } + if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) { ImGui::SetScrollHereY(1.f); } diff --git a/src/chat_gui4.hpp b/src/chat_gui4.hpp index 837c8bd..4433e1d 100644 --- a/src/chat_gui4.hpp +++ b/src/chat_gui4.hpp @@ -10,6 +10,9 @@ #include "./file_selector.hpp" #include "./send_image_popup.hpp" +// HACK: move to public msg api? +#include "./fragment_store/message_fragment_store.hpp" + #include #include @@ -32,7 +35,12 @@ class ChatGui4 { FileSelector _fss; SendImagePopup _sip; + // TODO: refactor this to allow multiple open contacts std::optional _selected_contact; + // set to the ts of the newest rendered msg + Message3Handle _view_begin{}; + // set to the ts of the oldest rendered msg + Message3Handle _view_end{}; // TODO: per contact std::string _text_input_buffer; diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index e6a2117..fac6165 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -63,41 +63,51 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; // we only handle msg with ts } - if (!m.registry()->ctx().contains()) { - // first message in this reg - m.registry()->ctx().emplace(); - - // TODO: move this to async - // new reg -> load all fragments for this contact (for now, ranges later) - for (const auto& [fid, tsrange, fmc] : _fs._reg.view().each()) { - Contact3 frag_contact = entt::null; - // TODO: id lookup table, this is very inefficent - for (const auto& [c_it, id_it] : _cr.view().each()) { - if (fmc.id == id_it.data) { - //h.emplace_or_replace(c_it); - //return true; - frag_contact = c_it; - break; - } - } - if (!_cr.valid(frag_contact)) { - // unkown contact - continue; - } - - // registry is the same as the one the message event is for - if (static_cast(_rmm).get(frag_contact) == m.registry()) { - // TODO: make dirty instead (they already are) - loadFragment(*m.registry(), FragmentHandle{_fs._reg, fid}); - } - } + _potentially_dirty_contacts.emplace(m.registry()->ctx().get()); // always mark dirty here + if (m.any_of()) { + // not an actual message, but we probalby need to check and see if we need to load fragments + std::cout << "MFS: new or updated curser\n"; + return; } - auto& fuid_open = m.registry()->ctx().get().fuid_open; - - const auto msg_ts = m.get().ts; - if (!m.all_of()) { + std::cout << "MFS: new msg missing FUID\n"; + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); + + // TODO: move this to async + // TODO: move this to tick and just respect the dirty + FragmentHandle most_recent_fag; + uint64_t most_recent_ts{0}; + if (m.registry()->ctx().contains()) { + for (const auto fid : m.registry()->ctx().get().frags) { + auto fh = _fs.fragmentHandle(fid); + if (!static_cast(fh) || !fh.all_of()) { + // TODO: remove at this point? + continue; + } + + const uint64_t f_ts = fh.get().begin; + if (f_ts > most_recent_ts) { + if (m.registry()->ctx().contains()) { + if (m.registry()->ctx().get().frags.contains(fh)) { + continue; // already loaded + } + } + most_recent_ts = f_ts; + most_recent_fag = {_fs._reg, fid}; + } + } + } + + if (static_cast(most_recent_fag)) { + loadFragment(*m.registry(), most_recent_fag); + } + } + + auto& fuid_open = m.registry()->ctx().get().fuid_open; + + const auto msg_ts = m.get().ts; // missing fuid // find closesed non-sealed off fragment @@ -380,6 +390,23 @@ static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message return false; } +static bool isLess(const std::vector& lhs, const std::vector& rhs) { + size_t i = 0; + for (; i < lhs.size() && i < rhs.size(); i++) { + if (lhs[i] < rhs[i]) { + return true; + } else if (lhs[i] > rhs[i]) { + return false; + } + // else continue + } + + // here we have equality of common lenths + + // we define smaller arrays to be less + return lhs.size() < rhs.size(); +} + float MessageFragmentStore::tick(float time_delta) { // sync dirty fragments here if (!_fuid_save_queue.empty()) { @@ -497,21 +524,27 @@ float MessageFragmentStore::tick(float time_delta) { // first do collision check agains every contact associated fragment // that is not already loaded !! if (msg_reg->ctx().contains()) { + if (!msg_reg->ctx().contains()) { + msg_reg->ctx().emplace(); + } + const auto& loaded_frags = msg_reg->ctx().get().frags; + for (const FragmentID fid : msg_reg->ctx().get().frags) { - // TODO: better ctx caching code? - if (msg_reg->ctx().contains()) { - if (msg_reg->ctx().get().frags.contains(fid)) { - continue; - } + if (loaded_frags.contains(fid)) { + continue; } auto fh = _fs.fragmentHandle(fid); if (!static_cast(fh)) { + // WHAT + msg_reg->ctx().get().frags.erase(fid); return 0.05f; } if (!fh.all_of()) { + // ???? + msg_reg->ctx().get().frags.erase(fid); return 0.05f; } @@ -526,7 +559,113 @@ float MessageFragmentStore::tick(float time_delta) { // no new visible fragment // now, finally, check for adjecent fragments that need to be loaded - // we do this by finding a fragment in a rage + // we do this by finding the outermost fragment in a rage, and extend it by one + + // TODO: this is all extreamly unperformant code !!!!!! + // rewrite using some bounding range tree to perform collision checks !!! + + // for each view + auto c_b_view = msg_reg->view(); + c_b_view.use(); + for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) { + // track down both in the same run + FragmentID frag_newest {entt::null}; + uint64_t frag_newest_ts {}; + FragmentID frag_oldest {entt::null}; + uint64_t frag_oldest_ts {}; + + // find newest frag in range + for (const FragmentID fid : msg_reg->ctx().get().frags) { + // we want to find the last and first fragment of the range (if not all hits are loaded, we did something wrong) + if (!loaded_frags.contains(fid)) { + continue; + } + + // not checking frags for validity here, we checked above + auto fh = _fs.fragmentHandle(fid); + + const auto [range_begin, range_end] = fh.get(); + + // perf, only check begin curser fist + if (ts_begin_comp.ts < range_end) { + //if (ts_begin_comp.ts < range_end || ts_end > range_begin) { + continue; + } + + if (ts_begin_comp.ts < range_begin) { + // begin curser does not hit the frag, but end might still hit/contain it + // if has curser end, check that + if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of(vcb.curser_end)) { + // no end, save no hit + continue; + } + const auto ts_end = msg_reg->get(vcb.curser_end).ts; + + if (ts_end > range_begin) { + continue; + } + } + + // save hit + if (!_fs._reg.valid(frag_newest)) { + frag_newest = fid; + frag_newest_ts = range_begin; // new only compare against begin + } else { + // now we check if >= prev + if (range_begin < frag_newest_ts) { + continue; + } + + if (range_begin == frag_newest_ts) { + // equal ts -> fallback to id + if (isLess(fh.get().v, _fs._reg.get(frag_newest).v)) { + // we dont "care" about equality here, since that *should* never happen + continue; + } + } + + frag_newest = fid; + frag_newest_ts = range_begin; // new only compare against begin + } + + if (!_fs._reg.valid(frag_oldest)) { + frag_oldest = fid; + frag_oldest_ts = range_end; // old only compare against end + } + + if (fid != frag_oldest && fid != frag_newest) { + // now check against old + if (range_end > frag_oldest_ts) { + continue; + } + + if (range_end == frag_oldest_ts) { + // equal ts -> fallback to id + if (!isLess(fh.get().v, _fs._reg.get(frag_oldest).v)) { + // we dont "care" about equality here, since that *should* never happen + continue; + } + } + + frag_oldest = fid; + frag_oldest_ts = range_end; // old only compare against end + } + } + + auto frag_after = _fs.fragmentHandle(fragmentAfter(frag_newest)); + if (static_cast(frag_after) && !loaded_frags.contains(frag_after)) { + std::cout << "MFS: loading frag after newest\n"; + loadFragment(*msg_reg, frag_after); + return 0.05f; + } + + auto frag_before = _fs.fragmentHandle(fragmentBefore(frag_oldest)); + if (static_cast(frag_before) && !loaded_frags.contains(frag_before)) { + std::cout << "MFS: loading frag before oldest\n"; + loadFragment(*msg_reg, frag_before); + return 0.05f; + } + } } else { // contact has no fragments, skip } @@ -544,23 +683,6 @@ void MessageFragmentStore::triggerScan(void) { _fs.scanStoragePath("test_message_store/"); } -static bool isLess(const std::vector& lhs, const std::vector& rhs) { - size_t i = 0; - for (; i < lhs.size() && i < rhs.size(); i++) { - if (lhs[i] < rhs[i]) { - return true; - } else if (lhs[i] > rhs[i]) { - return false; - } - // else continue - } - - // here we have equality of common lenths - - // we define smaller arrays to be less - return lhs.size() < rhs.size(); -} - FragmentID MessageFragmentStore::fragmentBefore(FragmentID fid) { auto fh = _fs.fragmentHandle(fid); if (!fh.all_of()) { From 461a4f1aa730539d1028c345b274b89eaa25fb2d Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 25 Feb 2024 18:56:11 +0100 Subject: [PATCH 34/98] make inital curser a range --- src/chat_gui4.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/chat_gui4.cpp b/src/chat_gui4.cpp index 7d91f0f..91623a5 100644 --- a/src/chat_gui4.cpp +++ b/src/chat_gui4.cpp @@ -556,25 +556,37 @@ float ChatGui4::render(float time_delta) { if (!static_cast(message_view_oldest)) { // no message in view? should we setup a view at current time? - if (static_cast(_view_end)) { - // TODO: throwEventDestroy - _view_end.destroy(); - } + //if (static_cast(_view_end)) { + //// TODO: throwEventDestroy + //_view_end.destroy(); + //} //if (static_cast(_view_begin)) { //// TODO: throwEventDestroy //_view_begin.destroy(); //} - // HACK: create begin curser with current time until someone else manages that + // no message loaded, so we create an virtual empty view, so the next frags are loaded if (!static_cast(_view_begin) || _view_begin.registry() != msg_reg_ptr) { - _view_begin = {msg_reg, msg_reg.create()}; + if (static_cast(_view_begin)) { + _view_begin.destroy(); + } + if (static_cast(_view_end)) { + _view_end.destroy(); + } - _view_begin.emplace_or_replace(entt::null); + _view_begin = {msg_reg, msg_reg.create()}; + _view_end = {msg_reg, msg_reg.create()}; + + _view_begin.emplace_or_replace(_view_end); + _view_end.emplace_or_replace(_view_begin); // TODO: this needs to be saved somewhere? _view_begin.get_or_emplace().ts = Message::getTimeMS(); + _view_end.get_or_emplace().ts = Message::getTimeMS(); std::cout << "CG: created view FRONT begin ts\n"; _rmm.throwEventConstruct(_view_begin); + std::cout << "CG: created view FRONT end ts\n"; + _rmm.throwEventConstruct(_view_end); } } else { From 2e3c779bec42d304952f8b1d6ba9e69a53cbe80e Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 26 Feb 2024 01:22:49 +0100 Subject: [PATCH 35/98] stop ignoring mfs interval and sort after --- src/main_screen.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 29ff026..6ed720d 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -427,8 +427,8 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { tdch.tick(time_delta); // compute - mts.iterate(); // compute - mfs.tick(time_delta); // TODO: use delta + const float mfs_interval = mfs.tick(time_delta); + mts.iterate(); // compute (after mfs) _min_tick_interval = std::min( // HACK: pow by 1.6 to increase 50 -> ~500 (~522) @@ -440,6 +440,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { _min_tick_interval, fo_interval ); + _min_tick_interval = std::min( + _min_tick_interval, + mfs_interval + ); //std::cout << "MS: min tick interval: " << _min_tick_interval << "\n"; From bdf4e60f2f517c57b74c5602250cf13f76d3b0a7 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 26 Feb 2024 01:23:31 +0100 Subject: [PATCH 36/98] fix one inverted comparator --- src/fragment_store/message_fragment_store.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index fac6165..353787d 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -33,9 +33,12 @@ namespace Message::Components { struct ContactFragments final { // kept up-to-date by events entt::dense_set frags; + + // add 2 sorted contact lists for both range begin and end }; // all LOADED message fragments + // TODO: merge into ContactFragments (and pull in openfrags) struct LoadedContactFragments final { // kept up-to-date by events entt::dense_set frags; @@ -481,9 +484,10 @@ float MessageFragmentStore::tick(float time_delta) { // load needed fragments here // last check event frags - // only checks if it collides with ranges, not adjacency ??? + // only checks if it collides with ranges, not adjacent // bc ~range~ msgreg will be marked dirty and checked next tick if (!_event_check_queue.empty()) { + std::cout << "MFS: event check\n"; auto fh = _fs.fragmentHandle(_event_check_queue.front().fid); auto c = _event_check_queue.front().c; _event_check_queue.pop(); @@ -509,10 +513,12 @@ float MessageFragmentStore::tick(float time_delta) { _potentially_dirty_contacts.emplace(c); } + std::cout << "MFS: event check none\n"; return 0.05f; // only one but soon again } if (!_potentially_dirty_contacts.empty()) { + std::cout << "MFS: pdc\n"; // here we check if any view of said contact needs frag loading // only once per tick tho @@ -537,12 +543,14 @@ float MessageFragmentStore::tick(float time_delta) { auto fh = _fs.fragmentHandle(fid); if (!static_cast(fh)) { + std::cerr << "MFS error: frag is invalid\n"; // WHAT msg_reg->ctx().get().frags.erase(fid); return 0.05f; } if (!fh.all_of()) { + std::cerr << "MFS error: frag has no range\n"; // ???? msg_reg->ctx().get().frags.erase(fid); return 0.05f; @@ -552,11 +560,13 @@ float MessageFragmentStore::tick(float time_delta) { const auto& [range_begin, range_end] = fh.get(); if (rangeVisible(range_begin, range_end, *msg_reg)) { + std::cout << "MFS: frag hit by vis range\n"; loadFragment(*msg_reg, fh); return 0.05f; } } // no new visible fragment + std::cout << "MFS: no new frag directly visible\n"; // now, finally, check for adjecent fragments that need to be loaded // we do this by finding the outermost fragment in a rage, and extend it by one @@ -592,7 +602,7 @@ float MessageFragmentStore::tick(float time_delta) { continue; } - if (ts_begin_comp.ts < range_begin) { + if (ts_begin_comp.ts > range_begin) { // begin curser does not hit the frag, but end might still hit/contain it // if has curser end, check that if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of(vcb.curser_end)) { @@ -605,6 +615,7 @@ float MessageFragmentStore::tick(float time_delta) { continue; } } + //std::cout << "------------ hit\n"; // save hit if (!_fs._reg.valid(frag_newest)) { From eaa316a2aa8c09aa07d8faf0d6cd3fda81ad837c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 27 Feb 2024 17:47:17 +0100 Subject: [PATCH 37/98] rework cursers for cg, keep views between switching. will be refactored later --- src/chat_gui4.cpp | 136 +++++++++++++++++++++++----------------------- src/chat_gui4.hpp | 4 -- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/src/chat_gui4.cpp b/src/chat_gui4.cpp index 91623a5..0172d98 100644 --- a/src/chat_gui4.cpp +++ b/src/chat_gui4.cpp @@ -47,6 +47,18 @@ namespace Components { } // Components +namespace Context { + + // TODO: move back to chat log window and keep per window instead of per contact + struct CGView { + // set to the ts of the newest rendered msg + Message3Handle begin{}; + // set to the ts of the oldest rendered msg + Message3Handle end{}; + }; + +} // Context + static constexpr float lerp(float a, float b, float t) { return a + t * (b - a); } @@ -552,90 +564,80 @@ float ChatGui4::render(float time_delta) { //ImGui::TableNextRow(0, TEXT_BASE_HEIGHT); { // update view cursers + if (!msg_reg.ctx().contains()) { + msg_reg.ctx().emplace(); + } + + auto& cg_view = msg_reg.ctx().get(); + // any message in view if (!static_cast(message_view_oldest)) { - // no message in view? should we setup a view at current time? - - //if (static_cast(_view_end)) { - //// TODO: throwEventDestroy - //_view_end.destroy(); - //} - //if (static_cast(_view_begin)) { - //// TODO: throwEventDestroy - //_view_begin.destroy(); - //} - - // no message loaded, so we create an virtual empty view, so the next frags are loaded - if (!static_cast(_view_begin) || _view_begin.registry() != msg_reg_ptr) { - if (static_cast(_view_begin)) { - _view_begin.destroy(); + // no message in view, we setup a view at current time, so the next frags are loaded + if (!static_cast(cg_view.begin) || !static_cast(cg_view.end)) { + // fix invalid state + if (static_cast(cg_view.begin)) { + cg_view.begin.destroy(); + _rmm.throwEventDestroy(cg_view.begin); } - if (static_cast(_view_end)) { - _view_end.destroy(); + if (static_cast(cg_view.end)) { + cg_view.end.destroy(); + _rmm.throwEventDestroy(cg_view.end); } - _view_begin = {msg_reg, msg_reg.create()}; - _view_end = {msg_reg, msg_reg.create()}; + // create new + cg_view.begin = {msg_reg, msg_reg.create()}; + cg_view.end = {msg_reg, msg_reg.create()}; - _view_begin.emplace_or_replace(_view_end); - _view_end.emplace_or_replace(_view_begin); - // TODO: this needs to be saved somewhere? - _view_begin.get_or_emplace().ts = Message::getTimeMS(); - _view_end.get_or_emplace().ts = Message::getTimeMS(); + cg_view.begin.emplace_or_replace(cg_view.end); + cg_view.end.emplace_or_replace(cg_view.begin); + + cg_view.begin.get_or_emplace().ts = Message::getTimeMS(); + cg_view.end.get_or_emplace().ts = Message::getTimeMS(); std::cout << "CG: created view FRONT begin ts\n"; - _rmm.throwEventConstruct(_view_begin); + _rmm.throwEventConstruct(cg_view.begin); std::cout << "CG: created view FRONT end ts\n"; - _rmm.throwEventConstruct(_view_end); - } - + _rmm.throwEventConstruct(cg_view.end); + } // else? we do nothing? } else { - // clean up old view - // TODO: properly handle this on contact transition (will be removed once multi chat refactor lands) - if (static_cast(_view_end) && _view_end.registry() != msg_reg_ptr) { - // TODO: throwEventDestroy - _view_end.destroy(); - } - if (static_cast(_view_begin) && _view_begin.registry() != msg_reg_ptr) { - // TODO: throwEventDestroy - _view_begin.destroy(); - } - - bool end_created {false}; - if (!static_cast(_view_end)) { - _view_end = {msg_reg, msg_reg.create()}; - end_created = true; - } bool begin_created {false}; - if (!static_cast(_view_begin)) { - _view_begin = {msg_reg, msg_reg.create()}; + if (!static_cast(cg_view.begin)) { + cg_view.begin = {msg_reg, msg_reg.create()}; begin_created = true; } - _view_end.emplace_or_replace(_view_begin); - _view_begin.emplace_or_replace(_view_end); + bool end_created {false}; + if (!static_cast(cg_view.end)) { + cg_view.end = {msg_reg, msg_reg.create()}; + end_created = true; + } + cg_view.begin.emplace_or_replace(cg_view.end); + cg_view.end.emplace_or_replace(cg_view.begin); - auto& old_end_ts = _view_end.get_or_emplace().ts; - auto& old_begin_ts = _view_begin.get_or_emplace().ts; - - if (old_end_ts != message_view_oldest.get().ts) { - old_end_ts = message_view_oldest.get().ts; - if (end_created) { - std::cout << "CG: created view end ts with " << old_end_ts << "\n"; - _rmm.throwEventConstruct(_view_end); - } else { - std::cout << "CG: updated view end ts to " << old_end_ts << "\n"; - _rmm.throwEventUpdate(_view_end); + { + auto& old_begin_ts = cg_view.begin.get_or_emplace().ts; + if (old_begin_ts != message_view_newest.get().ts) { + old_begin_ts = message_view_newest.get().ts; + if (begin_created) { + std::cout << "CG: created view begin ts with " << old_begin_ts << "\n"; + _rmm.throwEventConstruct(cg_view.begin); + } else { + std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n"; + _rmm.throwEventUpdate(cg_view.begin); + } } } - if (old_begin_ts != message_view_newest.get().ts) { - old_begin_ts = message_view_newest.get().ts; - if (begin_created) { - std::cout << "CG: created view begin ts with " << old_begin_ts << "\n"; - _rmm.throwEventConstruct(_view_begin); - } else { - std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n"; - _rmm.throwEventUpdate(_view_begin); + { + auto& old_end_ts = cg_view.end.get_or_emplace().ts; + if (old_end_ts != message_view_oldest.get().ts) { + old_end_ts = message_view_oldest.get().ts; + if (end_created) { + std::cout << "CG: created view end ts with " << old_end_ts << "\n"; + _rmm.throwEventConstruct(cg_view.end); + } else { + std::cout << "CG: updated view end ts to " << old_end_ts << "\n"; + _rmm.throwEventUpdate(cg_view.end); + } } } } diff --git a/src/chat_gui4.hpp b/src/chat_gui4.hpp index 4433e1d..283d5fd 100644 --- a/src/chat_gui4.hpp +++ b/src/chat_gui4.hpp @@ -37,10 +37,6 @@ class ChatGui4 { // TODO: refactor this to allow multiple open contacts std::optional _selected_contact; - // set to the ts of the newest rendered msg - Message3Handle _view_begin{}; - // set to the ts of the oldest rendered msg - Message3Handle _view_end{}; // TODO: per contact std::string _text_input_buffer; From 52e95ca654fb16076115660a4e8babb939626359 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 28 Feb 2024 01:07:17 +0100 Subject: [PATCH 38/98] forgot to check contact --- src/fragment_store/message_fragment_store.cpp | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 353787d..11124ed 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -701,12 +701,13 @@ FragmentID MessageFragmentStore::fragmentBefore(FragmentID fid) { } const auto& mtsrange = fh.get(); + const auto& m_c_id = fh.get().id; const auto& fuid = fh.get(); FragmentHandle current; - auto mts_view = fh.registry()->view(); - for (const auto& [it_f, it_mtsrange, it_fuid] : mts_view.each()) { + auto mts_view = fh.registry()->view(); + for (const auto& [it_f, it_mtsrange, it_m_c_id, it_fuid] : mts_view.each()) { // before means we compare end, so we dont jump over any if (it_mtsrange.end > mtsrange.end) { @@ -721,6 +722,11 @@ FragmentID MessageFragmentStore::fragmentBefore(FragmentID fid) { } } + // now we check contact (might be less cheap than range check) + if (it_m_c_id.id != m_c_id) { + continue; + } + // here we know that "it" is before // now we check for the largest, so the closest @@ -756,12 +762,13 @@ FragmentID MessageFragmentStore::fragmentAfter(FragmentID fid) { } const auto& mtsrange = fh.get(); + const auto& m_c_id = fh.get().id; const auto& fuid = fh.get(); FragmentHandle current; - auto mts_view = fh.registry()->view(); - for (const auto& [it_f, it_mtsrange, it_fuid] : mts_view.each()) { + auto mts_view = fh.registry()->view(); + for (const auto& [it_f, it_mtsrange, it_m_c_id, it_fuid] : mts_view.each()) { // after means we compare begin if (it_mtsrange.begin < mtsrange.begin) { @@ -777,6 +784,11 @@ FragmentID MessageFragmentStore::fragmentAfter(FragmentID fid) { } } + // now we check contact (might be less cheap than range check) + if (it_m_c_id.id != m_c_id) { + continue; + } + // here we know that "it" is after // now we check for the smallest, so the closest @@ -834,8 +846,6 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) // TODO: id lookup table, this is very inefficent for (const auto& [c_it, id_it] : _cr.view().each()) { if (frag_contact_id == id_it.data) { - //h.emplace_or_replace(c_it); - //return true; frag_contact = c_it; break; } @@ -850,6 +860,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) auto* msg_reg = _rmm.get(frag_contact); if (msg_reg == nullptr) { // msg reg not created yet + // TODO: this is an erroious path return false; } From 89f065a6106970cd358ae129bbdd7407f9933f5e Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 28 Feb 2024 18:34:26 +0100 Subject: [PATCH 39/98] impl new acceleration structure for components, not exploited yet disable funky load at first msg --- src/fragment_store/message_fragment_store.cpp | 120 ++++++++++++++++-- 1 file changed, 111 insertions(+), 9 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 11124ed..f026780 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -10,6 +10,7 @@ #include +#include #include #include #include @@ -17,6 +18,8 @@ // https://youtu.be/CU2exyhYPfA +// everything assumes a single fragment registry + namespace Message::Components { // ctx @@ -32,9 +35,23 @@ namespace Message::Components { // all message fragments of this contact struct ContactFragments final { // kept up-to-date by events - entt::dense_set frags; + //entt::dense_set frags; + struct InternalEntry { + // indecies into the sorted arrays + size_t i_b; + size_t i_e; + }; + entt::dense_map frags; // add 2 sorted contact lists for both range begin and end + std::vector sorted_begin; + std::vector sorted_end; + + // api + // return true if it was actually inserted + bool insert(FragmentHandle frag); + bool erase(FragmentID frag); + // update? (just erase() + insert()) }; // all LOADED message fragments @@ -78,12 +95,13 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (!m.registry()->ctx().contains()) { m.registry()->ctx().emplace(); +#if 0 // TODO: move this to async // TODO: move this to tick and just respect the dirty FragmentHandle most_recent_fag; uint64_t most_recent_ts{0}; if (m.registry()->ctx().contains()) { - for (const auto fid : m.registry()->ctx().get().frags) { + for (const auto& [fid, si] : m.registry()->ctx().get().frags) { auto fh = _fs.fragmentHandle(fid); if (!static_cast(fh) || !fh.all_of()) { // TODO: remove at this point? @@ -92,6 +110,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { const uint64_t f_ts = fh.get().begin; if (f_ts > most_recent_ts) { + // this makes no sense, we retry to load the first fragment on every new message and bail here, bc it was already if (m.registry()->ctx().contains()) { if (m.registry()->ctx().get().frags.contains(fh)) { continue; // already loaded @@ -106,6 +125,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (static_cast(most_recent_fag)) { loadFragment(*m.registry(), most_recent_fag); } +#endif } auto& fuid_open = m.registry()->ctx().get().fuid_open; @@ -193,7 +213,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (!m.registry()->ctx().contains()) { m.registry()->ctx().emplace(); } - m.registry()->ctx().get().frags.emplace(fh); + m.registry()->ctx().get().insert(fh); // loaded contact frag if (!m.registry()->ctx().contains()) { @@ -252,7 +272,7 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh if (!reg.ctx().contains()) { reg.ctx().emplace(); } - reg.ctx().get().frags.emplace(fh); + reg.ctx().get().insert(fh); // mark loaded if (!reg.ctx().contains()) { @@ -535,7 +555,7 @@ float MessageFragmentStore::tick(float time_delta) { } const auto& loaded_frags = msg_reg->ctx().get().frags; - for (const FragmentID fid : msg_reg->ctx().get().frags) { + for (const auto& [fid, si] : msg_reg->ctx().get().frags) { if (loaded_frags.contains(fid)) { continue; } @@ -545,14 +565,14 @@ float MessageFragmentStore::tick(float time_delta) { if (!static_cast(fh)) { std::cerr << "MFS error: frag is invalid\n"; // WHAT - msg_reg->ctx().get().frags.erase(fid); + msg_reg->ctx().get().erase(fid); return 0.05f; } if (!fh.all_of()) { std::cerr << "MFS error: frag has no range\n"; // ???? - msg_reg->ctx().get().frags.erase(fid); + msg_reg->ctx().get().erase(fid); return 0.05f; } @@ -585,7 +605,7 @@ float MessageFragmentStore::tick(float time_delta) { uint64_t frag_oldest_ts {}; // find newest frag in range - for (const FragmentID fid : msg_reg->ctx().get().frags) { + for (const auto& [fid, si] : msg_reg->ctx().get().frags) { // we want to find the last and first fragment of the range (if not all hits are loaded, we did something wrong) if (!loaded_frags.contains(fid)) { continue; @@ -867,10 +887,92 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) if (!msg_reg->ctx().contains()) { msg_reg->ctx().emplace(); } - msg_reg->ctx().get().frags.emplace(e.e); + msg_reg->ctx().get().insert(e.e); _event_check_queue.push(ECQueueEntry{e.e, frag_contact}); return false; } +bool Message::Components::ContactFragments::insert(FragmentHandle frag) { + if (frags.contains(frag)) { + return false; + } + + // both sorted arrays are sorted ascending + // so for insertion we search for the last index that is <= and insert after it + // or we search for the first > (or end) and insert before it <--- + // since equal fragments are UB, we can assume they are only > or < + + size_t begin_index {0}; + { // begin + const auto pos = std::find_if( + sorted_begin.cbegin(), + sorted_begin.cend(), + [frag](const FragmentID a) -> bool { + const auto begin_a = frag.registry()->get(a).begin; + const auto begin_frag = frag.get().begin; + if (begin_a > begin_frag) { + return true; + } else if (begin_a < begin_frag) { + return false; + } else { + // equal ts, we need to fall back to id (id can not be equal) + return isLess(frag.get().v, frag.registry()->get(a).v); + } + } + ); + + begin_index = std::distance(sorted_begin.cbegin(), pos); + + // we need to insert before pos (end is valid here) + sorted_begin.insert(pos, frag); + } + + size_t end_index {0}; + { // end + const auto pos = std::find_if_not( + sorted_end.cbegin(), + sorted_end.cend(), + [frag](const FragmentID a) -> bool { + const auto end_a = frag.registry()->get(a).end; + const auto end_frag = frag.get().end; + if (end_a > end_frag) { + return true; + } else if (end_a < end_frag) { + return false; + } else { + // equal ts, we need to fall back to id (id can not be equal) + return isLess(frag.get().v, frag.registry()->get(a).v); + } + } + ); + + end_index = std::distance(sorted_end.cbegin(), pos); + + // we need to insert before pos (end is valid here) + sorted_end.insert(pos, frag); + } + + frags.emplace(frag, InternalEntry{begin_index, end_index}); + + return true; +} + +bool Message::Components::ContactFragments::erase(FragmentID frag) { + auto frags_it = frags.find(frag); + if (frags_it == frags.end()) { + return false; + } + + assert(sorted_begin.size() == sorted_end.size()); + assert(sorted_begin.size() > frags_it->second.i_b); + + sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b); + sorted_end.erase(sorted_end.begin() + frags_it->second.i_e); + + frags.erase(frags_it); + + return true; +} + From 6a6de77ae95db48c6ba07686b13b7f1e4b9ebd52 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 28 Feb 2024 23:49:32 +0100 Subject: [PATCH 40/98] smaller contact frag fixes --- src/fragment_store/message_fragment_store.cpp | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index f026780..bf5c950 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -174,6 +174,12 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // assuming ts range exists fts_comp.begin = msg_ts; // extend into the past + if (m.registry()->ctx().contains()) { + // should be the case + m.registry()->ctx().get().erase(fh); + m.registry()->ctx().get().insert(fh); + } + // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty @@ -185,6 +191,12 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // assuming ts range exists fts_comp.end = msg_ts; // extend into the future + if (m.registry()->ctx().contains()) { + // should be the case + m.registry()->ctx().get().erase(fh); + m.registry()->ctx().get().insert(fh); + } + // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty } @@ -209,6 +221,15 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { new_ts_range.begin = msg_ts; new_ts_range.end = msg_ts; + { + const auto msg_reg_contact = m.registry()->ctx().get(); + if (_cr.all_of(msg_reg_contact)) { + fh.emplace(_cr.get(msg_reg_contact).data); + } else { + // ? rage quit? + } + } + // contact frag if (!m.registry()->ctx().contains()) { m.registry()->ctx().emplace(); @@ -221,15 +242,6 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } m.registry()->ctx().get().frags.emplace(fh); - { - const auto msg_reg_contact = m.registry()->ctx().get(); - if (_cr.all_of(msg_reg_contact)) { - fh.emplace(_cr.get(msg_reg_contact).data); - } else { - // ? rage quit? - } - } - fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{fragment_uid}); std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n"; @@ -887,6 +899,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) if (!msg_reg->ctx().contains()) { msg_reg->ctx().emplace(); } + msg_reg->ctx().get().erase(e.e); // TODO: check/update/fragment update msg_reg->ctx().get().insert(e.e); _event_check_queue.push(ECQueueEntry{e.e, frag_contact}); From 93f60bd07358942a31534dc54b481e7834bde63c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 29 Feb 2024 00:19:55 +0100 Subject: [PATCH 41/98] replace old bad prev/next code with way better code --- src/fragment_store/message_fragment_store.cpp | 165 +++++------------- src/fragment_store/message_fragment_store.hpp | 4 - 2 files changed, 40 insertions(+), 129 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index bf5c950..c5882f4 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -52,6 +52,11 @@ namespace Message::Components { bool insert(FragmentHandle frag); bool erase(FragmentID frag); // update? (just erase() + insert()) + + // uses range begin to go back in time + FragmentID prev(FragmentID frag) const; + // uses range end to go forward in time + FragmentID next(FragmentID frag) const; }; // all LOADED message fragments @@ -695,14 +700,15 @@ float MessageFragmentStore::tick(float time_delta) { } } - auto frag_after = _fs.fragmentHandle(fragmentAfter(frag_newest)); + const auto& cf = msg_reg->ctx().get(); + auto frag_after = _fs.fragmentHandle(cf.next(frag_newest)); if (static_cast(frag_after) && !loaded_frags.contains(frag_after)) { std::cout << "MFS: loading frag after newest\n"; loadFragment(*msg_reg, frag_after); return 0.05f; } - auto frag_before = _fs.fragmentHandle(fragmentBefore(frag_oldest)); + auto frag_before = _fs.fragmentHandle(cf.prev(frag_oldest)); if (static_cast(frag_before) && !loaded_frags.contains(frag_before)) { std::cout << "MFS: loading frag before oldest\n"; loadFragment(*msg_reg, frag_before); @@ -726,129 +732,6 @@ void MessageFragmentStore::triggerScan(void) { _fs.scanStoragePath("test_message_store/"); } -FragmentID MessageFragmentStore::fragmentBefore(FragmentID fid) { - auto fh = _fs.fragmentHandle(fid); - if (!fh.all_of()) { - return entt::null; - } - - const auto& mtsrange = fh.get(); - const auto& m_c_id = fh.get().id; - const auto& fuid = fh.get(); - - FragmentHandle current; - - auto mts_view = fh.registry()->view(); - for (const auto& [it_f, it_mtsrange, it_m_c_id, it_fuid] : mts_view.each()) { - // before means we compare end, so we dont jump over any - - if (it_mtsrange.end > mtsrange.end) { - continue; - } - - if (it_mtsrange.end == mtsrange.end) { - // equal ts -> fallback to id - if (!isLess(it_fuid.v, fuid.v)) { - // we dont "care" about equality here, since that *should* never happen - continue; - } - } - - // now we check contact (might be less cheap than range check) - if (it_m_c_id.id != m_c_id) { - continue; - } - - // here we know that "it" is before - // now we check for the largest, so the closest - - if (!static_cast(current)) { - current = {*fh.registry(), it_f}; - continue; - } - - const auto& curr_mtsrange = fh.get(); - if (it_mtsrange.end < curr_mtsrange.end) { - continue; // it is older than current selection - } - - if (it_mtsrange.end == curr_mtsrange.end) { - const auto& curr_fuid = fh.get(); - // it needs to be larger to be considered (ignoring equality again) - if (isLess(it_fuid.v, curr_fuid.v)) { - continue; - } - } - - // new best fit - current = {*fh.registry(), it_f}; - } - - return current; -} - -FragmentID MessageFragmentStore::fragmentAfter(FragmentID fid) { - auto fh = _fs.fragmentHandle(fid); - if (!fh.all_of()) { - return entt::null; - } - - const auto& mtsrange = fh.get(); - const auto& m_c_id = fh.get().id; - const auto& fuid = fh.get(); - - FragmentHandle current; - - auto mts_view = fh.registry()->view(); - for (const auto& [it_f, it_mtsrange, it_m_c_id, it_fuid] : mts_view.each()) { - // after means we compare begin - - if (it_mtsrange.begin < mtsrange.begin) { - continue; - } - - if (it_mtsrange.begin == mtsrange.begin) { - // equal ts -> fallback to id - // id needs to be larger - if (isLess(it_fuid.v, fuid.v)) { - // we dont "care" about equality here, since that *should* never happen - continue; - } - } - - // now we check contact (might be less cheap than range check) - if (it_m_c_id.id != m_c_id) { - continue; - } - - // here we know that "it" is after - // now we check for the smallest, so the closest - - if (!static_cast(current)) { - current = {*fh.registry(), it_f}; - continue; - } - - const auto& curr_mtsrange = fh.get(); - if (it_mtsrange.begin > curr_mtsrange.begin) { - continue; // it is newer than current selection - } - - if (it_mtsrange.begin == curr_mtsrange.begin) { - const auto& curr_fuid = fh.get(); - // it needs to be smaller to be considered (ignoring equality again) - if (!isLess(it_fuid.v, curr_fuid.v)) { - continue; - } - } - - // new best fit - current = {*fh.registry(), it_f}; - } - - return current; -} - bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) { handleMessage(e.e); return false; @@ -989,3 +872,35 @@ bool Message::Components::ContactFragments::erase(FragmentID frag) { return true; } +FragmentID Message::Components::ContactFragments::prev(FragmentID frag) const { + // uses range begin to go back in time + + auto it = frags.find(frag); + if (it == frags.end()) { + return entt::null; + } + + const auto src_i = it->second.i_b; + if (src_i > 0) { + return sorted_begin[src_i-1]; + } + + return entt::null; +} + +FragmentID Message::Components::ContactFragments::next(FragmentID frag) const { + // uses range end to go forward in time + + auto it = frags.find(frag); + if (it == frags.end()) { + return entt::null; + } + + const auto src_i = it->second.i_e; + if (src_i+1 < sorted_end.size()) { + return sorted_end[src_i+1]; + } + + return entt::null; +} + diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 9e64e75..7a68f59 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -116,10 +116,6 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS void triggerScan(void); - // uses ts ranges and id to create strict ordering - FragmentID fragmentBefore(FragmentID fid); - FragmentID fragmentAfter(FragmentID fid); - protected: // rmm bool onEvent(const Message::Events::MessageConstruct& e) override; bool onEvent(const Message::Events::MessageUpdated& e) override; From 592a4cb9cffa7af9fa3f52d1f4e83277967f0b15 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 29 Feb 2024 15:31:50 +0100 Subject: [PATCH 42/98] make adjacency loading work, extend range and use loops --- src/fragment_store/message_fragment_store.cpp | 250 +++++++++--------- 1 file changed, 119 insertions(+), 131 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index c5882f4..33ef466 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -35,7 +35,6 @@ namespace Message::Components { // all message fragments of this contact struct ContactFragments final { // kept up-to-date by events - //entt::dense_set frags; struct InternalEntry { // indecies into the sorted arrays size_t i_b; @@ -44,6 +43,7 @@ namespace Message::Components { entt::dense_map frags; // add 2 sorted contact lists for both range begin and end + // TODO: adding and removing becomes expensive with enough frags, consider splitting or heap std::vector sorted_begin; std::vector sorted_end; @@ -523,7 +523,8 @@ float MessageFragmentStore::tick(float time_delta) { // last check event frags // only checks if it collides with ranges, not adjacent // bc ~range~ msgreg will be marked dirty and checked next tick - if (!_event_check_queue.empty()) { + const bool had_events = !_event_check_queue.empty(); + for (size_t i = 0; i < 10 && !_event_check_queue.empty(); i++) { std::cout << "MFS: event check\n"; auto fh = _fs.fragmentHandle(_event_check_queue.front().fid); auto c = _event_check_queue.front().c; @@ -548,10 +549,12 @@ float MessageFragmentStore::tick(float time_delta) { if (rangeVisible(frag_range.begin, frag_range.end, !msg_reg)) { loadFragment(*msg_reg, fh); _potentially_dirty_contacts.emplace(c); + return 0.05f; // only one but soon again } - + } + if (had_events) { std::cout << "MFS: event check none\n"; - return 0.05f; // only one but soon again + return 0.05f; // only check events, even if non where hit } if (!_potentially_dirty_contacts.empty()) { @@ -567,153 +570,138 @@ float MessageFragmentStore::tick(float time_delta) { // first do collision check agains every contact associated fragment // that is not already loaded !! if (msg_reg->ctx().contains()) { - if (!msg_reg->ctx().contains()) { - msg_reg->ctx().emplace(); - } - const auto& loaded_frags = msg_reg->ctx().get().frags; - - for (const auto& [fid, si] : msg_reg->ctx().get().frags) { - if (loaded_frags.contains(fid)) { - continue; + const auto& cf = msg_reg->ctx().get(); + if (!cf.frags.empty()) { + if (!msg_reg->ctx().contains()) { + msg_reg->ctx().emplace(); } + const auto& loaded_frags = msg_reg->ctx().get().frags; - auto fh = _fs.fragmentHandle(fid); - - if (!static_cast(fh)) { - std::cerr << "MFS error: frag is invalid\n"; - // WHAT - msg_reg->ctx().get().erase(fid); - return 0.05f; - } - - if (!fh.all_of()) { - std::cerr << "MFS error: frag has no range\n"; - // ???? - msg_reg->ctx().get().erase(fid); - return 0.05f; - } - - // get ts range of frag and collide with all curser(s/ranges) - const auto& [range_begin, range_end] = fh.get(); - - if (rangeVisible(range_begin, range_end, *msg_reg)) { - std::cout << "MFS: frag hit by vis range\n"; - loadFragment(*msg_reg, fh); - return 0.05f; - } - } - // no new visible fragment - std::cout << "MFS: no new frag directly visible\n"; - - // now, finally, check for adjecent fragments that need to be loaded - // we do this by finding the outermost fragment in a rage, and extend it by one - - // TODO: this is all extreamly unperformant code !!!!!! - // rewrite using some bounding range tree to perform collision checks !!! - - // for each view - auto c_b_view = msg_reg->view(); - c_b_view.use(); - for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) { - // track down both in the same run - FragmentID frag_newest {entt::null}; - uint64_t frag_newest_ts {}; - FragmentID frag_oldest {entt::null}; - uint64_t frag_oldest_ts {}; - - // find newest frag in range for (const auto& [fid, si] : msg_reg->ctx().get().frags) { - // we want to find the last and first fragment of the range (if not all hits are loaded, we did something wrong) - if (!loaded_frags.contains(fid)) { + if (loaded_frags.contains(fid)) { continue; } - // not checking frags for validity here, we checked above auto fh = _fs.fragmentHandle(fid); - const auto [range_begin, range_end] = fh.get(); + if (!static_cast(fh)) { + std::cerr << "MFS error: frag is invalid\n"; + // WHAT + msg_reg->ctx().get().erase(fid); + return 0.05f; + } - // perf, only check begin curser fist - if (ts_begin_comp.ts < range_end) { - //if (ts_begin_comp.ts < range_end || ts_end > range_begin) { + if (!fh.all_of()) { + std::cerr << "MFS error: frag has no range\n"; + // ???? + msg_reg->ctx().get().erase(fid); + return 0.05f; + } + + // get ts range of frag and collide with all curser(s/ranges) + const auto& [range_begin, range_end] = fh.get(); + + if (rangeVisible(range_begin, range_end, *msg_reg)) { + std::cout << "MFS: frag hit by vis range\n"; + loadFragment(*msg_reg, fh); + return 0.05f; + } + } + // no new visible fragment + std::cout << "MFS: no new frag directly visible\n"; + + // now, finally, check for adjecent fragments that need to be loaded + // we do this by finding the outermost fragment in a rage, and extend it by one + + // TODO: rewrite using some bounding range tree to perform collision checks !!! + // (this is now performing better, but still) + + + // for each view + auto c_b_view = msg_reg->view(); + c_b_view.use(); + for (const auto& [_, ts_begin_comp, vcb] : c_b_view.each()) { + // aka "scroll down" + { // find newest(-ish) frag in range + // or in reverse frag end <= range begin + + + // lower bound of frag end and range begin + const auto right = std::lower_bound( + cf.sorted_end.crbegin(), + cf.sorted_end.crend(), + ts_begin_comp.ts, + [&](const FragmentID element, const auto& value) -> bool { + return _fs._reg.get(element).end >= value; + } + ); + + FragmentID next_frag{entt::null}; + if (right != cf.sorted_end.crend()) { + next_frag = cf.next(*right); + } + // we checked earlier that cf is not empty + if (!_fs._reg.valid(next_frag)) { + // fall back to closest, cf is not empty + next_frag = cf.sorted_end.front(); + } + + // a single adjacent frag is often not enough + // only ok bc next is cheap + for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); i++) { + if (!loaded_frags.contains(next_frag)) { + std::cout << "MFS: next frag of range\n"; + loadFragment(*msg_reg, {_fs._reg, next_frag}); + return 0.05f; + } + + next_frag = cf.next(next_frag); + } + } + + // curser end + if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of(vcb.curser_end)) { continue; } + const auto ts_end = msg_reg->get(vcb.curser_end).ts; - if (ts_begin_comp.ts > range_begin) { - // begin curser does not hit the frag, but end might still hit/contain it - // if has curser end, check that - if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of(vcb.curser_end)) { - // no end, save no hit - continue; - } - const auto ts_end = msg_reg->get(vcb.curser_end).ts; + // aka "scroll up" + { // find oldest(-ish) frag in range + // frag begin >= range end - if (ts_end > range_begin) { - continue; - } - } - //std::cout << "------------ hit\n"; - - // save hit - if (!_fs._reg.valid(frag_newest)) { - frag_newest = fid; - frag_newest_ts = range_begin; // new only compare against begin - } else { - // now we check if >= prev - if (range_begin < frag_newest_ts) { - continue; - } - - if (range_begin == frag_newest_ts) { - // equal ts -> fallback to id - if (isLess(fh.get().v, _fs._reg.get(frag_newest).v)) { - // we dont "care" about equality here, since that *should* never happen - continue; + // lower bound of frag begin and range end + const auto left = std::lower_bound( + cf.sorted_begin.cbegin(), + cf.sorted_begin.cend(), + ts_end, + [&](const FragmentID element, const auto& value) -> bool { + return _fs._reg.get(element).begin < value; } + ); + + FragmentID prev_frag{entt::null}; + if (left != cf.sorted_begin.cend()) { + prev_frag = cf.prev(*left); + } + // we checked earlier that cf is not empty + if (!_fs._reg.valid(prev_frag)) { + // fall back to closest, cf is not empty + prev_frag = cf.sorted_begin.back(); } - frag_newest = fid; - frag_newest_ts = range_begin; // new only compare against begin - } - - if (!_fs._reg.valid(frag_oldest)) { - frag_oldest = fid; - frag_oldest_ts = range_end; // old only compare against end - } - - if (fid != frag_oldest && fid != frag_newest) { - // now check against old - if (range_end > frag_oldest_ts) { - continue; - } - - if (range_end == frag_oldest_ts) { - // equal ts -> fallback to id - if (!isLess(fh.get().v, _fs._reg.get(frag_oldest).v)) { - // we dont "care" about equality here, since that *should* never happen - continue; + // a single adjacent frag is often not enough + // only ok bc next is cheap + for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); i++) { + if (!loaded_frags.contains(prev_frag)) { + std::cout << "MFS: prev frag of range\n"; + loadFragment(*msg_reg, {_fs._reg, prev_frag}); + return 0.05f; } + + prev_frag = cf.prev(prev_frag); } - - frag_oldest = fid; - frag_oldest_ts = range_end; // old only compare against end } } - - const auto& cf = msg_reg->ctx().get(); - auto frag_after = _fs.fragmentHandle(cf.next(frag_newest)); - if (static_cast(frag_after) && !loaded_frags.contains(frag_after)) { - std::cout << "MFS: loading frag after newest\n"; - loadFragment(*msg_reg, frag_after); - return 0.05f; - } - - auto frag_before = _fs.fragmentHandle(cf.prev(frag_oldest)); - if (static_cast(frag_before) && !loaded_frags.contains(frag_before)) { - std::cout << "MFS: loading frag before oldest\n"; - loadFragment(*msg_reg, frag_before); - return 0.05f; - } } } else { // contact has no fragments, skip From 0e0e81720b204a5e10e5a5c57543f23aaed5c9e4 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 29 Feb 2024 20:52:47 +0100 Subject: [PATCH 43/98] dont sync messages we dont know enough about --- src/tox_friend_faux_offline_messaging.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tox_friend_faux_offline_messaging.cpp b/src/tox_friend_faux_offline_messaging.cpp index dcd960e..4397bf7 100644 --- a/src/tox_friend_faux_offline_messaging.cpp +++ b/src/tox_friend_faux_offline_messaging.cpp @@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM // require if (!mr->all_of< Message::Components::MessageText, // text only for now - Message::Components::ContactTo + Message::Components::ContactTo, + Message::Components::ToxFriendMessageID // yes, needs fake ids >(msg) ) { continue; // skip From 5bf4640d61feb3ee0cef2e0250997817a5eb188b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 1 Mar 2024 12:18:06 +0100 Subject: [PATCH 44/98] forgot to throw update on read --- src/chat_gui4.cpp | 3 +++ src/fragment_store/message_fragment_store.cpp | 5 ++++- src/fragment_store/message_fragment_store.hpp | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/chat_gui4.cpp b/src/chat_gui4.cpp index 0172d98..e220c3d 100644 --- a/src/chat_gui4.cpp +++ b/src/chat_gui4.cpp @@ -389,6 +389,9 @@ float ChatGui4::render(float time_delta) { msg_reg.emplace_or_replace(e, ts_now); msg_reg.remove(e); msg_reg.emplace_or_replace(e, 1.f); + + // we remove the unread tag here + _rmm.throwEventUpdate(msg_reg, e); } // track view diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 33ef466..7797b10 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -95,6 +95,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; } + // TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag) if (!m.all_of()) { std::cout << "MFS: new msg missing FUID\n"; if (!m.registry()->ctx().contains()) { @@ -268,7 +269,9 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { _fuid_save_queue.push({Message::getTimeMS(), fragment_uid, m.registry()}); } - // TODO: do we use fid? + // TODO: save updates, and not only new messages (read state etc) + // new fragment?, since we dont write to others fragments? + // on new message: assign fuid // on new and update: mark as fragment dirty diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 7a68f59..ea854e9 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -29,7 +29,7 @@ namespace Message::Components { // points to the front/newer message // together they define a range that is, // eg the first(end) and last(begin) message being rendered - // MFS requires there to be atleast one other message after/before, + // MFS requires there to be atleast one other fragment after/before, // if not loaded fragment with fitting tsrange(direction) available // uses fragmentAfter/Before() // they can exist standalone @@ -42,6 +42,9 @@ namespace Message::Components { Message3 curser_begin{entt::null}; }; + // TODO: add adjacency range comp or inside curser + + // TODO: unused // mfs will only load a limited number of fragments per tick (1), // so this tag will be set if we loaded a fragment and // every tick we check all cursers for this tag and continue From 2b8cee6a2956fd3f89f1a12ae2568e8daa8a66b6 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 1 Mar 2024 13:53:32 +0100 Subject: [PATCH 45/98] remove old code --- src/fragment_store/message_fragment_store.cpp | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 7797b10..61fb968 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -100,38 +100,6 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { std::cout << "MFS: new msg missing FUID\n"; if (!m.registry()->ctx().contains()) { m.registry()->ctx().emplace(); - -#if 0 - // TODO: move this to async - // TODO: move this to tick and just respect the dirty - FragmentHandle most_recent_fag; - uint64_t most_recent_ts{0}; - if (m.registry()->ctx().contains()) { - for (const auto& [fid, si] : m.registry()->ctx().get().frags) { - auto fh = _fs.fragmentHandle(fid); - if (!static_cast(fh) || !fh.all_of()) { - // TODO: remove at this point? - continue; - } - - const uint64_t f_ts = fh.get().begin; - if (f_ts > most_recent_ts) { - // this makes no sense, we retry to load the first fragment on every new message and bail here, bc it was already - if (m.registry()->ctx().contains()) { - if (m.registry()->ctx().get().frags.contains(fh)) { - continue; // already loaded - } - } - most_recent_ts = f_ts; - most_recent_fag = {_fs._reg, fid}; - } - } - } - - if (static_cast(most_recent_fag)) { - loadFragment(*m.registry(), most_recent_fag); - } -#endif } auto& fuid_open = m.registry()->ctx().get().fuid_open; From 71be5c3c6eba41cb4043edceb8159f66b92a14b1 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 3 Mar 2024 11:57:30 +0100 Subject: [PATCH 46/98] reduce log spam --- src/chat_gui4.cpp | 4 ++-- src/fragment_store/message_fragment_store.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/chat_gui4.cpp b/src/chat_gui4.cpp index e220c3d..1fc847b 100644 --- a/src/chat_gui4.cpp +++ b/src/chat_gui4.cpp @@ -624,7 +624,7 @@ float ChatGui4::render(float time_delta) { std::cout << "CG: created view begin ts with " << old_begin_ts << "\n"; _rmm.throwEventConstruct(cg_view.begin); } else { - std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n"; + //std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n"; _rmm.throwEventUpdate(cg_view.begin); } } @@ -638,7 +638,7 @@ float ChatGui4::render(float time_delta) { std::cout << "CG: created view end ts with " << old_end_ts << "\n"; _rmm.throwEventConstruct(cg_view.end); } else { - std::cout << "CG: updated view end ts to " << old_end_ts << "\n"; + //std::cout << "CG: updated view end ts to " << old_end_ts << "\n"; _rmm.throwEventUpdate(cg_view.end); } } diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 61fb968..ade8203 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -91,7 +91,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { _potentially_dirty_contacts.emplace(m.registry()->ctx().get()); // always mark dirty here if (m.any_of()) { // not an actual message, but we probalby need to check and see if we need to load fragments - std::cout << "MFS: new or updated curser\n"; + //std::cout << "MFS: new or updated curser\n"; return; } From 7b8e93eec3eee2d001cb3f23ecc89c0d14c84f38 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 3 Mar 2024 12:20:41 +0100 Subject: [PATCH 47/98] refactor message fuid -> fid save alot of memory by using fid instead of fuid --- src/fragment_store/message_fragment_store.cpp | 64 ++++++++++--------- src/fragment_store/message_fragment_store.hpp | 7 +- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index ade8203..1af217d 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -24,12 +24,12 @@ namespace Message::Components { // ctx struct OpenFragments { - struct OpenFrag final { - std::vector uid; - }; - // only contains fragments with <1024 messages and <28h tsrage - // TODO: this needs to move into the message reg - std::vector fuid_open; + //struct OpenFrag final { + ////std::vector uid; + //FragmentID id; + //}; + // only contains fragments with <1024 messages and <28h tsrage (or whatever) + entt::dense_set fid_open; }; // all message fragments of this contact @@ -96,39 +96,39 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } // TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag) - if (!m.all_of()) { - std::cout << "MFS: new msg missing FUID\n"; + if (!m.all_of()) { + std::cout << "MFS: new msg missing FID\n"; if (!m.registry()->ctx().contains()) { m.registry()->ctx().emplace(); } - auto& fuid_open = m.registry()->ctx().get().fuid_open; + auto& fid_open = m.registry()->ctx().get().fid_open; const auto msg_ts = m.get().ts; // missing fuid // find closesed non-sealed off fragment - std::vector fragment_uid; + FragmentID fragment_id{entt::null}; // first search for fragment where the ts falls into the range - for (const auto& fuid : fuid_open) { - auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fuid.uid)); + for (const auto& fid : fid_open) { + auto fh = _fs.fragmentHandle(fid); assert(static_cast(fh)); // assuming ts range exists auto& fts_comp = fh.get(); if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) { - fragment_uid = fuid.uid; + fragment_id = fid; // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty } } // if it did not fit into an existing fragment, we next look for fragments that could be extended - if (fragment_uid.empty()) { - for (const auto& fuid : fuid_open) { - auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fuid.uid)); + if (!_fs._reg.valid(fragment_id)) { + for (const auto& fid : fid_open) { + auto fh = _fs.fragmentHandle(fid); assert(static_cast(fh)); // assuming ts range exists @@ -141,7 +141,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // which direction if ((fts_comp.begin - possible_extention) <= msg_ts && fts_comp.begin > msg_ts) { - fragment_uid = fuid.uid; + fragment_id = fid; std::cout << "MFS: extended begin from " << fts_comp.begin << " to " << msg_ts << "\n"; @@ -158,7 +158,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // TODO: check conditions for open here // TODO: mark msg (and frag?) dirty } else if ((fts_comp.end + possible_extention) >= msg_ts && fts_comp.end < msg_ts) { - fragment_uid = fuid.uid; + fragment_id = fid; std::cout << "MFS: extended end from " << fts_comp.end << " to " << msg_ts << "\n"; @@ -178,7 +178,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } // if its still not found, we need a new fragment - if (fragment_uid.empty()) { + if (!_fs._reg.valid(fragment_id)) { const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::BINARY_MSGPACK); auto fh = _fs.fragmentHandle(new_fid); if (!static_cast(fh)) { @@ -186,7 +186,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; } - fragment_uid = fh.get().v; + fragment_id = fh; fh.emplace_or_replace().comp = Compression::ZSTD; fh.emplace_or_replace().comp = Compression::ZSTD; @@ -216,9 +216,9 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } m.registry()->ctx().get().frags.emplace(fh); - fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{fragment_uid}); + fid_open.emplace(fragment_id); - std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n"; + std::cout << "MFS: created new fragment " << bin2hex(fh.get().v) << "\n"; _fs_ignore_event = true; _fs.throwEventConstruct(fh); @@ -226,17 +226,20 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } // if this is still empty, something is very wrong and we exit here - if (fragment_uid.empty()) { + if (!_fs._reg.valid(fragment_id)) { std::cout << "MFS error: failed to find/create fragment for message\n"; return; } - m.emplace(fragment_uid); + m.emplace(fragment_id); // in this case we know the fragment needs an update - _fuid_save_queue.push({Message::getTimeMS(), fragment_uid, m.registry()}); + _fuid_save_queue.push({Message::getTimeMS(), fragment_id, m.registry()}); + return; // done } + //m.get(); + // TODO: save updates, and not only new messages (read state etc) // new fragment?, since we dont write to others fragments? @@ -288,7 +291,7 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh } } - new_real_msg.emplace_or_replace(fh.get()); + new_real_msg.emplace_or_replace(fh); // dup check (hacky, specific to protocols) Message3 dup_msg {entt::null}; @@ -421,11 +424,10 @@ static bool isLess(const std::vector& lhs, const std::vector& float MessageFragmentStore::tick(float time_delta) { // sync dirty fragments here if (!_fuid_save_queue.empty()) { - const auto fid = _fs.getFragmentByID(_fuid_save_queue.front().id); + auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id); auto* reg = _fuid_save_queue.front().reg; assert(reg != nullptr); - auto fh = _fs.fragmentHandle(fid); auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); auto j = nlohmann::json::array(); @@ -436,7 +438,7 @@ float MessageFragmentStore::tick(float time_delta) { for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) { const Message3 m = *it; - if (!reg->all_of(m)) { + if (!reg->all_of(m)) { continue; } @@ -445,7 +447,7 @@ float MessageFragmentStore::tick(float time_delta) { continue; } - if (_fuid_save_queue.front().id != reg->get(m).v) { + if (_fuid_save_queue.front().id != reg->get(m).fid) { continue; // not ours } @@ -482,7 +484,7 @@ float MessageFragmentStore::tick(float time_delta) { // if save as binary //nlohmann::json::to_msgpack(j); auto j_dump = j.dump(2, ' ', true); - if (_fs.syncToStorage(fid, reinterpret_cast(j_dump.data()), j_dump.size())) { + if (_fs.syncToStorage(fh, reinterpret_cast(j_dump.data()), j_dump.size())) { //std::cout << "MFS: dumped " << j_dump << "\n"; // succ _fuid_save_queue.pop(); diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index ea854e9..e1b08dc 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -19,9 +19,9 @@ namespace Message::Components { - using FUID = FragComp::ID; + // unused, consumes too much memory (highly compressable) + //using FUID = FragComp::ID; - // unused struct FID { FragmentID fid {entt::null}; }; @@ -89,7 +89,8 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS struct SaveQueueEntry final { uint64_t ts_since_dirty{0}; - std::vector id; + //std::vector id; + FragmentID id; Message3Registry* reg{nullptr}; }; std::queue _fuid_save_queue; From bc22451524f00e38b96027dfa0ae9768bf431208 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 3 Mar 2024 12:50:39 +0100 Subject: [PATCH 48/98] dirty frag on message updates (if still open) --- src/fragment_store/message_fragment_store.cpp | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 1af217d..7fd73b6 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -231,16 +231,33 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; } - m.emplace(fragment_id); + m.emplace_or_replace(fragment_id); // in this case we know the fragment needs an update _fuid_save_queue.push({Message::getTimeMS(), fragment_id, m.registry()}); return; // done } - //m.get(); + const auto msg_fh = _fs.fragmentHandle(m.get().fid); + if (!static_cast(msg_fh)) { + std::cerr << "MFS error: fid in message is invalid\n"; + return; // TODO: properly handle this case + } - // TODO: save updates, and not only new messages (read state etc) + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); + } + + auto& fid_open = m.registry()->ctx().get().fid_open; + + if (fid_open.contains(msg_fh)) { + // TODO: dedup events + // TODO: cooldown per fragsave + _fuid_save_queue.push({Message::getTimeMS(), msg_fh, m.registry()}); + return; + } + + // TODO: save updates to old fragments, but writing them to a new fragment that would overwrite on merge // new fragment?, since we dont write to others fragments? From 88ea3e177d5a8a64c39dacdb28ba39a7c70221d3 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 3 Mar 2024 15:16:01 +0100 Subject: [PATCH 49/98] refactor saving and save on exit --- src/fragment_store/message_fragment_store.cpp | 140 ++++++++++-------- src/fragment_store/message_fragment_store.hpp | 2 + 2 files changed, 80 insertions(+), 62 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 7fd73b6..081090d 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -336,7 +336,7 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh } if (reg.valid(dup_msg)) { - // -> merge with preexisting + // -> merge with preexisting (needs to be order independent) // -> throw update reg.destroy(new_real_msg); //_rmm.throwEventUpdate(reg, new_real_msg); @@ -354,6 +354,75 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh } } +bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry& reg) { + auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); + + auto j = nlohmann::json::array(); + + // TODO: does every message have ts? + auto msg_view = reg.view(); + // we also assume all messages have fid + for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) { + const Message3 m = *it; + + if (!reg.all_of(m)) { + continue; + } + + // require msg for now + if (!reg.any_of(m)) { + continue; + } + + if (_fuid_save_queue.front().id != reg.get(m).fid) { + continue; // not ours + } + + { // potentially adjust tsrange (some external processes can change timestamps) + const auto msg_ts = msg_view.get(m).ts; + if (ftsrange.begin > msg_ts) { + ftsrange.begin = msg_ts; + } else if (ftsrange.end < msg_ts) { + ftsrange.end = msg_ts; + } + } + + auto& j_entry = j.emplace_back(nlohmann::json::object()); + + for (const auto& [type_id, storage] : reg.storage()) { + if (!storage.contains(m)) { + continue; + } + + //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; + + // use type_id to find serializer + auto s_cb_it = _sc._serl_json.find(type_id); + if (s_cb_it == _sc._serl_json.end()) { + // could not find serializer, not saving + //std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n"; + continue; + } + + s_cb_it->second(_sc, {reg, m}, j_entry[storage.type().name()]); + } + } + + // we cant skip if array is empty (in theory it will not be empty later on) + + // if save as binary + //nlohmann::json::to_msgpack(j); + auto j_dump = j.dump(2, ' ', true); + if (_fs.syncToStorage(fh, reinterpret_cast(j_dump.data()), j_dump.size())) { + //std::cout << "MFS: dumped " << j_dump << "\n"; + // succ + return true; + } + + // TODO: error + return false; +} + MessageFragmentStore::MessageFragmentStore( Contact3Registry& cr, RegistryMessageModel& rmm, @@ -372,7 +441,13 @@ MessageFragmentStore::MessageFragmentStore( } MessageFragmentStore::~MessageFragmentStore(void) { - // TODO: sync all dirty fragments + while (!_fuid_save_queue.empty()) { + auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id); + auto* reg = _fuid_save_queue.front().reg; + assert(reg != nullptr); + syncFragToStorage(fh, *reg); + _fuid_save_queue.pop(); // pop unconditionally + } } MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) { @@ -444,66 +519,7 @@ float MessageFragmentStore::tick(float time_delta) { auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id); auto* reg = _fuid_save_queue.front().reg; assert(reg != nullptr); - - auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); - - auto j = nlohmann::json::array(); - - // TODO: does every message have ts? - auto msg_view = reg->view(); - // we also assume all messages have fuid (hack: call handle when not?) - for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) { - const Message3 m = *it; - - if (!reg->all_of(m)) { - continue; - } - - // require msg for now - if (!reg->any_of(m)) { - continue; - } - - if (_fuid_save_queue.front().id != reg->get(m).fid) { - continue; // not ours - } - - { // potentially adjust tsrange (some external processes can change timestamps) - const auto msg_ts = msg_view.get(m).ts; - if (ftsrange.begin > msg_ts) { - ftsrange.begin = msg_ts; - } else if (ftsrange.end < msg_ts) { - ftsrange.end = msg_ts; - } - } - - auto& j_entry = j.emplace_back(nlohmann::json::object()); - - for (const auto& [type_id, storage] : reg->storage()) { - if (!storage.contains(m)) { - continue; - } - - //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; - - // use type_id to find serializer - auto s_cb_it = _sc._serl_json.find(type_id); - if (s_cb_it == _sc._serl_json.end()) { - // could not find serializer, not saving - //std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n"; - continue; - } - - s_cb_it->second(_sc, {*reg, m}, j_entry[storage.type().name()]); - } - } - - // if save as binary - //nlohmann::json::to_msgpack(j); - auto j_dump = j.dump(2, ' ', true); - if (_fs.syncToStorage(fh, reinterpret_cast(j_dump.data()), j_dump.size())) { - //std::cout << "MFS: dumped " << j_dump << "\n"; - // succ + if (syncFragToStorage(fh, *reg)) { _fuid_save_queue.pop(); } } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index e1b08dc..3ad7628 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -87,6 +87,8 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS void loadFragment(Message3Registry& reg, FragmentHandle fh); + bool syncFragToStorage(FragmentHandle fh, Message3Registry& reg); + struct SaveQueueEntry final { uint64_t ts_since_dirty{0}; //std::vector id; From 7879a0927b47a41777b2581321972c5b42ade2b8 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 6 Mar 2024 12:30:07 +0100 Subject: [PATCH 50/98] combat memory leaks with smart pointers --- src/fragment_store/fragment_store.cpp | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 6397e73..7ac7164 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -390,9 +390,9 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function compressed_buffer(ZSTD_CStreamOutSize()); uint64_t buffer_actual_size {0}; - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 0); // default (3) - ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) + std::unique_ptr cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, 0); // default (3) + ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) do { buffer_actual_size = data_cb(buffer.data(), buffer.size()); //if (buffer_actual_size == 0) { @@ -410,7 +410,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function in_buffer(ZSTD_DStreamInSize()); std::vector out_buffer(ZSTD_DStreamOutSize()); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + std::unique_ptr dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; uint64_t buffer_actual_size {0}; do { @@ -528,7 +528,7 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0}; ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0}; do { - size_t const ret = ZSTD_decompressStream(dctx, &output , &input); + size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input); if (ZSTD_isError(ret)) { // error <.< std::cerr << "FS error: decompression error\n"; - meta_data_decomp.clear(); + //meta_data_decomp.clear(); + output.pos = 0; // resize after loop break; } } while (input.pos < input.size); meta_data_decomp.resize(output.pos); - - ZSTD_freeDCtx(dctx); } else { assert(false && "implement me"); } From 77a0ae6acd21f65265120177bf231f5acf0e238e Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 11 Mar 2024 16:26:13 +0100 Subject: [PATCH 51/98] fix accel structure being wrong and mark empty frags and dont count them --- src/fragment_store/message_fragment_store.cpp | 64 +++++++++++++++++-- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 081090d..49e3f80 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -71,7 +71,13 @@ namespace Message::Components { namespace Fragment::Components { NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id) -} // Fragment::Components + + namespace Ephemeral { + // does not contain any messges + // (recheck on frag update) + struct MessagesEmptyTag {}; + } +} // Fragment::Component void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (_fs_ignore_event) { @@ -95,6 +101,12 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; } + // TODO: this is bad, we need a non persistence tag instead + if (!m.any_of()) { + // skip everything else for now + return; + } + // TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag) if (!m.all_of()) { std::cout << "MFS: new msg missing FID\n"; @@ -273,6 +285,13 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh if (!j.is_array()) { // wrong data + fh.emplace_or_replace(); + return; + } + + if (j.size() == 0) { + // empty array + fh.emplace_or_replace(); return; } @@ -288,6 +307,7 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh } reg.ctx().get().frags.emplace(fh); + size_t messages_new_or_updated {0}; for (const auto& j_entry : j) { auto new_real_msg = Message3Handle{reg, reg.create()}; // load into staging reg @@ -339,6 +359,7 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh // -> merge with preexisting (needs to be order independent) // -> throw update reg.destroy(new_real_msg); + //messages_new_or_updated++; // TODO: how do i know on merging, if data was useful //_rmm.throwEventUpdate(reg, new_real_msg); } else { if (!new_real_msg.all_of()) { @@ -348,10 +369,17 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh continue; } + messages_new_or_updated++; // -> throw create _rmm.throwEventConstruct(reg, new_real_msg); } } + + if (messages_new_or_updated == 0) { + // useless frag + // TODO: unload? + fh.emplace_or_replace(); + } } bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry& reg) { @@ -604,6 +632,10 @@ float MessageFragmentStore::tick(float time_delta) { return 0.05f; } + if (fh.all_of()) { + continue; // skip known empty + } + // get ts range of frag and collide with all curser(s/ranges) const auto& [range_begin, range_end] = fh.get(); @@ -654,14 +686,19 @@ float MessageFragmentStore::tick(float time_delta) { // a single adjacent frag is often not enough // only ok bc next is cheap - for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); i++) { + for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); next_frag = cf.next(next_frag)) { + auto fh = _fs.fragmentHandle(next_frag); + if (fh.any_of()) { + continue; // skip known empty + } + if (!loaded_frags.contains(next_frag)) { std::cout << "MFS: next frag of range\n"; - loadFragment(*msg_reg, {_fs._reg, next_frag}); + loadFragment(*msg_reg, fh); return 0.05f; } - next_frag = cf.next(next_frag); + i++; } } @@ -697,14 +734,19 @@ float MessageFragmentStore::tick(float time_delta) { // a single adjacent frag is often not enough // only ok bc next is cheap - for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); i++) { + for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); prev_frag = cf.prev(prev_frag)) { + auto fh = _fs.fragmentHandle(prev_frag); + if (fh.any_of()) { + continue; // skip known empty + } + if (!loaded_frags.contains(prev_frag)) { std::cout << "MFS: prev frag of range\n"; - loadFragment(*msg_reg, {_fs._reg, prev_frag}); + loadFragment(*msg_reg, fh); return 0.05f; } - prev_frag = cf.prev(prev_frag); + i++; } } } @@ -846,6 +888,14 @@ bool Message::Components::ContactFragments::insert(FragmentHandle frag) { frags.emplace(frag, InternalEntry{begin_index, end_index}); + // now adjust all indicies of fragments coming after the insert position + for (size_t i = begin_index + 1; i < sorted_begin.size(); i++) { + frags.at(sorted_begin[i]).i_b = i; + } + for (size_t i = end_index + 1; i < sorted_end.size(); i++) { + frags.at(sorted_end[i]).i_e = i; + } + return true; } From eac2927379fcac3e94d30405792206a43a03e93b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 11 Mar 2024 16:29:45 +0100 Subject: [PATCH 52/98] try to tame log spam --- src/fragment_store/message_fragment_store.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 49e3f80..f375e60 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -592,7 +592,7 @@ float MessageFragmentStore::tick(float time_delta) { } if (!_potentially_dirty_contacts.empty()) { - std::cout << "MFS: pdc\n"; + //std::cout << "MFS: pdc\n"; // here we check if any view of said contact needs frag loading // only once per tick tho @@ -646,7 +646,7 @@ float MessageFragmentStore::tick(float time_delta) { } } // no new visible fragment - std::cout << "MFS: no new frag directly visible\n"; + //std::cout << "MFS: no new frag directly visible\n"; // now, finally, check for adjecent fragments that need to be loaded // we do this by finding the outermost fragment in a rage, and extend it by one From 2772c8ee6944b108d86147e4d0e6aa09eb75b185 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 12 Mar 2024 11:58:23 +0100 Subject: [PATCH 53/98] reduce excessive message frag saving (queue dedup + waiting 10sec) prepare for frag updates --- src/fragment_store/fragment_store_i.hpp | 4 - src/fragment_store/message_fragment_store.cpp | 96 ++++++++++++++++--- src/fragment_store/message_fragment_store.hpp | 7 +- 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/src/fragment_store/fragment_store_i.hpp b/src/fragment_store/fragment_store_i.hpp index 8dffdf3..68d96a4 100644 --- a/src/fragment_store/fragment_store_i.hpp +++ b/src/fragment_store/fragment_store_i.hpp @@ -42,10 +42,6 @@ struct FragmentStoreEventI { virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; } virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; } //virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; } - - // mm3 - // send text - // send file path }; using FragmentStoreEventProviderI = EventProviderI; diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index f375e60..9b66638 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -76,6 +76,11 @@ namespace Fragment::Components { // does not contain any messges // (recheck on frag update) struct MessagesEmptyTag {}; + + // cache the contact for faster lookups + struct MessagesContactEntity { + Contact3 e {entt::null}; + }; } } // Fragment::Component @@ -246,7 +251,13 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { m.emplace_or_replace(fragment_id); // in this case we know the fragment needs an update - _fuid_save_queue.push({Message::getTimeMS(), fragment_id, m.registry()}); + for (const auto& it : _fuid_save_queue) { + if (it.id == fragment_id) { + // already in queue + return; // done + } + } + _fuid_save_queue.push_back({Message::getTimeMS(), fragment_id, m.registry()}); return; // done } @@ -265,7 +276,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (fid_open.contains(msg_fh)) { // TODO: dedup events // TODO: cooldown per fragsave - _fuid_save_queue.push({Message::getTimeMS(), msg_fh, m.registry()}); + _fuid_save_queue.push_back({Message::getTimeMS(), msg_fh, m.registry()}); return; } @@ -466,6 +477,7 @@ MessageFragmentStore::MessageFragmentStore( _fs._sc.registerDeSerializerJson(); _fs.subscribe(this, FragmentStore_Event::fragment_construct); + _fs.subscribe(this, FragmentStore_Event::fragment_updated); } MessageFragmentStore::~MessageFragmentStore(void) { @@ -474,7 +486,7 @@ MessageFragmentStore::~MessageFragmentStore(void) { auto* reg = _fuid_save_queue.front().reg; assert(reg != nullptr); syncFragToStorage(fh, *reg); - _fuid_save_queue.pop(); // pop unconditionally + _fuid_save_queue.pop_front(); // pop unconditionally } } @@ -541,14 +553,18 @@ static bool isLess(const std::vector& lhs, const std::vector& return lhs.size() < rhs.size(); } -float MessageFragmentStore::tick(float time_delta) { +float MessageFragmentStore::tick(float) { + const auto ts_now = Message::getTimeMS(); // sync dirty fragments here if (!_fuid_save_queue.empty()) { - auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id); - auto* reg = _fuid_save_queue.front().reg; - assert(reg != nullptr); - if (syncFragToStorage(fh, *reg)) { - _fuid_save_queue.pop(); + // wait 10sec before saving + if (_fuid_save_queue.front().ts_since_dirty + 10*1000 <= ts_now) { + auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id); + auto* reg = _fuid_save_queue.front().reg; + assert(reg != nullptr); + if (syncFragToStorage(fh, *reg)) { + _fuid_save_queue.pop_front(); + } } } @@ -562,7 +578,7 @@ float MessageFragmentStore::tick(float time_delta) { std::cout << "MFS: event check\n"; auto fh = _fs.fragmentHandle(_event_check_queue.front().fid); auto c = _event_check_queue.front().c; - _event_check_queue.pop(); + _event_check_queue.pop_front(); if (!static_cast(fh)) { return 0.05f; @@ -805,6 +821,63 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) // unkown contact return false; } + e.e.emplace_or_replace(frag_contact); + } + + // create if not exist + auto* msg_reg = _rmm.get(frag_contact); + if (msg_reg == nullptr) { + // msg reg not created yet + // TODO: this is an erroious path + return false; + } + + if (!msg_reg->ctx().contains()) { + msg_reg->ctx().emplace(); + } + msg_reg->ctx().get().erase(e.e); // TODO: can this happen? update + msg_reg->ctx().get().insert(e.e); + + _event_check_queue.push_back(ECQueueEntry{e.e, frag_contact}); + + return false; +} + +bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentUpdated& e) { + if (_fs_ignore_event) { + return false; // skip self + } + + if (!e.e.all_of()) { + return false; // not for us + } + + // since its an update, we might have it associated, or not + // its also possible it was tagged as empty + e.e.remove(); + + Contact3 frag_contact = entt::null; + { // get contact + // probably cached already + if (e.e.all_of()) { + frag_contact = e.e.get().e; + } + + if (!_cr.valid(frag_contact)) { + const auto& frag_contact_id = e.e.get().id; + // TODO: id lookup table, this is very inefficent + for (const auto& [c_it, id_it] : _cr.view().each()) { + if (frag_contact_id == id_it.data) { + frag_contact = c_it; + break; + } + } + if (!_cr.valid(frag_contact)) { + // unkown contact + return false; + } + e.e.emplace_or_replace(frag_contact); + } } // create if not exist @@ -821,7 +894,8 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) msg_reg->ctx().get().erase(e.e); // TODO: check/update/fragment update msg_reg->ctx().get().insert(e.e); - _event_check_queue.push(ECQueueEntry{e.e, frag_contact}); + // TODO: actually load it + //_event_check_queue.push_back(ECQueueEntry{e.e, frag_contact}); return false; } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 3ad7628..ac3813a 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include #include @@ -95,13 +95,13 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS FragmentID id; Message3Registry* reg{nullptr}; }; - std::queue _fuid_save_queue; + std::deque _fuid_save_queue; struct ECQueueEntry final { FragmentID fid; Contact3 c; }; - std::queue _event_check_queue; + std::deque _event_check_queue; // range changed or fragment loaded. // we only load a limited number of fragments at once, @@ -128,5 +128,6 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS protected: // fs bool onEvent(const Fragment::Events::FragmentConstruct& e) override; + bool onEvent(const Fragment::Events::FragmentUpdated& e) override; }; From 318be9cd6285c3d6d267919e9aa0ec19160c5d97 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 15 Mar 2024 14:51:30 +0100 Subject: [PATCH 54/98] throw update --- src/fragment_store/message_fragment_store.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 9b66638..a189972 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -453,6 +453,11 @@ bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry //nlohmann::json::to_msgpack(j); auto j_dump = j.dump(2, ' ', true); if (_fs.syncToStorage(fh, reinterpret_cast(j_dump.data()), j_dump.size())) { + // TODO: make this better, should this be called on fail? should this be called before sync? (prob not) + _fs_ignore_event = true; + _fs.throwEventUpdate(fh); + _fs_ignore_event = false; + //std::cout << "MFS: dumped " << j_dump << "\n"; // succ return true; From def7fc1959811753177b9deb28fa97e2724dedcb Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 28 Mar 2024 15:36:14 +0100 Subject: [PATCH 55/98] add file2 impl for zstd (lightly tested and not integrated yet) --- src/CMakeLists.txt | 14 ++ src/fragment_store/file2_zstd.cpp | 200 ++++++++++++++++++++++++++ src/fragment_store/file2_zstd.hpp | 51 +++++++ src/fragment_store/test_file_zstd.cpp | 151 +++++++++++++++++++ 4 files changed, 416 insertions(+) create mode 100644 src/fragment_store/file2_zstd.cpp create mode 100644 src/fragment_store/file2_zstd.hpp create mode 100644 src/fragment_store/test_file_zstd.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4c85e70..ea5ef87 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,9 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) add_library(fragment_store + ./fragment_store/file2_zstd.hpp + ./fragment_store/file2_zstd.cpp + ./fragment_store/fragment_store_i.hpp ./fragment_store/fragment_store_i.cpp ./fragment_store/types.hpp @@ -19,6 +22,7 @@ target_link_libraries(fragment_store PUBLIC EnTT::EnTT solanaceae_util + solanaceae_file2 zstd::zstd solanaceae_tox_messages # TODO: move @@ -26,6 +30,16 @@ target_link_libraries(fragment_store PUBLIC ######################################## +add_executable(test_file_zstd + fragment_store/test_file_zstd.cpp +) + +target_link_libraries(test_file_zstd PUBLIC + fragment_store +) + +######################################## + add_library(message_fragment_store ./fragment_store/message_serializer.hpp ./fragment_store/message_serializer.cpp diff --git a/src/fragment_store/file2_zstd.cpp b/src/fragment_store/file2_zstd.cpp new file mode 100644 index 0000000..f43e2fd --- /dev/null +++ b/src/fragment_store/file2_zstd.cpp @@ -0,0 +1,200 @@ +#include "./file2_zstd.hpp" + +#include +#include +#include +#include + +#include + +File2ZSTDW::File2ZSTDW(File2I& real) : + File2I(true, false), + _real_file(real) +{ + ZSTD_CCtx_setParameter(_cctx.get(), ZSTD_c_compressionLevel, 0); // default (3) + ZSTD_CCtx_setParameter(_cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) +} + +bool File2ZSTDW::isGood(void) { + return _real_file.isGood(); +} + +bool File2ZSTDW::write(const ByteSpan data, int64_t pos) { + if (pos != -1) { + return false; + } + + if (data.empty()) { + return false; // return true? + } + + if (data.size < 16) { + std::cout << "F2ZSTD warning: each write is a zstd frame and compression suffers significantly for small frames.\n"; + } + + std::vector compressed_buffer(ZSTD_CStreamOutSize()); + + ZSTD_inBuffer input = { data.ptr, data.size, 0 }; + + size_t remaining_ret {0}; + do { + // remaining data in input < compressed_buffer size (heuristic) + bool const lastChunk = (input.size - input.pos) <= compressed_buffer.size(); + + ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; + + ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 }; + + remaining_ret = ZSTD_compressStream2(_cctx.get(), &output , &input, mode); + if (ZSTD_isError(remaining_ret)) { + std::cerr << "F2WRZSTD error: compressing data failed\n"; + break; + } + + _real_file.write(ByteSpan{compressed_buffer.data(), output.pos}); + } while (input.pos < input.size && remaining_ret != 0 && _real_file.isGood()); + + return _real_file.isGood(); +} + +std::variant> File2ZSTDW::read(uint64_t, int64_t) { + return {}; +} + +// ######################################### decompression + +File2ZSTDR::File2ZSTDR(File2I& real) : + File2I(false, true), + _real_file(real), + + // 64kib + _in_buffer(ZSTD_DStreamInSize()), + _out_buffer(ZSTD_DStreamOutSize()) +{ +} + +bool File2ZSTDR::isGood(void) { + return _real_file.isGood(); +} + +bool File2ZSTDR::write(const ByteSpan, int64_t) { + return false; +} + +std::variant> File2ZSTDR::read(uint64_t size, int64_t pos) { + if (pos != -1) { + // error, only support streaming (for now) + return {}; + } + + std::vector ret_data; + + // actually first we check previous data + if (!_decompressed_buffer.empty()) { + uint64_t required_size = std::min(size, _decompressed_buffer.size()); + ret_data.insert(ret_data.end(), _decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size); + _decompressed_buffer.erase(_decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size); + } + + bool eof {false}; + // outerloop here + while (ret_data.size() < size && !eof) { + // first make sure we have data in input + if (_z_input.src == nullptr || _z_input.pos == _z_input.size) { + const auto request_size = _in_buffer.size(); + if (!feedInput(_real_file.read(request_size, -1))) { + return ret_data; + } + std::cout << "---- fed input " << _z_input.size << "bytes\n"; + // if _z_input.size < _in_buffer.size() -> assume eof? + if (_z_input.size < request_size) { + eof = true; + std::cout << "---- eof\n"; + } + } + + do { + ZSTD_outBuffer output = { _out_buffer.data(), _out_buffer.size(), 0 }; + size_t const ret = ZSTD_decompressStream(_dctx.get(), &output , &_z_input); + if (ZSTD_isError(ret)) { + // error <.< + std::cerr << "---- error: decompression error\n"; + return ret_data; + } + + // no new decomp data? + if (output.pos == 0) { + assert(eof || ret == 0); + break; + } + + int64_t returning_size = std::min(int64_t(size) - int64_t(ret_data.size()), output.pos); + assert(returning_size >= 0); + if (returning_size > 0) { + ret_data.insert( + ret_data.end(), + reinterpret_cast(output.dst), + reinterpret_cast(output.dst) + returning_size + ); + } + + // make sure we keep excess decompressed data + if (returning_size < int64_t(output.pos)) { + //const auto remaining_size = output.pos - returning_size; + _decompressed_buffer.insert( + _decompressed_buffer.cend(), + reinterpret_cast(output.dst) + returning_size, + reinterpret_cast(output.dst) + output.pos + ); + } + } while (_z_input.pos < _z_input.size); + } + + return ret_data; +} + +bool File2ZSTDR::feedInput(std::variant>&& read_buff) { + // TODO: optimize, we copy the buffer, but we might not need to + if (std::holds_alternative(read_buff)) { + const auto& span = std::get(read_buff); + std::cout << "---- feedInput got span " << span.size << "\n"; + if (span.size > _in_buffer.size()) { + // error, how did we read more than we asked for?? + return {}; + } + + if (span.empty()) { + _z_input = { _in_buffer.data(), 0, 0 }; + } else { + // cpy + _in_buffer = static_cast>(span); + _z_input = { + _in_buffer.data(), + span.size, + 0 + }; + } + } else if (std::holds_alternative>(read_buff)) { + auto& vec = std::get>(read_buff); + std::cout << "---- feedInput got vec " << vec.size() << "\n"; + if (vec.size() > _in_buffer.size()) { + // error, how did we read more than we asked for?? + return {}; + } + + // cpy + _in_buffer = vec; + + _z_input = { + _in_buffer.data(), + _in_buffer.size(), + 0 + }; + } else { + // error, unsupported return value of read?? + return false; + } + + return true; +} + diff --git a/src/fragment_store/file2_zstd.hpp b/src/fragment_store/file2_zstd.hpp new file mode 100644 index 0000000..7646374 --- /dev/null +++ b/src/fragment_store/file2_zstd.hpp @@ -0,0 +1,51 @@ +#pragma once + +#include + +#include + +#include + +// zstd compression wrapper over another file +// WARNING: only supports sequential writes +struct File2ZSTDW : public File2I { + File2I& _real_file; + + // TODO: hide this detail? + std::unique_ptr _cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + + File2ZSTDW(File2I& real); + virtual ~File2ZSTDW(void) {} + + bool isGood(void) override; + + // for simplicity and potential future seekability each write is its own frame + bool write(const ByteSpan data, int64_t pos = -1) override; + std::variant> read(uint64_t size, int64_t pos = -1) override; +}; + +// zstd decompression wrapper over another file +// WARNING: only supports sequential reads +// TODO: add seeking support (use frames) +struct File2ZSTDR : public File2I { + File2I& _real_file; + + // TODO: hide this detail? + std::unique_ptr _dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; + std::vector _in_buffer; + std::vector _out_buffer; + std::vector _decompressed_buffer; // retains decompressed unread data between read() calls + ZSTD_inBuffer _z_input{nullptr, 0, 0}; + + File2ZSTDR(File2I& real); + virtual ~File2ZSTDR(void) {} + + bool isGood(void) override; + + bool write(const ByteSpan data, int64_t pos = -1) override; + std::variant> read(uint64_t size, int64_t pos = -1) override; + + private: + bool feedInput(std::variant>&& read_buff); +}; + diff --git a/src/fragment_store/test_file_zstd.cpp b/src/fragment_store/test_file_zstd.cpp new file mode 100644 index 0000000..b5a1369 --- /dev/null +++ b/src/fragment_store/test_file_zstd.cpp @@ -0,0 +1,151 @@ +#include "./file2_zstd.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +const static std::string_view test_text1{"test1 1234 1234 :) 1234 5678 88888888\n"}; +const static ByteSpan data_test_text1{ + reinterpret_cast(test_text1.data()), + test_text1.size() +}; + +const static std::string_view test_text2{"test2 1234 1234 :) 1234 5678 88888888\n"}; +const static ByteSpan data_test_text2{ + reinterpret_cast(test_text2.data()), + test_text2.size() +}; + +const static std::string_view test_text3{ + "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" + "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" + "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" + "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" + "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" + "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" +}; +const static ByteSpan data_test_text3{ + reinterpret_cast(test_text3.data()), + test_text3.size() +}; + +int main(void) { + const auto temp_dir = std::filesystem::temp_directory_path() / "file2wzstdtests"; + + std::filesystem::create_directories(temp_dir); // making sure + assert(std::filesystem::exists(temp_dir)); + std::cout << "test temp dir: " << temp_dir << "\n"; + + const auto test1_file_path = temp_dir / "testfile1.zstd"; + { // simple write test + File2WFile f_w_file{test1_file_path.c_str(), true}; + assert(f_w_file.isGood()); + + File2ZSTDW f_w_zstd{f_w_file}; + assert(f_w_zstd.isGood()); + assert(f_w_file.isGood()); + + //bool res = f_w_file.write(data_test_text1); + bool res = f_w_zstd.write(data_test_text1); + assert(res); + assert(f_w_zstd.isGood()); + assert(f_w_file.isGood()); + + // write another frame of the same data + res = f_w_zstd.write(data_test_text2); + assert(res); + assert(f_w_zstd.isGood()); + assert(f_w_file.isGood()); + + // write larger frame + res = f_w_zstd.write(data_test_text3); + assert(res); + assert(f_w_zstd.isGood()); + assert(f_w_file.isGood()); + } + + // after flush + assert(std::filesystem::file_size(test1_file_path) != 0); + + { // simple read test (using write test created file) + File2RFile f_r_file{test1_file_path.c_str()}; + assert(f_r_file.isGood()); + + File2ZSTDR f_r_zstd{f_r_file}; + assert(f_r_zstd.isGood()); + assert(f_r_file.isGood()); + + // reads return owning buffers + + { // readback data_test_text1 + auto r_res_var = f_r_zstd.read(data_test_text1.size); + + //assert(f_r_zstd.isGood()); + //assert(f_r_file.isGood()); + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + + //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; + + assert(std::get>(r_res_var).size() == data_test_text1.size); + assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), std::get>(r_res_var).cbegin())); + } + + { // readback data_test_text2 + auto r_res_var = f_r_zstd.read(data_test_text2.size); + + //assert(f_r_zstd.isGood()); + //assert(f_r_file.isGood()); + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + + //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; + + assert(std::get>(r_res_var).size() == data_test_text2.size); + assert(std::equal( + data_test_text2.cbegin(), + data_test_text2.cend(), + std::get>(r_res_var).cbegin() + )); + } + + { // readback data_test_text3 + auto r_res_var = f_r_zstd.read(data_test_text3.size); + + //assert(f_r_zstd.isGood()); + //assert(f_r_file.isGood()); + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + + //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; + + assert(std::get>(r_res_var).size() == data_test_text3.size); + assert(std::equal( + data_test_text3.cbegin(), + data_test_text3.cend(), + r_res_vec.cbegin() + )); + } + + { // assert eof somehow + // since its eof, reading a single byte should return a zero sized buffer + auto r_res_var = f_r_zstd.read(1); + if (std::holds_alternative>(r_res_var)) { + assert(std::get>(r_res_var).empty()); + } else if (std::holds_alternative(r_res_var)) { + assert(std::get(r_res_var).empty()); + } else { + assert(false); + } + } + } + + // cleanup + std::filesystem::remove_all(temp_dir); +} + From 8b17ed195f2355f0115f65dac29c3b33ac8afa88 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 30 Mar 2024 13:50:31 +0100 Subject: [PATCH 56/98] more testing and file2 zstd now passes tests with varying frame sizes and 1.5gig files --- src/fragment_store/file2_zstd.cpp | 14 ++- src/fragment_store/test_file_zstd.cpp | 155 +++++++++++++++++++++++++- 2 files changed, 158 insertions(+), 11 deletions(-) diff --git a/src/fragment_store/file2_zstd.cpp b/src/fragment_store/file2_zstd.cpp index f43e2fd..b34b308 100644 --- a/src/fragment_store/file2_zstd.cpp +++ b/src/fragment_store/file2_zstd.cpp @@ -52,7 +52,7 @@ bool File2ZSTDW::write(const ByteSpan data, int64_t pos) { } _real_file.write(ByteSpan{compressed_buffer.data(), output.pos}); - } while (input.pos < input.size && remaining_ret != 0 && _real_file.isGood()); + } while ((input.pos < input.size || remaining_ret != 0) && _real_file.isGood()); return _real_file.isGood(); } @@ -105,11 +105,11 @@ std::variant> File2ZSTDR::read(uint64_t size, int if (!feedInput(_real_file.read(request_size, -1))) { return ret_data; } - std::cout << "---- fed input " << _z_input.size << "bytes\n"; + // if _z_input.size < _in_buffer.size() -> assume eof? if (_z_input.size < request_size) { eof = true; - std::cout << "---- eof\n"; + //std::cout << "---- eof\n"; } } @@ -124,6 +124,12 @@ std::variant> File2ZSTDR::read(uint64_t size, int // no new decomp data? if (output.pos == 0) { + if (ret != 0) { + // if not error and not 0, indicates that + // there is additional flushing needed + continue; + } + assert(eof || ret == 0); break; } @@ -157,7 +163,6 @@ bool File2ZSTDR::feedInput(std::variant>&& read_b // TODO: optimize, we copy the buffer, but we might not need to if (std::holds_alternative(read_buff)) { const auto& span = std::get(read_buff); - std::cout << "---- feedInput got span " << span.size << "\n"; if (span.size > _in_buffer.size()) { // error, how did we read more than we asked for?? return {}; @@ -176,7 +181,6 @@ bool File2ZSTDR::feedInput(std::variant>&& read_b } } else if (std::holds_alternative>(read_buff)) { auto& vec = std::get>(read_buff); - std::cout << "---- feedInput got vec " << vec.size() << "\n"; if (vec.size() > _in_buffer.size()) { // error, how did we read more than we asked for?? return {}; diff --git a/src/fragment_store/test_file_zstd.cpp b/src/fragment_store/test_file_zstd.cpp index b5a1369..08509b7 100644 --- a/src/fragment_store/test_file_zstd.cpp +++ b/src/fragment_store/test_file_zstd.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include const static std::string_view test_text1{"test1 1234 1234 :) 1234 5678 88888888\n"}; @@ -35,7 +37,7 @@ const static ByteSpan data_test_text3{ }; int main(void) { - const auto temp_dir = std::filesystem::temp_directory_path() / "file2wzstdtests"; + const auto temp_dir = std::filesystem::temp_directory_path() / "file2_zstd_tests"; std::filesystem::create_directories(temp_dir); // making sure assert(std::filesystem::exists(temp_dir)); @@ -92,8 +94,8 @@ int main(void) { //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(std::get>(r_res_var).size() == data_test_text1.size); - assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), std::get>(r_res_var).cbegin())); + assert(r_res_vec.size() == data_test_text1.size); + assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); } { // readback data_test_text2 @@ -106,11 +108,11 @@ int main(void) { //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(std::get>(r_res_var).size() == data_test_text2.size); + assert(r_res_vec.size() == data_test_text2.size); assert(std::equal( data_test_text2.cbegin(), data_test_text2.cend(), - std::get>(r_res_var).cbegin() + r_res_vec.cbegin() )); } @@ -124,7 +126,7 @@ int main(void) { //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - assert(std::get>(r_res_var).size() == data_test_text3.size); + assert(r_res_vec.size() == data_test_text3.size); assert(std::equal( data_test_text3.cbegin(), data_test_text3.cend(), @@ -145,6 +147,147 @@ int main(void) { } } + const auto test2_file_path = temp_dir / "testfile2.zstd"; + { // write and read a single frame with increasing size + for (size_t fslog = 1; fslog <= 25; fslog++) { + const size_t frame_size = 1< tmp_data(frame_size); + for (auto& e : tmp_data) { + e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough + } + assert(tmp_data.size() == frame_size); + + bool res = f_w_zstd.write(ByteSpan{tmp_data}); + assert(res); + assert(f_w_zstd.isGood()); + assert(f_w_file.isGood()); + } + + { // read + std::minstd_rand rng_data{11*1337}; + + File2RFile f_r_file{test2_file_path.c_str()}; + assert(f_r_file.isGood()); + + File2ZSTDR f_r_zstd{f_r_file}; + assert(f_r_zstd.isGood()); + assert(f_r_file.isGood()); + + { // read frame + auto r_res_var = f_r_zstd.read(frame_size); + + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + assert(r_res_vec.size() == frame_size); + + // assert equal + for (auto& e : r_res_vec) { + assert(e == uint8_t(rng_data() & 0xff)); + } + } + + { // eof test + auto r_res_var = f_r_zstd.read(1); + if (std::holds_alternative>(r_res_var)) { + assert(std::get>(r_res_var).empty()); + } else if (std::holds_alternative(r_res_var)) { + assert(std::get(r_res_var).empty()); + } else { + assert(false); + } + } + } + + // since we spam file, we immediatly remove them + std::filesystem::remove(test2_file_path); + } + } + + const auto test3_file_path = temp_dir / "testfile3.zstd"; + { // large file test write + File2WFile f_w_file{test3_file_path.c_str(), true}; + assert(f_w_file.isGood()); + + File2ZSTDW f_w_zstd{f_w_file}; + assert(f_w_zstd.isGood()); + assert(f_w_file.isGood()); + + std::minstd_rand rng{11*1337}; + std::minstd_rand rng_data{11*1337}; // make investigating easier + + size_t total_raw_size {0}; + for (size_t i = 0; i < 2000; i++) { + const size_t frame_size = (rng() % ((2<<19) - 1)) + 1; + + std::vector tmp_data(frame_size); + for (auto& e : tmp_data) { + e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough + } + + bool res = f_w_zstd.write(ByteSpan{tmp_data}); + assert(res); + assert(f_w_zstd.isGood()); + assert(f_w_file.isGood()); + total_raw_size += frame_size; + } + std::cout << "t3 total raw size: " << total_raw_size << "\n"; + } + + // after flush + std::cout << "t3 size on disk: " << std::filesystem::file_size(test3_file_path) << "\n"; + + { // large file test read + File2RFile f_r_file{test3_file_path.c_str()}; + assert(f_r_file.isGood()); + + File2ZSTDR f_r_zstd{f_r_file}; + assert(f_r_zstd.isGood()); + assert(f_r_file.isGood()); + + // using same rng state as write to compare + std::minstd_rand rng{11*1337}; + std::minstd_rand rng_data{11*1337}; + + for (size_t i = 0; i < 2000; i++) { + const size_t frame_size = (rng() % ((2<<19) - 1)) + 1; + //std::cerr << "f: " << i << " fs: " << frame_size << "\n"; + + auto r_res_var = f_r_zstd.read(frame_size); + + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + assert(r_res_vec.size() == frame_size); + + // assert equal + for (auto& e : r_res_vec) { + assert(e == uint8_t(rng_data() & 0xff)); + } + } + + { // eof test + auto r_res_var = f_r_zstd.read(1); + if (std::holds_alternative>(r_res_var)) { + assert(std::get>(r_res_var).empty()); + } else if (std::holds_alternative(r_res_var)) { + assert(std::get(r_res_var).empty()); + } else { + assert(false); + } + } + } + // cleanup std::filesystem::remove_all(temp_dir); } From f22f5237743569de989699fc65ecd01b8ea11848 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 30 Mar 2024 23:23:39 +0100 Subject: [PATCH 57/98] minor frag store refactor --- src/fragment_store/fragment_store.cpp | 4 ---- src/fragment_store/fragment_store.hpp | 3 --- src/fragment_store/fragment_store_i.cpp | 8 ++++++++ src/fragment_store/fragment_store_i.hpp | 4 ++++ 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 7ac7164..17830d8 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -67,10 +67,6 @@ FragmentStore::FragmentStore( registerSerializers(); } -FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) { - return {_reg, fid}; -} - std::vector FragmentStore::generateNewUID(std::array& uuid_namespace) { std::vector new_uid(uuid_namespace.cbegin(), uuid_namespace.cend()); new_uid.resize(new_uid.size() + 16); diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 811b4be..0e52e51 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -31,9 +31,6 @@ struct FragmentStore : public FragmentStoreI { FragmentStore(void); FragmentStore(std::array session_uuid_namespace); - // HACK: get access to the reg - FragmentHandle fragmentHandle(FragmentID fid); - // TODO: make the frags ref counted // TODO: check for exising diff --git a/src/fragment_store/fragment_store_i.cpp b/src/fragment_store/fragment_store_i.cpp index 75a9e44..3f757fd 100644 --- a/src/fragment_store/fragment_store_i.cpp +++ b/src/fragment_store/fragment_store_i.cpp @@ -2,6 +2,14 @@ #include +FragmentRegistry& FragmentStoreI::registry(void) { + return _reg; +} + +FragmentHandle FragmentStoreI::fragmentHandle(const FragmentID fid) { + return {_reg, fid}; +} + void FragmentStoreI::throwEventConstruct(const FragmentID fid) { std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n"; dispatch( diff --git a/src/fragment_store/fragment_store_i.hpp b/src/fragment_store/fragment_store_i.hpp index 68d96a4..774a766 100644 --- a/src/fragment_store/fragment_store_i.hpp +++ b/src/fragment_store/fragment_store_i.hpp @@ -52,8 +52,12 @@ struct FragmentStoreI : public FragmentStoreEventProviderI { virtual ~FragmentStoreI(void) {} + FragmentRegistry& registry(void); + FragmentHandle fragmentHandle(const FragmentID fid); + void throwEventConstruct(const FragmentID fid); void throwEventUpdate(const FragmentID fid); + // TODO //void throwEventDestroy(); }; From 19fd99f713928cd1ec0c88054df3011007be18f7 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 31 Mar 2024 14:23:03 +0200 Subject: [PATCH 58/98] refactor out uuidgenerator --- src/CMakeLists.txt | 3 ++ src/fragment_store/fragment_store.cpp | 62 +----------------------- src/fragment_store/fragment_store.hpp | 6 +-- src/fragment_store/uuid_generator.cpp | 68 +++++++++++++++++++++++++++ src/fragment_store/uuid_generator.hpp | 23 +++++++++ 5 files changed, 99 insertions(+), 63 deletions(-) create mode 100644 src/fragment_store/uuid_generator.cpp create mode 100644 src/fragment_store/uuid_generator.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ea5ef87..c26a40f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,9 @@ add_library(fragment_store ./fragment_store/file2_zstd.hpp ./fragment_store/file2_zstd.cpp + ./fragment_store/uuid_generator.hpp + ./fragment_store/uuid_generator.cpp + ./fragment_store/fragment_store_i.hpp ./fragment_store/fragment_store_i.cpp ./fragment_store/types.hpp diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 17830d8..450a016 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -32,75 +32,17 @@ static const char* metaFileTypeSuffix(MetaFileType mft) { } FragmentStore::FragmentStore(void) { - { // random namespace - const auto num0 = _rng(); - const auto num1 = _rng(); - const auto num2 = _rng(); - const auto num3 = _rng(); - - _session_uuid_namespace[0+0] = (num0 >> 0) & 0xff; - _session_uuid_namespace[0+1] = (num0 >> 8) & 0xff; - _session_uuid_namespace[0+2] = (num0 >> 16) & 0xff; - _session_uuid_namespace[0+3] = (num0 >> 24) & 0xff; - - _session_uuid_namespace[4+0] = (num1 >> 0) & 0xff; - _session_uuid_namespace[4+1] = (num1 >> 8) & 0xff; - _session_uuid_namespace[4+2] = (num1 >> 16) & 0xff; - _session_uuid_namespace[4+3] = (num1 >> 24) & 0xff; - - _session_uuid_namespace[8+0] = (num2 >> 0) & 0xff; - _session_uuid_namespace[8+1] = (num2 >> 8) & 0xff; - _session_uuid_namespace[8+2] = (num2 >> 16) & 0xff; - _session_uuid_namespace[8+3] = (num2 >> 24) & 0xff; - - _session_uuid_namespace[12+0] = (num3 >> 0) & 0xff; - _session_uuid_namespace[12+1] = (num3 >> 8) & 0xff; - _session_uuid_namespace[12+2] = (num3 >> 16) & 0xff; - _session_uuid_namespace[12+3] = (num3 >> 24) & 0xff; - } registerSerializers(); } FragmentStore::FragmentStore( std::array session_uuid_namespace -) : _session_uuid_namespace(std::move(session_uuid_namespace)) { +) : _session_uuid_gen(std::move(session_uuid_namespace)) { registerSerializers(); } -std::vector FragmentStore::generateNewUID(std::array& uuid_namespace) { - std::vector new_uid(uuid_namespace.cbegin(), uuid_namespace.cend()); - new_uid.resize(new_uid.size() + 16); - - const auto num0 = _rng(); - const auto num1 = _rng(); - const auto num2 = _rng(); - const auto num3 = _rng(); - - new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff; - new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff; - new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff; - new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff; - - new_uid[uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff; - new_uid[uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff; - new_uid[uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff; - new_uid[uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff; - - new_uid[uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff; - new_uid[uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff; - new_uid[uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff; - new_uid[uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff; - - new_uid[uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff; - new_uid[uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff; - new_uid[uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff; - new_uid[uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff; - - return new_uid; -} - std::vector FragmentStore::generateNewUID(void) { - return generateNewUID(_session_uuid_namespace); + return _session_uuid_gen(); } FragmentID FragmentStore::newFragmentMemoryOwned( diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 0e52e51..26c5204 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -7,6 +7,8 @@ #include "./serializer.hpp" +#include "./uuid_generator.hpp" + #include #include @@ -18,8 +20,7 @@ #include struct FragmentStore : public FragmentStoreI { - std::minstd_rand _rng{std::random_device{}()}; - std::array _session_uuid_namespace; + UUIDGenerator_128_128 _session_uuid_gen; std::string _default_store_path; @@ -34,7 +35,6 @@ struct FragmentStore : public FragmentStoreI { // TODO: make the frags ref counted // TODO: check for exising - std::vector generateNewUID(std::array& uuid_namespace); std::vector generateNewUID(void); // ========== new fragment ========== diff --git a/src/fragment_store/uuid_generator.cpp b/src/fragment_store/uuid_generator.cpp new file mode 100644 index 0000000..cc07985 --- /dev/null +++ b/src/fragment_store/uuid_generator.cpp @@ -0,0 +1,68 @@ +#include "./uuid_generator.hpp" + +UUIDGenerator_128_128::UUIDGenerator_128_128(void) { + { // random namespace + const auto num0 = _rng(); + const auto num1 = _rng(); + const auto num2 = _rng(); + const auto num3 = _rng(); + + _uuid_namespace[0+0] = (num0 >> 0) & 0xff; + _uuid_namespace[0+1] = (num0 >> 8) & 0xff; + _uuid_namespace[0+2] = (num0 >> 16) & 0xff; + _uuid_namespace[0+3] = (num0 >> 24) & 0xff; + + _uuid_namespace[4+0] = (num1 >> 0) & 0xff; + _uuid_namespace[4+1] = (num1 >> 8) & 0xff; + _uuid_namespace[4+2] = (num1 >> 16) & 0xff; + _uuid_namespace[4+3] = (num1 >> 24) & 0xff; + + _uuid_namespace[8+0] = (num2 >> 0) & 0xff; + _uuid_namespace[8+1] = (num2 >> 8) & 0xff; + _uuid_namespace[8+2] = (num2 >> 16) & 0xff; + _uuid_namespace[8+3] = (num2 >> 24) & 0xff; + + _uuid_namespace[12+0] = (num3 >> 0) & 0xff; + _uuid_namespace[12+1] = (num3 >> 8) & 0xff; + _uuid_namespace[12+2] = (num3 >> 16) & 0xff; + _uuid_namespace[12+3] = (num3 >> 24) & 0xff; + } +} + +UUIDGenerator_128_128::UUIDGenerator_128_128(const std::array& uuid_namespace) : + _uuid_namespace(uuid_namespace) +{ +} + +std::vector UUIDGenerator_128_128::operator()(void) { + std::vector new_uid(_uuid_namespace.cbegin(), _uuid_namespace.cend()); + new_uid.resize(new_uid.size() + 16); + + const auto num0 = _rng(); + const auto num1 = _rng(); + const auto num2 = _rng(); + const auto num3 = _rng(); + + new_uid[_uuid_namespace.size()+0] = (num0 >> 0) & 0xff; + new_uid[_uuid_namespace.size()+1] = (num0 >> 8) & 0xff; + new_uid[_uuid_namespace.size()+2] = (num0 >> 16) & 0xff; + new_uid[_uuid_namespace.size()+3] = (num0 >> 24) & 0xff; + + new_uid[_uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff; + new_uid[_uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff; + new_uid[_uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff; + new_uid[_uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff; + + new_uid[_uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff; + new_uid[_uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff; + new_uid[_uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff; + new_uid[_uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff; + + new_uid[_uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff; + new_uid[_uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff; + new_uid[_uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff; + new_uid[_uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff; + + return new_uid; +} + diff --git a/src/fragment_store/uuid_generator.hpp b/src/fragment_store/uuid_generator.hpp new file mode 100644 index 0000000..755e3be --- /dev/null +++ b/src/fragment_store/uuid_generator.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include +#include + +struct UUIDGeneratorI { + virtual std::vector operator()(void) = 0; +}; + +struct UUIDGenerator_128_128 final : public UUIDGeneratorI { + private: + std::array _uuid_namespace; + std::minstd_rand _rng{std::random_device{}()}; + + public: + UUIDGenerator_128_128(void); // default randomly initializes namespace + UUIDGenerator_128_128(const std::array& uuid_namespace); + + std::vector operator()(void) override; +}; + From 19844a9423d8d4594c7efb48f49ea0d1e4e4e8d6 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 31 Mar 2024 15:43:01 +0200 Subject: [PATCH 59/98] use file2 zstd wrapper to read frag data (untested) --- src/CMakeLists.txt | 4 +- src/fragment_store/fragment_store.cpp | 134 ++++++++++++++++---------- src/fragment_store/fragment_store.hpp | 4 +- src/fragment_store/test_fragstore.cpp | 2 +- src/fragment_store/uuid_generator.hpp | 1 + 5 files changed, 92 insertions(+), 53 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c26a40f..28c7938 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,11 +62,11 @@ target_link_libraries(message_fragment_store PUBLIC ######################################## -add_executable(fragment_store_test +add_executable(test_fragment_store fragment_store/test_fragstore.cpp ) -target_link_libraries(fragment_store_test PUBLIC +target_link_libraries(test_fragment_store PUBLIC fragment_store ) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 450a016..66d6f04 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -8,6 +8,9 @@ #include +#include +#include "./file2_zstd.hpp" + #include #include @@ -18,9 +21,12 @@ #include #include #include +#include +#include +#include +#include #include -#include static const char* metaFileTypeSuffix(MetaFileType mft) { switch (mft) { @@ -125,7 +131,7 @@ FragmentID FragmentStore::newFragmentFile( _reg.emplace(new_frag, mft); // meta needs to be synced to file - std::function empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; }; + std::function empty_data_cb = [](auto*, auto) -> uint64_t { return 0; }; if (!syncToStorage(new_frag, empty_data_cb)) { std::cerr << "FS error: syncToStorage failed while creating new fragment file\n"; _reg.destroy(new_frag); @@ -421,72 +427,102 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function> data_file_stack; + data_file_stack.push(std::make_unique(std::string_view{frag_path})); - if (!data_file.is_open()) { + //std::ifstream data_file{ + //frag_path, + //std::ios::in | std::ios::binary // always binary, also for text + //}; + + if (!data_file_stack.top()->isGood()) { std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; // error return false; } + // TODO: decrypt here + Compression data_comp = Compression::NONE; if (_reg.all_of(fid)) { data_comp = _reg.get(fid).comp; } - if (data_comp == Compression::NONE) { - std::array buffer; - uint64_t buffer_actual_size {0}; - do { - data_file.read(reinterpret_cast(buffer.data()), buffer.size()); - buffer_actual_size = data_file.gcount(); - - if (buffer_actual_size == 0) { - break; - } - - data_cb(buffer.data(), buffer_actual_size); - } while (buffer_actual_size == buffer.size() && !data_file.eof()); - } else if (data_comp == Compression::ZSTD) { - std::vector in_buffer(ZSTD_DStreamInSize()); - std::vector out_buffer(ZSTD_DStreamOutSize()); - std::unique_ptr dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; - - uint64_t buffer_actual_size {0}; - do { - data_file.read(reinterpret_cast(in_buffer.data()), in_buffer.size()); - buffer_actual_size = data_file.gcount(); - if (buffer_actual_size == 0) { - break; - } - - ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 }; - do { - ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 }; - size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input); - if (ZSTD_isError(ret)) { - // error <.< - std::cerr << "FS error: decompression error\n"; - break; - } - - data_cb(out_buffer.data(), output.pos); - } while (input.pos < input.size); - } while (buffer_actual_size == in_buffer.size() && !data_file.eof()); + // add layer based on enum + if (data_comp == Compression::ZSTD) { + data_file_stack.push(std::make_unique(*data_file_stack.top().get())); + if (!data_file_stack.top()->isGood()) { + std::cerr << "FS error: fragment data file failed to add zstd decompression layer '" << frag_path << "'\n"; + // error + return false; + } } else { - assert(false && "implement me"); + assert(data_comp == Compression::NONE); } + //if (data_comp == Compression::NONE) { + static constexpr int64_t chunk_size {1024 * 1024}; + do { + auto data_var = data_file_stack.top()->read(chunk_size); + ByteSpan data; + if (std::holds_alternative>(data_var)) { + auto& vec = std::get>(data_var); + data = {vec.data(), vec.size()}; + } else if (std::holds_alternative(data_var)) { + data = std::get(data_var); + } else { + assert(false); + } + + if (data.empty()) { + // error or probably eof + break; + } + + data_cb(data); + + if (data.size < chunk_size) { + // eof + break; + } + } while (data_file_stack.top()->isGood()); + //} else if (data_comp == Compression::ZSTD) { + //std::vector in_buffer(ZSTD_DStreamInSize()); + //std::vector out_buffer(ZSTD_DStreamOutSize()); + //std::unique_ptr dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; + + //uint64_t buffer_actual_size {0}; + //do { + //data_file.read(reinterpret_cast(in_buffer.data()), in_buffer.size()); + //buffer_actual_size = data_file.gcount(); + //if (buffer_actual_size == 0) { + //break; + //} + + //ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 }; + //do { + //ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 }; + //size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input); + //if (ZSTD_isError(ret)) { + //// error <.< + //std::cerr << "FS error: decompression error\n"; + //break; + //} + + //data_cb(out_buffer.data(), output.pos); + //} while (input.pos < input.size); + //} while (buffer_actual_size == in_buffer.size() && !data_file.eof()); + //} else { + //assert(false && "implement me"); + //} + return true; } nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) { std::vector tmp_buffer; - std::function cb = [&tmp_buffer](const uint8_t* buffer, const uint64_t buffer_size) { - tmp_buffer.insert(tmp_buffer.end(), buffer, buffer+buffer_size); + std::function cb = [&tmp_buffer](const ByteSpan buffer) { + tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); }; if (!loadFromStorage(fid, cb)) { diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 26c5204..d29bff6 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include "./fragment_store_i.hpp" #include "./types.hpp" @@ -80,7 +82,7 @@ struct FragmentStore : public FragmentStoreI { bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size); // ========== load fragment data from storage ========== - using read_from_storage_put_data_cb = void(const uint8_t* buffer, const uint64_t buffer_size); + using read_from_storage_put_data_cb = void(const ByteSpan buffer); bool loadFromStorage(FragmentID fid, std::function& data_cb); // convenience function nlohmann::json loadFromStorageNJ(FragmentID fid); diff --git a/src/fragment_store/test_fragstore.cpp b/src/fragment_store/test_fragstore.cpp index 7136643..1319e29 100644 --- a/src/fragment_store/test_fragstore.cpp +++ b/src/fragment_store/test_fragstore.cpp @@ -50,7 +50,7 @@ int main(void) { { auto frag1h = fs.fragmentHandle(frag1); - frag1h.emplace_or_replace(); + frag1h.emplace_or_replace().comp = Compression::ZSTD; frag1h.emplace_or_replace(); std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { diff --git a/src/fragment_store/uuid_generator.hpp b/src/fragment_store/uuid_generator.hpp index 755e3be..b0a1f99 100644 --- a/src/fragment_store/uuid_generator.hpp +++ b/src/fragment_store/uuid_generator.hpp @@ -9,6 +9,7 @@ struct UUIDGeneratorI { virtual std::vector operator()(void) = 0; }; +// TODO: templates? struct UUIDGenerator_128_128 final : public UUIDGeneratorI { private: std::array _uuid_namespace; From 16d2238f353de75f00c6d36984166faf3b57f99f Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 31 Mar 2024 22:03:55 +0200 Subject: [PATCH 60/98] fixes for ci --- src/fragment_store/file2_zstd.cpp | 2 +- src/fragment_store/test_file_zstd.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/fragment_store/file2_zstd.cpp b/src/fragment_store/file2_zstd.cpp index b34b308..4ccb58b 100644 --- a/src/fragment_store/file2_zstd.cpp +++ b/src/fragment_store/file2_zstd.cpp @@ -91,7 +91,7 @@ std::variant> File2ZSTDR::read(uint64_t size, int // actually first we check previous data if (!_decompressed_buffer.empty()) { - uint64_t required_size = std::min(size, _decompressed_buffer.size()); + uint64_t required_size = std::min(size, _decompressed_buffer.size()); ret_data.insert(ret_data.end(), _decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size); _decompressed_buffer.erase(_decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size); } diff --git a/src/fragment_store/test_file_zstd.cpp b/src/fragment_store/test_file_zstd.cpp index 08509b7..817639d 100644 --- a/src/fragment_store/test_file_zstd.cpp +++ b/src/fragment_store/test_file_zstd.cpp @@ -45,7 +45,7 @@ int main(void) { const auto test1_file_path = temp_dir / "testfile1.zstd"; { // simple write test - File2WFile f_w_file{test1_file_path.c_str(), true}; + File2WFile f_w_file{std::string_view{test1_file_path.u8string()}, true}; assert(f_w_file.isGood()); File2ZSTDW f_w_zstd{f_w_file}; @@ -75,7 +75,7 @@ int main(void) { assert(std::filesystem::file_size(test1_file_path) != 0); { // simple read test (using write test created file) - File2RFile f_r_file{test1_file_path.c_str()}; + File2RFile f_r_file{std::string_view{test1_file_path.u8string()}}; assert(f_r_file.isGood()); File2ZSTDR f_r_zstd{f_r_file}; @@ -156,7 +156,7 @@ int main(void) { { // write std::minstd_rand rng_data{11*1337}; - File2WFile f_w_file{test2_file_path.c_str(), true}; + File2WFile f_w_file{std::string_view{test2_file_path.u8string()}, true}; assert(f_w_file.isGood()); File2ZSTDW f_w_zstd{f_w_file}; @@ -178,7 +178,7 @@ int main(void) { { // read std::minstd_rand rng_data{11*1337}; - File2RFile f_r_file{test2_file_path.c_str()}; + File2RFile f_r_file{std::string_view{test2_file_path.u8string()}}; assert(f_r_file.isGood()); File2ZSTDR f_r_zstd{f_r_file}; @@ -217,7 +217,7 @@ int main(void) { const auto test3_file_path = temp_dir / "testfile3.zstd"; { // large file test write - File2WFile f_w_file{test3_file_path.c_str(), true}; + File2WFile f_w_file{std::string_view{test3_file_path.u8string()}, true}; assert(f_w_file.isGood()); File2ZSTDW f_w_zstd{f_w_file}; @@ -249,7 +249,7 @@ int main(void) { std::cout << "t3 size on disk: " << std::filesystem::file_size(test3_file_path) << "\n"; { // large file test read - File2RFile f_r_file{test3_file_path.c_str()}; + File2RFile f_r_file{std::string_view{test3_file_path.u8string()}}; assert(f_r_file.isGood()); File2ZSTDR f_r_zstd{f_r_file}; From 1b9363e7b5b84e6ff4c1c84579e063ed1f447881 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 1 Apr 2024 12:32:22 +0200 Subject: [PATCH 61/98] tested and works, cleaning up commented code --- src/fragment_store/fragment_store.cpp | 40 ++------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 66d6f04..5d41853 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -430,14 +430,8 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function> data_file_stack; data_file_stack.push(std::make_unique(std::string_view{frag_path})); - //std::ifstream data_file{ - //frag_path, - //std::ios::in | std::ios::binary // always binary, also for text - //}; - if (!data_file_stack.top()->isGood()) { std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; - // error return false; } @@ -453,15 +447,14 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function(*data_file_stack.top().get())); if (!data_file_stack.top()->isGood()) { std::cerr << "FS error: fragment data file failed to add zstd decompression layer '" << frag_path << "'\n"; - // error return false; } } else { assert(data_comp == Compression::NONE); } - //if (data_comp == Compression::NONE) { - static constexpr int64_t chunk_size {1024 * 1024}; + // TODO: make it read in a single chunk instead? + static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read do { auto data_var = data_file_stack.top()->read(chunk_size); ByteSpan data; @@ -486,35 +479,6 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::functionisGood()); - //} else if (data_comp == Compression::ZSTD) { - //std::vector in_buffer(ZSTD_DStreamInSize()); - //std::vector out_buffer(ZSTD_DStreamOutSize()); - //std::unique_ptr dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; - - //uint64_t buffer_actual_size {0}; - //do { - //data_file.read(reinterpret_cast(in_buffer.data()), in_buffer.size()); - //buffer_actual_size = data_file.gcount(); - //if (buffer_actual_size == 0) { - //break; - //} - - //ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 }; - //do { - //ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 }; - //size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input); - //if (ZSTD_isError(ret)) { - //// error <.< - //std::cerr << "FS error: decompression error\n"; - //break; - //} - - //data_cb(out_buffer.data(), output.pos); - //} while (input.pos < input.size); - //} while (buffer_actual_size == in_buffer.size() && !data_file.eof()); - //} else { - //assert(false && "implement me"); - //} return true; } From b640b5a06b4caaf776684bbdb9180a0cbab8a4b2 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 1 Apr 2024 12:43:21 +0200 Subject: [PATCH 62/98] variant to span helper --- src/fragment_store/fragment_store.cpp | 33 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 5d41853..3af7f69 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -37,6 +37,19 @@ static const char* metaFileTypeSuffix(MetaFileType mft) { return ""; // .unk? } +// TODO: move to ... somewhere. (span? file2i?) +static ByteSpan spanFromRead(const std::variant>& data_var) { + if (std::holds_alternative>(data_var)) { + auto& vec = std::get>(data_var); + return {vec.data(), vec.size()}; + } else if (std::holds_alternative(data_var)) { + return std::get(data_var); + } else { + assert(false); + return {}; + } +} + FragmentStore::FragmentStore(void) { registerSerializers(); } @@ -445,27 +458,21 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function(*data_file_stack.top().get())); - if (!data_file_stack.top()->isGood()) { - std::cerr << "FS error: fragment data file failed to add zstd decompression layer '" << frag_path << "'\n"; - return false; - } } else { + // TODO: make error instead assert(data_comp == Compression::NONE); } + if (!data_file_stack.top()->isGood()) { + std::cerr << "FS error: fragment data file failed to add " << (int)data_comp << " decompression layer '" << frag_path << "'\n"; + return false; + } + // TODO: make it read in a single chunk instead? static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read do { auto data_var = data_file_stack.top()->read(chunk_size); - ByteSpan data; - if (std::holds_alternative>(data_var)) { - auto& vec = std::get>(data_var); - data = {vec.data(), vec.size()}; - } else if (std::holds_alternative(data_var)) { - data = std::get(data_var); - } else { - assert(false); - } + ByteSpan data = spanFromRead(data_var); if (data.empty()) { // error or probably eof From c737715c6603e4d1f05184dd284642dd7d29169a Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 1 Apr 2024 13:28:33 +0200 Subject: [PATCH 63/98] more refactor and transition meta write --- src/fragment_store/fragment_store.cpp | 125 +++++++++++++++++++------- 1 file changed, 95 insertions(+), 30 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 3af7f69..1b00d64 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -50,6 +50,63 @@ static ByteSpan spanFromRead(const std::variant>& } } +// TODO: these stacks are file specific +static std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for reading '" << file_path << "'\n"; + return {}; + } + + // TODO: decrypt here + assert(encryption == Encryption::NONE); + + // add layer based on enum + if (compression == Compression::ZSTD) { + file_stack.push(std::make_unique(*file_stack.top().get())); + } else { + // TODO: make error instead + assert(compression == Compression::NONE); + } + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer '" << file_path << "'\n"; + return {}; + } + + return file_stack; +} + +static std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for writing '" << file_path << "'\n"; + return {}; + } + + // TODO: decrypt here + assert(encryption == Encryption::NONE); + + // add layer based on enum + if (compression == Compression::ZSTD) { + file_stack.push(std::make_unique(*file_stack.top().get())); + } else { + // TODO: make error instead + assert(compression == Compression::NONE); + } + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: file failed to add " << (int)compression << " compression layer '" << file_path << "'\n"; + return {}; + } + + return file_stack; +} + FragmentStore::FragmentStore(void) { registerSerializers(); } @@ -227,13 +284,12 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); - std::ofstream meta_file{ - meta_tmp_path, - std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text - }; + auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); + //meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()}, true)); - if (!meta_file.is_open()) { - std::cerr << "FS error: failed to create temporary meta file\n"; + if (meta_file_stack.empty()) { + std::cerr << "FS error: failed to create temporary meta file stack\n"; + std::filesystem::remove(meta_tmp_path); // might have created an empty file return false; } @@ -250,7 +306,8 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(meta_header_data.data()), meta_header_data.size()); + //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); + meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); } else if (meta_type == MetaFileType::TEXT_JSON) { // cant be compressed or encrypted - meta_file << meta_data_j.dump(2, ' ', true); + const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); + meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); } // now data @@ -385,8 +444,9 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function> data_file_stack; - data_file_stack.push(std::make_unique(std::string_view{frag_path})); - - if (!data_file_stack.top()->isGood()) { - std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; - return false; - } - - // TODO: decrypt here - Compression data_comp = Compression::NONE; if (_reg.all_of(fid)) { data_comp = _reg.get(fid).comp; } - // add layer based on enum - if (data_comp == Compression::ZSTD) { - data_file_stack.push(std::make_unique(*data_file_stack.top().get())); - } else { - // TODO: make error instead - assert(data_comp == Compression::NONE); - } + //std::stack> data_file_stack; + //data_file_stack.push(std::make_unique(std::string_view{frag_path})); - if (!data_file_stack.top()->isGood()) { - std::cerr << "FS error: fragment data file failed to add " << (int)data_comp << " decompression layer '" << frag_path << "'\n"; + //if (!data_file_stack.top()->isGood()) { + //std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; + //return false; + //} + + //// TODO: decrypt here + + + //// add layer based on enum + //if (data_comp == Compression::ZSTD) { + //data_file_stack.push(std::make_unique(*data_file_stack.top().get())); + //} else { + //// TODO: make error instead + //assert(data_comp == Compression::NONE); + //} + + //if (!data_file_stack.top()->isGood()) { + //std::cerr << "FS error: fragment data file failed to add " << (int)data_comp << " decompression layer '" << frag_path << "'\n"; + //return false; + //} + auto data_file_stack = buildFileStackRead(std::string_view{frag_path}, Encryption::NONE, data_comp); + if (data_file_stack.empty()) { return false; } From 6d150ba44154798b1716d30eba2d81ee4135d9b5 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 1 Apr 2024 17:57:14 +0200 Subject: [PATCH 64/98] roll back meta comp (did it wrong) and enable data comp --- src/fragment_store/fragment_store.cpp | 117 +++++++++++++------------- 1 file changed, 60 insertions(+), 57 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 1b00d64..c144a32 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -270,22 +270,23 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - _reg.emplace_or_replace(fid, Encryption::NONE); - //} - //if (_reg.all_of(fid)) { - _reg.emplace_or_replace(fid, Compression::NONE); - //} + _reg.emplace_or_replace(fid, Encryption::NONE); + _reg.emplace_or_replace(fid, Compression::NONE); } std::filesystem::path meta_tmp_path = _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); - auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); - //meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()}, true)); + // TODO: make meta comp work with mem compressor + //auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); + std::stack> meta_file_stack; + meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()})); + + if (!meta_file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for writing '" << meta_tmp_path << "'\n"; + return {}; + } if (meta_file_stack.empty()) { std::cerr << "FS error: failed to create temporary meta file stack\n"; @@ -293,20 +294,19 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { + data_enc = _reg.get(fid).enc; + } if (_reg.all_of(fid)) { data_comp = _reg.get(fid).comp; } std::filesystem::path data_tmp_path = _reg.get(fid).path + ".tmp"; data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); - std::ofstream data_file{ - data_tmp_path, - std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text - }; - - if (!data_file.is_open()) { - //meta_file.close(); + auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp); + if (!data_file_stack.empty()) { while (!meta_file_stack.empty()) { meta_file_stack.pop(); } std::filesystem::remove(meta_tmp_path); std::cerr << "FS error: failed to create temporary data file\n"; @@ -386,8 +386,10 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function buffer; + //if (data_comp == Compression::NONE) { + // for zstd compression, chunk size is frame size. (no cross frame referencing) + static constexpr int64_t chunk_size{1024*1024*10}; + std::array buffer; uint64_t buffer_actual_size {0}; do { buffer_actual_size = data_cb(buffer.data(), buffer.size()); @@ -399,56 +401,57 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(buffer.data()), buffer_actual_size); + data_file_stack.top()->write({buffer.data(), buffer_actual_size}); } while (buffer_actual_size == buffer.size()); - } else if (data_comp == Compression::ZSTD) { - std::vector buffer(ZSTD_CStreamInSize()); - std::vector compressed_buffer(ZSTD_CStreamOutSize()); - uint64_t buffer_actual_size {0}; + //} else if (data_comp == Compression::ZSTD) { + //std::vector buffer(ZSTD_CStreamInSize()); + //std::vector compressed_buffer(ZSTD_CStreamOutSize()); + //uint64_t buffer_actual_size {0}; - std::unique_ptr cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; - ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, 0); // default (3) - ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) - do { - buffer_actual_size = data_cb(buffer.data(), buffer.size()); - //if (buffer_actual_size == 0) { + //std::unique_ptr cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; + //ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, 0); // default (3) + //ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) + //do { + //buffer_actual_size = data_cb(buffer.data(), buffer.size()); + ////if (buffer_actual_size == 0) { + ////break; + ////} + //if (buffer_actual_size > buffer.size()) { + //// wtf //break; //} - if (buffer_actual_size > buffer.size()) { - // wtf - break; - } - bool const lastChunk = (buffer_actual_size < buffer.size()); + //bool const lastChunk = (buffer_actual_size < buffer.size()); - ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 }; + //ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; + //ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 }; - while (input.pos < input.size) { - ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 }; + //while (input.pos < input.size) { + //ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 }; - size_t const remaining = ZSTD_compressStream2(cctx.get(), &output , &input, mode); - if (ZSTD_isError(remaining)) { - std::cerr << "FS error: compressing data failed\n"; - break; - } + //size_t const remaining = ZSTD_compressStream2(cctx.get(), &output , &input, mode); + //if (ZSTD_isError(remaining)) { + //std::cerr << "FS error: compressing data failed\n"; + //break; + //} - data_file.write(reinterpret_cast(compressed_buffer.data()), output.pos); + //data_file.write(reinterpret_cast(compressed_buffer.data()), output.pos); - if (remaining == 0) { - break; - } - } - // same as if lastChunk break; - } while (buffer_actual_size == buffer.size()); - } else { - assert(false && "implement me"); - } + //if (remaining == 0) { + //break; + //} + //} + //// same as if lastChunk break; + //} while (buffer_actual_size == buffer.size()); + //} else { + //assert(false && "implement me"); + //} //meta_file.flush(); //meta_file.close(); while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - data_file.flush(); - data_file.close(); + //data_file.flush(); + //data_file.close(); + while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? std::filesystem::rename( meta_tmp_path, From 8d0518c2e337a6cb38cfe94ef9894cdc955de315 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Mon, 1 Apr 2024 18:06:41 +0200 Subject: [PATCH 65/98] harden against some parsing exceptions (by disabling them, since the error case is already handled) --- src/fragment_store/fragment_store.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index c144a32..8f5d193 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -722,7 +722,7 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { file.read(reinterpret_cast(full_meta_data.data()), full_meta_data.size()); } - const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data); + const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data, true, false); if (!meta_header_j.is_array() || meta_header_j.size() < 4) { std::cerr << "FS error: broken binary meta " << it.frag_path << "\n"; @@ -780,9 +780,9 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { // TODO: enc if (!meta_data_decomp.empty()) { - j = nlohmann::json::from_msgpack(meta_data_decomp); + j = nlohmann::json::from_msgpack(meta_data_decomp, true, false); } else { - j = nlohmann::json::from_msgpack(meta_data_ref); + j = nlohmann::json::from_msgpack(meta_data_ref, true, false); } } else if (it.meta_ext == ".meta.json") { std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); From 84bd24807d39dafd7648cf01f397c31a6b977520 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 2 Apr 2024 17:48:15 +0200 Subject: [PATCH 66/98] small change to flush work space. writing new frags is broken rn --- src/fragment_store/file2_zstd.cpp | 5 +++++ src/fragment_store/file2_zstd.hpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/fragment_store/file2_zstd.cpp b/src/fragment_store/file2_zstd.cpp index 4ccb58b..f3ba0ab 100644 --- a/src/fragment_store/file2_zstd.cpp +++ b/src/fragment_store/file2_zstd.cpp @@ -15,6 +15,11 @@ File2ZSTDW::File2ZSTDW(File2I& real) : ZSTD_CCtx_setParameter(_cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) } +File2ZSTDW::~File2ZSTDW(void) { + // flush remaining data (and maybe header) + // actually nvm, write will always flush all data, so only on empty files this would be an issue +} + bool File2ZSTDW::isGood(void) { return _real_file.isGood(); } diff --git a/src/fragment_store/file2_zstd.hpp b/src/fragment_store/file2_zstd.hpp index 7646374..110b5a3 100644 --- a/src/fragment_store/file2_zstd.hpp +++ b/src/fragment_store/file2_zstd.hpp @@ -15,7 +15,7 @@ struct File2ZSTDW : public File2I { std::unique_ptr _cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; File2ZSTDW(File2I& real); - virtual ~File2ZSTDW(void) {} + virtual ~File2ZSTDW(void); bool isGood(void) override; From 53ce292e82a689d9d15b21d25cf352499ac9c0ad Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 2 Apr 2024 18:01:29 +0200 Subject: [PATCH 67/98] fix stack overflow (i am stupid) and some file stack error handling --- src/fragment_store/fragment_store.cpp | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 8f5d193..bc35aa2 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -283,11 +283,6 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function> meta_file_stack; meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()})); - if (!meta_file_stack.top()->isGood()) { - std::cerr << "FS error: opening file for writing '" << meta_tmp_path << "'\n"; - return {}; - } - if (meta_file_stack.empty()) { std::cerr << "FS error: failed to create temporary meta file stack\n"; std::filesystem::remove(meta_tmp_path); // might have created an empty file @@ -306,10 +301,10 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path + ".tmp"; data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp); - if (!data_file_stack.empty()) { + if (data_file_stack.empty()) { while (!meta_file_stack.empty()) { meta_file_stack.pop(); } std::filesystem::remove(meta_tmp_path); - std::cerr << "FS error: failed to create temporary data file\n"; + std::cerr << "FS error: failed to create temporary data file stack\n"; return false; } @@ -389,7 +384,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function buffer; + std::vector buffer(chunk_size); uint64_t buffer_actual_size {0}; do { buffer_actual_size = data_cb(buffer.data(), buffer.size()); From 5767834f71eed9ff37f39d35582d2936a52b6383 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 2 Apr 2024 18:27:47 +0200 Subject: [PATCH 68/98] since its working, remove old commented code --- src/fragment_store/fragment_store.cpp | 73 ++++++--------------------- 1 file changed, 15 insertions(+), 58 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index bc35aa2..31a1876 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -381,65 +381,22 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function buffer(chunk_size); - uint64_t buffer_actual_size {0}; - do { - buffer_actual_size = data_cb(buffer.data(), buffer.size()); - if (buffer_actual_size == 0) { - break; - } - if (buffer_actual_size > buffer.size()) { - // wtf - break; - } + // for zstd compression, chunk size is frame size. (no cross frame referencing) + static constexpr int64_t chunk_size{1024*1024}; // 1MiB should be enough + std::vector buffer(chunk_size); + uint64_t buffer_actual_size {0}; + do { + buffer_actual_size = data_cb(buffer.data(), buffer.size()); + if (buffer_actual_size == 0) { + break; + } + if (buffer_actual_size > buffer.size()) { + // wtf + break; + } - data_file_stack.top()->write({buffer.data(), buffer_actual_size}); - } while (buffer_actual_size == buffer.size()); - //} else if (data_comp == Compression::ZSTD) { - //std::vector buffer(ZSTD_CStreamInSize()); - //std::vector compressed_buffer(ZSTD_CStreamOutSize()); - //uint64_t buffer_actual_size {0}; - - //std::unique_ptr cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; - //ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, 0); // default (3) - //ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) - //do { - //buffer_actual_size = data_cb(buffer.data(), buffer.size()); - ////if (buffer_actual_size == 0) { - ////break; - ////} - //if (buffer_actual_size > buffer.size()) { - //// wtf - //break; - //} - //bool const lastChunk = (buffer_actual_size < buffer.size()); - - //ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - //ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 }; - - //while (input.pos < input.size) { - //ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 }; - - //size_t const remaining = ZSTD_compressStream2(cctx.get(), &output , &input, mode); - //if (ZSTD_isError(remaining)) { - //std::cerr << "FS error: compressing data failed\n"; - //break; - //} - - //data_file.write(reinterpret_cast(compressed_buffer.data()), output.pos); - - //if (remaining == 0) { - //break; - //} - //} - //// same as if lastChunk break; - //} while (buffer_actual_size == buffer.size()); - //} else { - //assert(false && "implement me"); - //} + data_file_stack.top()->write({buffer.data(), buffer_actual_size}); + } while (buffer_actual_size == buffer.size()); //meta_file.flush(); //meta_file.close(); From 7e285290fe03a01708eb79a2fbf41d9acf9f1403 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 3 Apr 2024 21:03:35 +0200 Subject: [PATCH 69/98] test in memory comp --- src/fragment_store/test_file_zstd.cpp | 100 ++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/src/fragment_store/test_file_zstd.cpp b/src/fragment_store/test_file_zstd.cpp index 817639d..cecddd7 100644 --- a/src/fragment_store/test_file_zstd.cpp +++ b/src/fragment_store/test_file_zstd.cpp @@ -1,6 +1,7 @@ #include "./file2_zstd.hpp" #include +#include #include #include @@ -37,6 +38,105 @@ const static ByteSpan data_test_text3{ }; int main(void) { + { // first do a simple mem backed test + std::vector buffer; + { // write + File2MemW f_w_mem{buffer}; + assert(f_w_mem.isGood()); + + File2ZSTDW f_w_zstd{f_w_mem}; + assert(f_w_zstd.isGood()); + + bool res = f_w_zstd.write(data_test_text1); + assert(res); + assert(f_w_zstd.isGood()); + + // write another frame of the same data + res = f_w_zstd.write(data_test_text2); + assert(res); + assert(f_w_zstd.isGood()); + + // write larger frame + res = f_w_zstd.write(data_test_text3); + assert(res); + assert(f_w_zstd.isGood()); + } + + std::cout << "in mem size: " << buffer.size() << "\n"; + + { // read + File2MemR f_r_mem{ByteSpan{buffer}}; + assert(f_r_mem.isGood()); + + File2ZSTDR f_r_zstd{f_r_mem}; + assert(f_r_zstd.isGood()); + + // reads return owning buffers + + { // readback data_test_text1 + auto r_res_var = f_r_zstd.read(data_test_text1.size); + + //assert(f_r_zstd.isGood()); + //assert(f_r_file.isGood()); + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + + //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; + + assert(r_res_vec.size() == data_test_text1.size); + assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); + } + + { // readback data_test_text2 + auto r_res_var = f_r_zstd.read(data_test_text2.size); + + //assert(f_r_zstd.isGood()); + //assert(f_r_file.isGood()); + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + + //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; + + assert(r_res_vec.size() == data_test_text2.size); + assert(std::equal( + data_test_text2.cbegin(), + data_test_text2.cend(), + r_res_vec.cbegin() + )); + } + + { // readback data_test_text3 + auto r_res_var = f_r_zstd.read(data_test_text3.size); + + //assert(f_r_zstd.isGood()); + //assert(f_r_file.isGood()); + assert(std::holds_alternative>(r_res_var)); + const auto& r_res_vec = std::get>(r_res_var); + + //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; + + assert(r_res_vec.size() == data_test_text3.size); + assert(std::equal( + data_test_text3.cbegin(), + data_test_text3.cend(), + r_res_vec.cbegin() + )); + } + + { // assert eof somehow + // since its eof, reading a single byte should return a zero sized buffer + auto r_res_var = f_r_zstd.read(1); + if (std::holds_alternative>(r_res_var)) { + assert(std::get>(r_res_var).empty()); + } else if (std::holds_alternative(r_res_var)) { + assert(std::get(r_res_var).empty()); + } else { + assert(false); + } + } + } + } + const auto temp_dir = std::filesystem::temp_directory_path() / "file2_zstd_tests"; std::filesystem::create_directories(temp_dir); // making sure From 3b010bd16fbcfb524b31aa76c1b819500d9e39ca Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 5 Apr 2024 18:18:21 +0200 Subject: [PATCH 70/98] catch write errors (eg unsanitzed strings) --- src/fragment_store/fragment_store.cpp | 157 ++++++++++++-------------- 1 file changed, 72 insertions(+), 85 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 31a1876..bdb92cd 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -308,76 +308,86 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::functionsecond(storage.value(fid), meta_data[storage.type().hash()]); - //} else if (meta_type == MetaFileType::TEXT_JSON) { - s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]); - //} - } - - if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file - const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); - std::vector meta_data_compressed; // empty if none - //std::vector meta_data_encrypted; // empty if none - - if (meta_comp == Compression::ZSTD) { - meta_data_compressed.resize(ZSTD_compressBound(meta_data.size())); - - size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3 - if (ZSTD_isError(cSize)) { - std::cerr << "FS error: compressing meta failed\n"; - meta_data_compressed.clear(); - meta_comp = Compression::NONE; - } else { - meta_data_compressed.resize(cSize); + for (const auto& [type_id, storage] : _reg.storage()) { + if (!storage.contains(fid)) { + continue; } - } else if (meta_comp == Compression::NONE) { - // do nothing - } else { - assert(false && "implement me"); + + //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; + + // use type_id to find serializer + auto s_cb_it = _sc._serl_json.find(type_id); + if (s_cb_it == _sc._serl_json.end()) { + // could not find serializer, not saving + continue; + } + + // noooo, why cant numbers be keys + //if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead + //s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]); + //} else if (meta_type == MetaFileType::TEXT_JSON) { + s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]); + //} } - // TODO: encryption + if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file + const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); + std::vector meta_data_compressed; // empty if none + //std::vector meta_data_encrypted; // empty if none - // the meta file is itself msgpack data - nlohmann::json meta_header_j = nlohmann::json::array(); - meta_header_j.emplace_back() = "SOLMET"; - meta_header_j.push_back(meta_enc); - meta_header_j.push_back(meta_comp); + if (meta_comp == Compression::ZSTD) { + meta_data_compressed.resize(ZSTD_compressBound(meta_data.size())); - if (false) { // TODO: encryption - } else if (!meta_data_compressed.empty()) { - meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed)); - } else { - meta_header_j.push_back(nlohmann::json::binary(meta_data)); + size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3 + if (ZSTD_isError(cSize)) { + std::cerr << "FS error: compressing meta failed\n"; + meta_data_compressed.clear(); + meta_comp = Compression::NONE; + } else { + meta_data_compressed.resize(cSize); + } + } else if (meta_comp == Compression::NONE) { + // do nothing + } else { + assert(false && "implement me"); + } + + // TODO: encryption + + // the meta file is itself msgpack data + nlohmann::json meta_header_j = nlohmann::json::array(); + meta_header_j.emplace_back() = "SOLMET"; + meta_header_j.push_back(meta_enc); + meta_header_j.push_back(meta_comp); + + if (false) { // TODO: encryption + } else if (!meta_data_compressed.empty()) { + meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed)); + } else { + meta_header_j.push_back(nlohmann::json::binary(meta_data)); + } + + const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); + //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); + meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); + } else if (meta_type == MetaFileType::TEXT_JSON) { + // cant be compressed or encrypted + const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); + meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); } - const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); - //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); - meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); - } else if (meta_type == MetaFileType::TEXT_JSON) { - // cant be compressed or encrypted - const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); - meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); + } catch (...) { + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: failed to serialize json data\n"; + return false; } // now data @@ -460,29 +470,6 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function(fid).comp; } - //std::stack> data_file_stack; - //data_file_stack.push(std::make_unique(std::string_view{frag_path})); - - //if (!data_file_stack.top()->isGood()) { - //std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n"; - //return false; - //} - - //// TODO: decrypt here - - - //// add layer based on enum - //if (data_comp == Compression::ZSTD) { - //data_file_stack.push(std::make_unique(*data_file_stack.top().get())); - //} else { - //// TODO: make error instead - //assert(data_comp == Compression::NONE); - //} - - //if (!data_file_stack.top()->isGood()) { - //std::cerr << "FS error: fragment data file failed to add " << (int)data_comp << " decompression layer '" << frag_path << "'\n"; - //return false; - //} auto data_file_stack = buildFileStackRead(std::string_view{frag_path}, Encryption::NONE, data_comp); if (data_file_stack.empty()) { return false; From 248f68f6a2060319fa18dda2df8c47205db1219a Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 6 Apr 2024 11:57:48 +0200 Subject: [PATCH 71/98] minimal stack creation refactor (wip) --- src/fragment_store/fragment_store.cpp | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index bdb92cd..5caa8cc 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -51,14 +51,11 @@ static ByteSpan spanFromRead(const std::variant>& } // TODO: these stacks are file specific -static std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { - std::stack> file_stack; - file_stack.push(std::make_unique(file_path)); - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: opening file for reading '" << file_path << "'\n"; - return {}; - } +// add enc and comp file layers +// assumes a file is already in the stack +static bool buildStackRead(std::stack>& file_stack, Encryption encryption, Compression compression) { + assert(!file_stack.empty()); // TODO: decrypt here assert(encryption == Encryption::NONE); @@ -72,7 +69,24 @@ static std::stack> buildFileStackRead(std::string_view f } if (!file_stack.top()->isGood()) { - std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer '" << file_path << "'\n"; + std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer\n"; + return false; + } + + return true; +} + +static std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for reading '" << file_path << "'\n"; + return {}; + } + + if (!buildStackRead(file_stack, encryption, compression)) { + std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; return {}; } From 31bb0d3e611c170f56010f104a7b58c814f36b37 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 6 Apr 2024 13:03:09 +0200 Subject: [PATCH 72/98] refactor meta data decompression using file2 and mem --- src/fragment_store/fragment_store.cpp | 73 +++++++++++++-------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 5caa8cc..963f67a 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -9,6 +9,7 @@ #include #include +#include #include "./file2_zstd.hpp" #include @@ -76,6 +77,7 @@ static bool buildStackRead(std::stack>& file_stack, Encr return true; } +// do i need this? static std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { std::stack> file_stack; file_stack.push(std::make_unique(file_path)); @@ -93,16 +95,12 @@ static std::stack> buildFileStackRead(std::string_view f return file_stack; } -static std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) { - std::stack> file_stack; - file_stack.push(std::make_unique(file_path)); +// add enc and comp file layers +// assumes a file is already in the stack +static bool buildStackWrite(std::stack>& file_stack, Encryption encryption, Compression compression) { + assert(!file_stack.empty()); - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: opening file for writing '" << file_path << "'\n"; - return {}; - } - - // TODO: decrypt here + // TODO: encrypt here assert(encryption == Encryption::NONE); // add layer based on enum @@ -114,7 +112,25 @@ static std::stack> buildFileStackWrite(std::string_view } if (!file_stack.top()->isGood()) { - std::cerr << "FS error: file failed to add " << (int)compression << " compression layer '" << file_path << "'\n"; + std::cerr << "FS error: file failed to add " << (int)compression << " compression layer\n"; + return false; + } + + return true; +} + +// do i need this? +static std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for writing '" << file_path << "'\n"; + return {}; + } + + if (!buildStackWrite(file_stack, encryption, compression)) { + std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; return {}; } @@ -706,37 +722,20 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { } const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3); - std::vector meta_data_decomp; - if (meta_comp == Compression::NONE) { - // do nothing - } else if (meta_comp == Compression::ZSTD) { - meta_data_decomp.resize(ZSTD_DStreamOutSize()); - std::unique_ptr dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; + std::stack> binary_reader_stack; + binary_reader_stack.push(std::make_unique(ByteSpan{meta_data_ref.data(), meta_data_ref.size()})); - ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0}; - ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0}; - do { - size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input); - if (ZSTD_isError(ret)) { - // error <.< - std::cerr << "FS error: decompression error\n"; - //meta_data_decomp.clear(); - output.pos = 0; // resize after loop - break; - } - } while (input.pos < input.size); - meta_data_decomp.resize(output.pos); - } else { - assert(false && "implement me"); + if (!buildStackRead(binary_reader_stack, meta_enc, meta_comp)) { + std::cerr << "FS error: binary reader creation failed " << it.frag_path << "\n"; + continue; } - // TODO: enc + // HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through + auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta? + const auto binary_read_span = spanFromRead(binary_read_value); + assert(binary_read_span.size < 10*1024*1024); - if (!meta_data_decomp.empty()) { - j = nlohmann::json::from_msgpack(meta_data_decomp, true, false); - } else { - j = nlohmann::json::from_msgpack(meta_data_ref, true, false); - } + j = nlohmann::json::from_msgpack(binary_read_span, true, false); } else if (it.meta_ext == ".meta.json") { std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); if (!file.is_open()) { From 268cbe137e890e2131de488f8a12d0beedba14bc Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 6 Apr 2024 13:21:26 +0200 Subject: [PATCH 73/98] move file stack creation to separate file, removing some scope --- src/CMakeLists.txt | 2 + src/fragment_store/file2_stack.cpp | 93 +++++++++++++++++++++++++++ src/fragment_store/file2_stack.hpp | 23 +++++++ src/fragment_store/fragment_store.cpp | 89 +------------------------ 4 files changed, 120 insertions(+), 87 deletions(-) create mode 100644 src/fragment_store/file2_stack.cpp create mode 100644 src/fragment_store/file2_stack.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 28c7938..4643d2b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,6 +13,8 @@ add_library(fragment_store ./fragment_store/meta_components.hpp ./fragment_store/meta_components_id.inl ./fragment_store/serializer.hpp + ./fragment_store/file2_stack.hpp + ./fragment_store/file2_stack.cpp ./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.cpp diff --git a/src/fragment_store/file2_stack.cpp b/src/fragment_store/file2_stack.cpp new file mode 100644 index 0000000..70b599a --- /dev/null +++ b/src/fragment_store/file2_stack.cpp @@ -0,0 +1,93 @@ +#include "./file2_stack.hpp" + +#include +#include "./file2_zstd.hpp" + +#include + +#include + +// add enc and comp file layers +// assumes a file is already in the stack +bool buildStackRead(std::stack>& file_stack, Encryption encryption, Compression compression) { + assert(!file_stack.empty()); + + // TODO: decrypt here + assert(encryption == Encryption::NONE); + + // add layer based on enum + if (compression == Compression::ZSTD) { + file_stack.push(std::make_unique(*file_stack.top().get())); + } else { + // TODO: make error instead + assert(compression == Compression::NONE); + } + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer\n"; + return false; + } + + return true; +} + +// do i need this? +std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for reading '" << file_path << "'\n"; + return {}; + } + + if (!buildStackRead(file_stack, encryption, compression)) { + std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; + return {}; + } + + return file_stack; +} + +// add enc and comp file layers +// assumes a file is already in the stack +bool buildStackWrite(std::stack>& file_stack, Encryption encryption, Compression compression) { + assert(!file_stack.empty()); + + // TODO: encrypt here + assert(encryption == Encryption::NONE); + + // add layer based on enum + if (compression == Compression::ZSTD) { + file_stack.push(std::make_unique(*file_stack.top().get())); + } else { + // TODO: make error instead + assert(compression == Compression::NONE); + } + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: file failed to add " << (int)compression << " compression layer\n"; + return false; + } + + return true; +} + +// do i need this? +std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) { + std::stack> file_stack; + file_stack.push(std::make_unique(file_path)); + + if (!file_stack.top()->isGood()) { + std::cerr << "FS error: opening file for writing '" << file_path << "'\n"; + return {}; + } + + if (!buildStackWrite(file_stack, encryption, compression)) { + std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; + return {}; + } + + return file_stack; +} + diff --git a/src/fragment_store/file2_stack.hpp b/src/fragment_store/file2_stack.hpp new file mode 100644 index 0000000..c1e878c --- /dev/null +++ b/src/fragment_store/file2_stack.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include "./types.hpp" + +#include + +#include +#include +#include + +// add enc and comp file layers +// assumes a file is already in the stack +[[nodiscard]] bool buildStackRead(std::stack>& file_stack, Encryption encryption, Compression compression); + +// do i need this? +[[nodiscard]] std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression); + +// add enc and comp file layers +// assumes a file is already in the stack +[[nodiscard]] bool buildStackWrite(std::stack>& file_stack, Encryption encryption, Compression compression); + +// do i need this? +[[nodiscard]] std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression); diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index 963f67a..d8fe80d 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -10,7 +10,8 @@ #include #include -#include "./file2_zstd.hpp" + +#include "./file2_stack.hpp" #include @@ -51,92 +52,6 @@ static ByteSpan spanFromRead(const std::variant>& } } -// TODO: these stacks are file specific - -// add enc and comp file layers -// assumes a file is already in the stack -static bool buildStackRead(std::stack>& file_stack, Encryption encryption, Compression compression) { - assert(!file_stack.empty()); - - // TODO: decrypt here - assert(encryption == Encryption::NONE); - - // add layer based on enum - if (compression == Compression::ZSTD) { - file_stack.push(std::make_unique(*file_stack.top().get())); - } else { - // TODO: make error instead - assert(compression == Compression::NONE); - } - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer\n"; - return false; - } - - return true; -} - -// do i need this? -static std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { - std::stack> file_stack; - file_stack.push(std::make_unique(file_path)); - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: opening file for reading '" << file_path << "'\n"; - return {}; - } - - if (!buildStackRead(file_stack, encryption, compression)) { - std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; - return {}; - } - - return file_stack; -} - -// add enc and comp file layers -// assumes a file is already in the stack -static bool buildStackWrite(std::stack>& file_stack, Encryption encryption, Compression compression) { - assert(!file_stack.empty()); - - // TODO: encrypt here - assert(encryption == Encryption::NONE); - - // add layer based on enum - if (compression == Compression::ZSTD) { - file_stack.push(std::make_unique(*file_stack.top().get())); - } else { - // TODO: make error instead - assert(compression == Compression::NONE); - } - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: file failed to add " << (int)compression << " compression layer\n"; - return false; - } - - return true; -} - -// do i need this? -static std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) { - std::stack> file_stack; - file_stack.push(std::make_unique(file_path)); - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: opening file for writing '" << file_path << "'\n"; - return {}; - } - - if (!buildStackWrite(file_stack, encryption, compression)) { - std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; - return {}; - } - - return file_stack; -} - FragmentStore::FragmentStore(void) { registerSerializers(); } From fd0b210bbb6af9a0e923d7d28e119f7fea28bfd3 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 6 Apr 2024 14:23:43 +0200 Subject: [PATCH 74/98] fragment store fully file2 zstd. no more zstd.h in fragment store --- src/fragment_store/fragment_store.cpp | 57 +++++++++++++-------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index d8fe80d..fc2b0e3 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -13,8 +13,6 @@ #include "./file2_stack.hpp" -#include - #include #include #include @@ -281,44 +279,44 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function meta_data = nlohmann::json::to_msgpack(meta_data_j); - std::vector meta_data_compressed; // empty if none - //std::vector meta_data_encrypted; // empty if none + std::vector binary_meta_data; + { + std::stack> binary_writer_stack; + binary_writer_stack.push(std::make_unique(binary_meta_data)); - if (meta_comp == Compression::ZSTD) { - meta_data_compressed.resize(ZSTD_compressBound(meta_data.size())); - - size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3 - if (ZSTD_isError(cSize)) { - std::cerr << "FS error: compressing meta failed\n"; - meta_data_compressed.clear(); - meta_comp = Compression::NONE; - } else { - meta_data_compressed.resize(cSize); + if (!buildStackWrite(binary_writer_stack, meta_enc, meta_comp)) { + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: binary writer creation failed '" << _reg.get(fid).path << "'\n"; + return false; + } + + { + const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); + if (!binary_writer_stack.top()->write(ByteSpan{meta_data})) { + // i feel like exceptions or refactoring would be nice here + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: binary writer failed '" << _reg.get(fid).path << "'\n"; + return false; + } } - } else if (meta_comp == Compression::NONE) { - // do nothing - } else { - assert(false && "implement me"); } - // TODO: encryption - - // the meta file is itself msgpack data + //// the meta file is itself msgpack data nlohmann::json meta_header_j = nlohmann::json::array(); meta_header_j.emplace_back() = "SOLMET"; meta_header_j.push_back(meta_enc); meta_header_j.push_back(meta_comp); - if (false) { // TODO: encryption - } else if (!meta_data_compressed.empty()) { - meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed)); - } else { - meta_header_j.push_back(nlohmann::json::binary(meta_data)); - } + // with a custom msgpack impl like cmp, we can be smarter here and dont need an extra buffer + meta_header_j.push_back(nlohmann::json::binary(binary_meta_data)); const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); - //meta_file.write(reinterpret_cast(meta_header_data.data()), meta_header_data.size()); meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); } else if (meta_type == MetaFileType::TEXT_JSON) { // cant be compressed or encrypted @@ -337,6 +335,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function buffer(chunk_size); uint64_t buffer_actual_size {0}; From 26d07b06db11d85b6999776efaf335a464e8016c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 6 Apr 2024 22:27:53 +0200 Subject: [PATCH 75/98] start refactoring in the name of object store --- src/CMakeLists.txt | 8 ++- src/fragment_store/object_store.cpp | 88 +++++++++++++++++++++++++++++ src/fragment_store/object_store.hpp | 67 ++++++++++++++++++++++ 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 src/fragment_store/object_store.cpp create mode 100644 src/fragment_store/object_store.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4643d2b..f83d286 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,16 +7,20 @@ add_library(fragment_store ./fragment_store/uuid_generator.hpp ./fragment_store/uuid_generator.cpp - ./fragment_store/fragment_store_i.hpp - ./fragment_store/fragment_store_i.cpp ./fragment_store/types.hpp ./fragment_store/meta_components.hpp ./fragment_store/meta_components_id.inl ./fragment_store/serializer.hpp ./fragment_store/file2_stack.hpp ./fragment_store/file2_stack.cpp + #old + ./fragment_store/fragment_store_i.hpp + ./fragment_store/fragment_store_i.cpp ./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.cpp + #new + ./fragment_store/object_store.hpp + ./fragment_store/object_store.cpp ./json/message_components.hpp # TODO: move ./json/tox_message_components.hpp # TODO: move diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp new file mode 100644 index 0000000..0806c61 --- /dev/null +++ b/src/fragment_store/object_store.cpp @@ -0,0 +1,88 @@ +#include "./object_store.hpp" + +#include "./meta_components.hpp" + +#include // this sucks + +#include + +static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { + out = static_cast>( + oh.get().enc + ); + return true; +} + +static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + +static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { + out = static_cast>( + oh.get().comp + ); + return true; +} + +static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + +ObjectStore2::ObjectStore2(void) { + _sc.registerSerializerJson(serl_json_data_enc_type); + _sc.registerDeSerializerJson(deserl_json_data_enc_type); + _sc.registerSerializerJson(serl_json_data_comp_type); + _sc.registerDeSerializerJson(deserl_json_data_comp_type); +} + +ObjectStore2::~ObjectStore2(void) { +} + +ObjectRegistry& ObjectStore2::registry(void) { + return _reg; +} + +ObjectHandle ObjectStore2::objectHandle(const Object o) { + return {_reg, o}; +} + +void ObjectStore2::throwEventConstruct(const Object o) { + std::cout << "OS debug: event construct " << entt::to_integral(o) << "\n"; + dispatch( + ObjectStore_Event::object_construct, + ObjectStore::Events::ObjectConstruct{ + ObjectHandle{_reg, o} + } + ); +} + +void ObjectStore2::throwEventUpdate(const Object o) { + std::cout << "OS debug: event update " << entt::to_integral(o) << "\n"; + dispatch( + ObjectStore_Event::object_update, + ObjectStore::Events::ObjectUpdate{ + ObjectHandle{_reg, o} + } + ); +} + +void ObjectStore2::throwEventDestroy(const Object o) { + std::cout << "OS debug: event destroy " << entt::to_integral(o) << "\n"; + dispatch( + ObjectStore_Event::object_destroy, + ObjectStore::Events::ObjectUpdate{ + ObjectHandle{_reg, o} + } + ); +} + diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp new file mode 100644 index 0000000..764a979 --- /dev/null +++ b/src/fragment_store/object_store.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include + +#include "./serializer.hpp" // TODO: get rid of the tight nljson integration + +#include +#include + +#include + +// internal id +enum class Object : uint32_t {}; +using ObjectRegistry = entt::basic_registry; +using ObjectHandle = entt::basic_handle; + +namespace ObjectStore::Events { + + struct ObjectConstruct { + const ObjectHandle e; + }; + struct ObjectUpdate { + const ObjectHandle e; + }; + struct ObjectDestory { + const ObjectHandle e; + }; + +} // ObjectStore::Events + +enum class ObjectStore_Event : uint16_t { + object_construct, + object_update, + object_destroy, + + MAX +}; + +struct ObjectStoreEventI { + using enumType = ObjectStore_Event; + + virtual ~ObjectStoreEventI(void) {} + + virtual bool onEvent(const ObjectStore::Events::ObjectConstruct&) { return false; } + virtual bool onEvent(const ObjectStore::Events::ObjectUpdate&) { return false; } + virtual bool onEvent(const ObjectStore::Events::ObjectDestory&) { return false; } +}; +using ObjectStoreEventProviderI = EventProviderI; + +struct ObjectStore2 : public ObjectStoreEventProviderI { + static constexpr const char* version {"2"}; + + ObjectRegistry _reg; + + SerializerCallbacks _sc; + + ObjectStore2(void); + virtual ~ObjectStore2(void); + + ObjectRegistry& registry(void); + ObjectHandle objectHandle(const Object o); + + void throwEventConstruct(const Object o); + void throwEventUpdate(const Object o); + void throwEventDestroy(const Object o); +}; + From 0610a6a64a8d14795862047e476537573bc57f53 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 9 Apr 2024 12:09:21 +0200 Subject: [PATCH 76/98] continue os refactor, start with fs backend --- src/CMakeLists.txt | 2 + .../backends/filesystem_storage.cpp | 237 ++++++++++++++++++ .../backends/filesystem_storage.hpp | 18 ++ src/fragment_store/object_store.cpp | 16 ++ src/fragment_store/object_store.hpp | 27 ++ 5 files changed, 300 insertions(+) create mode 100644 src/fragment_store/backends/filesystem_storage.cpp create mode 100644 src/fragment_store/backends/filesystem_storage.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f83d286..76b371e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,8 @@ add_library(fragment_store #new ./fragment_store/object_store.hpp ./fragment_store/object_store.cpp + ./fragment_store/backends/filesystem_storage.hpp + ./fragment_store/backends/filesystem_storage.cpp ./json/message_components.hpp # TODO: move ./json/tox_message_components.hpp # TODO: move diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp new file mode 100644 index 0000000..895cad9 --- /dev/null +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -0,0 +1,237 @@ +#include "./filesystem_storage.hpp" + +#include "../meta_components.hpp" + +#include + +#include +#include + +#include "../file2_stack.hpp" + +#include + +#include + +static const char* metaFileTypeSuffix(MetaFileType mft) { + switch (mft) { + case MetaFileType::TEXT_JSON: return ".json"; + //case MetaFileType::BINARY_ARB: return ".bin"; + case MetaFileType::BINARY_MSGPACK: return ".msgpack"; + } + return ""; // .unk? +} + +namespace backend { + +FilesystemStorage::FilesystemStorage(ObjectStore2& os) : StorageBackendI::StorageBackendI(os) { +} + +FilesystemStorage::~FilesystemStorage(void) { +} + +bool FilesystemStorage::write(Object o, std::function& data_cb) { + auto& reg = _os.registry(); + + if (!reg.valid(o)) { + return false; + } + + ObjectHandle oh {reg, o}; + + if (!oh.all_of()) { + // not a file fragment? + return false; + } + + // split object storage + + MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults + if (reg.all_of(o)) { + meta_type = oh.get().type; + } + + Encryption meta_enc = Encryption::NONE; // TODO: better defaults + Compression meta_comp = Compression::NONE; // TODO: better defaults + + if (meta_type != MetaFileType::TEXT_JSON) { + if (oh.all_of()) { + meta_enc = oh.get().enc; + } + + if (oh.all_of()) { + meta_comp = oh.get().comp; + } + } else { + // we cant have encryption or compression + // so we force NONE for TEXT JSON + + oh.emplace_or_replace(Encryption::NONE); + oh.emplace_or_replace(Compression::NONE); + } + + std::filesystem::path meta_tmp_path = oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; + meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); + // TODO: make meta comp work with mem compressor + //auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); + std::stack> meta_file_stack; + meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()})); + + if (meta_file_stack.empty()) { + std::cerr << "FS error: failed to create temporary meta file stack\n"; + std::filesystem::remove(meta_tmp_path); // might have created an empty file + return false; + } + + Encryption data_enc = Encryption::NONE; // TODO: better defaults + Compression data_comp = Compression::NONE; // TODO: better defaults + if (oh.all_of()) { + data_enc = oh.get().enc; + } + if (oh.all_of()) { + data_comp = oh.get().comp; + } + + std::filesystem::path data_tmp_path = oh.get().path + ".tmp"; + data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); + auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp); + if (data_file_stack.empty()) { + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } + std::filesystem::remove(meta_tmp_path); + std::cerr << "FS error: failed to create temporary data file stack\n"; + return false; + } + + try { // TODO: properly sanitize strings, so this cant throw + // sharing code between binary msgpack and text json for now + nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed + // metadata file + + for (const auto& [type_id, storage] : reg.storage()) { + if (!storage.contains(o)) { + continue; + } + + //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; + + // use type_id to find serializer + auto s_cb_it = _os._sc._serl_json.find(type_id); + if (s_cb_it == _os._sc._serl_json.end()) { + // could not find serializer, not saving + continue; + } + + // noooo, why cant numbers be keys + //if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead + //s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]); + //} else if (meta_type == MetaFileType::TEXT_JSON) { + s_cb_it->second(oh, meta_data_j[storage.type().name()]); + //} + } + + if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file + std::vector binary_meta_data; + { + std::stack> binary_writer_stack; + binary_writer_stack.push(std::make_unique(binary_meta_data)); + + if (!buildStackWrite(binary_writer_stack, meta_enc, meta_comp)) { + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: binary writer creation failed '" << oh.get().path << "'\n"; + return false; + } + + { + const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); + if (!binary_writer_stack.top()->write(ByteSpan{meta_data})) { + // i feel like exceptions or refactoring would be nice here + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: binary writer failed '" << oh.get().path << "'\n"; + return false; + } + } + } + + //// the meta file is itself msgpack data + nlohmann::json meta_header_j = nlohmann::json::array(); + meta_header_j.emplace_back() = "SOLMET"; + meta_header_j.push_back(meta_enc); + meta_header_j.push_back(meta_comp); + + // with a custom msgpack impl like cmp, we can be smarter here and dont need an extra buffer + meta_header_j.push_back(nlohmann::json::binary(binary_meta_data)); + + const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); + meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); + } else if (meta_type == MetaFileType::TEXT_JSON) { + // cant be compressed or encrypted + const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); + meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); + } + + } catch (...) { + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + std::filesystem::remove(meta_tmp_path); + while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + std::filesystem::remove(data_tmp_path); + std::cerr << "FS error: failed to serialize json data\n"; + return false; + } + + // now data + // for zstd compression, chunk size is frame size. (no cross frame referencing) + // TODO: add buffering steam layer + static constexpr int64_t chunk_size{1024*1024}; // 1MiB should be enough + std::vector buffer(chunk_size); + uint64_t buffer_actual_size {0}; + do { + buffer_actual_size = data_cb(buffer.data(), buffer.size()); + if (buffer_actual_size == 0) { + break; + } + if (buffer_actual_size > buffer.size()) { + // wtf + break; + } + + data_file_stack.top()->write({buffer.data(), buffer_actual_size}); + } while (buffer_actual_size == buffer.size()); + + //meta_file.flush(); + //meta_file.close(); + while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + //data_file.flush(); + //data_file.close(); + while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? + + std::filesystem::rename( + meta_tmp_path, + oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) + ); + + std::filesystem::rename( + data_tmp_path, + oh.get().path + ); + + // TODO: check return value of renames + + if (oh.all_of()) { + oh.remove(); + } + + return true; +} + +bool FilesystemStorage::read(Object o, std::function& data_cb) { + return false; +} + +} // backend + diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp new file mode 100644 index 0000000..715bf91 --- /dev/null +++ b/src/fragment_store/backends/filesystem_storage.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include "../object_store.hpp" + +namespace backend { + +struct FilesystemStorage : public StorageBackendI { + FilesystemStorage(ObjectStore2& os); + ~FilesystemStorage(void); + + bool write(Object o, std::function& data_cb) override; + bool read(Object o, std::function& data_cb) override; + + //// convenience function + //nlohmann::json loadFromStorageNJ(FragmentID fid); +}; + +} // backend diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp index 0806c61..4fc7786 100644 --- a/src/fragment_store/object_store.cpp +++ b/src/fragment_store/object_store.cpp @@ -6,6 +6,22 @@ #include +StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) { +} + +bool StorageBackendI::write(Object o, const ByteSpan data) { + std::function fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { + uint64_t i = 0; + for (; i+read < data.size && i < buffer_size; i++) { + request_buffer[i] = data[i+read]; + } + read += i; + + return i; + }; + return write(o, fn_cb); +} + static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { out = static_cast>( oh.get().enc diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp index 764a979..bcfff21 100644 --- a/src/fragment_store/object_store.hpp +++ b/src/fragment_store/object_store.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include "./serializer.hpp" // TODO: get rid of the tight nljson integration @@ -14,6 +15,28 @@ enum class Object : uint32_t {}; using ObjectRegistry = entt::basic_registry; using ObjectHandle = entt::basic_handle; +// fwd +struct ObjectStore2; + +struct StorageBackendI { + // OR or OS ? + ObjectStore2& _os; + + StorageBackendI(ObjectStore2& os); + + // ========== write object to storage ========== + using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); + // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. + virtual bool write(Object o, std::function& data_cb) = 0; + //virtual bool write(Object o, const uint8_t* data, const uint64_t data_size); // default impl + virtual bool write(Object o, const ByteSpan data); // default impl + + // ========== read object from storage ========== + using read_from_storage_put_data_cb = void(const ByteSpan buffer); + virtual bool read(Object o, std::function& data_cb) = 0; + +}; + namespace ObjectStore::Events { struct ObjectConstruct { @@ -54,12 +77,16 @@ struct ObjectStore2 : public ObjectStoreEventProviderI { SerializerCallbacks _sc; + // TODO: default backend? + ObjectStore2(void); virtual ~ObjectStore2(void); ObjectRegistry& registry(void); ObjectHandle objectHandle(const Object o); + // sync? + void throwEventConstruct(const Object o); void throwEventUpdate(const Object o); void throwEventDestroy(const Object o); From 3cede91aa01d84d2f0bf779fc2e2c2518d3d3c2c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 9 Apr 2024 19:36:35 +0200 Subject: [PATCH 77/98] more work on backend and moving frags to objs --- src/CMakeLists.txt | 9 + .../backends/filesystem_storage.cpp | 377 ++++++++++++++++-- .../backends/filesystem_storage.hpp | 16 +- src/fragment_store/convert_frag_to_obj.cpp | 36 ++ src/fragment_store/fragment_store.cpp | 44 +- src/fragment_store/fragment_store.hpp | 4 - src/fragment_store/message_fragment_store.cpp | 88 ++-- src/fragment_store/message_fragment_store.hpp | 8 +- src/fragment_store/meta_components.hpp | 42 +- src/fragment_store/meta_components_id.inl | 4 + src/fragment_store/object_store.cpp | 36 -- src/fragment_store/object_store.hpp | 2 - 12 files changed, 514 insertions(+), 152 deletions(-) create mode 100644 src/fragment_store/convert_frag_to_obj.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 76b371e..09d10aa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -80,6 +80,15 @@ target_link_libraries(test_fragment_store PUBLIC ######################################## +add_executable(convert_frag_to_obj + fragment_store/convert_frag_to_obj.cpp +) + +target_link_libraries(convert_frag_to_obj PUBLIC + fragment_store +) + +######################################## add_executable(tomato ./main.cpp ./icon.rc diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp index 895cad9..8b8685b 100644 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -2,6 +2,9 @@ #include "../meta_components.hpp" +#include + +#include #include #include @@ -22,9 +25,65 @@ static const char* metaFileTypeSuffix(MetaFileType mft) { return ""; // .unk? } +// TODO: move to ... somewhere. (span? file2i?) +static ByteSpan spanFromRead(const std::variant>& data_var) { + if (std::holds_alternative>(data_var)) { + auto& vec = std::get>(data_var); + return {vec.data(), vec.size()}; + } else if (std::holds_alternative(data_var)) { + return std::get(data_var); + } else { + assert(false); + return {}; + } +} + +// TODO: move somewhere else +static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { + out = static_cast>( + oh.get().enc + ); + return true; +} + +static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + +static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { + out = static_cast>( + oh.get().comp + ); + return true; +} + +static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + namespace backend { -FilesystemStorage::FilesystemStorage(ObjectStore2& os) : StorageBackendI::StorageBackendI(os) { +FilesystemStorage::FilesystemStorage(ObjectStore2& os, std::string_view storage_path) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path) { + _sc.registerSerializerJson(serl_json_data_enc_type); + _sc.registerDeSerializerJson(deserl_json_data_enc_type); + _sc.registerSerializerJson(serl_json_data_comp_type); + _sc.registerDeSerializerJson(deserl_json_data_comp_type); + + // old stuff + _sc.registerSerializerJson(serl_json_data_enc_type); + _sc.registerDeSerializerJson(deserl_json_data_enc_type); + _sc.registerSerializerJson(serl_json_data_comp_type); + _sc.registerDeSerializerJson(deserl_json_data_comp_type); } FilesystemStorage::~FilesystemStorage(void) { @@ -39,38 +98,38 @@ bool FilesystemStorage::write(Object o, std::function()) { + if (!oh.all_of()) { // not a file fragment? return false; } - // split object storage + // split object storage (meta and data are 2 files) - MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults - if (reg.all_of(o)) { - meta_type = oh.get().type; + MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults? + if (reg.all_of(o)) { + meta_type = oh.get().type; } - Encryption meta_enc = Encryption::NONE; // TODO: better defaults - Compression meta_comp = Compression::NONE; // TODO: better defaults + Encryption meta_enc = Encryption::NONE; // TODO: better defaults? + Compression meta_comp = Compression::NONE; // TODO: better defaults? if (meta_type != MetaFileType::TEXT_JSON) { - if (oh.all_of()) { - meta_enc = oh.get().enc; + if (oh.all_of()) { + meta_enc = oh.get().enc; } - if (oh.all_of()) { - meta_comp = oh.get().comp; + if (oh.all_of()) { + meta_comp = oh.get().comp; } } else { // we cant have encryption or compression // so we force NONE for TEXT JSON - oh.emplace_or_replace(Encryption::NONE); - oh.emplace_or_replace(Compression::NONE); + oh.emplace_or_replace(Encryption::NONE); + oh.emplace_or_replace(Compression::NONE); } - std::filesystem::path meta_tmp_path = oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; + std::filesystem::path meta_tmp_path = oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); // TODO: make meta comp work with mem compressor //auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); @@ -85,14 +144,14 @@ bool FilesystemStorage::write(Object o, std::function()) { - data_enc = oh.get().enc; + if (oh.all_of()) { + data_enc = oh.get().enc; } - if (oh.all_of()) { - data_comp = oh.get().comp; + if (oh.all_of()) { + data_comp = oh.get().comp; } - std::filesystem::path data_tmp_path = oh.get().path + ".tmp"; + std::filesystem::path data_tmp_path = oh.get().path + ".tmp"; data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp); if (data_file_stack.empty()) { @@ -107,6 +166,7 @@ bool FilesystemStorage::write(Object o, std::function().path << "'\n"; + std::cerr << "FS error: binary writer creation failed '" << oh.get().path << "'\n"; return false; } @@ -152,7 +212,7 @@ bool FilesystemStorage::write(Object o, std::function().path << "'\n"; + std::cerr << "FS error: binary writer failed '" << oh.get().path << "'\n"; return false; } } @@ -203,27 +263,24 @@ bool FilesystemStorage::write(Object o, std::functionwrite({buffer.data(), buffer_actual_size}); } while (buffer_actual_size == buffer.size()); - //meta_file.flush(); - //meta_file.close(); + // flush // TODO: use scope while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - //data_file.flush(); - //data_file.close(); while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? std::filesystem::rename( meta_tmp_path, - oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) + oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) ); std::filesystem::rename( data_tmp_path, - oh.get().path + oh.get().path ); // TODO: check return value of renames - if (oh.all_of()) { - oh.remove(); + if (oh.all_of()) { + oh.remove(); } return true; @@ -233,5 +290,261 @@ bool FilesystemStorage::read(Object o, std::function file_obj_list; + + std::filesystem::path storage_path{path}; + + auto handle_file = [&](const std::filesystem::path& file_path) { + if (!std::filesystem::is_regular_file(file_path)) { + return; + } + // handle file + + if (file_path.has_extension()) { + // skip over metadata, assuming only metafiles have extentions (might be wrong?) + // also skips temps + return; + } + + auto relative_path = std::filesystem::proximate(file_path, storage_path); + std::string id_str = relative_path.generic_u8string(); + // delete all '/' + id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end()); + if (id_str.size() % 2 != 0) { + std::cerr << "FS error: non hex object uid detected: '" << id_str << "'\n"; + } + + if (file_obj_list.contains(ObjFileEntry{id_str, {}, ""})) { + std::cerr << "FS error: object duplicate detected: '" << id_str << "'\n"; + return; // skip + } + + const char* meta_ext = ".meta.msgpack"; + { // find meta + // TODO: this as to know all possible extentions + bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack"); + bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json"); + const size_t meta_sum = + (has_meta_msgpack?1:0) + + (has_meta_json?1:0) + ; + + if (meta_sum > 1) { // has multiple + std::cerr << "FS error: object with multiple meta files detected: " << id_str << "\n"; + return; // skip + } + + if (meta_sum == 0) { + std::cerr << "FS error: object missing meta file detected: " << id_str << "\n"; + return; // skip + } + + if (has_meta_json) { + meta_ext = ".meta.json"; + } + } + + file_obj_list.emplace(ObjFileEntry{ + std::move(id_str), + file_path, + meta_ext + }); + }; + + for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) { + if (std::filesystem::is_regular_file(outer_path)) { + handle_file(outer_path); + } else if (std::filesystem::is_directory(outer_path)) { + // subdir, part of id + for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) { + //if (std::filesystem::is_regular_file(inner_path)) { + + //// handle file + //} // TODO: support deeper recursion? + handle_file(inner_path); + } + } + } + + std::cout << "FS: scan found:\n"; + for (const auto& it : file_obj_list) { + std::cout << " " << it.id_str << "\n"; + } + + // step 2: check if files preexist in reg + // main thread + // (merge into step 3 and do more error checking?) + // TODO: properly handle duplicates, dups form different backends might be legit + // important +#if 0 + for (auto it = file_obj_list.begin(); it != file_obj_list.end();) { + auto id = hex2bin(it->id_str); + auto fid = getFragmentByID(id); + if (_reg.valid(fid)) { + // pre exising (handle differently??) + // check if store differs? + it = file_obj_list.erase(it); + } else { + it++; + } + } +#endif + + std::vector scanned_objs; + // step 3: parse meta and insert into reg of non preexising + // main thread + // TODO: check timestamps of preexisting and reload? mark external/remote dirty? + for (const auto& it : file_obj_list) { + nlohmann::json j; + if (it.meta_ext == ".meta.msgpack") { + std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); + if (!file.is_open()) { + std::cout << "FS error: failed opening meta " << it.obj_path << "\n"; + continue; + } + + // file is a msgpack within a msgpack + + std::vector full_meta_data; + { // read meta file + // figure out size + file.seekg(0, file.end); + uint64_t file_size = file.tellg(); + file.seekg(0, file.beg); + + full_meta_data.resize(file_size); + + file.read(reinterpret_cast(full_meta_data.data()), full_meta_data.size()); + } + + const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data, true, false); + + if (!meta_header_j.is_array() || meta_header_j.size() < 4) { + std::cerr << "FS error: broken binary meta " << it.obj_path << "\n"; + continue; + } + + if (meta_header_j.at(0) != "SOLMET") { + std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.obj_path << "\n"; + continue; + } + + Encryption meta_enc = meta_header_j.at(1); + if (meta_enc != Encryption::NONE) { + std::cerr << "FS error: unknown encryption " << it.obj_path << "\n"; + continue; + } + + Compression meta_comp = meta_header_j.at(2); + if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) { + std::cerr << "FS error: unknown compression " << it.obj_path << "\n"; + continue; + } + + //const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data"); + if (!meta_header_j.at(3).is_binary()) { + std::cerr << "FS error: meta data not binary " << it.obj_path << "\n"; + continue; + } + const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3); + + std::stack> binary_reader_stack; + binary_reader_stack.push(std::make_unique(ByteSpan{meta_data_ref.data(), meta_data_ref.size()})); + + if (!buildStackRead(binary_reader_stack, meta_enc, meta_comp)) { + std::cerr << "FS error: binary reader creation failed " << it.obj_path << "\n"; + continue; + } + + // HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through + auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta? + const auto binary_read_span = spanFromRead(binary_read_value); + assert(binary_read_span.size < 10*1024*1024); + + j = nlohmann::json::from_msgpack(binary_read_span, true, false); + } else if (it.meta_ext == ".meta.json") { + std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); + if (!file.is_open()) { + std::cerr << "FS error: failed opening meta " << it.obj_path << "\n"; + continue; + } + + file >> j; + } else { + assert(false); + } + + if (!j.is_object()) { + std::cerr << "FS error: json in meta is broken " << it.id_str << "\n"; + continue; + } + + // TODO: existing fragment file + //newFragmentFile(); + ObjectHandle oh{_os.registry(), _os.registry().create()}; + oh.emplace(this); + oh.emplace(hex2bin(it.id_str)); + + oh.emplace(it.obj_path.generic_u8string()); + + for (const auto& [k, v] : j.items()) { + // type id from string hash + const auto type_id = entt::hashed_string(k.data(), k.size()); + const auto deserl_fn_it = _sc._deserl_json.find(type_id); + if (deserl_fn_it != _sc._deserl_json.cend()) { + // TODO: check return value + deserl_fn_it->second(oh, v); + } else { + std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; + } + } + scanned_objs.push_back(oh); + } + + // TODO: mutex and move code to async and return this list ? + + // throw new frag event here, after loading them all + for (const Object o : scanned_objs) { + _os.throwEventConstruct(o); + } + + return scanned_objs.size(); +} + +void FilesystemStorage::scanPathAsync(std::string path) { + // add path to queue + // HACK: if path is known/any fragment is in the path, this operation blocks (non async) + scanPath(path); // TODO: make async and post result +} + } // backend diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp index 715bf91..dc0c803 100644 --- a/src/fragment_store/backends/filesystem_storage.hpp +++ b/src/fragment_store/backends/filesystem_storage.hpp @@ -5,14 +5,28 @@ namespace backend { struct FilesystemStorage : public StorageBackendI { - FilesystemStorage(ObjectStore2& os); + FilesystemStorage(ObjectStore2& os, std::string_view storage_path = "test_obj_store"); ~FilesystemStorage(void); + // TODO: fix the path for this specific fs? + std::string _storage_path; + bool write(Object o, std::function& data_cb) override; bool read(Object o, std::function& data_cb) override; //// convenience function //nlohmann::json loadFromStorageNJ(FragmentID fid); + + void scanAsync(void); + + private: + size_t scanPath(std::string_view path); + void scanPathAsync(std::string path); + + private: + // this thing needs to change and be facilitated over the OS + // but the json serializer are specific to the backend + SerializerCallbacks _sc; }; } // backend diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp new file mode 100644 index 0000000..772d7bf --- /dev/null +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -0,0 +1,36 @@ +#include "./object_store.hpp" +#include "./backends/filesystem_storage.hpp" + +#include +#include + +int main(int argc, const char** argv) { + if (argc != 3) { + std::cerr << "wrong paramter count, do " << argv[0] << " \n"; + return 1; + } + + if (!std::filesystem::is_directory(argv[1])) { + std::cerr << "input folder is no folder\n"; + } + + std::filesystem::create_directories(argv[2]); + + // we are going to use 2 different OS for convineance, but could be done with 1 too + ObjectStore2 os_src; + ObjectStore2 os_dst; + + backend::FilesystemStorage fsb_src(os_src, argv[1]); + backend::FilesystemStorage fsb_dst(os_dst, argv[2]); + + // add message fragment store too (adds meta?) + + // hookup events + // perform scan (which triggers events) + fsb_dst.scanAsync(); // fill with existing? + fsb_src.scanAsync(); // the scan + + // done + return 0; +} + diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index fc2b0e3..2a34930 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -139,9 +139,9 @@ FragmentID FragmentStore::newFragmentFile( _reg.emplace(new_frag, id); // file (info) comp - _reg.emplace(new_frag, fragment_file_path.generic_u8string()); + _reg.emplace(new_frag, fragment_file_path.generic_u8string()); - _reg.emplace(new_frag, mft); + _reg.emplace(new_frag, mft); // meta needs to be synced to file std::function empty_data_cb = [](auto*, auto) -> uint64_t { return 0; }; @@ -188,7 +188,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { + if (!_reg.all_of(fid)) { // not a file fragment? return false; } @@ -196,30 +196,30 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - meta_type = _reg.get(fid).type; + if (_reg.all_of(fid)) { + meta_type = _reg.get(fid).type; } Encryption meta_enc = Encryption::NONE; // TODO: better defaults Compression meta_comp = Compression::NONE; // TODO: better defaults if (meta_type != MetaFileType::TEXT_JSON) { - if (_reg.all_of(fid)) { - meta_enc = _reg.get(fid).enc; + if (_reg.all_of(fid)) { + meta_enc = _reg.get(fid).enc; } - if (_reg.all_of(fid)) { - meta_comp = _reg.get(fid).comp; + if (_reg.all_of(fid)) { + meta_comp = _reg.get(fid).comp; } } else { // we cant have encryption or compression // so we force NONE for TEXT JSON - _reg.emplace_or_replace(fid, Encryption::NONE); - _reg.emplace_or_replace(fid, Compression::NONE); + _reg.emplace_or_replace(fid, Encryption::NONE); + _reg.emplace_or_replace(fid, Compression::NONE); } - std::filesystem::path meta_tmp_path = _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; + std::filesystem::path meta_tmp_path = _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); // TODO: make meta comp work with mem compressor //auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); @@ -241,7 +241,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).comp; } - std::filesystem::path data_tmp_path = _reg.get(fid).path + ".tmp"; + std::filesystem::path data_tmp_path = _reg.get(fid).path + ".tmp"; data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp); if (data_file_stack.empty()) { @@ -289,7 +289,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path << "'\n"; + std::cerr << "FS error: binary writer creation failed '" << _reg.get(fid).path << "'\n"; return false; } @@ -301,7 +301,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path << "'\n"; + std::cerr << "FS error: binary writer failed '" << _reg.get(fid).path << "'\n"; return false; } } @@ -361,18 +361,18 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type) ); std::filesystem::rename( data_tmp_path, - _reg.get(fid).path + _reg.get(fid).path ); // TODO: check return value of renames - if (_reg.all_of(fid)) { - _reg.remove(fid); + if (_reg.all_of(fid)) { + _reg.remove(fid); } return true; @@ -396,13 +396,13 @@ bool FragmentStore::loadFromStorage(FragmentID fid, std::function(fid)) { + if (!_reg.all_of(fid)) { // not a file fragment? // TODO: memory fragments return false; } - const auto& frag_path = _reg.get(fid).path; + const auto& frag_path = _reg.get(fid).path; // TODO: check if metadata dirty? // TODO: what if file changed on disk? @@ -672,7 +672,7 @@ size_t FragmentStore::scanStoragePath(std::string_view path) { FragmentHandle fh{_reg, _reg.create()}; fh.emplace(hex2bin(it.id_str)); - fh.emplace(it.frag_path.generic_u8string()); + fh.emplace(it.frag_path.generic_u8string()); for (const auto& [k, v] : j.items()) { // type id from string hash diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index d29bff6..5982f70 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -94,9 +94,5 @@ struct FragmentStore : public FragmentStoreI { private: void registerSerializers(void); // internal comps - // internal actual backends - // TODO: seperate out - bool syncToMemory(FragmentID fid, std::function& data_cb); - bool syncToFile(FragmentID fid, std::function& data_cb); }; diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index a189972..ce0b1eb 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -68,7 +68,7 @@ namespace Message::Components { } // Message::Components -namespace Fragment::Components { +namespace ObjectStore::Components { NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id) @@ -82,7 +82,7 @@ namespace Fragment::Components { Contact3 e {entt::null}; }; } -} // Fragment::Component +} // ObjectStore::Component void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (_fs_ignore_event) { @@ -133,7 +133,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { assert(static_cast(fh)); // assuming ts range exists - auto& fts_comp = fh.get(); + auto& fts_comp = fh.get(); if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) { fragment_id = fid; @@ -149,7 +149,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { assert(static_cast(fh)); // assuming ts range exists - auto& fts_comp = fh.get(); + auto& fts_comp = fh.get(); const int64_t frag_range = int64_t(fts_comp.end) - int64_t(fts_comp.begin); constexpr static int64_t max_frag_ts_extent {1000*60*60}; @@ -205,17 +205,17 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { fragment_id = fh; - fh.emplace_or_replace().comp = Compression::ZSTD; - fh.emplace_or_replace().comp = Compression::ZSTD; + fh.emplace_or_replace().comp = Compression::ZSTD; + fh.emplace_or_replace().comp = Compression::ZSTD; - auto& new_ts_range = fh.emplace_or_replace(); + auto& new_ts_range = fh.emplace_or_replace(); new_ts_range.begin = msg_ts; new_ts_range.end = msg_ts; { const auto msg_reg_contact = m.registry()->ctx().get(); if (_cr.all_of(msg_reg_contact)) { - fh.emplace(_cr.get(msg_reg_contact).data); + fh.emplace(_cr.get(msg_reg_contact).data); } else { // ? rage quit? } @@ -235,7 +235,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { fid_open.emplace(fragment_id); - std::cout << "MFS: created new fragment " << bin2hex(fh.get().v) << "\n"; + std::cout << "MFS: created new fragment " << bin2hex(fh.get().v) << "\n"; _fs_ignore_event = true; _fs.throwEventConstruct(fh); @@ -296,13 +296,13 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh if (!j.is_array()) { // wrong data - fh.emplace_or_replace(); + fh.emplace_or_replace(); return; } if (j.size() == 0) { // empty array - fh.emplace_or_replace(); + fh.emplace_or_replace(); return; } @@ -389,12 +389,12 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh if (messages_new_or_updated == 0) { // useless frag // TODO: unload? - fh.emplace_or_replace(); + fh.emplace_or_replace(); } } bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry& reg) { - auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); + auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); auto j = nlohmann::json::array(); @@ -476,10 +476,16 @@ MessageFragmentStore::MessageFragmentStore( _rmm.subscribe(this, RegistryMessageModel_Event::message_updated); _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); - _fs._sc.registerSerializerJson(); - _fs._sc.registerDeSerializerJson(); - _fs._sc.registerSerializerJson(); - _fs._sc.registerDeSerializerJson(); + _fs._sc.registerSerializerJson(); + _fs._sc.registerDeSerializerJson(); + _fs._sc.registerSerializerJson(); + _fs._sc.registerDeSerializerJson(); + + // old + _fs._sc.registerSerializerJson(_fs._sc.component_get_json); + _fs._sc.registerDeSerializerJson(_fs._sc.component_emplace_or_replace_json); + _fs._sc.registerSerializerJson(_fs._sc.component_get_json); + _fs._sc.registerDeSerializerJson(_fs._sc.component_emplace_or_replace_json); _fs.subscribe(this, FragmentStore_Event::fragment_construct); _fs.subscribe(this, FragmentStore_Event::fragment_updated); @@ -589,12 +595,12 @@ float MessageFragmentStore::tick(float) { return 0.05f; } - if (!fh.all_of()) { + if (!fh.all_of()) { return 0.05f; } // get ts range of frag and collide with all curser(s/ranges) - const auto& frag_range = fh.get(); + const auto& frag_range = fh.get(); auto* msg_reg = _rmm.get(c); if (msg_reg == nullptr) { @@ -646,19 +652,19 @@ float MessageFragmentStore::tick(float) { return 0.05f; } - if (!fh.all_of()) { + if (!fh.all_of()) { std::cerr << "MFS error: frag has no range\n"; // ???? msg_reg->ctx().get().erase(fid); return 0.05f; } - if (fh.all_of()) { + if (fh.all_of()) { continue; // skip known empty } // get ts range of frag and collide with all curser(s/ranges) - const auto& [range_begin, range_end] = fh.get(); + const auto& [range_begin, range_end] = fh.get(); if (rangeVisible(range_begin, range_end, *msg_reg)) { std::cout << "MFS: frag hit by vis range\n"; @@ -691,7 +697,7 @@ float MessageFragmentStore::tick(float) { cf.sorted_end.crend(), ts_begin_comp.ts, [&](const FragmentID element, const auto& value) -> bool { - return _fs._reg.get(element).end >= value; + return _fs._reg.get(element).end >= value; } ); @@ -709,7 +715,7 @@ float MessageFragmentStore::tick(float) { // only ok bc next is cheap for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); next_frag = cf.next(next_frag)) { auto fh = _fs.fragmentHandle(next_frag); - if (fh.any_of()) { + if (fh.any_of()) { continue; // skip known empty } @@ -739,7 +745,7 @@ float MessageFragmentStore::tick(float) { cf.sorted_begin.cend(), ts_end, [&](const FragmentID element, const auto& value) -> bool { - return _fs._reg.get(element).begin < value; + return _fs._reg.get(element).begin < value; } ); @@ -757,7 +763,7 @@ float MessageFragmentStore::tick(float) { // only ok bc next is cheap for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); prev_frag = cf.prev(prev_frag)) { auto fh = _fs.fragmentHandle(prev_frag); - if (fh.any_of()) { + if (fh.any_of()) { continue; // skip known empty } @@ -806,7 +812,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) return false; // skip self } - if (!e.e.all_of()) { + if (!e.e.all_of()) { return false; // not for us } @@ -814,7 +820,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) Contact3 frag_contact = entt::null; { // get contact - const auto& frag_contact_id = e.e.get().id; + const auto& frag_contact_id = e.e.get().id; // TODO: id lookup table, this is very inefficent for (const auto& [c_it, id_it] : _cr.view().each()) { if (frag_contact_id == id_it.data) { @@ -826,7 +832,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) // unkown contact return false; } - e.e.emplace_or_replace(frag_contact); + e.e.emplace_or_replace(frag_contact); } // create if not exist @@ -853,23 +859,23 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentUpdated& e) { return false; // skip self } - if (!e.e.all_of()) { + if (!e.e.all_of()) { return false; // not for us } // since its an update, we might have it associated, or not // its also possible it was tagged as empty - e.e.remove(); + e.e.remove(); Contact3 frag_contact = entt::null; { // get contact // probably cached already - if (e.e.all_of()) { - frag_contact = e.e.get().e; + if (e.e.all_of()) { + frag_contact = e.e.get().e; } if (!_cr.valid(frag_contact)) { - const auto& frag_contact_id = e.e.get().id; + const auto& frag_contact_id = e.e.get().id; // TODO: id lookup table, this is very inefficent for (const auto& [c_it, id_it] : _cr.view().each()) { if (frag_contact_id == id_it.data) { @@ -881,7 +887,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentUpdated& e) { // unkown contact return false; } - e.e.emplace_or_replace(frag_contact); + e.e.emplace_or_replace(frag_contact); } } @@ -921,15 +927,15 @@ bool Message::Components::ContactFragments::insert(FragmentHandle frag) { sorted_begin.cbegin(), sorted_begin.cend(), [frag](const FragmentID a) -> bool { - const auto begin_a = frag.registry()->get(a).begin; - const auto begin_frag = frag.get().begin; + const auto begin_a = frag.registry()->get(a).begin; + const auto begin_frag = frag.get().begin; if (begin_a > begin_frag) { return true; } else if (begin_a < begin_frag) { return false; } else { // equal ts, we need to fall back to id (id can not be equal) - return isLess(frag.get().v, frag.registry()->get(a).v); + return isLess(frag.get().v, frag.registry()->get(a).v); } } ); @@ -946,15 +952,15 @@ bool Message::Components::ContactFragments::insert(FragmentHandle frag) { sorted_end.cbegin(), sorted_end.cend(), [frag](const FragmentID a) -> bool { - const auto end_a = frag.registry()->get(a).end; - const auto end_frag = frag.get().end; + const auto end_a = frag.registry()->get(a).end; + const auto end_frag = frag.get().end; if (end_a > end_frag) { return true; } else if (end_a < end_frag) { return false; } else { // equal ts, we need to fall back to id (id can not be equal) - return isLess(frag.get().v, frag.registry()->get(a).v); + return isLess(frag.get().v, frag.registry()->get(a).v); } } ); diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index ac3813a..179ece1 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -54,7 +54,7 @@ namespace Message::Components { } // Message::Components -namespace Fragment::Components { +namespace ObjectStore::Components { struct MessagesTSRange { // timestamp range within the fragment uint64_t begin {0}; // newer msg -> higher number @@ -67,6 +67,12 @@ namespace Fragment::Components { // TODO: add src contact (self id) +} // ObjectStore::Components + +// old +namespace Fragment::Components { + struct MessagesTSRange : public ObjComp::MessagesTSRange {}; + struct MessagesContact : public ObjComp::MessagesContact {}; } // Fragment::Components // handles fragments for messages diff --git a/src/fragment_store/meta_components.hpp b/src/fragment_store/meta_components.hpp index a8ec20e..f0dcf57 100644 --- a/src/fragment_store/meta_components.hpp +++ b/src/fragment_store/meta_components.hpp @@ -1,12 +1,13 @@ #pragma once #include "./types.hpp" +#include "object_store.hpp" #include #include #include -namespace Fragment::Components { +namespace ObjectStore::Components { // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) struct ID { @@ -25,18 +26,6 @@ namespace Fragment::Components { // meta that is not written to (meta-)file namespace Ephemeral { - // excluded from file meta - struct FilePath { - // contains store path, if any - std::string path; - }; - - // TODO: seperate into remote and local? - // (remote meaning eg. the file on disk was changed by another program) - struct DirtyTag {}; - - - // type as comp struct MetaFileType { ::MetaFileType type {::MetaFileType::TEXT_JSON}; }; @@ -49,11 +38,38 @@ namespace Fragment::Components { Compression comp {Compression::NONE}; }; + struct Backend { + // TODO: shared_ptr instead?? + StorageBackendI* ptr; + }; + + // excluded from file meta + // TODO: move to backend specific + struct FilePath { + // contains store path, if any + std::string path; + }; + + // TODO: seperate into remote and local? + // (remote meaning eg. the file on disk was changed by another program) + struct DirtyTag {}; + } // Ephemeral } // Components // shortened to save bytes (until I find a way to save by ID in msgpack) +namespace ObjComp = ObjectStore::Components; + +// old names +namespace Fragment::Components { + //struct ID {}; + //struct DataEncryptionType {}; + //struct DataCompressionType {}; + struct ID : public ObjComp::ID {}; + struct DataEncryptionType : public ObjComp::DataEncryptionType {}; + struct DataCompressionType : public ObjComp::DataCompressionType {}; +} namespace FragComp = Fragment::Components; #include "./meta_components_id.inl" diff --git a/src/fragment_store/meta_components_id.inl b/src/fragment_store/meta_components_id.inl index c912cce..d9f7de3 100644 --- a/src/fragment_store/meta_components_id.inl +++ b/src/fragment_store/meta_components_id.inl @@ -18,6 +18,10 @@ constexpr std::string_view entt::type_name::value() noexcept { \ // cross compiler stable ids +DEFINE_COMP_ID(ObjComp::DataEncryptionType) +DEFINE_COMP_ID(ObjComp::DataCompressionType) + +// old stuff DEFINE_COMP_ID(FragComp::DataEncryptionType) DEFINE_COMP_ID(FragComp::DataCompressionType) diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp index 4fc7786..146f677 100644 --- a/src/fragment_store/object_store.cpp +++ b/src/fragment_store/object_store.cpp @@ -22,43 +22,7 @@ bool StorageBackendI::write(Object o, const ByteSpan data) { return write(o, fn_cb); } -static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { - out = static_cast>( - oh.get().enc - ); - return true; -} - -static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) { - oh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} - -static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { - out = static_cast>( - oh.get().comp - ); - return true; -} - -static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) { - oh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} - ObjectStore2::ObjectStore2(void) { - _sc.registerSerializerJson(serl_json_data_enc_type); - _sc.registerDeSerializerJson(deserl_json_data_enc_type); - _sc.registerSerializerJson(serl_json_data_comp_type); - _sc.registerDeSerializerJson(deserl_json_data_comp_type); } ObjectStore2::~ObjectStore2(void) { diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp index bcfff21..3ce56bb 100644 --- a/src/fragment_store/object_store.hpp +++ b/src/fragment_store/object_store.hpp @@ -75,8 +75,6 @@ struct ObjectStore2 : public ObjectStoreEventProviderI { ObjectRegistry _reg; - SerializerCallbacks _sc; - // TODO: default backend? ObjectStore2(void); From 8a580e2fbbf9f348ce37aded3d88372ab179e81d Mon Sep 17 00:00:00 2001 From: Green Sky Date: Tue, 9 Apr 2024 20:58:47 +0200 Subject: [PATCH 78/98] backend fs read --- .../backends/filesystem_storage.cpp | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp index 8b8685b..64049bf 100644 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -287,7 +287,56 @@ bool FilesystemStorage::write(Object o, std::function& data_cb) { - return false; + auto& reg = _os.registry(); + + if (!reg.valid(o)) { + return false; + } + + ObjectHandle oh {reg, o}; + + if (!oh.all_of()) { + // not a file + return false; + } + + const auto& obj_path = oh.get().path; + + // TODO: check if metadata dirty? + // TODO: what if file changed on disk? + + std::cout << "FS: loading fragment '" << obj_path << "'\n"; + + Compression data_comp = Compression::NONE; + if (oh.all_of()) { + data_comp = oh.get().comp; + } + + auto data_file_stack = buildFileStackRead(std::string_view{obj_path}, Encryption::NONE, data_comp); + if (data_file_stack.empty()) { + return false; + } + + // TODO: make it read in a single chunk instead? + static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read + do { + auto data_var = data_file_stack.top()->read(chunk_size); + ByteSpan data = spanFromRead(data_var); + + if (data.empty()) { + // error or probably eof + break; + } + + data_cb(data); + + if (data.size < chunk_size) { + // eof + break; + } + } while (data_file_stack.top()->isGood()); + + return true; } void FilesystemStorage::scanAsync(void) { From 5c3b797a9912d205c4ec070f7cc872b9c7ec88c0 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 10 Apr 2024 13:27:45 +0200 Subject: [PATCH 79/98] new object (os + fsb mostly done) --- .../backends/filesystem_storage.cpp | 59 +++++++++++++++++-- .../backends/filesystem_storage.hpp | 4 ++ src/fragment_store/object_store.cpp | 12 ++++ src/fragment_store/object_store.hpp | 3 + src/tox_client.hpp | 1 - 5 files changed, 74 insertions(+), 5 deletions(-) diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp index 64049bf..45bfe00 100644 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -89,6 +89,59 @@ FilesystemStorage::FilesystemStorage(ObjectStore2& os, std::string_view storage_ FilesystemStorage::~FilesystemStorage(void) { } +ObjectHandle FilesystemStorage::newObject(MetaFileType mft, ByteSpan id) { + { // first check if id is already used (TODO: solve the multi obj/backend problem) + auto exising_oh = _os.getOneObjectByID(id); + if (static_cast(exising_oh)) { + return {}; + } + } + + std::filesystem::create_directories(_storage_path); + + if (!std::filesystem::is_directory(_storage_path)) { + std::cerr << "FS error: failed to create storage path dir '" << _storage_path << "'\n"; + return {}; + } + + const auto id_hex = bin2hex(id); + std::filesystem::path object_file_path; + + // TODO: refactor this magic somehow + if (id_hex.size() < 6) { + object_file_path = std::filesystem::path{_storage_path}/id_hex; + } else { + // use the first 2hex (1byte) as a subfolder + std::filesystem::create_directories(std::string{_storage_path} + id_hex.substr(0, 2)); + object_file_path = std::filesystem::path{std::string{_storage_path} + id_hex.substr(0, 2)} / id_hex.substr(2); + } + + if (std::filesystem::exists(object_file_path)) { + std::cerr << "FS error: object already exists in path '" << id_hex << "'\n"; + return {}; + } + + ObjectHandle oh{_os.registry(), _os.registry().create()}; + + oh.emplace(this); + oh.emplace(std::vector{id}); + oh.emplace(object_file_path.generic_u8string()); + oh.emplace(mft); + + // meta needs to be synced to file + std::function empty_data_cb = [](auto*, auto) -> uint64_t { return 0; }; + if (!write(oh, empty_data_cb)) { + std::cerr << "FS error: write failed while creating new object file\n"; + oh.destroy(); + return {}; + } + + // while new metadata might be created here, making sure the file could be created is more important + _os.throwEventConstruct(oh); + + return oh; +} + bool FilesystemStorage::write(Object o, std::function& data_cb) { auto& reg = _os.registry(); @@ -454,11 +507,10 @@ size_t FilesystemStorage::scanPath(std::string_view path) { // (merge into step 3 and do more error checking?) // TODO: properly handle duplicates, dups form different backends might be legit // important -#if 0 for (auto it = file_obj_list.begin(); it != file_obj_list.end();) { auto id = hex2bin(it->id_str); - auto fid = getFragmentByID(id); - if (_reg.valid(fid)) { + auto oh = _os.getOneObjectByID(ByteSpan{id}); + if (static_cast(oh)) { // pre exising (handle differently??) // check if store differs? it = file_obj_list.erase(it); @@ -466,7 +518,6 @@ size_t FilesystemStorage::scanPath(std::string_view path) { it++; } } -#endif std::vector scanned_objs; // step 3: parse meta and insert into reg of non preexising diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp index dc0c803..17c1c48 100644 --- a/src/fragment_store/backends/filesystem_storage.hpp +++ b/src/fragment_store/backends/filesystem_storage.hpp @@ -1,5 +1,6 @@ #pragma once +#include "../types.hpp" #include "../object_store.hpp" namespace backend { @@ -9,8 +10,11 @@ struct FilesystemStorage : public StorageBackendI { ~FilesystemStorage(void); // TODO: fix the path for this specific fs? + // for now we assume a single storage path per backend (there can be multiple per type) std::string _storage_path; + ObjectHandle newObject(MetaFileType mft, ByteSpan id); + bool write(Object o, std::function& data_cb) override; bool read(Object o, std::function& data_cb) override; diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp index 146f677..663d1f9 100644 --- a/src/fragment_store/object_store.cpp +++ b/src/fragment_store/object_store.cpp @@ -36,6 +36,18 @@ ObjectHandle ObjectStore2::objectHandle(const Object o) { return {_reg, o}; } +ObjectHandle ObjectStore2::getOneObjectByID(const ByteSpan id) { + // TODO: accelerate + // maybe keep it sorted and binary search? hash table lookup? + for (const auto& [obj, id_comp] : _reg.view().each()) { + if (id == ByteSpan{id_comp.v}) { + return {_reg, obj}; + } + } + + return {_reg, entt::null}; +} + void ObjectStore2::throwEventConstruct(const Object o) { std::cout << "OS debug: event construct " << entt::to_integral(o) << "\n"; dispatch( diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp index 3ce56bb..6b48ade 100644 --- a/src/fragment_store/object_store.hpp +++ b/src/fragment_store/object_store.hpp @@ -83,6 +83,9 @@ struct ObjectStore2 : public ObjectStoreEventProviderI { ObjectRegistry& registry(void); ObjectHandle objectHandle(const Object o); + // TODO: properly think about multiple objects witht he same id / different backends + ObjectHandle getOneObjectByID(const ByteSpan id); + // sync? void throwEventConstruct(const Object o); diff --git a/src/tox_client.hpp b/src/tox_client.hpp index dc412b6..f09e5e0 100644 --- a/src/tox_client.hpp +++ b/src/tox_client.hpp @@ -48,4 +48,3 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase { void saveToxProfile(void); }; - From 854ed851b40e49982c5afa5f3958afd88fe263cb Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 10 Apr 2024 16:26:24 +0200 Subject: [PATCH 80/98] fixes missing comps and conversion tool is almost working (most of meta is transfered) --- .../backends/filesystem_storage.cpp | 22 +++++- .../backends/filesystem_storage.hpp | 2 +- src/fragment_store/convert_frag_to_obj.cpp | 76 +++++++++++++++++++ src/fragment_store/object_store.hpp | 2 +- 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp index 45bfe00..222a971 100644 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -40,6 +40,10 @@ static ByteSpan spanFromRead(const std::variant>& // TODO: move somewhere else static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { + if (!oh.all_of()) { + return false; + } + out = static_cast>( oh.get().enc ); @@ -56,6 +60,10 @@ static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) } static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { + if (!oh.all_of()) { + return false; + } + out = static_cast>( oh.get().comp ); @@ -524,6 +532,9 @@ size_t FilesystemStorage::scanPath(std::string_view path) { // main thread // TODO: check timestamps of preexisting and reload? mark external/remote dirty? for (const auto& it : file_obj_list) { + MetaFileType mft {MetaFileType::TEXT_JSON}; + Encryption meta_enc {Encryption::NONE}; + Compression meta_comp {Compression::NONE}; nlohmann::json j; if (it.meta_ext == ".meta.msgpack") { std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); @@ -532,6 +543,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) { continue; } + mft = MetaFileType::BINARY_MSGPACK; + // file is a msgpack within a msgpack std::vector full_meta_data; @@ -558,13 +571,13 @@ size_t FilesystemStorage::scanPath(std::string_view path) { continue; } - Encryption meta_enc = meta_header_j.at(1); + meta_enc = meta_header_j.at(1); if (meta_enc != Encryption::NONE) { std::cerr << "FS error: unknown encryption " << it.obj_path << "\n"; continue; } - Compression meta_comp = meta_header_j.at(2); + meta_comp = meta_header_j.at(2); if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) { std::cerr << "FS error: unknown compression " << it.obj_path << "\n"; continue; @@ -598,6 +611,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) { continue; } + mft = MetaFileType::TEXT_JSON; + file >> j; } else { assert(false); @@ -613,6 +628,9 @@ size_t FilesystemStorage::scanPath(std::string_view path) { ObjectHandle oh{_os.registry(), _os.registry().create()}; oh.emplace(this); oh.emplace(hex2bin(it.id_str)); + oh.emplace(mft); + oh.emplace(meta_enc); + oh.emplace(meta_comp); oh.emplace(it.obj_path.generic_u8string()); diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp index 17c1c48..ef5c111 100644 --- a/src/fragment_store/backends/filesystem_storage.hpp +++ b/src/fragment_store/backends/filesystem_storage.hpp @@ -27,7 +27,7 @@ struct FilesystemStorage : public StorageBackendI { size_t scanPath(std::string_view path); void scanPathAsync(std::string path); - private: + public: // TODO: private? // this thing needs to change and be facilitated over the OS // but the json serializer are specific to the backend SerializerCallbacks _sc; diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 772d7bf..5b4cd4a 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -1,9 +1,14 @@ #include "./object_store.hpp" #include "./backends/filesystem_storage.hpp" +#include "./meta_components.hpp" + +#include #include #include +#include + int main(int argc, const char** argv) { if (argc != 3) { std::cerr << "wrong paramter count, do " << argv[0] << " \n"; @@ -26,6 +31,77 @@ int main(int argc, const char** argv) { // add message fragment store too (adds meta?) // hookup events + struct EventListener : public ObjectStoreEventI { + ObjectStore2& _os_src; + backend::FilesystemStorage& _fsb_src; + + ObjectStore2& _os_dst; + backend::FilesystemStorage& _fsb_dst; + + EventListener( + ObjectStore2& os_src, + backend::FilesystemStorage& fsb_src, + ObjectStore2& os_dst, + backend::FilesystemStorage& fsb_dst + ) : + _os_src(os_src), + _fsb_src(fsb_src), + _os_dst(os_dst), + _fsb_dst(fsb_dst) + { + _os_src.subscribe(this, ObjectStore_Event::object_construct); + _os_src.subscribe(this, ObjectStore_Event::object_update); + } + + protected: // os + bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override { + assert(e.e.all_of()); + assert(e.e.all_of()); + auto oh = _fsb_dst.newObject(e.e.get().type, ByteSpan{e.e.get().v}); + + if (!static_cast(oh)) { + // already exists + return false; + } + + { // sync meta + // some hardcoded ehpemeral (besides mft/id) + oh.emplace_or_replace(e.e.get_or_emplace()); + oh.emplace_or_replace(e.e.get_or_emplace()); + + // serializable + for (const auto& [type, fn] : _fsb_src._sc._serl_json) { + //if (!e.e.registry()->storage(type)->contains(e.e)) { + //continue; + //} + + // this is hacky but we serialize and then deserialize the component + // raw copy might be better in the future + nlohmann::json tmp_j; + if (fn(e.e, tmp_j)) { + _fsb_dst._sc._deserl_json.at(type)(oh, tmp_j); + } + } + } + + // read src and write dst data + static_cast(_fsb_dst).write(oh, ByteSpan{}); + + return false; + } + + bool onEvent(const ObjectStore::Events::ObjectUpdate&) override { + std::cerr << "Update called\n"; + assert(false); + return false; + } + } el { + os_src, + fsb_src, + os_dst, + fsb_dst, + }; + // perform scan (which triggers events) fsb_dst.scanAsync(); // fill with existing? fsb_src.scanAsync(); // the scan diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp index 6b48ade..fcc4456 100644 --- a/src/fragment_store/object_store.hpp +++ b/src/fragment_store/object_store.hpp @@ -29,7 +29,7 @@ struct StorageBackendI { // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. virtual bool write(Object o, std::function& data_cb) = 0; //virtual bool write(Object o, const uint8_t* data, const uint64_t data_size); // default impl - virtual bool write(Object o, const ByteSpan data); // default impl + bool write(Object o, const ByteSpan data); // ========== read object from storage ========== using read_from_storage_put_data_cb = void(const ByteSpan buffer); From 379684196181b2edf811df057a9a679d22a1487b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 10 Apr 2024 21:18:27 +0200 Subject: [PATCH 81/98] conversion improvements --- src/fragment_store/convert_frag_to_obj.cpp | 30 ++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 5b4cd4a..0329212 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -2,6 +2,8 @@ #include "./backends/filesystem_storage.hpp" #include "./meta_components.hpp" +#include + #include #include @@ -57,6 +59,31 @@ int main(int argc, const char** argv) { bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override { assert(e.e.all_of()); assert(e.e.all_of()); + + // !! we read the obj first, so we can discard empty objects + // technically we could just copy the file, but meh + // read src and write dst data + std::vector tmp_buffer; + std::function cb = [&tmp_buffer](const ByteSpan buffer) { + tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); + }; + if (!_fsb_src.read(e.e, cb)) { + std::cerr << "failed to read obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + + if (tmp_buffer.empty()) { + std::cerr << "discarded empty obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + { // try getting lucky and see if its an empty json + const auto j = nlohmann::json::parse(tmp_buffer, nullptr, false); + if (j.is_array() && j.empty()) { + std::cerr << "discarded empty json array obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + } + auto oh = _fsb_dst.newObject(e.e.get().type, ByteSpan{e.e.get().v}); if (!static_cast(oh)) { @@ -84,8 +111,7 @@ int main(int argc, const char** argv) { } } - // read src and write dst data - static_cast(_fsb_dst).write(oh, ByteSpan{}); + static_cast(_fsb_dst).write(oh, ByteSpan{tmp_buffer}); return false; } From 10b689ca95aa5390ba8631eba4fd20dc967216c5 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Wed, 10 Apr 2024 22:27:01 +0200 Subject: [PATCH 82/98] refactor the serializer again --- src/CMakeLists.txt | 3 +- .../backends/filesystem_storage.cpp | 63 +++-------------- .../backends/filesystem_storage.hpp | 5 -- src/fragment_store/convert_frag_to_obj.cpp | 5 +- src/fragment_store/object_store.cpp | 55 +++++++++++++++ src/fragment_store/object_store.hpp | 2 - src/fragment_store/serializer_json.hpp | 67 +++++++++++++++++++ 7 files changed, 136 insertions(+), 64 deletions(-) create mode 100644 src/fragment_store/serializer_json.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 09d10aa..ffd0531 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,15 +10,16 @@ add_library(fragment_store ./fragment_store/types.hpp ./fragment_store/meta_components.hpp ./fragment_store/meta_components_id.inl - ./fragment_store/serializer.hpp ./fragment_store/file2_stack.hpp ./fragment_store/file2_stack.cpp #old + ./fragment_store/serializer.hpp ./fragment_store/fragment_store_i.hpp ./fragment_store/fragment_store_i.cpp ./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.cpp #new + ./fragment_store/serializer_json.hpp ./fragment_store/object_store.hpp ./fragment_store/object_store.cpp ./fragment_store/backends/filesystem_storage.hpp diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp index 222a971..b5ae420 100644 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -1,6 +1,7 @@ #include "./filesystem_storage.hpp" #include "../meta_components.hpp" +#include "../serializer_json.hpp" #include @@ -38,60 +39,10 @@ static ByteSpan spanFromRead(const std::variant>& } } -// TODO: move somewhere else -static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { - if (!oh.all_of()) { - return false; - } - - out = static_cast>( - oh.get().enc - ); - return true; -} - -static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) { - oh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} - -static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { - if (!oh.all_of()) { - return false; - } - - out = static_cast>( - oh.get().comp - ); - return true; -} - -static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) { - oh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} namespace backend { FilesystemStorage::FilesystemStorage(ObjectStore2& os, std::string_view storage_path) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path) { - _sc.registerSerializerJson(serl_json_data_enc_type); - _sc.registerDeSerializerJson(deserl_json_data_enc_type); - _sc.registerSerializerJson(serl_json_data_comp_type); - _sc.registerDeSerializerJson(deserl_json_data_comp_type); - - // old stuff - _sc.registerSerializerJson(serl_json_data_enc_type); - _sc.registerDeSerializerJson(deserl_json_data_enc_type); - _sc.registerSerializerJson(serl_json_data_comp_type); - _sc.registerDeSerializerJson(deserl_json_data_comp_type); } FilesystemStorage::~FilesystemStorage(void) { @@ -227,6 +178,8 @@ bool FilesystemStorage::write(Object o, std::function>(); + // TODO: refactor extract to OS for (const auto& [type_id, storage] : reg.storage()) { if (!storage.contains(o)) { @@ -236,8 +189,8 @@ bool FilesystemStorage::write(Object o, std::function>(); + std::vector scanned_objs; // step 3: parse meta and insert into reg of non preexising // main thread @@ -637,8 +592,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) { for (const auto& [k, v] : j.items()) { // type id from string hash const auto type_id = entt::hashed_string(k.data(), k.size()); - const auto deserl_fn_it = _sc._deserl_json.find(type_id); - if (deserl_fn_it != _sc._deserl_json.cend()) { + const auto deserl_fn_it = sjc._deserl.find(type_id); + if (deserl_fn_it != sjc._deserl.cend()) { // TODO: check return value deserl_fn_it->second(oh, v); } else { diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp index ef5c111..626f000 100644 --- a/src/fragment_store/backends/filesystem_storage.hpp +++ b/src/fragment_store/backends/filesystem_storage.hpp @@ -26,11 +26,6 @@ struct FilesystemStorage : public StorageBackendI { private: size_t scanPath(std::string_view path); void scanPathAsync(std::string path); - - public: // TODO: private? - // this thing needs to change and be facilitated over the OS - // but the json serializer are specific to the backend - SerializerCallbacks _sc; }; } // backend diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 0329212..7d36bfa 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -1,6 +1,7 @@ #include "./object_store.hpp" #include "./backends/filesystem_storage.hpp" #include "./meta_components.hpp" +#include "./serializer_json.hpp" #include @@ -97,7 +98,7 @@ int main(int argc, const char** argv) { oh.emplace_or_replace(e.e.get_or_emplace()); // serializable - for (const auto& [type, fn] : _fsb_src._sc._serl_json) { + for (const auto& [type, fn] : _os_src.registry().ctx().get>()._serl) { //if (!e.e.registry()->storage(type)->contains(e.e)) { //continue; //} @@ -106,7 +107,7 @@ int main(int argc, const char** argv) { // raw copy might be better in the future nlohmann::json tmp_j; if (fn(e.e, tmp_j)) { - _fsb_dst._sc._deserl_json.at(type)(oh, tmp_j); + _os_dst.registry().ctx().get>()._deserl.at(type)(oh, tmp_j); } } } diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp index 663d1f9..d5e2dae 100644 --- a/src/fragment_store/object_store.cpp +++ b/src/fragment_store/object_store.cpp @@ -2,10 +2,53 @@ #include "./meta_components.hpp" +#include "./serializer_json.hpp" + #include // this sucks #include +// TODO: move somewhere else +static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { + if (!oh.all_of()) { + return false; + } + + out = static_cast>( + oh.get().enc + ); + return true; +} + +static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + +static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { + if (!oh.all_of()) { + return false; + } + + out = static_cast>( + oh.get().comp + ); + return true; +} + +static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) { } @@ -23,6 +66,18 @@ bool StorageBackendI::write(Object o, const ByteSpan data) { } ObjectStore2::ObjectStore2(void) { + // HACK: set them up independently + auto& sjc = _reg.ctx().emplace>(); + sjc.registerSerializer(serl_json_data_enc_type); + sjc.registerDeSerializer(deserl_json_data_enc_type); + sjc.registerSerializer(serl_json_data_comp_type); + sjc.registerDeSerializer(deserl_json_data_comp_type); + + // old stuff + sjc.registerSerializer(serl_json_data_enc_type); + sjc.registerDeSerializer(deserl_json_data_enc_type); + sjc.registerSerializer(serl_json_data_comp_type); + sjc.registerDeSerializer(deserl_json_data_comp_type); } ObjectStore2::~ObjectStore2(void) { diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp index fcc4456..03bac9b 100644 --- a/src/fragment_store/object_store.hpp +++ b/src/fragment_store/object_store.hpp @@ -3,8 +3,6 @@ #include #include -#include "./serializer.hpp" // TODO: get rid of the tight nljson integration - #include #include diff --git a/src/fragment_store/serializer_json.hpp b/src/fragment_store/serializer_json.hpp new file mode 100644 index 0000000..cd74540 --- /dev/null +++ b/src/fragment_store/serializer_json.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include + +#include + +// nlohmann +template +struct SerializerJsonCallbacks { + using Registry = entt::basic_registry; + using Handle = entt::basic_handle; + + using serialize_fn = bool(*)(const Handle h, nlohmann::json& out); + entt::dense_map _serl; + + using deserialize_fn = bool(*)(Handle h, const nlohmann::json& in); + entt::dense_map _deserl; + + template + static bool component_get_json(const Handle h, nlohmann::json& j) { + if (h.template all_of()) { + if constexpr (!std::is_empty_v) { + j = h.template get(); + } + return true; + } + + return false; + } + + template + static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) { + if constexpr (std::is_empty_v) { + h.template emplace_or_replace(); // assert empty json? + } else { + h.template emplace_or_replace(static_cast(j)); + } + return true; + } + + void registerSerializer(serialize_fn fn, const entt::type_info& type_info) { + _serl[type_info.hash()] = fn; + } + + template + void registerSerializer( + serialize_fn fn = component_get_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerSerializer(fn, type_info); + } + + void registerDeSerializer(deserialize_fn fn, const entt::type_info& type_info) { + _deserl[type_info.hash()] = fn; + } + + template + void registerDeSerializer( + deserialize_fn fn = component_emplace_or_replace_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerDeSerializer(fn, type_info); + } +}; + From 73180195fedd71b011811bb752225d74cd9decb8 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 11 Apr 2024 11:10:19 +0200 Subject: [PATCH 83/98] some more backend interface changes i realized i had to do --- src/fragment_store/backends/filesystem_storage.cpp | 10 +++++++--- src/fragment_store/backends/filesystem_storage.hpp | 13 +++++++++++-- src/fragment_store/convert_frag_to_obj.cpp | 3 ++- src/fragment_store/object_store.cpp | 5 +++++ src/fragment_store/object_store.hpp | 4 +++- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp index b5ae420..4a13663 100644 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -42,13 +42,17 @@ static ByteSpan spanFromRead(const std::variant>& namespace backend { -FilesystemStorage::FilesystemStorage(ObjectStore2& os, std::string_view storage_path) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path) { +FilesystemStorage::FilesystemStorage( + ObjectStore2& os, + std::string_view storage_path, + MetaFileType mft_new +) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path), _mft_new(mft_new) { } FilesystemStorage::~FilesystemStorage(void) { } -ObjectHandle FilesystemStorage::newObject(MetaFileType mft, ByteSpan id) { +ObjectHandle FilesystemStorage::newObject(ByteSpan id) { { // first check if id is already used (TODO: solve the multi obj/backend problem) auto exising_oh = _os.getOneObjectByID(id); if (static_cast(exising_oh)) { @@ -85,7 +89,7 @@ ObjectHandle FilesystemStorage::newObject(MetaFileType mft, ByteSpan id) { oh.emplace(this); oh.emplace(std::vector{id}); oh.emplace(object_file_path.generic_u8string()); - oh.emplace(mft); + oh.emplace(_mft_new); // meta needs to be synced to file std::function empty_data_cb = [](auto*, auto) -> uint64_t { return 0; }; diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp index 626f000..b8ddafd 100644 --- a/src/fragment_store/backends/filesystem_storage.hpp +++ b/src/fragment_store/backends/filesystem_storage.hpp @@ -3,17 +3,26 @@ #include "../types.hpp" #include "../object_store.hpp" +#include + namespace backend { struct FilesystemStorage : public StorageBackendI { - FilesystemStorage(ObjectStore2& os, std::string_view storage_path = "test_obj_store"); + FilesystemStorage( + ObjectStore2& os, + std::string_view storage_path = "test_obj_store", + MetaFileType mft_new = MetaFileType::BINARY_MSGPACK + ); ~FilesystemStorage(void); // TODO: fix the path for this specific fs? // for now we assume a single storage path per backend (there can be multiple per type) std::string _storage_path; - ObjectHandle newObject(MetaFileType mft, ByteSpan id); + // meta file type for new objects + MetaFileType _mft_new {MetaFileType::BINARY_MSGPACK}; + + ObjectHandle newObject(ByteSpan id) override; bool write(Object o, std::function& data_cb) override; bool read(Object o, std::function& data_cb) override; diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 7d36bfa..eccc9e9 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -85,7 +85,8 @@ int main(int argc, const char** argv) { } } - auto oh = _fsb_dst.newObject(e.e.get().type, ByteSpan{e.e.get().v}); + // we dont copy meta file type, it will be the same for all "new" objects + auto oh = _fsb_dst.newObject(ByteSpan{e.e.get().v}); if (!static_cast(oh)) { // already exists diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp index d5e2dae..e459e94 100644 --- a/src/fragment_store/object_store.cpp +++ b/src/fragment_store/object_store.cpp @@ -52,6 +52,11 @@ static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) { } +ObjectHandle StorageBackendI::newObject(ByteSpan) { + //return {_os.registry(), entt::null}; + return {}; +} + bool StorageBackendI::write(Object o, const ByteSpan data) { std::function fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { uint64_t i = 0; diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp index 03bac9b..ebbb8f7 100644 --- a/src/fragment_store/object_store.hpp +++ b/src/fragment_store/object_store.hpp @@ -22,11 +22,13 @@ struct StorageBackendI { StorageBackendI(ObjectStore2& os); + // default impl fails, acting like a read only store + virtual ObjectHandle newObject(ByteSpan id); + // ========== write object to storage ========== using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. virtual bool write(Object o, std::function& data_cb) = 0; - //virtual bool write(Object o, const uint8_t* data, const uint64_t data_size); // default impl bool write(Object o, const ByteSpan data); // ========== read object from storage ========== From a9f6a5d7633b3da5c0be9f24e65e847ef247d24c Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 11 Apr 2024 11:54:00 +0200 Subject: [PATCH 84/98] move mfs to os, works, convert tool still incomplete --- src/CMakeLists.txt | 17 - src/fragment_store/fragment_store.cpp | 745 ------------------ src/fragment_store/fragment_store.hpp | 98 --- src/fragment_store/fragment_store_i.cpp | 31 - src/fragment_store/fragment_store_i.hpp | 63 -- src/fragment_store/message_fragment_store.cpp | 168 ++-- src/fragment_store/message_fragment_store.hpp | 34 +- src/fragment_store/serializer.hpp | 68 -- src/fragment_store/test_fragstore.cpp | 80 -- src/main_screen.cpp | 9 +- src/main_screen.hpp | 6 +- 11 files changed, 121 insertions(+), 1198 deletions(-) delete mode 100644 src/fragment_store/fragment_store.cpp delete mode 100644 src/fragment_store/fragment_store.hpp delete mode 100644 src/fragment_store/fragment_store_i.cpp delete mode 100644 src/fragment_store/fragment_store_i.hpp delete mode 100644 src/fragment_store/serializer.hpp delete mode 100644 src/fragment_store/test_fragstore.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ffd0531..5edd7c8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,13 +12,6 @@ add_library(fragment_store ./fragment_store/meta_components_id.inl ./fragment_store/file2_stack.hpp ./fragment_store/file2_stack.cpp - #old - ./fragment_store/serializer.hpp - ./fragment_store/fragment_store_i.hpp - ./fragment_store/fragment_store_i.cpp - ./fragment_store/fragment_store.hpp - ./fragment_store/fragment_store.cpp - #new ./fragment_store/serializer_json.hpp ./fragment_store/object_store.hpp ./fragment_store/object_store.cpp @@ -71,16 +64,6 @@ target_link_libraries(message_fragment_store PUBLIC ######################################## -add_executable(test_fragment_store - fragment_store/test_fragstore.cpp -) - -target_link_libraries(test_fragment_store PUBLIC - fragment_store -) - -######################################## - add_executable(convert_frag_to_obj fragment_store/convert_frag_to_obj.cpp ) diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp deleted file mode 100644 index 2a34930..0000000 --- a/src/fragment_store/fragment_store.cpp +++ /dev/null @@ -1,745 +0,0 @@ -#include "./fragment_store.hpp" - -#include - -#include -#include -#include - -#include - -#include -#include - -#include "./file2_stack.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -static const char* metaFileTypeSuffix(MetaFileType mft) { - switch (mft) { - case MetaFileType::TEXT_JSON: return ".json"; - //case MetaFileType::BINARY_ARB: return ".bin"; - case MetaFileType::BINARY_MSGPACK: return ".msgpack"; - } - return ""; // .unk? -} - -// TODO: move to ... somewhere. (span? file2i?) -static ByteSpan spanFromRead(const std::variant>& data_var) { - if (std::holds_alternative>(data_var)) { - auto& vec = std::get>(data_var); - return {vec.data(), vec.size()}; - } else if (std::holds_alternative(data_var)) { - return std::get(data_var); - } else { - assert(false); - return {}; - } -} - -FragmentStore::FragmentStore(void) { - registerSerializers(); -} - -FragmentStore::FragmentStore( - std::array session_uuid_namespace -) : _session_uuid_gen(std::move(session_uuid_namespace)) { - registerSerializers(); -} - -std::vector FragmentStore::generateNewUID(void) { - return _session_uuid_gen(); -} - -FragmentID FragmentStore::newFragmentMemoryOwned( - const std::vector& id, - size_t initial_size -) { - { // first check if id is already used - auto exising_id = getFragmentByID(id); - if (_reg.valid(exising_id)) { - return entt::null; - } - } - - { // next check if space in memory budget - const auto free_memory = _memory_budget - _memory_usage; - if (initial_size > free_memory) { - return entt::null; - } - } - - // actually allocate and create - auto new_data = std::make_unique>(initial_size); - if (!static_cast(new_data)) { - // allocation failure - return entt::null; - } - _memory_usage += initial_size; - - const auto new_frag = _reg.create(); - - _reg.emplace(new_frag, id); - // TODO: memory comp - _reg.emplace>>(new_frag) = std::move(new_data); - - throwEventConstruct(new_frag); - - return new_frag; -} - -FragmentID FragmentStore::newFragmentFile( - std::string_view store_path, - MetaFileType mft, - const std::vector& id -) { - { // first check if id is already used - const auto exising_id = getFragmentByID(id); - if (_reg.valid(exising_id)) { - return entt::null; - } - } - - if (store_path.empty()) { - store_path = _default_store_path; - } - - std::filesystem::create_directories(store_path); - - const auto id_hex = bin2hex(id); - std::filesystem::path fragment_file_path; - - if (id_hex.size() < 6) { - fragment_file_path = std::filesystem::path{store_path}/id_hex; - } else { - // use the first 2hex (1byte) as a subfolder - std::filesystem::create_directories(std::string{store_path} + id_hex.substr(0, 2)); - fragment_file_path = std::filesystem::path{std::string{store_path} + id_hex.substr(0, 2)} / id_hex.substr(2); - } - - if (std::filesystem::exists(fragment_file_path)) { - return entt::null; - } - - const auto new_frag = _reg.create(); - - _reg.emplace(new_frag, id); - - // file (info) comp - _reg.emplace(new_frag, fragment_file_path.generic_u8string()); - - _reg.emplace(new_frag, mft); - - // meta needs to be synced to file - std::function empty_data_cb = [](auto*, auto) -> uint64_t { return 0; }; - if (!syncToStorage(new_frag, empty_data_cb)) { - std::cerr << "FS error: syncToStorage failed while creating new fragment file\n"; - _reg.destroy(new_frag); - return entt::null; - } - - // while new metadata might be created here, making sure the file could be created is more important - throwEventConstruct(new_frag); - - return new_frag; -} -FragmentID FragmentStore::newFragmentFile( - std::string_view store_path, - MetaFileType mft -) { - return newFragmentFile(store_path, mft, generateNewUID()); -} - -FragmentID FragmentStore::getFragmentByID( - const std::vector& id -) { - // TODO: accelerate - // maybe keep it sorted and binary search? hash table lookup? - for (const auto& [frag, id_comp] : _reg.view().each()) { - if (id == id_comp.v) { - return frag; - } - } - - return entt::null; -} - -FragmentID FragmentStore::getFragmentCustomMatcher( - std::function& fn -) { - return entt::null; -} - -bool FragmentStore::syncToStorage(FragmentID fid, std::function& data_cb) { - if (!_reg.valid(fid)) { - return false; - } - - if (!_reg.all_of(fid)) { - // not a file fragment? - return false; - } - - // split object storage - - MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults - if (_reg.all_of(fid)) { - meta_type = _reg.get(fid).type; - } - - Encryption meta_enc = Encryption::NONE; // TODO: better defaults - Compression meta_comp = Compression::NONE; // TODO: better defaults - - if (meta_type != MetaFileType::TEXT_JSON) { - if (_reg.all_of(fid)) { - meta_enc = _reg.get(fid).enc; - } - - if (_reg.all_of(fid)) { - meta_comp = _reg.get(fid).comp; - } - } else { - // we cant have encryption or compression - // so we force NONE for TEXT JSON - - _reg.emplace_or_replace(fid, Encryption::NONE); - _reg.emplace_or_replace(fid, Compression::NONE); - } - - std::filesystem::path meta_tmp_path = _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; - meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); - // TODO: make meta comp work with mem compressor - //auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); - std::stack> meta_file_stack; - meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()})); - - if (meta_file_stack.empty()) { - std::cerr << "FS error: failed to create temporary meta file stack\n"; - std::filesystem::remove(meta_tmp_path); // might have created an empty file - return false; - } - - Encryption data_enc = Encryption::NONE; // TODO: better defaults - Compression data_comp = Compression::NONE; // TODO: better defaults - if (_reg.all_of(fid)) { - data_enc = _reg.get(fid).enc; - } - if (_reg.all_of(fid)) { - data_comp = _reg.get(fid).comp; - } - - std::filesystem::path data_tmp_path = _reg.get(fid).path + ".tmp"; - data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); - auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp); - if (data_file_stack.empty()) { - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } - std::filesystem::remove(meta_tmp_path); - std::cerr << "FS error: failed to create temporary data file stack\n"; - return false; - } - - try { // TODO: properly sanitize strings, so this cant throw - // sharing code between binary msgpack and text json for now - nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed - // metadata file - - for (const auto& [type_id, storage] : _reg.storage()) { - if (!storage.contains(fid)) { - continue; - } - - //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; - - // use type_id to find serializer - auto s_cb_it = _sc._serl_json.find(type_id); - if (s_cb_it == _sc._serl_json.end()) { - // could not find serializer, not saving - continue; - } - - // noooo, why cant numbers be keys - //if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead - //s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]); - //} else if (meta_type == MetaFileType::TEXT_JSON) { - s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]); - //} - } - - if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file - std::vector binary_meta_data; - { - std::stack> binary_writer_stack; - binary_writer_stack.push(std::make_unique(binary_meta_data)); - - if (!buildStackWrite(binary_writer_stack, meta_enc, meta_comp)) { - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } - std::filesystem::remove(meta_tmp_path); - while (!data_file_stack.empty()) { data_file_stack.pop(); } - std::filesystem::remove(data_tmp_path); - std::cerr << "FS error: binary writer creation failed '" << _reg.get(fid).path << "'\n"; - return false; - } - - { - const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); - if (!binary_writer_stack.top()->write(ByteSpan{meta_data})) { - // i feel like exceptions or refactoring would be nice here - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } - std::filesystem::remove(meta_tmp_path); - while (!data_file_stack.empty()) { data_file_stack.pop(); } - std::filesystem::remove(data_tmp_path); - std::cerr << "FS error: binary writer failed '" << _reg.get(fid).path << "'\n"; - return false; - } - } - } - - //// the meta file is itself msgpack data - nlohmann::json meta_header_j = nlohmann::json::array(); - meta_header_j.emplace_back() = "SOLMET"; - meta_header_j.push_back(meta_enc); - meta_header_j.push_back(meta_comp); - - // with a custom msgpack impl like cmp, we can be smarter here and dont need an extra buffer - meta_header_j.push_back(nlohmann::json::binary(binary_meta_data)); - - const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); - meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); - } else if (meta_type == MetaFileType::TEXT_JSON) { - // cant be compressed or encrypted - const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); - meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); - } - - } catch (...) { - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - std::filesystem::remove(meta_tmp_path); - while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - std::filesystem::remove(data_tmp_path); - std::cerr << "FS error: failed to serialize json data\n"; - return false; - } - - // now data - // for zstd compression, chunk size is frame size. (no cross frame referencing) - // TODO: add buffering steam layer - static constexpr int64_t chunk_size{1024*1024}; // 1MiB should be enough - std::vector buffer(chunk_size); - uint64_t buffer_actual_size {0}; - do { - buffer_actual_size = data_cb(buffer.data(), buffer.size()); - if (buffer_actual_size == 0) { - break; - } - if (buffer_actual_size > buffer.size()) { - // wtf - break; - } - - data_file_stack.top()->write({buffer.data(), buffer_actual_size}); - } while (buffer_actual_size == buffer.size()); - - //meta_file.flush(); - //meta_file.close(); - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - //data_file.flush(); - //data_file.close(); - while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - - std::filesystem::rename( - meta_tmp_path, - _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type) - ); - - std::filesystem::rename( - data_tmp_path, - _reg.get(fid).path - ); - - // TODO: check return value of renames - - if (_reg.all_of(fid)) { - _reg.remove(fid); - } - - return true; -} - -bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) { - std::function fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { - uint64_t i = 0; - for (; i+read < data_size && i < buffer_size; i++) { - request_buffer[i] = data[i+read]; - } - read += i; - - return i; - }; - return syncToStorage(fid, fn_cb); -} - -bool FragmentStore::loadFromStorage(FragmentID fid, std::function& data_cb) { - if (!_reg.valid(fid)) { - return false; - } - - if (!_reg.all_of(fid)) { - // not a file fragment? - // TODO: memory fragments - return false; - } - - const auto& frag_path = _reg.get(fid).path; - - // TODO: check if metadata dirty? - // TODO: what if file changed on disk? - - std::cout << "FS: loading fragment '" << frag_path << "'\n"; - - Compression data_comp = Compression::NONE; - if (_reg.all_of(fid)) { - data_comp = _reg.get(fid).comp; - } - - auto data_file_stack = buildFileStackRead(std::string_view{frag_path}, Encryption::NONE, data_comp); - if (data_file_stack.empty()) { - return false; - } - - // TODO: make it read in a single chunk instead? - static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read - do { - auto data_var = data_file_stack.top()->read(chunk_size); - ByteSpan data = spanFromRead(data_var); - - if (data.empty()) { - // error or probably eof - break; - } - - data_cb(data); - - if (data.size < chunk_size) { - // eof - break; - } - } while (data_file_stack.top()->isGood()); - - return true; -} - -nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) { - std::vector tmp_buffer; - std::function cb = [&tmp_buffer](const ByteSpan buffer) { - tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); - }; - - if (!loadFromStorage(fid, cb)) { - return nullptr; - } - - return nlohmann::json::parse(tmp_buffer); -} - -size_t FragmentStore::scanStoragePath(std::string_view path) { - if (path.empty()) { - path = _default_store_path; - } - // TODO: extract so async can work (or/and make iteratable generator) - - if (!std::filesystem::is_directory(path)) { - std::cerr << "FS error: scan path not a directory '" << path << "'\n"; - return 0; - } - - // step 1: make snapshot of files, validate metafiles and save id/path+meta.ext - // can be extra thread (if non vfs) - struct FragFileEntry { - std::string id_str; - std::filesystem::path frag_path; - std::string meta_ext; - - bool operator==(const FragFileEntry& other) const { - // only compare by id - return id_str == other.id_str; - } - }; - struct FragFileEntryHash { - size_t operator()(const FragFileEntry& it) const { - return entt::hashed_string(it.id_str.data(), it.id_str.size()); - } - }; - entt::dense_set file_frag_list; - - std::filesystem::path storage_path{path}; - - auto handle_file = [&](const std::filesystem::path& file_path) { - if (!std::filesystem::is_regular_file(file_path)) { - return; - } - // handle file - - if (file_path.has_extension()) { - // skip over metadata, assuming only metafiles have extentions (might be wrong?) - // also skips temps - return; - } - - auto relative_path = std::filesystem::proximate(file_path, storage_path); - std::string id_str = relative_path.generic_u8string(); - // delete all '/' - id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end()); - if (id_str.size() % 2 != 0) { - std::cerr << "FS error: non hex fragment uid detected: '" << id_str << "'\n"; - } - - if (file_frag_list.contains(FragFileEntry{id_str, {}, ""})) { - std::cerr << "FS error: fragment duplicate detected: '" << id_str << "'\n"; - return; // skip - } - - const char* meta_ext = ".meta.msgpack"; - { // find meta - // TODO: this as to know all possible extentions - bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack"); - bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json"); - const size_t meta_sum = - (has_meta_msgpack?1:0) + - (has_meta_json?1:0) - ; - - if (meta_sum > 1) { // has multiple - std::cerr << "FS error: fragment with multiple meta files detected: " << id_str << "\n"; - return; // skip - } - - if (meta_sum == 0) { - std::cerr << "FS error: fragment missing meta file detected: " << id_str << "\n"; - return; // skip - } - - if (has_meta_json) { - meta_ext = ".meta.json"; - } - } - - file_frag_list.emplace(FragFileEntry{ - std::move(id_str), - file_path, - meta_ext - }); - }; - - for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) { - if (std::filesystem::is_regular_file(outer_path)) { - handle_file(outer_path); - } else if (std::filesystem::is_directory(outer_path)) { - // subdir, part of id - for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) { - //if (std::filesystem::is_regular_file(inner_path)) { - - //// handle file - //} // TODO: support deeper recursion? - handle_file(inner_path); - } - } - } - - std::cout << "FS: scan found:\n"; - for (const auto& it : file_frag_list) { - std::cout << " " << it.id_str << "\n"; - } - - // step 2: check if files preexist in reg - // main thread - // (merge into step 3 and do more error checking?) - for (auto it = file_frag_list.begin(); it != file_frag_list.end();) { - auto id = hex2bin(it->id_str); - auto fid = getFragmentByID(id); - if (_reg.valid(fid)) { - // pre exising (handle differently??) - // check if store differs? - it = file_frag_list.erase(it); - } else { - it++; - } - } - - std::vector scanned_frags; - // step 3: parse meta and insert into reg of non preexising - // main thread - // TODO: check timestamps of preexisting and reload? mark external/remote dirty? - for (const auto& it : file_frag_list) { - nlohmann::json j; - if (it.meta_ext == ".meta.msgpack") { - std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); - if (!file.is_open()) { - std::cout << "FS error: failed opening meta " << it.frag_path << "\n"; - continue; - } - - // file is a msgpack within a msgpack - - std::vector full_meta_data; - { // read meta file - // figure out size - file.seekg(0, file.end); - uint64_t file_size = file.tellg(); - file.seekg(0, file.beg); - - full_meta_data.resize(file_size); - - file.read(reinterpret_cast(full_meta_data.data()), full_meta_data.size()); - } - - const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data, true, false); - - if (!meta_header_j.is_array() || meta_header_j.size() < 4) { - std::cerr << "FS error: broken binary meta " << it.frag_path << "\n"; - continue; - } - - if (meta_header_j.at(0) != "SOLMET") { - std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.frag_path << "\n"; - continue; - } - - Encryption meta_enc = meta_header_j.at(1); - if (meta_enc != Encryption::NONE) { - std::cerr << "FS error: unknown encryption " << it.frag_path << "\n"; - continue; - } - - Compression meta_comp = meta_header_j.at(2); - if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) { - std::cerr << "FS error: unknown compression " << it.frag_path << "\n"; - continue; - } - - //const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data"); - if (!meta_header_j.at(3).is_binary()) { - std::cerr << "FS error: meta data not binary " << it.frag_path << "\n"; - continue; - } - const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3); - - std::stack> binary_reader_stack; - binary_reader_stack.push(std::make_unique(ByteSpan{meta_data_ref.data(), meta_data_ref.size()})); - - if (!buildStackRead(binary_reader_stack, meta_enc, meta_comp)) { - std::cerr << "FS error: binary reader creation failed " << it.frag_path << "\n"; - continue; - } - - // HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through - auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta? - const auto binary_read_span = spanFromRead(binary_read_value); - assert(binary_read_span.size < 10*1024*1024); - - j = nlohmann::json::from_msgpack(binary_read_span, true, false); - } else if (it.meta_ext == ".meta.json") { - std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); - if (!file.is_open()) { - std::cerr << "FS error: failed opening meta " << it.frag_path << "\n"; - continue; - } - - file >> j; - } else { - assert(false); - } - - if (!j.is_object()) { - std::cerr << "FS error: json in meta is broken " << it.id_str << "\n"; - continue; - } - - // TODO: existing fragment file - //newFragmentFile(); - FragmentHandle fh{_reg, _reg.create()}; - fh.emplace(hex2bin(it.id_str)); - - fh.emplace(it.frag_path.generic_u8string()); - - for (const auto& [k, v] : j.items()) { - // type id from string hash - const auto type_id = entt::hashed_string(k.data(), k.size()); - const auto deserl_fn_it = _sc._deserl_json.find(type_id); - if (deserl_fn_it != _sc._deserl_json.cend()) { - // TODO: check return value - deserl_fn_it->second(fh, v); - } else { - std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; - } - } - scanned_frags.push_back(fh); - } - - // TODO: mutex and move code to async and return this list ? - - // throw new frag event here, after loading them all - for (const FragmentID fid : scanned_frags) { - throwEventConstruct(fid); - } - - return scanned_frags.size(); -} - -void FragmentStore::scanStoragePathAsync(std::string path) { - // add path to queue - // HACK: if path is known/any fragment is in the path, this operation blocks (non async) - scanStoragePath(path); // TODO: make async and post result -} - -static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) { - out = static_cast>( - fh.get().enc - ); - return true; -} - -static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) { - fh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} - -static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) { - out = static_cast>( - fh.get().comp - ); - return true; -} - -static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) { - fh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} - -void FragmentStore::registerSerializers(void) { - _sc.registerSerializerJson(serl_json_data_enc_type); - _sc.registerDeSerializerJson(deserl_json_data_enc_type); - _sc.registerSerializerJson(serl_json_data_comp_type); - _sc.registerDeSerializerJson(deserl_json_data_comp_type); -} - diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp deleted file mode 100644 index 5982f70..0000000 --- a/src/fragment_store/fragment_store.hpp +++ /dev/null @@ -1,98 +0,0 @@ -#pragma once - -#include - -#include "./fragment_store_i.hpp" - -#include "./types.hpp" -#include "./meta_components.hpp" - -#include "./serializer.hpp" - -#include "./uuid_generator.hpp" - -#include -#include - -#include - -#include -#include -#include -#include - -struct FragmentStore : public FragmentStoreI { - UUIDGenerator_128_128 _session_uuid_gen; - - std::string _default_store_path; - - uint64_t _memory_budget {10u*1024u*1024u}; - uint64_t _memory_usage {0u}; - - SerializerCallbacks _sc; - - FragmentStore(void); - FragmentStore(std::array session_uuid_namespace); - - // TODO: make the frags ref counted - - // TODO: check for exising - std::vector generateNewUID(void); - - // ========== new fragment ========== - - // memory backed owned - FragmentID newFragmentMemoryOwned( - const std::vector& id, - size_t initial_size - ); - - // memory backed view (can only be added? not new?) - - // file backed (rw...) - // needs to know which store path to put into - FragmentID newFragmentFile( - std::string_view store_path, - MetaFileType mft, - const std::vector& id - ); - // this variant generate a new, mostly unique, id for us - FragmentID newFragmentFile( - std::string_view store_path, - MetaFileType mft - ); - - // ========== add fragment ========== - - // ========== get fragment ========== - FragmentID getFragmentByID( - const std::vector& id - ); - FragmentID getFragmentCustomMatcher( - std::function& fn - ); - - // remove fragment? - // unload? - - // ========== sync fragment to storage ========== - using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); - // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. - bool syncToStorage(FragmentID fid, std::function& data_cb); - bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size); - - // ========== load fragment data from storage ========== - using read_from_storage_put_data_cb = void(const ByteSpan buffer); - bool loadFromStorage(FragmentID fid, std::function& data_cb); - // convenience function - nlohmann::json loadFromStorageNJ(FragmentID fid); - - // fragment discovery? - // returns number of new fragments - size_t scanStoragePath(std::string_view path); - void scanStoragePathAsync(std::string path); - - private: - void registerSerializers(void); // internal comps -}; - diff --git a/src/fragment_store/fragment_store_i.cpp b/src/fragment_store/fragment_store_i.cpp deleted file mode 100644 index 3f757fd..0000000 --- a/src/fragment_store/fragment_store_i.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "./fragment_store_i.hpp" - -#include - -FragmentRegistry& FragmentStoreI::registry(void) { - return _reg; -} - -FragmentHandle FragmentStoreI::fragmentHandle(const FragmentID fid) { - return {_reg, fid}; -} - -void FragmentStoreI::throwEventConstruct(const FragmentID fid) { - std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n"; - dispatch( - FragmentStore_Event::fragment_construct, - Fragment::Events::FragmentConstruct{ - FragmentHandle{_reg, fid} - } - ); -} - -void FragmentStoreI::throwEventUpdate(const FragmentID fid) { - std::cout << "FSI debug: event updated " << entt::to_integral(fid) << "\n"; - dispatch( - FragmentStore_Event::fragment_updated, - Fragment::Events::FragmentUpdated{ - FragmentHandle{_reg, fid} - } - ); -} diff --git a/src/fragment_store/fragment_store_i.hpp b/src/fragment_store/fragment_store_i.hpp deleted file mode 100644 index 774a766..0000000 --- a/src/fragment_store/fragment_store_i.hpp +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include - -#include -#include - -#include - -// internal id -enum class FragmentID : uint32_t {}; -using FragmentRegistry = entt::basic_registry; -using FragmentHandle = entt::basic_handle; - -namespace Fragment::Events { - - struct FragmentConstruct { - const FragmentHandle e; - }; - struct FragmentUpdated { - const FragmentHandle e; - }; - //struct MessageDestory { - //const Message3Handle e; - //}; - -} // Fragment::Events - -enum class FragmentStore_Event : uint32_t { - fragment_construct, - fragment_updated, - //message_destroy, - - MAX -}; - -struct FragmentStoreEventI { - using enumType = FragmentStore_Event; - - virtual ~FragmentStoreEventI(void) {} - - virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; } - virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; } - //virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; } -}; -using FragmentStoreEventProviderI = EventProviderI; - -struct FragmentStoreI : public FragmentStoreEventProviderI { - static constexpr const char* version {"1"}; - - FragmentRegistry _reg; - - virtual ~FragmentStoreI(void) {} - - FragmentRegistry& registry(void); - FragmentHandle fragmentHandle(const FragmentID fid); - - void throwEventConstruct(const FragmentID fid); - void throwEventUpdate(const FragmentID fid); - // TODO - //void throwEventDestroy(); -}; - diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index ce0b1eb..c44b511 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -1,5 +1,7 @@ #include "./message_fragment_store.hpp" +#include "./serializer_json.hpp" + #include "../json/message_components.hpp" #include @@ -28,8 +30,8 @@ namespace Message::Components { ////std::vector uid; //FragmentID id; //}; - // only contains fragments with <1024 messages and <28h tsrage (or whatever) - entt::dense_set fid_open; + // only contains fragments with <1024 messages and <2h tsrage (or whatever) + entt::dense_set fid_open; }; // all message fragments of this contact @@ -40,30 +42,30 @@ namespace Message::Components { size_t i_b; size_t i_e; }; - entt::dense_map frags; + entt::dense_map frags; // add 2 sorted contact lists for both range begin and end // TODO: adding and removing becomes expensive with enough frags, consider splitting or heap - std::vector sorted_begin; - std::vector sorted_end; + std::vector sorted_begin; + std::vector sorted_end; // api // return true if it was actually inserted - bool insert(FragmentHandle frag); - bool erase(FragmentID frag); + bool insert(ObjectHandle frag); + bool erase(Object frag); // update? (just erase() + insert()) // uses range begin to go back in time - FragmentID prev(FragmentID frag) const; + Object prev(Object frag) const; // uses range end to go forward in time - FragmentID next(FragmentID frag) const; + Object next(Object frag) const; }; // all LOADED message fragments // TODO: merge into ContactFragments (and pull in openfrags) struct LoadedContactFragments final { // kept up-to-date by events - entt::dense_set frags; + entt::dense_set frags; }; } // Message::Components @@ -84,6 +86,23 @@ namespace ObjectStore::Components { } } // ObjectStore::Component +static nlohmann::json loadFromStorageNJ(ObjectHandle oh) { + assert(oh.all_of()); + auto* backend = oh.get().ptr; + assert(backend != nullptr); + + std::vector tmp_buffer; + std::function cb = [&tmp_buffer](const ByteSpan buffer) { + tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); + }; + if (!backend->read(oh, cb)) { + std::cerr << "failed to read obj '" << bin2hex(oh.get().v) << "'\n"; + return false; + } + + return nlohmann::json::parse(tmp_buffer, nullptr, false); +} + void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (_fs_ignore_event) { // message event because of us loading a fragment, ignore @@ -113,8 +132,8 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } // TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag) - if (!m.all_of()) { - std::cout << "MFS: new msg missing FID\n"; + if (!m.all_of()) { + std::cout << "MFS: new msg missing Object\n"; if (!m.registry()->ctx().contains()) { m.registry()->ctx().emplace(); } @@ -125,11 +144,11 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // missing fuid // find closesed non-sealed off fragment - FragmentID fragment_id{entt::null}; + Object fragment_id{entt::null}; // first search for fragment where the ts falls into the range for (const auto& fid : fid_open) { - auto fh = _fs.fragmentHandle(fid); + auto fh = _os.objectHandle(fid); assert(static_cast(fh)); // assuming ts range exists @@ -143,9 +162,9 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } // if it did not fit into an existing fragment, we next look for fragments that could be extended - if (!_fs._reg.valid(fragment_id)) { + if (!_os._reg.valid(fragment_id)) { for (const auto& fid : fid_open) { - auto fh = _fs.fragmentHandle(fid); + auto fh = _os.objectHandle(fid); assert(static_cast(fh)); // assuming ts range exists @@ -195,11 +214,13 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } // if its still not found, we need a new fragment - if (!_fs._reg.valid(fragment_id)) { - const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::BINARY_MSGPACK); - auto fh = _fs.fragmentHandle(new_fid); + if (!_os.registry().valid(fragment_id)) { + const auto new_uuid = _session_uuid_gen(); + _fs_ignore_event = true; + auto fh = _sb.newObject(ByteSpan{new_uuid}); + _fs_ignore_event = false; if (!static_cast(fh)) { - std::cout << "MFS error: failed to create new fragment for message\n"; + std::cout << "MFS error: failed to create new object for message\n"; return; } @@ -238,17 +259,17 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { std::cout << "MFS: created new fragment " << bin2hex(fh.get().v) << "\n"; _fs_ignore_event = true; - _fs.throwEventConstruct(fh); + _os.throwEventConstruct(fh); _fs_ignore_event = false; } // if this is still empty, something is very wrong and we exit here - if (!_fs._reg.valid(fragment_id)) { + if (!_os.registry().valid(fragment_id)) { std::cout << "MFS error: failed to find/create fragment for message\n"; return; } - m.emplace_or_replace(fragment_id); + m.emplace_or_replace(fragment_id); // in this case we know the fragment needs an update for (const auto& it : _fuid_save_queue) { @@ -257,11 +278,11 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; // done } } - _fuid_save_queue.push_back({Message::getTimeMS(), fragment_id, m.registry()}); + _fuid_save_queue.push_back({Message::getTimeMS(), {_os.registry(), fragment_id}, m.registry()}); return; // done } - const auto msg_fh = _fs.fragmentHandle(m.get().fid); + const auto msg_fh = _os.objectHandle(m.get().o); if (!static_cast(msg_fh)) { std::cerr << "MFS error: fid in message is invalid\n"; return; // TODO: properly handle this case @@ -290,9 +311,9 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // assumes not loaded frag // need update from frag -void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) { +void MessageFragmentStore::loadFragment(Message3Registry& reg, ObjectHandle fh) { std::cout << "MFS: loadFragment\n"; - const auto j = _fs.loadFromStorageNJ(fh); + const auto j = loadFromStorageNJ(fh); if (!j.is_array()) { // wrong data @@ -339,7 +360,7 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh } } - new_real_msg.emplace_or_replace(fh); + new_real_msg.emplace_or_replace(fh); // dup check (hacky, specific to protocols) Message3 dup_msg {entt::null}; @@ -393,7 +414,7 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh } } -bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry& reg) { +bool MessageFragmentStore::syncFragToStorage(ObjectHandle fh, Message3Registry& reg) { auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); auto j = nlohmann::json::array(); @@ -404,7 +425,7 @@ bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) { const Message3 m = *it; - if (!reg.all_of(m)) { + if (!reg.all_of(m)) { continue; } @@ -413,7 +434,7 @@ bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry continue; } - if (_fuid_save_queue.front().id != reg.get(m).fid) { + if (_fuid_save_queue.front().id != reg.get(m).o) { continue; // not ours } @@ -452,10 +473,13 @@ bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry // if save as binary //nlohmann::json::to_msgpack(j); auto j_dump = j.dump(2, ' ', true); - if (_fs.syncToStorage(fh, reinterpret_cast(j_dump.data()), j_dump.size())) { + assert(fh.all_of()); + auto* backend = fh.get().ptr; + //if (_os.syncToStorage(fh, reinterpret_cast(j_dump.data()), j_dump.size())) { + if (backend->write(fh, {reinterpret_cast(j_dump.data()), j_dump.size()})) { // TODO: make this better, should this be called on fail? should this be called before sync? (prob not) _fs_ignore_event = true; - _fs.throwEventUpdate(fh); + _os.throwEventUpdate(fh); _fs_ignore_event = false; //std::cout << "MFS: dumped " << j_dump << "\n"; @@ -470,30 +494,32 @@ bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry MessageFragmentStore::MessageFragmentStore( Contact3Registry& cr, RegistryMessageModel& rmm, - FragmentStore& fs -) : _cr(cr), _rmm(rmm), _fs(fs), _sc{_cr, {}, {}} { + ObjectStore2& os, + StorageBackendI& sb +) : _cr(cr), _rmm(rmm), _os(os), _sb(sb), _sc{_cr, {}, {}} { _rmm.subscribe(this, RegistryMessageModel_Event::message_construct); _rmm.subscribe(this, RegistryMessageModel_Event::message_updated); _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); - _fs._sc.registerSerializerJson(); - _fs._sc.registerDeSerializerJson(); - _fs._sc.registerSerializerJson(); - _fs._sc.registerDeSerializerJson(); + auto& sjc = _os.registry().ctx().get>(); + sjc.registerSerializer(); + sjc.registerDeSerializer(); + sjc.registerSerializer(); + sjc.registerDeSerializer(); // old - _fs._sc.registerSerializerJson(_fs._sc.component_get_json); - _fs._sc.registerDeSerializerJson(_fs._sc.component_emplace_or_replace_json); - _fs._sc.registerSerializerJson(_fs._sc.component_get_json); - _fs._sc.registerDeSerializerJson(_fs._sc.component_emplace_or_replace_json); + sjc.registerSerializer(sjc.component_get_json); + sjc.registerDeSerializer(sjc.component_emplace_or_replace_json); + sjc.registerSerializer(sjc.component_get_json); + sjc.registerDeSerializer(sjc.component_emplace_or_replace_json); - _fs.subscribe(this, FragmentStore_Event::fragment_construct); - _fs.subscribe(this, FragmentStore_Event::fragment_updated); + _os.subscribe(this, ObjectStore_Event::object_construct); + _os.subscribe(this, ObjectStore_Event::object_update); } MessageFragmentStore::~MessageFragmentStore(void) { while (!_fuid_save_queue.empty()) { - auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id); + auto fh = _fuid_save_queue.front().id; auto* reg = _fuid_save_queue.front().reg; assert(reg != nullptr); syncFragToStorage(fh, *reg); @@ -570,7 +596,7 @@ float MessageFragmentStore::tick(float) { if (!_fuid_save_queue.empty()) { // wait 10sec before saving if (_fuid_save_queue.front().ts_since_dirty + 10*1000 <= ts_now) { - auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id); + auto fh = _fuid_save_queue.front().id; auto* reg = _fuid_save_queue.front().reg; assert(reg != nullptr); if (syncFragToStorage(fh, *reg)) { @@ -587,7 +613,7 @@ float MessageFragmentStore::tick(float) { const bool had_events = !_event_check_queue.empty(); for (size_t i = 0; i < 10 && !_event_check_queue.empty(); i++) { std::cout << "MFS: event check\n"; - auto fh = _fs.fragmentHandle(_event_check_queue.front().fid); + auto fh = _event_check_queue.front().fid; auto c = _event_check_queue.front().c; _event_check_queue.pop_front(); @@ -643,7 +669,7 @@ float MessageFragmentStore::tick(float) { continue; } - auto fh = _fs.fragmentHandle(fid); + auto fh = _os.objectHandle(fid); if (!static_cast(fh)) { std::cerr << "MFS error: frag is invalid\n"; @@ -696,25 +722,25 @@ float MessageFragmentStore::tick(float) { cf.sorted_end.crbegin(), cf.sorted_end.crend(), ts_begin_comp.ts, - [&](const FragmentID element, const auto& value) -> bool { - return _fs._reg.get(element).end >= value; + [&](const Object element, const auto& value) -> bool { + return _os.registry().get(element).end >= value; } ); - FragmentID next_frag{entt::null}; + Object next_frag{entt::null}; if (right != cf.sorted_end.crend()) { next_frag = cf.next(*right); } // we checked earlier that cf is not empty - if (!_fs._reg.valid(next_frag)) { + if (!_os.registry().valid(next_frag)) { // fall back to closest, cf is not empty next_frag = cf.sorted_end.front(); } // a single adjacent frag is often not enough // only ok bc next is cheap - for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); next_frag = cf.next(next_frag)) { - auto fh = _fs.fragmentHandle(next_frag); + for (size_t i = 0; i < 5 && _os.registry().valid(next_frag); next_frag = cf.next(next_frag)) { + auto fh = _os.objectHandle(next_frag); if (fh.any_of()) { continue; // skip known empty } @@ -744,25 +770,25 @@ float MessageFragmentStore::tick(float) { cf.sorted_begin.cbegin(), cf.sorted_begin.cend(), ts_end, - [&](const FragmentID element, const auto& value) -> bool { - return _fs._reg.get(element).begin < value; + [&](const Object element, const auto& value) -> bool { + return _os.registry().get(element).begin < value; } ); - FragmentID prev_frag{entt::null}; + Object prev_frag{entt::null}; if (left != cf.sorted_begin.cend()) { prev_frag = cf.prev(*left); } // we checked earlier that cf is not empty - if (!_fs._reg.valid(prev_frag)) { + if (!_os.registry().valid(prev_frag)) { // fall back to closest, cf is not empty prev_frag = cf.sorted_begin.back(); } // a single adjacent frag is often not enough // only ok bc next is cheap - for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); prev_frag = cf.prev(prev_frag)) { - auto fh = _fs.fragmentHandle(prev_frag); + for (size_t i = 0; i < 5 && _os.registry().valid(prev_frag); prev_frag = cf.prev(prev_frag)) { + auto fh = _os.objectHandle(prev_frag); if (fh.any_of()) { continue; // skip known empty } @@ -791,10 +817,6 @@ float MessageFragmentStore::tick(float) { return 1000.f*60.f*60.f; } -void MessageFragmentStore::triggerScan(void) { - _fs.scanStoragePath("test_message_store/"); -} - bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) { handleMessage(e.e); return false; @@ -807,7 +829,7 @@ bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) { // TODO: handle deletes? diff between unload? -bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) { +bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectConstruct& e) { if (_fs_ignore_event) { return false; // skip self } @@ -854,7 +876,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) return false; } -bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentUpdated& e) { +bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectUpdate& e) { if (_fs_ignore_event) { return false; // skip self } @@ -911,7 +933,7 @@ bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentUpdated& e) { return false; } -bool Message::Components::ContactFragments::insert(FragmentHandle frag) { +bool Message::Components::ContactFragments::insert(ObjectHandle frag) { if (frags.contains(frag)) { return false; } @@ -926,7 +948,7 @@ bool Message::Components::ContactFragments::insert(FragmentHandle frag) { const auto pos = std::find_if( sorted_begin.cbegin(), sorted_begin.cend(), - [frag](const FragmentID a) -> bool { + [frag](const Object a) -> bool { const auto begin_a = frag.registry()->get(a).begin; const auto begin_frag = frag.get().begin; if (begin_a > begin_frag) { @@ -951,7 +973,7 @@ bool Message::Components::ContactFragments::insert(FragmentHandle frag) { const auto pos = std::find_if_not( sorted_end.cbegin(), sorted_end.cend(), - [frag](const FragmentID a) -> bool { + [frag](const Object a) -> bool { const auto end_a = frag.registry()->get(a).end; const auto end_frag = frag.get().end; if (end_a > end_frag) { @@ -984,7 +1006,7 @@ bool Message::Components::ContactFragments::insert(FragmentHandle frag) { return true; } -bool Message::Components::ContactFragments::erase(FragmentID frag) { +bool Message::Components::ContactFragments::erase(Object frag) { auto frags_it = frags.find(frag); if (frags_it == frags.end()) { return false; @@ -1001,7 +1023,7 @@ bool Message::Components::ContactFragments::erase(FragmentID frag) { return true; } -FragmentID Message::Components::ContactFragments::prev(FragmentID frag) const { +Object Message::Components::ContactFragments::prev(Object frag) const { // uses range begin to go back in time auto it = frags.find(frag); @@ -1017,7 +1039,7 @@ FragmentID Message::Components::ContactFragments::prev(FragmentID frag) const { return entt::null; } -FragmentID Message::Components::ContactFragments::next(FragmentID frag) const { +Object Message::Components::ContactFragments::next(Object frag) const { // uses range end to go forward in time auto it = frags.find(frag); diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 179ece1..91f355d 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -1,12 +1,12 @@ #pragma once #include "./meta_components.hpp" -#include "./fragment_store_i.hpp" -#include "./fragment_store.hpp" +#include "./object_store.hpp" + +#include "./uuid_generator.hpp" #include "./message_serializer.hpp" -#include #include #include @@ -22,8 +22,8 @@ namespace Message::Components { // unused, consumes too much memory (highly compressable) //using FUID = FragComp::ID; - struct FID { - FragmentID fid {entt::null}; + struct Obj { + Object o {entt::null}; }; // points to the front/newer message @@ -79,32 +79,35 @@ namespace Fragment::Components { // on new message: assign fuid // on new and update: mark as fragment dirty // on delete: mark as fragment dirty? -class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentStoreEventI { +class MessageFragmentStore : public RegistryMessageModelEventI, public ObjectStoreEventI { protected: Contact3Registry& _cr; RegistryMessageModel& _rmm; - FragmentStore& _fs; + ObjectStore2& _os; + StorageBackendI& _sb; bool _fs_ignore_event {false}; + UUIDGenerator_128_128 _session_uuid_gen; + // for message components only MessageSerializerCallbacks _sc; void handleMessage(const Message3Handle& m); - void loadFragment(Message3Registry& reg, FragmentHandle fh); + void loadFragment(Message3Registry& reg, ObjectHandle oh); - bool syncFragToStorage(FragmentHandle fh, Message3Registry& reg); + bool syncFragToStorage(ObjectHandle oh, Message3Registry& reg); struct SaveQueueEntry final { uint64_t ts_since_dirty{0}; //std::vector id; - FragmentID id; + ObjectHandle id; Message3Registry* reg{nullptr}; }; std::deque _fuid_save_queue; struct ECQueueEntry final { - FragmentID fid; + ObjectHandle fid; Contact3 c; }; std::deque _event_check_queue; @@ -118,7 +121,8 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS MessageFragmentStore( Contact3Registry& cr, RegistryMessageModel& rmm, - FragmentStore& fs + ObjectStore2& os, + StorageBackendI& sb ); virtual ~MessageFragmentStore(void); @@ -126,14 +130,12 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentS float tick(float time_delta); - void triggerScan(void); - protected: // rmm bool onEvent(const Message::Events::MessageConstruct& e) override; bool onEvent(const Message::Events::MessageUpdated& e) override; protected: // fs - bool onEvent(const Fragment::Events::FragmentConstruct& e) override; - bool onEvent(const Fragment::Events::FragmentUpdated& e) override; + bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override; + bool onEvent(const ObjectStore::Events::ObjectUpdate& e) override; }; diff --git a/src/fragment_store/serializer.hpp b/src/fragment_store/serializer.hpp deleted file mode 100644 index a1daa98..0000000 --- a/src/fragment_store/serializer.hpp +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -template -struct SerializerCallbacks { - using Registry = entt::basic_registry; - using Handle = entt::basic_handle; - - // nlohmann - // json/msgpack - using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out); - entt::dense_map _serl_json; - - using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in); - entt::dense_map _deserl_json; - - template - static bool component_get_json(const Handle h, nlohmann::json& j) { - if (h.template all_of()) { - if constexpr (!std::is_empty_v) { - j = h.template get(); - } - return true; - } - - return false; - } - - template - static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) { - if constexpr (std::is_empty_v) { - h.template emplace_or_replace(); // assert empty json? - } else { - h.template emplace_or_replace(static_cast(j)); - } - return true; - } - - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { - _serl_json[type_info.hash()] = fn; - } - - template - void registerSerializerJson( - serialize_json_fn fn = component_get_json, - const entt::type_info& type_info = entt::type_id() - ) { - registerSerializerJson(fn, type_info); - } - - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { - _deserl_json[type_info.hash()] = fn; - } - - template - void registerDeSerializerJson( - deserialize_json_fn fn = component_emplace_or_replace_json, - const entt::type_info& type_info = entt::type_id() - ) { - registerDeSerializerJson(fn, type_info); - } -}; - diff --git a/src/fragment_store/test_fragstore.cpp b/src/fragment_store/test_fragstore.cpp deleted file mode 100644 index 1319e29..0000000 --- a/src/fragment_store/test_fragstore.cpp +++ /dev/null @@ -1,80 +0,0 @@ -#include -#include - -#include "./fragment_store.hpp" - -#include -#include - -namespace Components { - struct MessagesTimestampRange { - uint64_t begin {0}; - uint64_t end {1000}; - }; - - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end) -} // Components - - -int main(void) { - FragmentStore fs; - fs._default_store_path = "test_store/"; - fs._sc.registerSerializerJson(); - fs._sc.registerDeSerializerJson(); - - const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9}); - - const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK); - - const auto frag2 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK); - - { - auto frag0h = fs.fragmentHandle(frag0); - - frag0h.emplace_or_replace(); - frag0h.emplace_or_replace(); - frag0h.emplace_or_replace(); - - std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { - uint64_t i = 0; - for (; i+read < 3000 && i < buffer_size; i++) { - request_buffer[i] = uint8_t((i+read) & 0xff); - } - read += i; - - return i; - }; - fs.syncToStorage(frag0, fn_cb); - } - - { - auto frag1h = fs.fragmentHandle(frag1); - - frag1h.emplace_or_replace().comp = Compression::ZSTD; - frag1h.emplace_or_replace(); - - std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { - static constexpr std::string_view text = "This is some random data"; - uint64_t i = 0; - for (; i+read < text.size() && i < buffer_size; i++) { - request_buffer[i] = text[i+read]; - } - read += i; - - return i; - }; - fs.syncToStorage(frag1, fn_cb); - } - - { - auto frag2h = fs.fragmentHandle(frag2); - - frag2h.emplace_or_replace(); - frag2h.emplace_or_replace(); - - static constexpr std::string_view text = "This is more random data"; - fs.syncToStorage(frag2, reinterpret_cast(text.data()), text.size()); - } - return 0; -} - diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 6ed720d..c588f35 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -16,7 +16,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri renderer(renderer_), rmm(cr), mts(rmm), - mfs(cr, rmm, fs), + mfsb(os, "test2_message_store/"), + mfs(cr, rmm, os, mfsb), tc(save_path, save_password), tpi(tc.getTox()), ad(tc), @@ -55,9 +56,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n"; { // setup plugin instances - // TODO: make interface useful - g_provideInstance("FragmentStoreI", "host", &fs); - g_provideInstance("FragmentStore", "host", &fs); + g_provideInstance("ObjectStore2", "host", &os); g_provideInstance("ConfigModelI", "host", &conf); g_provideInstance("Contact3Registry", "1", "host", &cr); @@ -85,7 +84,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri conf.dump(); - mfs.triggerScan(); // HACK: after plugins and tox contacts got loaded + mfsb.scanAsync(); // HACK: after plugins and tox contacts got loaded } MainScreen::~MainScreen(void) { diff --git a/src/main_screen.hpp b/src/main_screen.hpp index 39baf7e..abe77f3 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -2,7 +2,8 @@ #include "./screen.hpp" -#include "./fragment_store/fragment_store.hpp" +#include "./fragment_store/object_store.hpp" +#include "./fragment_store/backends/filesystem_storage.hpp" #include #include #include @@ -45,12 +46,13 @@ extern "C" { struct MainScreen final : public Screen { SDL_Renderer* renderer; - FragmentStore fs; + ObjectStore2 os; SimpleConfigModel conf; Contact3Registry cr; RegistryMessageModel rmm; MessageTimeSort mts; + backend::FilesystemStorage mfsb; // message fsb // TODO: make configurable MessageFragmentStore mfs; ToxEventLogger tel{std::cout}; From 85a29372f4a5425bf0cbf06b0b3f9bfec57fa198 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 11 Apr 2024 16:17:47 +0200 Subject: [PATCH 85/98] adding message frag object version and conversion work, but comp name changes incoming --- src/CMakeLists.txt | 5 ++-- src/fragment_store/convert_frag_to_obj.cpp | 8 ++++++ src/fragment_store/message_fragment_store.cpp | 26 +++++++++++++++++-- src/fragment_store/message_fragment_store.hpp | 6 +++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5edd7c8..8fc3573 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -64,12 +64,13 @@ target_link_libraries(message_fragment_store PUBLIC ######################################## -add_executable(convert_frag_to_obj +add_executable(convert_message_object_store fragment_store/convert_frag_to_obj.cpp ) -target_link_libraries(convert_frag_to_obj PUBLIC +target_link_libraries(convert_message_object_store PUBLIC fragment_store + message_fragment_store ) ######################################## diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index eccc9e9..41a69bd 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -2,6 +2,7 @@ #include "./backends/filesystem_storage.hpp" #include "./meta_components.hpp" #include "./serializer_json.hpp" +#include "./message_fragment_store.hpp" #include @@ -31,6 +32,13 @@ int main(int argc, const char** argv) { backend::FilesystemStorage fsb_src(os_src, argv[1]); backend::FilesystemStorage fsb_dst(os_dst, argv[2]); + Contact3Registry cr; // dummy + RegistryMessageModel rmm(cr); // dummy + // they only exist for the serializers (for now) + // TODO: version + MessageFragmentStore mfs_src(cr, rmm, os_src, fsb_src); + MessageFragmentStore mfs_dst(cr, rmm, os_dst, fsb_dst); + // add message fragment store too (adds meta?) // hookup events diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index c44b511..85f47b4 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -71,6 +71,7 @@ namespace Message::Components { } // Message::Components namespace ObjectStore::Components { + NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesVersion, v) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id) @@ -228,6 +229,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { fh.emplace_or_replace().comp = Compression::ZSTD; fh.emplace_or_replace().comp = Compression::ZSTD; + fh.emplace_or_replace(); // default is current auto& new_ts_range = fh.emplace_or_replace(); new_ts_range.begin = msg_ts; @@ -464,7 +466,11 @@ bool MessageFragmentStore::syncFragToStorage(ObjectHandle fh, Message3Registry& continue; } - s_cb_it->second(_sc, {reg, m}, j_entry[storage.type().name()]); + try { + s_cb_it->second(_sc, {reg, m}, j_entry[storage.type().name()]); + } catch (...) { + std::cerr << "MFS error: failed to serialize " << storage.type().name() << "(" << type_id << ")\n"; + } } } @@ -502,12 +508,14 @@ MessageFragmentStore::MessageFragmentStore( _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); auto& sjc = _os.registry().ctx().get>(); + sjc.registerSerializer(); + sjc.registerDeSerializer(); sjc.registerSerializer(); sjc.registerDeSerializer(); sjc.registerSerializer(); sjc.registerDeSerializer(); - // old + // old frag names sjc.registerSerializer(sjc.component_get_json); sjc.registerDeSerializer(sjc.component_emplace_or_replace_json); sjc.registerSerializer(sjc.component_get_json); @@ -625,6 +633,15 @@ float MessageFragmentStore::tick(float) { return 0.05f; } + if (!fh.all_of()) { + // missing version, adding + fh.emplace(); + } + if (fh.get().v != 1) { + std::cerr << "MFS: object with version mismatch\n"; + return 0.05f; + } + // get ts range of frag and collide with all curser(s/ranges) const auto& frag_range = fh.get(); @@ -837,6 +854,11 @@ bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectConstruct& e if (!e.e.all_of()) { return false; // not for us } + if (!e.e.all_of()) { + // missing version, adding + // version check is later + e.e.emplace(); + } // TODO: are we sure it is a *new* fragment? diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 91f355d..b23d054 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -55,6 +55,12 @@ namespace Message::Components { } // Message::Components namespace ObjectStore::Components { + struct MessagesVersion { + // messages Object version + // 1 -> text_json + uint16_t v {1}; + }; + struct MessagesTSRange { // timestamp range within the fragment uint64_t begin {0}; // newer msg -> higher number From 2597edd5797b6b984f662a5d2793e019cc8bdbad Mon Sep 17 00:00:00 2001 From: Green Sky Date: Thu, 11 Apr 2024 17:04:06 +0200 Subject: [PATCH 86/98] new messages objcomp names --- src/CMakeLists.txt | 2 ++ src/fragment_store/message_fragment_store.hpp | 29 ++-------------- .../messages_meta_components.hpp | 33 +++++++++++++++++++ .../messages_meta_components_id.inl | 31 +++++++++++++++++ src/fragment_store/meta_components.hpp | 5 +-- 5 files changed, 71 insertions(+), 29 deletions(-) create mode 100644 src/fragment_store/messages_meta_components.hpp create mode 100644 src/fragment_store/messages_meta_components_id.inl diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8fc3573..390ea26 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,6 +48,8 @@ target_link_libraries(test_file_zstd PUBLIC add_library(message_fragment_store ./fragment_store/message_serializer.hpp ./fragment_store/message_serializer.cpp + ./fragment_store/messages_meta_components.hpp + ./fragment_store/messages_meta_components_id.inl ./fragment_store/message_fragment_store.hpp ./fragment_store/message_fragment_store.cpp diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index b23d054..ba2d7c0 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -7,6 +7,8 @@ #include "./message_serializer.hpp" +#include "./messages_meta_components.hpp" + #include #include @@ -54,33 +56,6 @@ namespace Message::Components { } // Message::Components -namespace ObjectStore::Components { - struct MessagesVersion { - // messages Object version - // 1 -> text_json - uint16_t v {1}; - }; - - struct MessagesTSRange { - // timestamp range within the fragment - uint64_t begin {0}; // newer msg -> higher number - uint64_t end {0}; - }; - - struct MessagesContact { - std::vector id; - }; - - // TODO: add src contact (self id) - -} // ObjectStore::Components - -// old -namespace Fragment::Components { - struct MessagesTSRange : public ObjComp::MessagesTSRange {}; - struct MessagesContact : public ObjComp::MessagesContact {}; -} // Fragment::Components - // handles fragments for messages // on new message: assign fuid // on new and update: mark as fragment dirty diff --git a/src/fragment_store/messages_meta_components.hpp b/src/fragment_store/messages_meta_components.hpp new file mode 100644 index 0000000..d2ea019 --- /dev/null +++ b/src/fragment_store/messages_meta_components.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include "./meta_components.hpp" + +namespace ObjectStore::Components { + struct MessagesVersion { + // messages Object version + // 1 -> text_json + uint16_t v {1}; + }; + + struct MessagesTSRange { + // timestamp range within the fragment + uint64_t begin {0}; // newer msg -> higher number + uint64_t end {0}; + }; + + struct MessagesContact { + std::vector id; + }; + + // TODO: add src contact (self id) + +} // ObjectStore::Components + +// old +namespace Fragment::Components { + struct MessagesTSRange : public ObjComp::MessagesTSRange {}; + struct MessagesContact : public ObjComp::MessagesContact {}; +} // Fragment::Components + +#include "./messages_meta_components_id.inl" + diff --git a/src/fragment_store/messages_meta_components_id.inl b/src/fragment_store/messages_meta_components_id.inl new file mode 100644 index 0000000..4713637 --- /dev/null +++ b/src/fragment_store/messages_meta_components_id.inl @@ -0,0 +1,31 @@ +#pragma once + +#include "./messages_meta_components.hpp" + +#include + +// TODO: move more central +#define DEFINE_COMP_ID(x) \ +template<> \ +constexpr entt::id_type entt::type_hash::value() noexcept { \ + using namespace entt::literals; \ + return #x##_hs; \ +} \ +template<> \ +constexpr std::string_view entt::type_name::value() noexcept { \ + return #x; \ +} + +// cross compiler stable ids + +DEFINE_COMP_ID(ObjComp::MessagesVersion) +DEFINE_COMP_ID(ObjComp::MessagesTSRange) +DEFINE_COMP_ID(ObjComp::MessagesContact) + +// old stuff +//DEFINE_COMP_ID(FragComp::MessagesTSRange) +//DEFINE_COMP_ID(FragComp::MessagesContact) + +#undef DEFINE_COMP_ID + + diff --git a/src/fragment_store/meta_components.hpp b/src/fragment_store/meta_components.hpp index f0dcf57..0363caa 100644 --- a/src/fragment_store/meta_components.hpp +++ b/src/fragment_store/meta_components.hpp @@ -1,7 +1,7 @@ #pragma once #include "./types.hpp" -#include "object_store.hpp" +#include "./object_store.hpp" #include #include @@ -26,6 +26,7 @@ namespace ObjectStore::Components { // meta that is not written to (meta-)file namespace Ephemeral { + // TODO: move, backend specific struct MetaFileType { ::MetaFileType type {::MetaFileType::TEXT_JSON}; }; @@ -61,7 +62,7 @@ namespace ObjectStore::Components { // shortened to save bytes (until I find a way to save by ID in msgpack) namespace ObjComp = ObjectStore::Components; -// old names +// old names from frag era namespace Fragment::Components { //struct ID {}; //struct DataEncryptionType {}; From dfbb1dea6821927051a690e96ae4ffea2a91d971 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 12 Apr 2024 13:34:20 +0200 Subject: [PATCH 87/98] move os and backend to sub --- .gitmodules | 3 + external/CMakeLists.txt | 24 +- external/solanaceae_object_store | 1 + src/CMakeLists.txt | 58 +- .../backends/filesystem_storage.cpp | 627 ------------------ .../backends/filesystem_storage.hpp | 40 -- src/fragment_store/convert_frag_to_obj.cpp | 8 +- src/fragment_store/file2_stack.cpp | 93 --- src/fragment_store/file2_stack.hpp | 23 - src/fragment_store/file2_zstd.cpp | 209 ------ src/fragment_store/file2_zstd.hpp | 51 -- src/fragment_store/message_fragment_store.cpp | 2 +- src/fragment_store/message_fragment_store.hpp | 4 +- .../messages_meta_components.hpp | 2 +- src/fragment_store/meta_components.hpp | 77 --- src/fragment_store/meta_components_id.inl | 30 - src/fragment_store/object_store.cpp | 140 ---- src/fragment_store/object_store.hpp | 95 --- src/fragment_store/serializer_json.hpp | 67 -- src/fragment_store/test_file_zstd.cpp | 394 ----------- src/fragment_store/types.hpp | 19 - src/main_screen.hpp | 4 +- 22 files changed, 28 insertions(+), 1943 deletions(-) create mode 160000 external/solanaceae_object_store delete mode 100644 src/fragment_store/backends/filesystem_storage.cpp delete mode 100644 src/fragment_store/backends/filesystem_storage.hpp delete mode 100644 src/fragment_store/file2_stack.cpp delete mode 100644 src/fragment_store/file2_stack.hpp delete mode 100644 src/fragment_store/file2_zstd.cpp delete mode 100644 src/fragment_store/file2_zstd.hpp delete mode 100644 src/fragment_store/meta_components.hpp delete mode 100644 src/fragment_store/meta_components_id.inl delete mode 100644 src/fragment_store/object_store.cpp delete mode 100644 src/fragment_store/object_store.hpp delete mode 100644 src/fragment_store/serializer_json.hpp delete mode 100644 src/fragment_store/test_file_zstd.cpp delete mode 100644 src/fragment_store/types.hpp diff --git a/.gitmodules b/.gitmodules index 06f552c..4d17134 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,3 +20,6 @@ [submodule "external/solanaceae_plugin"] path = external/solanaceae_plugin url = https://github.com/Green-Sky/solanaceae_plugin.git +[submodule "external/solanaceae_object_store"] + path = external/solanaceae_object_store + url = https://github.com/Green-Sky/solanaceae_object_store.git diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index a7066be..5019deb 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -12,6 +12,8 @@ add_subdirectory(./toxcore) add_subdirectory(./solanaceae_toxcore) add_subdirectory(./solanaceae_tox) +add_subdirectory(./solanaceae_object_store) + add_subdirectory(./sdl) add_subdirectory(./imgui) @@ -28,25 +30,3 @@ if (NOT TARGET nlohmann_json::nlohmann_json) FetchContent_MakeAvailable(json) endif() -if (NOT TARGET zstd::zstd) - # TODO: try find_package() first - # TODO: try pkg-config next (will work on most distros) - - set(ZSTD_BUILD_STATIC ON) - set(ZSTD_BUILD_SHARED OFF) - set(ZSTD_BUILD_PROGRAMS OFF) - set(ZSTD_BUILD_CONTRIB OFF) - set(ZSTD_BUILD_TESTS OFF) - FetchContent_Declare(zstd - URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz" - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - SOURCE_SUBDIR build/cmake - EXCLUDE_FROM_ALL - ) - FetchContent_MakeAvailable(zstd) - - add_library(zstd INTERFACE) # somehow zstd fkd this up - target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/) - target_link_libraries(zstd INTERFACE libzstd_static) - add_library(zstd::zstd ALIAS zstd) -endif() diff --git a/external/solanaceae_object_store b/external/solanaceae_object_store new file mode 160000 index 0000000..4d3ffb8 --- /dev/null +++ b/external/solanaceae_object_store @@ -0,0 +1 @@ +Subproject commit 4d3ffb8192623740f6e170855ee1cffd428b78da diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 390ea26..7715e3e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,51 +1,14 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) -add_library(fragment_store - ./fragment_store/file2_zstd.hpp - ./fragment_store/file2_zstd.cpp - - ./fragment_store/uuid_generator.hpp - ./fragment_store/uuid_generator.cpp - - ./fragment_store/types.hpp - ./fragment_store/meta_components.hpp - ./fragment_store/meta_components_id.inl - ./fragment_store/file2_stack.hpp - ./fragment_store/file2_stack.cpp - ./fragment_store/serializer_json.hpp - ./fragment_store/object_store.hpp - ./fragment_store/object_store.cpp - ./fragment_store/backends/filesystem_storage.hpp - ./fragment_store/backends/filesystem_storage.cpp - - ./json/message_components.hpp # TODO: move - ./json/tox_message_components.hpp # TODO: move -) - -target_link_libraries(fragment_store PUBLIC - nlohmann_json::nlohmann_json - EnTT::EnTT - solanaceae_util - - solanaceae_file2 - zstd::zstd - - solanaceae_tox_messages # TODO: move -) - -######################################## - -add_executable(test_file_zstd - fragment_store/test_file_zstd.cpp -) - -target_link_libraries(test_file_zstd PUBLIC - fragment_store -) - ######################################## add_library(message_fragment_store + ./fragment_store/uuid_generator.hpp + ./fragment_store/uuid_generator.cpp + + ./json/message_components.hpp # TODO: move + ./json/tox_message_components.hpp # TODO: move + ./fragment_store/message_serializer.hpp ./fragment_store/message_serializer.cpp ./fragment_store/messages_meta_components.hpp @@ -60,8 +23,9 @@ add_library(message_fragment_store ) target_compile_features(message_fragment_store PRIVATE cxx_std_20) target_link_libraries(message_fragment_store PUBLIC - fragment_store + solanaceae_object_store solanaceae_message3 + solanaceae_tox_messages # TODO: move ) ######################################## @@ -71,7 +35,8 @@ add_executable(convert_message_object_store ) target_link_libraries(convert_message_object_store PUBLIC - fragment_store + solanaceae_object_store + solanaceae_object_store_backend_filesystem message_fragment_store ) @@ -158,7 +123,8 @@ target_link_libraries(tomato PUBLIC solanaceae_tox_contacts solanaceae_tox_messages - fragment_store + solanaceae_object_store + solanaceae_object_store_backend_filesystem message_fragment_store SDL3::SDL3 diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp deleted file mode 100644 index 4a13663..0000000 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ /dev/null @@ -1,627 +0,0 @@ -#include "./filesystem_storage.hpp" - -#include "../meta_components.hpp" -#include "../serializer_json.hpp" - -#include - -#include -#include - -#include -#include - -#include "../file2_stack.hpp" - -#include - -#include - -static const char* metaFileTypeSuffix(MetaFileType mft) { - switch (mft) { - case MetaFileType::TEXT_JSON: return ".json"; - //case MetaFileType::BINARY_ARB: return ".bin"; - case MetaFileType::BINARY_MSGPACK: return ".msgpack"; - } - return ""; // .unk? -} - -// TODO: move to ... somewhere. (span? file2i?) -static ByteSpan spanFromRead(const std::variant>& data_var) { - if (std::holds_alternative>(data_var)) { - auto& vec = std::get>(data_var); - return {vec.data(), vec.size()}; - } else if (std::holds_alternative(data_var)) { - return std::get(data_var); - } else { - assert(false); - return {}; - } -} - - -namespace backend { - -FilesystemStorage::FilesystemStorage( - ObjectStore2& os, - std::string_view storage_path, - MetaFileType mft_new -) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path), _mft_new(mft_new) { -} - -FilesystemStorage::~FilesystemStorage(void) { -} - -ObjectHandle FilesystemStorage::newObject(ByteSpan id) { - { // first check if id is already used (TODO: solve the multi obj/backend problem) - auto exising_oh = _os.getOneObjectByID(id); - if (static_cast(exising_oh)) { - return {}; - } - } - - std::filesystem::create_directories(_storage_path); - - if (!std::filesystem::is_directory(_storage_path)) { - std::cerr << "FS error: failed to create storage path dir '" << _storage_path << "'\n"; - return {}; - } - - const auto id_hex = bin2hex(id); - std::filesystem::path object_file_path; - - // TODO: refactor this magic somehow - if (id_hex.size() < 6) { - object_file_path = std::filesystem::path{_storage_path}/id_hex; - } else { - // use the first 2hex (1byte) as a subfolder - std::filesystem::create_directories(std::string{_storage_path} + id_hex.substr(0, 2)); - object_file_path = std::filesystem::path{std::string{_storage_path} + id_hex.substr(0, 2)} / id_hex.substr(2); - } - - if (std::filesystem::exists(object_file_path)) { - std::cerr << "FS error: object already exists in path '" << id_hex << "'\n"; - return {}; - } - - ObjectHandle oh{_os.registry(), _os.registry().create()}; - - oh.emplace(this); - oh.emplace(std::vector{id}); - oh.emplace(object_file_path.generic_u8string()); - oh.emplace(_mft_new); - - // meta needs to be synced to file - std::function empty_data_cb = [](auto*, auto) -> uint64_t { return 0; }; - if (!write(oh, empty_data_cb)) { - std::cerr << "FS error: write failed while creating new object file\n"; - oh.destroy(); - return {}; - } - - // while new metadata might be created here, making sure the file could be created is more important - _os.throwEventConstruct(oh); - - return oh; -} - -bool FilesystemStorage::write(Object o, std::function& data_cb) { - auto& reg = _os.registry(); - - if (!reg.valid(o)) { - return false; - } - - ObjectHandle oh {reg, o}; - - if (!oh.all_of()) { - // not a file fragment? - return false; - } - - // split object storage (meta and data are 2 files) - - MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults? - if (reg.all_of(o)) { - meta_type = oh.get().type; - } - - Encryption meta_enc = Encryption::NONE; // TODO: better defaults? - Compression meta_comp = Compression::NONE; // TODO: better defaults? - - if (meta_type != MetaFileType::TEXT_JSON) { - if (oh.all_of()) { - meta_enc = oh.get().enc; - } - - if (oh.all_of()) { - meta_comp = oh.get().comp; - } - } else { - // we cant have encryption or compression - // so we force NONE for TEXT JSON - - oh.emplace_or_replace(Encryption::NONE); - oh.emplace_or_replace(Compression::NONE); - } - - std::filesystem::path meta_tmp_path = oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp"; - meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string()); - // TODO: make meta comp work with mem compressor - //auto meta_file_stack = buildFileStackWrite(std::string_view{meta_tmp_path.generic_u8string()}, meta_enc, meta_comp); - std::stack> meta_file_stack; - meta_file_stack.push(std::make_unique(std::string_view{meta_tmp_path.generic_u8string()})); - - if (meta_file_stack.empty()) { - std::cerr << "FS error: failed to create temporary meta file stack\n"; - std::filesystem::remove(meta_tmp_path); // might have created an empty file - return false; - } - - Encryption data_enc = Encryption::NONE; // TODO: better defaults - Compression data_comp = Compression::NONE; // TODO: better defaults - if (oh.all_of()) { - data_enc = oh.get().enc; - } - if (oh.all_of()) { - data_comp = oh.get().comp; - } - - std::filesystem::path data_tmp_path = oh.get().path + ".tmp"; - data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string()); - auto data_file_stack = buildFileStackWrite(std::string_view{data_tmp_path.generic_u8string()}, data_enc, data_comp); - if (data_file_stack.empty()) { - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } - std::filesystem::remove(meta_tmp_path); - std::cerr << "FS error: failed to create temporary data file stack\n"; - return false; - } - - try { // TODO: properly sanitize strings, so this cant throw - // sharing code between binary msgpack and text json for now - nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed - // metadata file - - auto& sjc = _os.registry().ctx().get>(); - - // TODO: refactor extract to OS - for (const auto& [type_id, storage] : reg.storage()) { - if (!storage.contains(o)) { - continue; - } - - //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; - - // use type_id to find serializer - auto s_cb_it = sjc._serl.find(type_id); - if (s_cb_it == sjc._serl.end()) { - // could not find serializer, not saving - continue; - } - - // noooo, why cant numbers be keys - //if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead - //s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]); - //} else if (meta_type == MetaFileType::TEXT_JSON) { - s_cb_it->second(oh, meta_data_j[storage.type().name()]); - //} - } - - if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file - std::vector binary_meta_data; - { - std::stack> binary_writer_stack; - binary_writer_stack.push(std::make_unique(binary_meta_data)); - - if (!buildStackWrite(binary_writer_stack, meta_enc, meta_comp)) { - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } - std::filesystem::remove(meta_tmp_path); - while (!data_file_stack.empty()) { data_file_stack.pop(); } - std::filesystem::remove(data_tmp_path); - std::cerr << "FS error: binary writer creation failed '" << oh.get().path << "'\n"; - return false; - } - - { - const std::vector meta_data = nlohmann::json::to_msgpack(meta_data_j); - if (!binary_writer_stack.top()->write(ByteSpan{meta_data})) { - // i feel like exceptions or refactoring would be nice here - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } - std::filesystem::remove(meta_tmp_path); - while (!data_file_stack.empty()) { data_file_stack.pop(); } - std::filesystem::remove(data_tmp_path); - std::cerr << "FS error: binary writer failed '" << oh.get().path << "'\n"; - return false; - } - } - } - - //// the meta file is itself msgpack data - nlohmann::json meta_header_j = nlohmann::json::array(); - meta_header_j.emplace_back() = "SOLMET"; - meta_header_j.push_back(meta_enc); - meta_header_j.push_back(meta_comp); - - // with a custom msgpack impl like cmp, we can be smarter here and dont need an extra buffer - meta_header_j.push_back(nlohmann::json::binary(binary_meta_data)); - - const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j); - meta_file_stack.top()->write({meta_header_data.data(), meta_header_data.size()}); - } else if (meta_type == MetaFileType::TEXT_JSON) { - // cant be compressed or encrypted - const auto meta_file_json_str = meta_data_j.dump(2, ' ', true); - meta_file_stack.top()->write({reinterpret_cast(meta_file_json_str.data()), meta_file_json_str.size()}); - } - - } catch (...) { - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - std::filesystem::remove(meta_tmp_path); - while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - std::filesystem::remove(data_tmp_path); - std::cerr << "FS error: failed to serialize json data\n"; - return false; - } - - // now data - // for zstd compression, chunk size is frame size. (no cross frame referencing) - // TODO: add buffering steam layer - static constexpr int64_t chunk_size{1024*1024}; // 1MiB should be enough - std::vector buffer(chunk_size); - uint64_t buffer_actual_size {0}; - do { - buffer_actual_size = data_cb(buffer.data(), buffer.size()); - if (buffer_actual_size == 0) { - break; - } - if (buffer_actual_size > buffer.size()) { - // wtf - break; - } - - data_file_stack.top()->write({buffer.data(), buffer_actual_size}); - } while (buffer_actual_size == buffer.size()); - - // flush // TODO: use scope - while (!meta_file_stack.empty()) { meta_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - while (!data_file_stack.empty()) { data_file_stack.pop(); } // destroy stack // TODO: maybe work with scope? - - std::filesystem::rename( - meta_tmp_path, - oh.get().path + ".meta" + metaFileTypeSuffix(meta_type) - ); - - std::filesystem::rename( - data_tmp_path, - oh.get().path - ); - - // TODO: check return value of renames - - if (oh.all_of()) { - oh.remove(); - } - - return true; -} - -bool FilesystemStorage::read(Object o, std::function& data_cb) { - auto& reg = _os.registry(); - - if (!reg.valid(o)) { - return false; - } - - ObjectHandle oh {reg, o}; - - if (!oh.all_of()) { - // not a file - return false; - } - - const auto& obj_path = oh.get().path; - - // TODO: check if metadata dirty? - // TODO: what if file changed on disk? - - std::cout << "FS: loading fragment '" << obj_path << "'\n"; - - Compression data_comp = Compression::NONE; - if (oh.all_of()) { - data_comp = oh.get().comp; - } - - auto data_file_stack = buildFileStackRead(std::string_view{obj_path}, Encryption::NONE, data_comp); - if (data_file_stack.empty()) { - return false; - } - - // TODO: make it read in a single chunk instead? - static constexpr int64_t chunk_size {1024 * 1024}; // 1MiB should be good for read - do { - auto data_var = data_file_stack.top()->read(chunk_size); - ByteSpan data = spanFromRead(data_var); - - if (data.empty()) { - // error or probably eof - break; - } - - data_cb(data); - - if (data.size < chunk_size) { - // eof - break; - } - } while (data_file_stack.top()->isGood()); - - return true; -} - -void FilesystemStorage::scanAsync(void) { - scanPathAsync(_storage_path); -} - -size_t FilesystemStorage::scanPath(std::string_view path) { - // TODO: extract so async can work (or/and make iteratable generator) - - if (path.empty() || !std::filesystem::is_directory(path)) { - std::cerr << "FS error: scan path not a directory '" << path << "'\n"; - return 0; - } - - // step 1: make snapshot of files, validate metafiles and save id/path+meta.ext - // can be extra thread (if non vfs) - struct ObjFileEntry { - std::string id_str; - std::filesystem::path obj_path; - std::string meta_ext; - - bool operator==(const ObjFileEntry& other) const { - // only compare by id - return id_str == other.id_str; - } - }; - struct ObjFileEntryHash { - size_t operator()(const ObjFileEntry& it) const { - return entt::hashed_string(it.id_str.data(), it.id_str.size()); - } - }; - entt::dense_set file_obj_list; - - std::filesystem::path storage_path{path}; - - auto handle_file = [&](const std::filesystem::path& file_path) { - if (!std::filesystem::is_regular_file(file_path)) { - return; - } - // handle file - - if (file_path.has_extension()) { - // skip over metadata, assuming only metafiles have extentions (might be wrong?) - // also skips temps - return; - } - - auto relative_path = std::filesystem::proximate(file_path, storage_path); - std::string id_str = relative_path.generic_u8string(); - // delete all '/' - id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end()); - if (id_str.size() % 2 != 0) { - std::cerr << "FS error: non hex object uid detected: '" << id_str << "'\n"; - } - - if (file_obj_list.contains(ObjFileEntry{id_str, {}, ""})) { - std::cerr << "FS error: object duplicate detected: '" << id_str << "'\n"; - return; // skip - } - - const char* meta_ext = ".meta.msgpack"; - { // find meta - // TODO: this as to know all possible extentions - bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack"); - bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json"); - const size_t meta_sum = - (has_meta_msgpack?1:0) + - (has_meta_json?1:0) - ; - - if (meta_sum > 1) { // has multiple - std::cerr << "FS error: object with multiple meta files detected: " << id_str << "\n"; - return; // skip - } - - if (meta_sum == 0) { - std::cerr << "FS error: object missing meta file detected: " << id_str << "\n"; - return; // skip - } - - if (has_meta_json) { - meta_ext = ".meta.json"; - } - } - - file_obj_list.emplace(ObjFileEntry{ - std::move(id_str), - file_path, - meta_ext - }); - }; - - for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) { - if (std::filesystem::is_regular_file(outer_path)) { - handle_file(outer_path); - } else if (std::filesystem::is_directory(outer_path)) { - // subdir, part of id - for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) { - //if (std::filesystem::is_regular_file(inner_path)) { - - //// handle file - //} // TODO: support deeper recursion? - handle_file(inner_path); - } - } - } - - std::cout << "FS: scan found:\n"; - for (const auto& it : file_obj_list) { - std::cout << " " << it.id_str << "\n"; - } - - // step 2: check if files preexist in reg - // main thread - // (merge into step 3 and do more error checking?) - // TODO: properly handle duplicates, dups form different backends might be legit - // important - for (auto it = file_obj_list.begin(); it != file_obj_list.end();) { - auto id = hex2bin(it->id_str); - auto oh = _os.getOneObjectByID(ByteSpan{id}); - if (static_cast(oh)) { - // pre exising (handle differently??) - // check if store differs? - it = file_obj_list.erase(it); - } else { - it++; - } - } - - auto& sjc = _os.registry().ctx().get>(); - - std::vector scanned_objs; - // step 3: parse meta and insert into reg of non preexising - // main thread - // TODO: check timestamps of preexisting and reload? mark external/remote dirty? - for (const auto& it : file_obj_list) { - MetaFileType mft {MetaFileType::TEXT_JSON}; - Encryption meta_enc {Encryption::NONE}; - Compression meta_comp {Compression::NONE}; - nlohmann::json j; - if (it.meta_ext == ".meta.msgpack") { - std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); - if (!file.is_open()) { - std::cout << "FS error: failed opening meta " << it.obj_path << "\n"; - continue; - } - - mft = MetaFileType::BINARY_MSGPACK; - - // file is a msgpack within a msgpack - - std::vector full_meta_data; - { // read meta file - // figure out size - file.seekg(0, file.end); - uint64_t file_size = file.tellg(); - file.seekg(0, file.beg); - - full_meta_data.resize(file_size); - - file.read(reinterpret_cast(full_meta_data.data()), full_meta_data.size()); - } - - const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data, true, false); - - if (!meta_header_j.is_array() || meta_header_j.size() < 4) { - std::cerr << "FS error: broken binary meta " << it.obj_path << "\n"; - continue; - } - - if (meta_header_j.at(0) != "SOLMET") { - std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.obj_path << "\n"; - continue; - } - - meta_enc = meta_header_j.at(1); - if (meta_enc != Encryption::NONE) { - std::cerr << "FS error: unknown encryption " << it.obj_path << "\n"; - continue; - } - - meta_comp = meta_header_j.at(2); - if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) { - std::cerr << "FS error: unknown compression " << it.obj_path << "\n"; - continue; - } - - //const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data"); - if (!meta_header_j.at(3).is_binary()) { - std::cerr << "FS error: meta data not binary " << it.obj_path << "\n"; - continue; - } - const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3); - - std::stack> binary_reader_stack; - binary_reader_stack.push(std::make_unique(ByteSpan{meta_data_ref.data(), meta_data_ref.size()})); - - if (!buildStackRead(binary_reader_stack, meta_enc, meta_comp)) { - std::cerr << "FS error: binary reader creation failed " << it.obj_path << "\n"; - continue; - } - - // HACK: read fixed amout of data, but this way if we have neither enc nor comp we pass the span through - auto binary_read_value = binary_reader_stack.top()->read(10*1024*1024); // is 10MiB large enough for meta? - const auto binary_read_span = spanFromRead(binary_read_value); - assert(binary_read_span.size < 10*1024*1024); - - j = nlohmann::json::from_msgpack(binary_read_span, true, false); - } else if (it.meta_ext == ".meta.json") { - std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary); - if (!file.is_open()) { - std::cerr << "FS error: failed opening meta " << it.obj_path << "\n"; - continue; - } - - mft = MetaFileType::TEXT_JSON; - - file >> j; - } else { - assert(false); - } - - if (!j.is_object()) { - std::cerr << "FS error: json in meta is broken " << it.id_str << "\n"; - continue; - } - - // TODO: existing fragment file - //newFragmentFile(); - ObjectHandle oh{_os.registry(), _os.registry().create()}; - oh.emplace(this); - oh.emplace(hex2bin(it.id_str)); - oh.emplace(mft); - oh.emplace(meta_enc); - oh.emplace(meta_comp); - - oh.emplace(it.obj_path.generic_u8string()); - - for (const auto& [k, v] : j.items()) { - // type id from string hash - const auto type_id = entt::hashed_string(k.data(), k.size()); - const auto deserl_fn_it = sjc._deserl.find(type_id); - if (deserl_fn_it != sjc._deserl.cend()) { - // TODO: check return value - deserl_fn_it->second(oh, v); - } else { - std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n"; - } - } - scanned_objs.push_back(oh); - } - - // TODO: mutex and move code to async and return this list ? - - // throw new frag event here, after loading them all - for (const Object o : scanned_objs) { - _os.throwEventConstruct(o); - } - - return scanned_objs.size(); -} - -void FilesystemStorage::scanPathAsync(std::string path) { - // add path to queue - // HACK: if path is known/any fragment is in the path, this operation blocks (non async) - scanPath(path); // TODO: make async and post result -} - -} // backend - diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp deleted file mode 100644 index b8ddafd..0000000 --- a/src/fragment_store/backends/filesystem_storage.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include "../types.hpp" -#include "../object_store.hpp" - -#include - -namespace backend { - -struct FilesystemStorage : public StorageBackendI { - FilesystemStorage( - ObjectStore2& os, - std::string_view storage_path = "test_obj_store", - MetaFileType mft_new = MetaFileType::BINARY_MSGPACK - ); - ~FilesystemStorage(void); - - // TODO: fix the path for this specific fs? - // for now we assume a single storage path per backend (there can be multiple per type) - std::string _storage_path; - - // meta file type for new objects - MetaFileType _mft_new {MetaFileType::BINARY_MSGPACK}; - - ObjectHandle newObject(ByteSpan id) override; - - bool write(Object o, std::function& data_cb) override; - bool read(Object o, std::function& data_cb) override; - - //// convenience function - //nlohmann::json loadFromStorageNJ(FragmentID fid); - - void scanAsync(void); - - private: - size_t scanPath(std::string_view path); - void scanPathAsync(std::string path); -}; - -} // backend diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 41a69bd..d7520c7 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -1,7 +1,7 @@ -#include "./object_store.hpp" -#include "./backends/filesystem_storage.hpp" -#include "./meta_components.hpp" -#include "./serializer_json.hpp" +#include +#include +#include +#include #include "./message_fragment_store.hpp" #include diff --git a/src/fragment_store/file2_stack.cpp b/src/fragment_store/file2_stack.cpp deleted file mode 100644 index 70b599a..0000000 --- a/src/fragment_store/file2_stack.cpp +++ /dev/null @@ -1,93 +0,0 @@ -#include "./file2_stack.hpp" - -#include -#include "./file2_zstd.hpp" - -#include - -#include - -// add enc and comp file layers -// assumes a file is already in the stack -bool buildStackRead(std::stack>& file_stack, Encryption encryption, Compression compression) { - assert(!file_stack.empty()); - - // TODO: decrypt here - assert(encryption == Encryption::NONE); - - // add layer based on enum - if (compression == Compression::ZSTD) { - file_stack.push(std::make_unique(*file_stack.top().get())); - } else { - // TODO: make error instead - assert(compression == Compression::NONE); - } - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: file failed to add " << (int)compression << " decompression layer\n"; - return false; - } - - return true; -} - -// do i need this? -std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression) { - std::stack> file_stack; - file_stack.push(std::make_unique(file_path)); - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: opening file for reading '" << file_path << "'\n"; - return {}; - } - - if (!buildStackRead(file_stack, encryption, compression)) { - std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; - return {}; - } - - return file_stack; -} - -// add enc and comp file layers -// assumes a file is already in the stack -bool buildStackWrite(std::stack>& file_stack, Encryption encryption, Compression compression) { - assert(!file_stack.empty()); - - // TODO: encrypt here - assert(encryption == Encryption::NONE); - - // add layer based on enum - if (compression == Compression::ZSTD) { - file_stack.push(std::make_unique(*file_stack.top().get())); - } else { - // TODO: make error instead - assert(compression == Compression::NONE); - } - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: file failed to add " << (int)compression << " compression layer\n"; - return false; - } - - return true; -} - -// do i need this? -std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression) { - std::stack> file_stack; - file_stack.push(std::make_unique(file_path)); - - if (!file_stack.top()->isGood()) { - std::cerr << "FS error: opening file for writing '" << file_path << "'\n"; - return {}; - } - - if (!buildStackWrite(file_stack, encryption, compression)) { - std::cerr << "FS error: file failed to add layers for '" << file_path << "'\n"; - return {}; - } - - return file_stack; -} - diff --git a/src/fragment_store/file2_stack.hpp b/src/fragment_store/file2_stack.hpp deleted file mode 100644 index c1e878c..0000000 --- a/src/fragment_store/file2_stack.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include "./types.hpp" - -#include - -#include -#include -#include - -// add enc and comp file layers -// assumes a file is already in the stack -[[nodiscard]] bool buildStackRead(std::stack>& file_stack, Encryption encryption, Compression compression); - -// do i need this? -[[nodiscard]] std::stack> buildFileStackRead(std::string_view file_path, Encryption encryption, Compression compression); - -// add enc and comp file layers -// assumes a file is already in the stack -[[nodiscard]] bool buildStackWrite(std::stack>& file_stack, Encryption encryption, Compression compression); - -// do i need this? -[[nodiscard]] std::stack> buildFileStackWrite(std::string_view file_path, Encryption encryption, Compression compression); diff --git a/src/fragment_store/file2_zstd.cpp b/src/fragment_store/file2_zstd.cpp deleted file mode 100644 index f3ba0ab..0000000 --- a/src/fragment_store/file2_zstd.cpp +++ /dev/null @@ -1,209 +0,0 @@ -#include "./file2_zstd.hpp" - -#include -#include -#include -#include - -#include - -File2ZSTDW::File2ZSTDW(File2I& real) : - File2I(true, false), - _real_file(real) -{ - ZSTD_CCtx_setParameter(_cctx.get(), ZSTD_c_compressionLevel, 0); // default (3) - ZSTD_CCtx_setParameter(_cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?) -} - -File2ZSTDW::~File2ZSTDW(void) { - // flush remaining data (and maybe header) - // actually nvm, write will always flush all data, so only on empty files this would be an issue -} - -bool File2ZSTDW::isGood(void) { - return _real_file.isGood(); -} - -bool File2ZSTDW::write(const ByteSpan data, int64_t pos) { - if (pos != -1) { - return false; - } - - if (data.empty()) { - return false; // return true? - } - - if (data.size < 16) { - std::cout << "F2ZSTD warning: each write is a zstd frame and compression suffers significantly for small frames.\n"; - } - - std::vector compressed_buffer(ZSTD_CStreamOutSize()); - - ZSTD_inBuffer input = { data.ptr, data.size, 0 }; - - size_t remaining_ret {0}; - do { - // remaining data in input < compressed_buffer size (heuristic) - bool const lastChunk = (input.size - input.pos) <= compressed_buffer.size(); - - ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue; - - ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 }; - - remaining_ret = ZSTD_compressStream2(_cctx.get(), &output , &input, mode); - if (ZSTD_isError(remaining_ret)) { - std::cerr << "F2WRZSTD error: compressing data failed\n"; - break; - } - - _real_file.write(ByteSpan{compressed_buffer.data(), output.pos}); - } while ((input.pos < input.size || remaining_ret != 0) && _real_file.isGood()); - - return _real_file.isGood(); -} - -std::variant> File2ZSTDW::read(uint64_t, int64_t) { - return {}; -} - -// ######################################### decompression - -File2ZSTDR::File2ZSTDR(File2I& real) : - File2I(false, true), - _real_file(real), - - // 64kib - _in_buffer(ZSTD_DStreamInSize()), - _out_buffer(ZSTD_DStreamOutSize()) -{ -} - -bool File2ZSTDR::isGood(void) { - return _real_file.isGood(); -} - -bool File2ZSTDR::write(const ByteSpan, int64_t) { - return false; -} - -std::variant> File2ZSTDR::read(uint64_t size, int64_t pos) { - if (pos != -1) { - // error, only support streaming (for now) - return {}; - } - - std::vector ret_data; - - // actually first we check previous data - if (!_decompressed_buffer.empty()) { - uint64_t required_size = std::min(size, _decompressed_buffer.size()); - ret_data.insert(ret_data.end(), _decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size); - _decompressed_buffer.erase(_decompressed_buffer.cbegin(), _decompressed_buffer.cbegin() + required_size); - } - - bool eof {false}; - // outerloop here - while (ret_data.size() < size && !eof) { - // first make sure we have data in input - if (_z_input.src == nullptr || _z_input.pos == _z_input.size) { - const auto request_size = _in_buffer.size(); - if (!feedInput(_real_file.read(request_size, -1))) { - return ret_data; - } - - // if _z_input.size < _in_buffer.size() -> assume eof? - if (_z_input.size < request_size) { - eof = true; - //std::cout << "---- eof\n"; - } - } - - do { - ZSTD_outBuffer output = { _out_buffer.data(), _out_buffer.size(), 0 }; - size_t const ret = ZSTD_decompressStream(_dctx.get(), &output , &_z_input); - if (ZSTD_isError(ret)) { - // error <.< - std::cerr << "---- error: decompression error\n"; - return ret_data; - } - - // no new decomp data? - if (output.pos == 0) { - if (ret != 0) { - // if not error and not 0, indicates that - // there is additional flushing needed - continue; - } - - assert(eof || ret == 0); - break; - } - - int64_t returning_size = std::min(int64_t(size) - int64_t(ret_data.size()), output.pos); - assert(returning_size >= 0); - if (returning_size > 0) { - ret_data.insert( - ret_data.end(), - reinterpret_cast(output.dst), - reinterpret_cast(output.dst) + returning_size - ); - } - - // make sure we keep excess decompressed data - if (returning_size < int64_t(output.pos)) { - //const auto remaining_size = output.pos - returning_size; - _decompressed_buffer.insert( - _decompressed_buffer.cend(), - reinterpret_cast(output.dst) + returning_size, - reinterpret_cast(output.dst) + output.pos - ); - } - } while (_z_input.pos < _z_input.size); - } - - return ret_data; -} - -bool File2ZSTDR::feedInput(std::variant>&& read_buff) { - // TODO: optimize, we copy the buffer, but we might not need to - if (std::holds_alternative(read_buff)) { - const auto& span = std::get(read_buff); - if (span.size > _in_buffer.size()) { - // error, how did we read more than we asked for?? - return {}; - } - - if (span.empty()) { - _z_input = { _in_buffer.data(), 0, 0 }; - } else { - // cpy - _in_buffer = static_cast>(span); - _z_input = { - _in_buffer.data(), - span.size, - 0 - }; - } - } else if (std::holds_alternative>(read_buff)) { - auto& vec = std::get>(read_buff); - if (vec.size() > _in_buffer.size()) { - // error, how did we read more than we asked for?? - return {}; - } - - // cpy - _in_buffer = vec; - - _z_input = { - _in_buffer.data(), - _in_buffer.size(), - 0 - }; - } else { - // error, unsupported return value of read?? - return false; - } - - return true; -} - diff --git a/src/fragment_store/file2_zstd.hpp b/src/fragment_store/file2_zstd.hpp deleted file mode 100644 index 110b5a3..0000000 --- a/src/fragment_store/file2_zstd.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include - -#include - -#include - -// zstd compression wrapper over another file -// WARNING: only supports sequential writes -struct File2ZSTDW : public File2I { - File2I& _real_file; - - // TODO: hide this detail? - std::unique_ptr _cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx}; - - File2ZSTDW(File2I& real); - virtual ~File2ZSTDW(void); - - bool isGood(void) override; - - // for simplicity and potential future seekability each write is its own frame - bool write(const ByteSpan data, int64_t pos = -1) override; - std::variant> read(uint64_t size, int64_t pos = -1) override; -}; - -// zstd decompression wrapper over another file -// WARNING: only supports sequential reads -// TODO: add seeking support (use frames) -struct File2ZSTDR : public File2I { - File2I& _real_file; - - // TODO: hide this detail? - std::unique_ptr _dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx}; - std::vector _in_buffer; - std::vector _out_buffer; - std::vector _decompressed_buffer; // retains decompressed unread data between read() calls - ZSTD_inBuffer _z_input{nullptr, 0, 0}; - - File2ZSTDR(File2I& real); - virtual ~File2ZSTDR(void) {} - - bool isGood(void) override; - - bool write(const ByteSpan data, int64_t pos = -1) override; - std::variant> read(uint64_t size, int64_t pos = -1) override; - - private: - bool feedInput(std::variant>&& read_buff); -}; - diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 85f47b4..90638a9 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -1,6 +1,6 @@ #include "./message_fragment_store.hpp" -#include "./serializer_json.hpp" +#include #include "../json/message_components.hpp" diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index ba2d7c0..6a26c4e 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -1,7 +1,7 @@ #pragma once -#include "./meta_components.hpp" -#include "./object_store.hpp" +#include +#include #include "./uuid_generator.hpp" diff --git a/src/fragment_store/messages_meta_components.hpp b/src/fragment_store/messages_meta_components.hpp index d2ea019..0324053 100644 --- a/src/fragment_store/messages_meta_components.hpp +++ b/src/fragment_store/messages_meta_components.hpp @@ -1,6 +1,6 @@ #pragma once -#include "./meta_components.hpp" +#include namespace ObjectStore::Components { struct MessagesVersion { diff --git a/src/fragment_store/meta_components.hpp b/src/fragment_store/meta_components.hpp deleted file mode 100644 index 0363caa..0000000 --- a/src/fragment_store/meta_components.hpp +++ /dev/null @@ -1,77 +0,0 @@ -#pragma once - -#include "./types.hpp" -#include "./object_store.hpp" - -#include -#include -#include - -namespace ObjectStore::Components { - - // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) - struct ID { - std::vector v; - }; - - struct DataEncryptionType { - Encryption enc {Encryption::NONE}; - }; - - struct DataCompressionType { - Compression comp {Compression::NONE}; - }; - - - // meta that is not written to (meta-)file - namespace Ephemeral { - - // TODO: move, backend specific - struct MetaFileType { - ::MetaFileType type {::MetaFileType::TEXT_JSON}; - }; - - struct MetaEncryptionType { - Encryption enc {Encryption::NONE}; - }; - - struct MetaCompressionType { - Compression comp {Compression::NONE}; - }; - - struct Backend { - // TODO: shared_ptr instead?? - StorageBackendI* ptr; - }; - - // excluded from file meta - // TODO: move to backend specific - struct FilePath { - // contains store path, if any - std::string path; - }; - - // TODO: seperate into remote and local? - // (remote meaning eg. the file on disk was changed by another program) - struct DirtyTag {}; - - } // Ephemeral - -} // Components - -// shortened to save bytes (until I find a way to save by ID in msgpack) -namespace ObjComp = ObjectStore::Components; - -// old names from frag era -namespace Fragment::Components { - //struct ID {}; - //struct DataEncryptionType {}; - //struct DataCompressionType {}; - struct ID : public ObjComp::ID {}; - struct DataEncryptionType : public ObjComp::DataEncryptionType {}; - struct DataCompressionType : public ObjComp::DataCompressionType {}; -} -namespace FragComp = Fragment::Components; - -#include "./meta_components_id.inl" - diff --git a/src/fragment_store/meta_components_id.inl b/src/fragment_store/meta_components_id.inl deleted file mode 100644 index d9f7de3..0000000 --- a/src/fragment_store/meta_components_id.inl +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include "./meta_components.hpp" - -#include - -// TODO: move more central -#define DEFINE_COMP_ID(x) \ -template<> \ -constexpr entt::id_type entt::type_hash::value() noexcept { \ - using namespace entt::literals; \ - return #x##_hs; \ -} \ -template<> \ -constexpr std::string_view entt::type_name::value() noexcept { \ - return #x; \ -} - -// cross compiler stable ids - -DEFINE_COMP_ID(ObjComp::DataEncryptionType) -DEFINE_COMP_ID(ObjComp::DataCompressionType) - -// old stuff -DEFINE_COMP_ID(FragComp::DataEncryptionType) -DEFINE_COMP_ID(FragComp::DataCompressionType) - -#undef DEFINE_COMP_ID - - diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp deleted file mode 100644 index e459e94..0000000 --- a/src/fragment_store/object_store.cpp +++ /dev/null @@ -1,140 +0,0 @@ -#include "./object_store.hpp" - -#include "./meta_components.hpp" - -#include "./serializer_json.hpp" - -#include // this sucks - -#include - -// TODO: move somewhere else -static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { - if (!oh.all_of()) { - return false; - } - - out = static_cast>( - oh.get().enc - ); - return true; -} - -static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) { - oh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} - -static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { - if (!oh.all_of()) { - return false; - } - - out = static_cast>( - oh.get().comp - ); - return true; -} - -static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) { - oh.emplace_or_replace( - static_cast( - static_cast>(in) - ) - ); - return true; -} - -StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) { -} - -ObjectHandle StorageBackendI::newObject(ByteSpan) { - //return {_os.registry(), entt::null}; - return {}; -} - -bool StorageBackendI::write(Object o, const ByteSpan data) { - std::function fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { - uint64_t i = 0; - for (; i+read < data.size && i < buffer_size; i++) { - request_buffer[i] = data[i+read]; - } - read += i; - - return i; - }; - return write(o, fn_cb); -} - -ObjectStore2::ObjectStore2(void) { - // HACK: set them up independently - auto& sjc = _reg.ctx().emplace>(); - sjc.registerSerializer(serl_json_data_enc_type); - sjc.registerDeSerializer(deserl_json_data_enc_type); - sjc.registerSerializer(serl_json_data_comp_type); - sjc.registerDeSerializer(deserl_json_data_comp_type); - - // old stuff - sjc.registerSerializer(serl_json_data_enc_type); - sjc.registerDeSerializer(deserl_json_data_enc_type); - sjc.registerSerializer(serl_json_data_comp_type); - sjc.registerDeSerializer(deserl_json_data_comp_type); -} - -ObjectStore2::~ObjectStore2(void) { -} - -ObjectRegistry& ObjectStore2::registry(void) { - return _reg; -} - -ObjectHandle ObjectStore2::objectHandle(const Object o) { - return {_reg, o}; -} - -ObjectHandle ObjectStore2::getOneObjectByID(const ByteSpan id) { - // TODO: accelerate - // maybe keep it sorted and binary search? hash table lookup? - for (const auto& [obj, id_comp] : _reg.view().each()) { - if (id == ByteSpan{id_comp.v}) { - return {_reg, obj}; - } - } - - return {_reg, entt::null}; -} - -void ObjectStore2::throwEventConstruct(const Object o) { - std::cout << "OS debug: event construct " << entt::to_integral(o) << "\n"; - dispatch( - ObjectStore_Event::object_construct, - ObjectStore::Events::ObjectConstruct{ - ObjectHandle{_reg, o} - } - ); -} - -void ObjectStore2::throwEventUpdate(const Object o) { - std::cout << "OS debug: event update " << entt::to_integral(o) << "\n"; - dispatch( - ObjectStore_Event::object_update, - ObjectStore::Events::ObjectUpdate{ - ObjectHandle{_reg, o} - } - ); -} - -void ObjectStore2::throwEventDestroy(const Object o) { - std::cout << "OS debug: event destroy " << entt::to_integral(o) << "\n"; - dispatch( - ObjectStore_Event::object_destroy, - ObjectStore::Events::ObjectUpdate{ - ObjectHandle{_reg, o} - } - ); -} - diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp deleted file mode 100644 index ebbb8f7..0000000 --- a/src/fragment_store/object_store.hpp +++ /dev/null @@ -1,95 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -#include - -// internal id -enum class Object : uint32_t {}; -using ObjectRegistry = entt::basic_registry; -using ObjectHandle = entt::basic_handle; - -// fwd -struct ObjectStore2; - -struct StorageBackendI { - // OR or OS ? - ObjectStore2& _os; - - StorageBackendI(ObjectStore2& os); - - // default impl fails, acting like a read only store - virtual ObjectHandle newObject(ByteSpan id); - - // ========== write object to storage ========== - using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); - // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. - virtual bool write(Object o, std::function& data_cb) = 0; - bool write(Object o, const ByteSpan data); - - // ========== read object from storage ========== - using read_from_storage_put_data_cb = void(const ByteSpan buffer); - virtual bool read(Object o, std::function& data_cb) = 0; - -}; - -namespace ObjectStore::Events { - - struct ObjectConstruct { - const ObjectHandle e; - }; - struct ObjectUpdate { - const ObjectHandle e; - }; - struct ObjectDestory { - const ObjectHandle e; - }; - -} // ObjectStore::Events - -enum class ObjectStore_Event : uint16_t { - object_construct, - object_update, - object_destroy, - - MAX -}; - -struct ObjectStoreEventI { - using enumType = ObjectStore_Event; - - virtual ~ObjectStoreEventI(void) {} - - virtual bool onEvent(const ObjectStore::Events::ObjectConstruct&) { return false; } - virtual bool onEvent(const ObjectStore::Events::ObjectUpdate&) { return false; } - virtual bool onEvent(const ObjectStore::Events::ObjectDestory&) { return false; } -}; -using ObjectStoreEventProviderI = EventProviderI; - -struct ObjectStore2 : public ObjectStoreEventProviderI { - static constexpr const char* version {"2"}; - - ObjectRegistry _reg; - - // TODO: default backend? - - ObjectStore2(void); - virtual ~ObjectStore2(void); - - ObjectRegistry& registry(void); - ObjectHandle objectHandle(const Object o); - - // TODO: properly think about multiple objects witht he same id / different backends - ObjectHandle getOneObjectByID(const ByteSpan id); - - // sync? - - void throwEventConstruct(const Object o); - void throwEventUpdate(const Object o); - void throwEventDestroy(const Object o); -}; - diff --git a/src/fragment_store/serializer_json.hpp b/src/fragment_store/serializer_json.hpp deleted file mode 100644 index cd74540..0000000 --- a/src/fragment_store/serializer_json.hpp +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -// nlohmann -template -struct SerializerJsonCallbacks { - using Registry = entt::basic_registry; - using Handle = entt::basic_handle; - - using serialize_fn = bool(*)(const Handle h, nlohmann::json& out); - entt::dense_map _serl; - - using deserialize_fn = bool(*)(Handle h, const nlohmann::json& in); - entt::dense_map _deserl; - - template - static bool component_get_json(const Handle h, nlohmann::json& j) { - if (h.template all_of()) { - if constexpr (!std::is_empty_v) { - j = h.template get(); - } - return true; - } - - return false; - } - - template - static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) { - if constexpr (std::is_empty_v) { - h.template emplace_or_replace(); // assert empty json? - } else { - h.template emplace_or_replace(static_cast(j)); - } - return true; - } - - void registerSerializer(serialize_fn fn, const entt::type_info& type_info) { - _serl[type_info.hash()] = fn; - } - - template - void registerSerializer( - serialize_fn fn = component_get_json, - const entt::type_info& type_info = entt::type_id() - ) { - registerSerializer(fn, type_info); - } - - void registerDeSerializer(deserialize_fn fn, const entt::type_info& type_info) { - _deserl[type_info.hash()] = fn; - } - - template - void registerDeSerializer( - deserialize_fn fn = component_emplace_or_replace_json, - const entt::type_info& type_info = entt::type_id() - ) { - registerDeSerializer(fn, type_info); - } -}; - diff --git a/src/fragment_store/test_file_zstd.cpp b/src/fragment_store/test_file_zstd.cpp deleted file mode 100644 index cecddd7..0000000 --- a/src/fragment_store/test_file_zstd.cpp +++ /dev/null @@ -1,394 +0,0 @@ -#include "./file2_zstd.hpp" - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -const static std::string_view test_text1{"test1 1234 1234 :) 1234 5678 88888888\n"}; -const static ByteSpan data_test_text1{ - reinterpret_cast(test_text1.data()), - test_text1.size() -}; - -const static std::string_view test_text2{"test2 1234 1234 :) 1234 5678 88888888\n"}; -const static ByteSpan data_test_text2{ - reinterpret_cast(test_text2.data()), - test_text2.size() -}; - -const static std::string_view test_text3{ - "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" - "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" - "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" - "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" - "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" - "00000000000000000000000000000000000000000000000000000 test 00000000000000000000000000000000000000\n" -}; -const static ByteSpan data_test_text3{ - reinterpret_cast(test_text3.data()), - test_text3.size() -}; - -int main(void) { - { // first do a simple mem backed test - std::vector buffer; - { // write - File2MemW f_w_mem{buffer}; - assert(f_w_mem.isGood()); - - File2ZSTDW f_w_zstd{f_w_mem}; - assert(f_w_zstd.isGood()); - - bool res = f_w_zstd.write(data_test_text1); - assert(res); - assert(f_w_zstd.isGood()); - - // write another frame of the same data - res = f_w_zstd.write(data_test_text2); - assert(res); - assert(f_w_zstd.isGood()); - - // write larger frame - res = f_w_zstd.write(data_test_text3); - assert(res); - assert(f_w_zstd.isGood()); - } - - std::cout << "in mem size: " << buffer.size() << "\n"; - - { // read - File2MemR f_r_mem{ByteSpan{buffer}}; - assert(f_r_mem.isGood()); - - File2ZSTDR f_r_zstd{f_r_mem}; - assert(f_r_zstd.isGood()); - - // reads return owning buffers - - { // readback data_test_text1 - auto r_res_var = f_r_zstd.read(data_test_text1.size); - - //assert(f_r_zstd.isGood()); - //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - - //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - - assert(r_res_vec.size() == data_test_text1.size); - assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); - } - - { // readback data_test_text2 - auto r_res_var = f_r_zstd.read(data_test_text2.size); - - //assert(f_r_zstd.isGood()); - //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - - //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - - assert(r_res_vec.size() == data_test_text2.size); - assert(std::equal( - data_test_text2.cbegin(), - data_test_text2.cend(), - r_res_vec.cbegin() - )); - } - - { // readback data_test_text3 - auto r_res_var = f_r_zstd.read(data_test_text3.size); - - //assert(f_r_zstd.isGood()); - //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - - //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - - assert(r_res_vec.size() == data_test_text3.size); - assert(std::equal( - data_test_text3.cbegin(), - data_test_text3.cend(), - r_res_vec.cbegin() - )); - } - - { // assert eof somehow - // since its eof, reading a single byte should return a zero sized buffer - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } - } - } - } - - const auto temp_dir = std::filesystem::temp_directory_path() / "file2_zstd_tests"; - - std::filesystem::create_directories(temp_dir); // making sure - assert(std::filesystem::exists(temp_dir)); - std::cout << "test temp dir: " << temp_dir << "\n"; - - const auto test1_file_path = temp_dir / "testfile1.zstd"; - { // simple write test - File2WFile f_w_file{std::string_view{test1_file_path.u8string()}, true}; - assert(f_w_file.isGood()); - - File2ZSTDW f_w_zstd{f_w_file}; - assert(f_w_zstd.isGood()); - assert(f_w_file.isGood()); - - //bool res = f_w_file.write(data_test_text1); - bool res = f_w_zstd.write(data_test_text1); - assert(res); - assert(f_w_zstd.isGood()); - assert(f_w_file.isGood()); - - // write another frame of the same data - res = f_w_zstd.write(data_test_text2); - assert(res); - assert(f_w_zstd.isGood()); - assert(f_w_file.isGood()); - - // write larger frame - res = f_w_zstd.write(data_test_text3); - assert(res); - assert(f_w_zstd.isGood()); - assert(f_w_file.isGood()); - } - - // after flush - assert(std::filesystem::file_size(test1_file_path) != 0); - - { // simple read test (using write test created file) - File2RFile f_r_file{std::string_view{test1_file_path.u8string()}}; - assert(f_r_file.isGood()); - - File2ZSTDR f_r_zstd{f_r_file}; - assert(f_r_zstd.isGood()); - assert(f_r_file.isGood()); - - // reads return owning buffers - - { // readback data_test_text1 - auto r_res_var = f_r_zstd.read(data_test_text1.size); - - //assert(f_r_zstd.isGood()); - //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - - //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - - assert(r_res_vec.size() == data_test_text1.size); - assert(std::equal(data_test_text1.cbegin(), data_test_text1.cend(), r_res_vec.cbegin())); - } - - { // readback data_test_text2 - auto r_res_var = f_r_zstd.read(data_test_text2.size); - - //assert(f_r_zstd.isGood()); - //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - - //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - - assert(r_res_vec.size() == data_test_text2.size); - assert(std::equal( - data_test_text2.cbegin(), - data_test_text2.cend(), - r_res_vec.cbegin() - )); - } - - { // readback data_test_text3 - auto r_res_var = f_r_zstd.read(data_test_text3.size); - - //assert(f_r_zstd.isGood()); - //assert(f_r_file.isGood()); - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - - //std::cout << "decomp: " << std::string_view{reinterpret_cast(r_res_vec.data()), r_res_vec.size()}; - - assert(r_res_vec.size() == data_test_text3.size); - assert(std::equal( - data_test_text3.cbegin(), - data_test_text3.cend(), - r_res_vec.cbegin() - )); - } - - { // assert eof somehow - // since its eof, reading a single byte should return a zero sized buffer - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } - } - } - - const auto test2_file_path = temp_dir / "testfile2.zstd"; - { // write and read a single frame with increasing size - for (size_t fslog = 1; fslog <= 25; fslog++) { - const size_t frame_size = 1< tmp_data(frame_size); - for (auto& e : tmp_data) { - e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough - } - assert(tmp_data.size() == frame_size); - - bool res = f_w_zstd.write(ByteSpan{tmp_data}); - assert(res); - assert(f_w_zstd.isGood()); - assert(f_w_file.isGood()); - } - - { // read - std::minstd_rand rng_data{11*1337}; - - File2RFile f_r_file{std::string_view{test2_file_path.u8string()}}; - assert(f_r_file.isGood()); - - File2ZSTDR f_r_zstd{f_r_file}; - assert(f_r_zstd.isGood()); - assert(f_r_file.isGood()); - - { // read frame - auto r_res_var = f_r_zstd.read(frame_size); - - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - assert(r_res_vec.size() == frame_size); - - // assert equal - for (auto& e : r_res_vec) { - assert(e == uint8_t(rng_data() & 0xff)); - } - } - - { // eof test - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } - } - } - - // since we spam file, we immediatly remove them - std::filesystem::remove(test2_file_path); - } - } - - const auto test3_file_path = temp_dir / "testfile3.zstd"; - { // large file test write - File2WFile f_w_file{std::string_view{test3_file_path.u8string()}, true}; - assert(f_w_file.isGood()); - - File2ZSTDW f_w_zstd{f_w_file}; - assert(f_w_zstd.isGood()); - assert(f_w_file.isGood()); - - std::minstd_rand rng{11*1337}; - std::minstd_rand rng_data{11*1337}; // make investigating easier - - size_t total_raw_size {0}; - for (size_t i = 0; i < 2000; i++) { - const size_t frame_size = (rng() % ((2<<19) - 1)) + 1; - - std::vector tmp_data(frame_size); - for (auto& e : tmp_data) { - e = uint8_t(rng_data() & 0xff); // cutoff bad but good enough - } - - bool res = f_w_zstd.write(ByteSpan{tmp_data}); - assert(res); - assert(f_w_zstd.isGood()); - assert(f_w_file.isGood()); - total_raw_size += frame_size; - } - std::cout << "t3 total raw size: " << total_raw_size << "\n"; - } - - // after flush - std::cout << "t3 size on disk: " << std::filesystem::file_size(test3_file_path) << "\n"; - - { // large file test read - File2RFile f_r_file{std::string_view{test3_file_path.u8string()}}; - assert(f_r_file.isGood()); - - File2ZSTDR f_r_zstd{f_r_file}; - assert(f_r_zstd.isGood()); - assert(f_r_file.isGood()); - - // using same rng state as write to compare - std::minstd_rand rng{11*1337}; - std::minstd_rand rng_data{11*1337}; - - for (size_t i = 0; i < 2000; i++) { - const size_t frame_size = (rng() % ((2<<19) - 1)) + 1; - //std::cerr << "f: " << i << " fs: " << frame_size << "\n"; - - auto r_res_var = f_r_zstd.read(frame_size); - - assert(std::holds_alternative>(r_res_var)); - const auto& r_res_vec = std::get>(r_res_var); - assert(r_res_vec.size() == frame_size); - - // assert equal - for (auto& e : r_res_vec) { - assert(e == uint8_t(rng_data() & 0xff)); - } - } - - { // eof test - auto r_res_var = f_r_zstd.read(1); - if (std::holds_alternative>(r_res_var)) { - assert(std::get>(r_res_var).empty()); - } else if (std::holds_alternative(r_res_var)) { - assert(std::get(r_res_var).empty()); - } else { - assert(false); - } - } - } - - // cleanup - std::filesystem::remove_all(temp_dir); -} - diff --git a/src/fragment_store/types.hpp b/src/fragment_store/types.hpp deleted file mode 100644 index 97cfb14..0000000 --- a/src/fragment_store/types.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include - -enum class Encryption : uint8_t { - NONE = 0x00, -}; -enum class Compression : uint8_t { - NONE = 0x00, - ZSTD = 0x01, - // TODO: zstd without magic - // TODO: zstd meta dict - // TODO: zstd data(message) dict -}; -enum class MetaFileType : uint8_t { - TEXT_JSON, - BINARY_MSGPACK, // msgpacked msgpack -}; - diff --git a/src/main_screen.hpp b/src/main_screen.hpp index abe77f3..5cbb608 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -2,8 +2,8 @@ #include "./screen.hpp" -#include "./fragment_store/object_store.hpp" -#include "./fragment_store/backends/filesystem_storage.hpp" +#include +#include #include #include #include From a5e67d0ee8c33ed4480458d591f2ea927a2ee04b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 12 Apr 2024 19:44:24 +0200 Subject: [PATCH 88/98] move uuid gen to util --- external/solanaceae_util | 2 +- src/CMakeLists.txt | 3 - src/fragment_store/message_fragment_store.hpp | 2 +- src/fragment_store/uuid_generator.cpp | 68 ------------------- src/fragment_store/uuid_generator.hpp | 24 ------- 5 files changed, 2 insertions(+), 97 deletions(-) delete mode 100644 src/fragment_store/uuid_generator.cpp delete mode 100644 src/fragment_store/uuid_generator.hpp diff --git a/external/solanaceae_util b/external/solanaceae_util index bee42d4..2420af4 160000 --- a/external/solanaceae_util +++ b/external/solanaceae_util @@ -1 +1 @@ -Subproject commit bee42d4688e873115d578032970f0d5b84c41605 +Subproject commit 2420af464f270afaf07391ab51cc94a8b979e00c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7715e3e..23d9138 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,9 +3,6 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) ######################################## add_library(message_fragment_store - ./fragment_store/uuid_generator.hpp - ./fragment_store/uuid_generator.cpp - ./json/message_components.hpp # TODO: move ./json/tox_message_components.hpp # TODO: move diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 6a26c4e..1ac4402 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -3,7 +3,7 @@ #include #include -#include "./uuid_generator.hpp" +#include #include "./message_serializer.hpp" diff --git a/src/fragment_store/uuid_generator.cpp b/src/fragment_store/uuid_generator.cpp deleted file mode 100644 index cc07985..0000000 --- a/src/fragment_store/uuid_generator.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include "./uuid_generator.hpp" - -UUIDGenerator_128_128::UUIDGenerator_128_128(void) { - { // random namespace - const auto num0 = _rng(); - const auto num1 = _rng(); - const auto num2 = _rng(); - const auto num3 = _rng(); - - _uuid_namespace[0+0] = (num0 >> 0) & 0xff; - _uuid_namespace[0+1] = (num0 >> 8) & 0xff; - _uuid_namespace[0+2] = (num0 >> 16) & 0xff; - _uuid_namespace[0+3] = (num0 >> 24) & 0xff; - - _uuid_namespace[4+0] = (num1 >> 0) & 0xff; - _uuid_namespace[4+1] = (num1 >> 8) & 0xff; - _uuid_namespace[4+2] = (num1 >> 16) & 0xff; - _uuid_namespace[4+3] = (num1 >> 24) & 0xff; - - _uuid_namespace[8+0] = (num2 >> 0) & 0xff; - _uuid_namespace[8+1] = (num2 >> 8) & 0xff; - _uuid_namespace[8+2] = (num2 >> 16) & 0xff; - _uuid_namespace[8+3] = (num2 >> 24) & 0xff; - - _uuid_namespace[12+0] = (num3 >> 0) & 0xff; - _uuid_namespace[12+1] = (num3 >> 8) & 0xff; - _uuid_namespace[12+2] = (num3 >> 16) & 0xff; - _uuid_namespace[12+3] = (num3 >> 24) & 0xff; - } -} - -UUIDGenerator_128_128::UUIDGenerator_128_128(const std::array& uuid_namespace) : - _uuid_namespace(uuid_namespace) -{ -} - -std::vector UUIDGenerator_128_128::operator()(void) { - std::vector new_uid(_uuid_namespace.cbegin(), _uuid_namespace.cend()); - new_uid.resize(new_uid.size() + 16); - - const auto num0 = _rng(); - const auto num1 = _rng(); - const auto num2 = _rng(); - const auto num3 = _rng(); - - new_uid[_uuid_namespace.size()+0] = (num0 >> 0) & 0xff; - new_uid[_uuid_namespace.size()+1] = (num0 >> 8) & 0xff; - new_uid[_uuid_namespace.size()+2] = (num0 >> 16) & 0xff; - new_uid[_uuid_namespace.size()+3] = (num0 >> 24) & 0xff; - - new_uid[_uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff; - new_uid[_uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff; - new_uid[_uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff; - new_uid[_uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff; - - new_uid[_uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff; - new_uid[_uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff; - new_uid[_uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff; - new_uid[_uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff; - - new_uid[_uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff; - new_uid[_uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff; - new_uid[_uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff; - new_uid[_uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff; - - return new_uid; -} - diff --git a/src/fragment_store/uuid_generator.hpp b/src/fragment_store/uuid_generator.hpp deleted file mode 100644 index b0a1f99..0000000 --- a/src/fragment_store/uuid_generator.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -struct UUIDGeneratorI { - virtual std::vector operator()(void) = 0; -}; - -// TODO: templates? -struct UUIDGenerator_128_128 final : public UUIDGeneratorI { - private: - std::array _uuid_namespace; - std::minstd_rand _rng{std::random_device{}()}; - - public: - UUIDGenerator_128_128(void); // default randomly initializes namespace - UUIDGenerator_128_128(const std::array& uuid_namespace); - - std::vector operator()(void) override; -}; - From 498b4435c75058d8ad3f70f33557fc1baaabd7b3 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 13 Apr 2024 11:38:13 +0200 Subject: [PATCH 89/98] refactor message contexts --- src/CMakeLists.txt | 2 + src/fragment_store/internal_mfs_contexts.cpp | 149 ++++++++++ src/fragment_store/internal_mfs_contexts.hpp | 53 ++++ src/fragment_store/message_fragment_store.cpp | 271 +++--------------- src/fragment_store/message_fragment_store.hpp | 1 + 5 files changed, 248 insertions(+), 228 deletions(-) create mode 100644 src/fragment_store/internal_mfs_contexts.cpp create mode 100644 src/fragment_store/internal_mfs_contexts.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 23d9138..91722ad 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,6 +10,8 @@ add_library(message_fragment_store ./fragment_store/message_serializer.cpp ./fragment_store/messages_meta_components.hpp ./fragment_store/messages_meta_components_id.inl + ./fragment_store/internal_mfs_contexts.hpp + ./fragment_store/internal_mfs_contexts.cpp ./fragment_store/message_fragment_store.hpp ./fragment_store/message_fragment_store.cpp diff --git a/src/fragment_store/internal_mfs_contexts.cpp b/src/fragment_store/internal_mfs_contexts.cpp new file mode 100644 index 0000000..a07ce4c --- /dev/null +++ b/src/fragment_store/internal_mfs_contexts.cpp @@ -0,0 +1,149 @@ +#include "./internal_mfs_contexts.hpp" + +#include "./message_fragment_store.hpp" + +#include +#include +#include + +#include + +static bool isLess(const std::vector& lhs, const std::vector& rhs) { + size_t i = 0; + for (; i < lhs.size() && i < rhs.size(); i++) { + if (lhs[i] < rhs[i]) { + return true; + } else if (lhs[i] > rhs[i]) { + return false; + } + // else continue + } + + // here we have equality of common lenths + + // we define smaller arrays to be less + return lhs.size() < rhs.size(); +} + +bool Message::Contexts::ContactFragments::insert(ObjectHandle frag) { + if (frags.contains(frag)) { + return false; + } + + // both sorted arrays are sorted ascending + // so for insertion we search for the last index that is <= and insert after it + // or we search for the first > (or end) and insert before it <--- + // since equal fragments are UB, we can assume they are only > or < + + size_t begin_index {0}; + { // begin + const auto pos = std::find_if( + sorted_begin.cbegin(), + sorted_begin.cend(), + [frag](const Object a) -> bool { + const auto begin_a = frag.registry()->get(a).begin; + const auto begin_frag = frag.get().begin; + if (begin_a > begin_frag) { + return true; + } else if (begin_a < begin_frag) { + return false; + } else { + // equal ts, we need to fall back to id (id can not be equal) + return isLess(frag.get().v, frag.registry()->get(a).v); + } + } + ); + + begin_index = std::distance(sorted_begin.cbegin(), pos); + + // we need to insert before pos (end is valid here) + sorted_begin.insert(pos, frag); + } + + size_t end_index {0}; + { // end + const auto pos = std::find_if_not( + sorted_end.cbegin(), + sorted_end.cend(), + [frag](const Object a) -> bool { + const auto end_a = frag.registry()->get(a).end; + const auto end_frag = frag.get().end; + if (end_a > end_frag) { + return true; + } else if (end_a < end_frag) { + return false; + } else { + // equal ts, we need to fall back to id (id can not be equal) + return isLess(frag.get().v, frag.registry()->get(a).v); + } + } + ); + + end_index = std::distance(sorted_end.cbegin(), pos); + + // we need to insert before pos (end is valid here) + sorted_end.insert(pos, frag); + } + + frags.emplace(frag, InternalEntry{begin_index, end_index}); + + // now adjust all indicies of fragments coming after the insert position + for (size_t i = begin_index + 1; i < sorted_begin.size(); i++) { + frags.at(sorted_begin[i]).i_b = i; + } + for (size_t i = end_index + 1; i < sorted_end.size(); i++) { + frags.at(sorted_end[i]).i_e = i; + } + + return true; +} + +bool Message::Contexts::ContactFragments::erase(Object frag) { + auto frags_it = frags.find(frag); + if (frags_it == frags.end()) { + return false; + } + + assert(sorted_begin.size() == sorted_end.size()); + assert(sorted_begin.size() > frags_it->second.i_b); + + sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b); + sorted_end.erase(sorted_end.begin() + frags_it->second.i_e); + + frags.erase(frags_it); + + return true; +} + +Object Message::Contexts::ContactFragments::prev(Object frag) const { + // uses range begin to go back in time + + auto it = frags.find(frag); + if (it == frags.end()) { + return entt::null; + } + + const auto src_i = it->second.i_b; + if (src_i > 0) { + return sorted_begin[src_i-1]; + } + + return entt::null; +} + +Object Message::Contexts::ContactFragments::next(Object frag) const { + // uses range end to go forward in time + + auto it = frags.find(frag); + if (it == frags.end()) { + return entt::null; + } + + const auto src_i = it->second.i_e; + if (src_i+1 < sorted_end.size()) { + return sorted_end[src_i+1]; + } + + return entt::null; +} + diff --git a/src/fragment_store/internal_mfs_contexts.hpp b/src/fragment_store/internal_mfs_contexts.hpp new file mode 100644 index 0000000..99aecd1 --- /dev/null +++ b/src/fragment_store/internal_mfs_contexts.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include + +#include +#include + +// everything assumes a single object registry (and unique objects) + +namespace Message::Contexts { + + // ctx + struct OpenFragments { + // only contains fragments with <1024 messages and <2h tsrage (or whatever) + entt::dense_set open_frags; + }; + + // all message fragments of this contact + struct ContactFragments final { + // kept up-to-date by events + struct InternalEntry { + // indecies into the sorted arrays + size_t i_b; + size_t i_e; + }; + entt::dense_map frags; + + // add 2 sorted contact lists for both range begin and end + // TODO: adding and removing becomes expensive with enough frags, consider splitting or heap + std::vector sorted_begin; + std::vector sorted_end; + + // api + // return true if it was actually inserted + bool insert(ObjectHandle frag); + bool erase(Object frag); + // update? (just erase() + insert()) + + // uses range begin to go back in time + Object prev(Object frag) const; + // uses range end to go forward in time + Object next(Object frag) const; + }; + + // all LOADED message fragments + // TODO: merge into ContactFragments (and pull in openfrags) + struct LoadedContactFragments final { + // kept up-to-date by events + entt::dense_set loaded_frags; + }; + +} // Message::Contexts + diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 90638a9..0d09965 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -1,5 +1,7 @@ #include "./message_fragment_store.hpp" +#include "./internal_mfs_contexts.hpp" + #include #include "../json/message_components.hpp" @@ -20,55 +22,7 @@ // https://youtu.be/CU2exyhYPfA -// everything assumes a single fragment registry - -namespace Message::Components { - - // ctx - struct OpenFragments { - //struct OpenFrag final { - ////std::vector uid; - //FragmentID id; - //}; - // only contains fragments with <1024 messages and <2h tsrage (or whatever) - entt::dense_set fid_open; - }; - - // all message fragments of this contact - struct ContactFragments final { - // kept up-to-date by events - struct InternalEntry { - // indecies into the sorted arrays - size_t i_b; - size_t i_e; - }; - entt::dense_map frags; - - // add 2 sorted contact lists for both range begin and end - // TODO: adding and removing becomes expensive with enough frags, consider splitting or heap - std::vector sorted_begin; - std::vector sorted_end; - - // api - // return true if it was actually inserted - bool insert(ObjectHandle frag); - bool erase(Object frag); - // update? (just erase() + insert()) - - // uses range begin to go back in time - Object prev(Object frag) const; - // uses range end to go forward in time - Object next(Object frag) const; - }; - - // all LOADED message fragments - // TODO: merge into ContactFragments (and pull in openfrags) - struct LoadedContactFragments final { - // kept up-to-date by events - entt::dense_set frags; - }; - -} // Message::Components +// everything assumes a single object registry (and unique objects) namespace ObjectStore::Components { NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesVersion, v) @@ -135,11 +89,11 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag) if (!m.all_of()) { std::cout << "MFS: new msg missing Object\n"; - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); } - auto& fid_open = m.registry()->ctx().get().fid_open; + auto& fid_open = m.registry()->ctx().get().open_frags; const auto msg_ts = m.get().ts; // missing fuid @@ -185,10 +139,10 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // assuming ts range exists fts_comp.begin = msg_ts; // extend into the past - if (m.registry()->ctx().contains()) { + if (m.registry()->ctx().contains()) { // should be the case - m.registry()->ctx().get().erase(fh); - m.registry()->ctx().get().insert(fh); + m.registry()->ctx().get().erase(fh); + m.registry()->ctx().get().insert(fh); } @@ -202,10 +156,10 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // assuming ts range exists fts_comp.end = msg_ts; // extend into the future - if (m.registry()->ctx().contains()) { + if (m.registry()->ctx().contains()) { // should be the case - m.registry()->ctx().get().erase(fh); - m.registry()->ctx().get().insert(fh); + m.registry()->ctx().get().erase(fh); + m.registry()->ctx().get().insert(fh); } // TODO: check conditions for open here @@ -245,16 +199,16 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { } // contact frag - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); } - m.registry()->ctx().get().insert(fh); + m.registry()->ctx().get().insert(fh); // loaded contact frag - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); } - m.registry()->ctx().get().frags.emplace(fh); + m.registry()->ctx().get().loaded_frags.emplace(fh); fid_open.emplace(fragment_id); @@ -290,11 +244,11 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { return; // TODO: properly handle this case } - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); + if (!m.registry()->ctx().contains()) { + m.registry()->ctx().emplace(); } - auto& fid_open = m.registry()->ctx().get().fid_open; + auto& fid_open = m.registry()->ctx().get().open_frags; if (fid_open.contains(msg_fh)) { // TODO: dedup events @@ -330,16 +284,16 @@ void MessageFragmentStore::loadFragment(Message3Registry& reg, ObjectHandle fh) } // TODO: this should probably never be the case, since we already know here that it is a msg frag - if (!reg.ctx().contains()) { - reg.ctx().emplace(); + if (!reg.ctx().contains()) { + reg.ctx().emplace(); } - reg.ctx().get().insert(fh); + reg.ctx().get().insert(fh); // mark loaded - if (!reg.ctx().contains()) { - reg.ctx().emplace(); + if (!reg.ctx().contains()) { + reg.ctx().emplace(); } - reg.ctx().get().frags.emplace(fh); + reg.ctx().get().loaded_frags.emplace(fh); size_t messages_new_or_updated {0}; for (const auto& j_entry : j) { @@ -581,23 +535,6 @@ static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message return false; } -static bool isLess(const std::vector& lhs, const std::vector& rhs) { - size_t i = 0; - for (; i < lhs.size() && i < rhs.size(); i++) { - if (lhs[i] < rhs[i]) { - return true; - } else if (lhs[i] > rhs[i]) { - return false; - } - // else continue - } - - // here we have equality of common lenths - - // we define smaller arrays to be less - return lhs.size() < rhs.size(); -} - float MessageFragmentStore::tick(float) { const auto ts_now = Message::getTimeMS(); // sync dirty fragments here @@ -673,15 +610,15 @@ float MessageFragmentStore::tick(float) { // first do collision check agains every contact associated fragment // that is not already loaded !! - if (msg_reg->ctx().contains()) { - const auto& cf = msg_reg->ctx().get(); + if (msg_reg->ctx().contains()) { + const auto& cf = msg_reg->ctx().get(); if (!cf.frags.empty()) { - if (!msg_reg->ctx().contains()) { - msg_reg->ctx().emplace(); + if (!msg_reg->ctx().contains()) { + msg_reg->ctx().emplace(); } - const auto& loaded_frags = msg_reg->ctx().get().frags; + const auto& loaded_frags = msg_reg->ctx().get().loaded_frags; - for (const auto& [fid, si] : msg_reg->ctx().get().frags) { + for (const auto& [fid, si] : msg_reg->ctx().get().frags) { if (loaded_frags.contains(fid)) { continue; } @@ -691,14 +628,14 @@ float MessageFragmentStore::tick(float) { if (!static_cast(fh)) { std::cerr << "MFS error: frag is invalid\n"; // WHAT - msg_reg->ctx().get().erase(fid); + msg_reg->ctx().get().erase(fid); return 0.05f; } if (!fh.all_of()) { std::cerr << "MFS error: frag has no range\n"; // ???? - msg_reg->ctx().get().erase(fid); + msg_reg->ctx().get().erase(fid); return 0.05f; } @@ -887,11 +824,11 @@ bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectConstruct& e return false; } - if (!msg_reg->ctx().contains()) { - msg_reg->ctx().emplace(); + if (!msg_reg->ctx().contains()) { + msg_reg->ctx().emplace(); } - msg_reg->ctx().get().erase(e.e); // TODO: can this happen? update - msg_reg->ctx().get().insert(e.e); + msg_reg->ctx().get().erase(e.e); // TODO: can this happen? update + msg_reg->ctx().get().insert(e.e); _event_check_queue.push_back(ECQueueEntry{e.e, frag_contact}); @@ -943,11 +880,11 @@ bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectUpdate& e) { return false; } - if (!msg_reg->ctx().contains()) { - msg_reg->ctx().emplace(); + if (!msg_reg->ctx().contains()) { + msg_reg->ctx().emplace(); } - msg_reg->ctx().get().erase(e.e); // TODO: check/update/fragment update - msg_reg->ctx().get().insert(e.e); + msg_reg->ctx().get().erase(e.e); // TODO: check/update/fragment update + msg_reg->ctx().get().insert(e.e); // TODO: actually load it //_event_check_queue.push_back(ECQueueEntry{e.e, frag_contact}); @@ -955,125 +892,3 @@ bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectUpdate& e) { return false; } -bool Message::Components::ContactFragments::insert(ObjectHandle frag) { - if (frags.contains(frag)) { - return false; - } - - // both sorted arrays are sorted ascending - // so for insertion we search for the last index that is <= and insert after it - // or we search for the first > (or end) and insert before it <--- - // since equal fragments are UB, we can assume they are only > or < - - size_t begin_index {0}; - { // begin - const auto pos = std::find_if( - sorted_begin.cbegin(), - sorted_begin.cend(), - [frag](const Object a) -> bool { - const auto begin_a = frag.registry()->get(a).begin; - const auto begin_frag = frag.get().begin; - if (begin_a > begin_frag) { - return true; - } else if (begin_a < begin_frag) { - return false; - } else { - // equal ts, we need to fall back to id (id can not be equal) - return isLess(frag.get().v, frag.registry()->get(a).v); - } - } - ); - - begin_index = std::distance(sorted_begin.cbegin(), pos); - - // we need to insert before pos (end is valid here) - sorted_begin.insert(pos, frag); - } - - size_t end_index {0}; - { // end - const auto pos = std::find_if_not( - sorted_end.cbegin(), - sorted_end.cend(), - [frag](const Object a) -> bool { - const auto end_a = frag.registry()->get(a).end; - const auto end_frag = frag.get().end; - if (end_a > end_frag) { - return true; - } else if (end_a < end_frag) { - return false; - } else { - // equal ts, we need to fall back to id (id can not be equal) - return isLess(frag.get().v, frag.registry()->get(a).v); - } - } - ); - - end_index = std::distance(sorted_end.cbegin(), pos); - - // we need to insert before pos (end is valid here) - sorted_end.insert(pos, frag); - } - - frags.emplace(frag, InternalEntry{begin_index, end_index}); - - // now adjust all indicies of fragments coming after the insert position - for (size_t i = begin_index + 1; i < sorted_begin.size(); i++) { - frags.at(sorted_begin[i]).i_b = i; - } - for (size_t i = end_index + 1; i < sorted_end.size(); i++) { - frags.at(sorted_end[i]).i_e = i; - } - - return true; -} - -bool Message::Components::ContactFragments::erase(Object frag) { - auto frags_it = frags.find(frag); - if (frags_it == frags.end()) { - return false; - } - - assert(sorted_begin.size() == sorted_end.size()); - assert(sorted_begin.size() > frags_it->second.i_b); - - sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b); - sorted_end.erase(sorted_end.begin() + frags_it->second.i_e); - - frags.erase(frags_it); - - return true; -} - -Object Message::Components::ContactFragments::prev(Object frag) const { - // uses range begin to go back in time - - auto it = frags.find(frag); - if (it == frags.end()) { - return entt::null; - } - - const auto src_i = it->second.i_b; - if (src_i > 0) { - return sorted_begin[src_i-1]; - } - - return entt::null; -} - -Object Message::Components::ContactFragments::next(Object frag) const { - // uses range end to go forward in time - - auto it = frags.find(frag); - if (it == frags.end()) { - return entt::null; - } - - const auto src_i = it->second.i_e; - if (src_i+1 < sorted_end.size()) { - return sorted_end[src_i+1]; - } - - return entt::null; -} - diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 1ac4402..2f13aca 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -25,6 +25,7 @@ namespace Message::Components { //using FUID = FragComp::ID; struct Obj { + // message fragment's object Object o {entt::null}; }; From f287348550f2e6299334d264ebab81b713a034f9 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 13 Apr 2024 19:13:18 +0200 Subject: [PATCH 90/98] introduce message fragments version 2 (msgpack) more smaller refactors --- src/fragment_store/internal_mfs_contexts.cpp | 22 ++--- src/fragment_store/internal_mfs_contexts.hpp | 2 +- src/fragment_store/message_fragment_store.cpp | 81 +++++++++++++------ src/fragment_store/message_fragment_store.hpp | 2 +- src/fragment_store/message_serializer.cpp | 18 ++++- .../messages_meta_components.hpp | 3 +- 6 files changed, 88 insertions(+), 40 deletions(-) diff --git a/src/fragment_store/internal_mfs_contexts.cpp b/src/fragment_store/internal_mfs_contexts.cpp index a07ce4c..cadb952 100644 --- a/src/fragment_store/internal_mfs_contexts.cpp +++ b/src/fragment_store/internal_mfs_contexts.cpp @@ -26,7 +26,7 @@ static bool isLess(const std::vector& lhs, const std::vector& } bool Message::Contexts::ContactFragments::insert(ObjectHandle frag) { - if (frags.contains(frag)) { + if (sorted_frags.contains(frag)) { return false; } @@ -85,22 +85,22 @@ bool Message::Contexts::ContactFragments::insert(ObjectHandle frag) { sorted_end.insert(pos, frag); } - frags.emplace(frag, InternalEntry{begin_index, end_index}); + sorted_frags.emplace(frag, InternalEntry{begin_index, end_index}); // now adjust all indicies of fragments coming after the insert position for (size_t i = begin_index + 1; i < sorted_begin.size(); i++) { - frags.at(sorted_begin[i]).i_b = i; + sorted_frags.at(sorted_begin[i]).i_b = i; } for (size_t i = end_index + 1; i < sorted_end.size(); i++) { - frags.at(sorted_end[i]).i_e = i; + sorted_frags.at(sorted_end[i]).i_e = i; } return true; } bool Message::Contexts::ContactFragments::erase(Object frag) { - auto frags_it = frags.find(frag); - if (frags_it == frags.end()) { + auto frags_it = sorted_frags.find(frag); + if (frags_it == sorted_frags.end()) { return false; } @@ -110,7 +110,7 @@ bool Message::Contexts::ContactFragments::erase(Object frag) { sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b); sorted_end.erase(sorted_end.begin() + frags_it->second.i_e); - frags.erase(frags_it); + sorted_frags.erase(frags_it); return true; } @@ -118,8 +118,8 @@ bool Message::Contexts::ContactFragments::erase(Object frag) { Object Message::Contexts::ContactFragments::prev(Object frag) const { // uses range begin to go back in time - auto it = frags.find(frag); - if (it == frags.end()) { + auto it = sorted_frags.find(frag); + if (it == sorted_frags.end()) { return entt::null; } @@ -134,8 +134,8 @@ Object Message::Contexts::ContactFragments::prev(Object frag) const { Object Message::Contexts::ContactFragments::next(Object frag) const { // uses range end to go forward in time - auto it = frags.find(frag); - if (it == frags.end()) { + auto it = sorted_frags.find(frag); + if (it == sorted_frags.end()) { return entt::null; } diff --git a/src/fragment_store/internal_mfs_contexts.hpp b/src/fragment_store/internal_mfs_contexts.hpp index 99aecd1..4283de8 100644 --- a/src/fragment_store/internal_mfs_contexts.hpp +++ b/src/fragment_store/internal_mfs_contexts.hpp @@ -23,7 +23,7 @@ namespace Message::Contexts { size_t i_b; size_t i_e; }; - entt::dense_map frags; + entt::dense_map sorted_frags; // add 2 sorted contact lists for both range begin and end // TODO: adding and removing becomes expensive with enough frags, consider splitting or heap diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 0d09965..7284432 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -5,6 +5,9 @@ #include #include "../json/message_components.hpp" +#include "messages_meta_components.hpp" +#include "nlohmann/json_fwd.hpp" +#include "solanaceae/util/span.hpp" #include @@ -55,7 +58,16 @@ static nlohmann::json loadFromStorageNJ(ObjectHandle oh) { return false; } - return nlohmann::json::parse(tmp_buffer, nullptr, false); + const auto obj_version = oh.get().v; + + if (obj_version == 1) { + return nlohmann::json::parse(tmp_buffer, nullptr, false); + } else if (obj_version == 2) { + return nlohmann::json::from_msgpack(tmp_buffer, true, false); + } else { + assert(false); + return {}; + } } void MessageFragmentStore::handleMessage(const Message3Handle& m) { @@ -228,13 +240,13 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { m.emplace_or_replace(fragment_id); // in this case we know the fragment needs an update - for (const auto& it : _fuid_save_queue) { + for (const auto& it : _frag_save_queue) { if (it.id == fragment_id) { // already in queue return; // done } } - _fuid_save_queue.push_back({Message::getTimeMS(), {_os.registry(), fragment_id}, m.registry()}); + _frag_save_queue.push_back({Message::getTimeMS(), {_os.registry(), fragment_id}, m.registry()}); return; // done } @@ -253,7 +265,7 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { if (fid_open.contains(msg_fh)) { // TODO: dedup events // TODO: cooldown per fragsave - _fuid_save_queue.push_back({Message::getTimeMS(), msg_fh, m.registry()}); + _frag_save_queue.push_back({Message::getTimeMS(), msg_fh, m.registry()}); return; } @@ -269,7 +281,20 @@ void MessageFragmentStore::handleMessage(const Message3Handle& m) { // need update from frag void MessageFragmentStore::loadFragment(Message3Registry& reg, ObjectHandle fh) { std::cout << "MFS: loadFragment\n"; - const auto j = loadFromStorageNJ(fh); + // version HAS to be set, or we just fail + if (!fh.all_of()) { + std::cerr << "MFS error: nope, object without version, cant load\n"; + return; + } + + nlohmann::json j; + const auto obj_version = fh.get().v; + if (obj_version == 1 || obj_version == 2) { + j = loadFromStorageNJ(fh); // also handles version and json/msgpack + } else { + std::cerr << "MFS error: nope, object with unknown version, cant load\n"; + return; + } if (!j.is_array()) { // wrong data @@ -377,7 +402,7 @@ bool MessageFragmentStore::syncFragToStorage(ObjectHandle fh, Message3Registry& // TODO: does every message have ts? auto msg_view = reg.view(); - // we also assume all messages have fid + // we also assume all messages have an associated object for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) { const Message3 m = *it; @@ -385,12 +410,13 @@ bool MessageFragmentStore::syncFragToStorage(ObjectHandle fh, Message3Registry& continue; } - // require msg for now + // filter: require msg for now + // this will be removed in the future if (!reg.any_of(m)) { continue; } - if (_fuid_save_queue.front().id != reg.get(m).o) { + if (_frag_save_queue.front().id != reg.get(m).o) { continue; // not ours } @@ -430,13 +456,20 @@ bool MessageFragmentStore::syncFragToStorage(ObjectHandle fh, Message3Registry& // we cant skip if array is empty (in theory it will not be empty later on) - // if save as binary - //nlohmann::json::to_msgpack(j); - auto j_dump = j.dump(2, ' ', true); + std::vector data_to_save; + const auto obj_version = fh.get_or_emplace().v; + if (obj_version == 1) { + auto j_dump = j.dump(2, ' ', true); + data_to_save = std::vector(j_dump.cbegin(), j_dump.cend()); + } else if (obj_version == 2) { + data_to_save = nlohmann::json::to_msgpack(j); + } else { + std::cerr << "MFS error: unknown object version\n"; + assert(false); + } assert(fh.all_of()); auto* backend = fh.get().ptr; - //if (_os.syncToStorage(fh, reinterpret_cast(j_dump.data()), j_dump.size())) { - if (backend->write(fh, {reinterpret_cast(j_dump.data()), j_dump.size()})) { + if (backend->write(fh, {reinterpret_cast(data_to_save.data()), data_to_save.size()})) { // TODO: make this better, should this be called on fail? should this be called before sync? (prob not) _fs_ignore_event = true; _os.throwEventUpdate(fh); @@ -480,12 +513,12 @@ MessageFragmentStore::MessageFragmentStore( } MessageFragmentStore::~MessageFragmentStore(void) { - while (!_fuid_save_queue.empty()) { - auto fh = _fuid_save_queue.front().id; - auto* reg = _fuid_save_queue.front().reg; + while (!_frag_save_queue.empty()) { + auto fh = _frag_save_queue.front().id; + auto* reg = _frag_save_queue.front().reg; assert(reg != nullptr); syncFragToStorage(fh, *reg); - _fuid_save_queue.pop_front(); // pop unconditionally + _frag_save_queue.pop_front(); // pop unconditionally } } @@ -538,14 +571,14 @@ static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message float MessageFragmentStore::tick(float) { const auto ts_now = Message::getTimeMS(); // sync dirty fragments here - if (!_fuid_save_queue.empty()) { + if (!_frag_save_queue.empty()) { // wait 10sec before saving - if (_fuid_save_queue.front().ts_since_dirty + 10*1000 <= ts_now) { - auto fh = _fuid_save_queue.front().id; - auto* reg = _fuid_save_queue.front().reg; + if (_frag_save_queue.front().ts_since_dirty + 10*1000 <= ts_now) { + auto fh = _frag_save_queue.front().id; + auto* reg = _frag_save_queue.front().reg; assert(reg != nullptr); if (syncFragToStorage(fh, *reg)) { - _fuid_save_queue.pop_front(); + _frag_save_queue.pop_front(); } } } @@ -612,13 +645,13 @@ float MessageFragmentStore::tick(float) { // that is not already loaded !! if (msg_reg->ctx().contains()) { const auto& cf = msg_reg->ctx().get(); - if (!cf.frags.empty()) { + if (!cf.sorted_frags.empty()) { if (!msg_reg->ctx().contains()) { msg_reg->ctx().emplace(); } const auto& loaded_frags = msg_reg->ctx().get().loaded_frags; - for (const auto& [fid, si] : msg_reg->ctx().get().frags) { + for (const auto& [fid, si] : msg_reg->ctx().get().sorted_frags) { if (loaded_frags.contains(fid)) { continue; } diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index 2f13aca..f7ec69f 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -86,7 +86,7 @@ class MessageFragmentStore : public RegistryMessageModelEventI, public ObjectSto ObjectHandle id; Message3Registry* reg{nullptr}; }; - std::deque _fuid_save_queue; + std::deque _frag_save_queue; struct ECQueueEntry final { ObjectHandle fid; diff --git a/src/fragment_store/message_serializer.cpp b/src/fragment_store/message_serializer.cpp index 35823e6..7b86c5a 100644 --- a/src/fragment_store/message_serializer.cpp +++ b/src/fragment_store/message_serializer.cpp @@ -1,5 +1,6 @@ #include "./message_serializer.hpp" +#include #include #include @@ -39,6 +40,7 @@ bool MessageSerializerCallbacks::component_get_json bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) { if (j.is_null()) { @@ -47,7 +49,12 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json id = j.is_binary()?j:j["bytes"]; + std::vector id; + if (j.is_binary()) { + id = j.get_binary(); + } else { + j["bytes"].get_to(id); + } Contact3 other_c = findContactByID(msc.cr, id); if (!msc.cr.valid(other_c)) { @@ -83,6 +90,7 @@ bool MessageSerializerCallbacks::component_get_json bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) { if (j.is_null()) { @@ -91,7 +99,12 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json id = j.is_binary()?j:j["bytes"]; + std::vector id; + if (j.is_binary()) { + id = j.get_binary(); + } else { + j["bytes"].get_to(id); + } Contact3 other_c = findContactByID(msc.cr, id); if (!msc.cr.valid(other_c)) { @@ -105,3 +118,4 @@ bool MessageSerializerCallbacks::component_emplace_or_replace_json text_json - uint16_t v {1}; + // 2 -> msgpack + uint16_t v {2}; }; struct MessagesTSRange { From 7b4af5854480e06fc1b79968fa7be3dfe1f9cdd0 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 13 Apr 2024 19:47:28 +0200 Subject: [PATCH 91/98] prep convert tool for msgpack transition --- src/fragment_store/convert_frag_to_obj.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index d7520c7..dc32637 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -121,6 +121,20 @@ int main(int argc, const char** argv) { } } + // WARNING: manual and hardcoded + // manually upconvert message json to msgpack (v1 to v2) + if (true && oh.get().v == 1) { + // TODO: error handling + const auto j = nlohmann::json::parse(tmp_buffer); + + if (false) { + oh.replace(uint16_t(2)); + + // overwrite og + tmp_buffer = nlohmann::json::to_msgpack(j); + } + } + static_cast(_fsb_dst).write(oh, ByteSpan{tmp_buffer}); return false; From de3b8f059ea4ed1b030672ce8a9b806be1f35e85 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 13 Apr 2024 21:06:51 +0200 Subject: [PATCH 92/98] import os and zstdfile2 bugfixes --- external/solanaceae_object_store | 2 +- src/fragment_store/convert_frag_to_obj.cpp | 30 ++++++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/external/solanaceae_object_store b/external/solanaceae_object_store index 4d3ffb8..4695579 160000 --- a/external/solanaceae_object_store +++ b/external/solanaceae_object_store @@ -1 +1 @@ -Subproject commit 4d3ffb8192623740f6e170855ee1cffd428b78da +Subproject commit 46955795b034ed4b058958bba620bd8965ce91f7 diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index dc32637..7881431 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -93,6 +93,20 @@ int main(int argc, const char** argv) { } } + // WARNING: manual and hardcoded + // manually upconvert message json to msgpack (v1 to v2) + if (true && e.e.get().v == 1) { + // TODO: error handling + const auto j = nlohmann::json::parse(tmp_buffer); + + if (false) { + e.e.replace(uint16_t(2)); + + // overwrite og + tmp_buffer = nlohmann::json::to_msgpack(j); + } + } + // we dont copy meta file type, it will be the same for all "new" objects auto oh = _fsb_dst.newObject(ByteSpan{e.e.get().v}); @@ -121,22 +135,10 @@ int main(int argc, const char** argv) { } } - // WARNING: manual and hardcoded - // manually upconvert message json to msgpack (v1 to v2) - if (true && oh.get().v == 1) { - // TODO: error handling - const auto j = nlohmann::json::parse(tmp_buffer); - - if (false) { - oh.replace(uint16_t(2)); - - // overwrite og - tmp_buffer = nlohmann::json::to_msgpack(j); - } - } - static_cast(_fsb_dst).write(oh, ByteSpan{tmp_buffer}); + //assert(std::filesystem::file_size(e.e.get().path) == std::filesystem::file_size(oh.get().path)); + return false; } From a845609660efb45a2bc56221b7430ee6d4573cc3 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 14 Apr 2024 10:23:13 +0200 Subject: [PATCH 93/98] actually allow loading v2 and enable in converter --- src/fragment_store/convert_frag_to_obj.cpp | 2 +- src/fragment_store/message_fragment_store.cpp | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 7881431..b3cee8c 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -99,7 +99,7 @@ int main(int argc, const char** argv) { // TODO: error handling const auto j = nlohmann::json::parse(tmp_buffer); - if (false) { + if (true) { e.e.replace(uint16_t(2)); // overwrite og diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp index 7284432..4042b1d 100644 --- a/src/fragment_store/message_fragment_store.cpp +++ b/src/fragment_store/message_fragment_store.cpp @@ -607,7 +607,9 @@ float MessageFragmentStore::tick(float) { // missing version, adding fh.emplace(); } - if (fh.get().v != 1) { + const auto object_version = fh.get().v; + // TODO: move this early version check somewhere else + if (object_version != 1 && object_version != 2) { std::cerr << "MFS: object with version mismatch\n"; return 0.05f; } From a6614e76cee3c48315f9d27c3e579b3b852f48a1 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 14 Apr 2024 11:13:05 +0200 Subject: [PATCH 94/98] move cursers to public api --- external/solanaceae_message3 | 2 +- src/chat_gui4.hpp | 3 --- src/fragment_store/message_fragment_store.hpp | 16 ---------------- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/external/solanaceae_message3 b/external/solanaceae_message3 index 7c28b23..f9f70a0 160000 --- a/external/solanaceae_message3 +++ b/external/solanaceae_message3 @@ -1 +1 @@ -Subproject commit 7c28b232a46ebede9d6f09bc6eafb49bacfa99ea +Subproject commit f9f70a05b1d248e84198c83abeda3579107d09bb diff --git a/src/chat_gui4.hpp b/src/chat_gui4.hpp index 283d5fd..9de29e2 100644 --- a/src/chat_gui4.hpp +++ b/src/chat_gui4.hpp @@ -10,9 +10,6 @@ #include "./file_selector.hpp" #include "./send_image_popup.hpp" -// HACK: move to public msg api? -#include "./fragment_store/message_fragment_store.hpp" - #include #include diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp index f7ec69f..c5f9f7e 100644 --- a/src/fragment_store/message_fragment_store.hpp +++ b/src/fragment_store/message_fragment_store.hpp @@ -29,22 +29,6 @@ namespace Message::Components { Object o {entt::null}; }; - // points to the front/newer message - // together they define a range that is, - // eg the first(end) and last(begin) message being rendered - // MFS requires there to be atleast one other fragment after/before, - // if not loaded fragment with fitting tsrange(direction) available - // uses fragmentAfter/Before() - // they can exist standalone - // if they are a pair, the inside is filled first - // cursers require a timestamp ??? - struct ViewCurserBegin { - Message3 curser_end{entt::null}; - }; - struct ViewCurserEnd { - Message3 curser_begin{entt::null}; - }; - // TODO: add adjacency range comp or inside curser // TODO: unused From da83065024cda223c40662d5bd9fbb8e0d5a4a74 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 14 Apr 2024 12:03:58 +0200 Subject: [PATCH 95/98] make sure we have the right json version in the flake --- flake.lock | 20 +++++++++++++++++++- flake.nix | 11 +++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 806ef1b..c4366bf 100644 --- a/flake.lock +++ b/flake.lock @@ -34,10 +34,28 @@ "type": "github" } }, + "nlohmann-json": { + "flake": false, + "locked": { + "lastModified": 1701207391, + "narHash": "sha256-7F0Jon+1oWL7uqet5i1IgHX0fUw/+z0QwEcA3zs5xHg=", + "owner": "nlohmann", + "repo": "json", + "rev": "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03", + "type": "github" + }, + "original": { + "owner": "nlohmann", + "ref": "v3.11.3", + "repo": "json", + "type": "github" + } + }, "root": { "inputs": { "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nlohmann-json": "nlohmann-json" } }, "systems": { diff --git a/flake.nix b/flake.nix index 934ef57..e01383f 100644 --- a/flake.nix +++ b/flake.nix @@ -6,9 +6,13 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; flake-utils.url = "github:numtide/flake-utils"; + nlohmann-json = { + url = "github:nlohmann/json/v3.11.3"; # TODO: read version from file + flake = false; + }; }; - outputs = { self, nixpkgs, flake-utils }: + outputs = { self, nixpkgs, flake-utils, nlohmann-json }: flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; }; @@ -61,9 +65,8 @@ "-DTOMATO_ASAN=OFF" "-DCMAKE_BUILD_TYPE=RelWithDebInfo" - "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here - # do we really care less about the version? do we need a stable abi? - "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" + "-DFETCHCONTENT_SOURCE_DIR_JSON=${nlohmann-json}" # we care about the version + "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # we dont care about the version (we use 1.4.x features) ]; # TODO: replace with install command From f932f5ffb4ed7539378207f58d18ea539e5f15e1 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 14 Apr 2024 13:58:31 +0200 Subject: [PATCH 96/98] mfs and ms moved to their own repo, now only a few files left to clean up --- .gitmodules | 3 + external/CMakeLists.txt | 1 + external/solanaceae_message_serializer | 1 + src/CMakeLists.txt | 41 +- src/fragment_store/README.md | 76 -- src/fragment_store/convert_frag_to_obj.cpp | 2 +- src/fragment_store/internal_mfs_contexts.cpp | 149 --- src/fragment_store/internal_mfs_contexts.hpp | 53 - src/fragment_store/message_fragment_store.cpp | 929 ------------------ src/fragment_store/message_fragment_store.hpp | 107 -- src/fragment_store/message_serializer.cpp | 121 --- src/fragment_store/message_serializer.hpp | 85 -- .../messages_meta_components.hpp | 34 - .../messages_meta_components_id.inl | 31 - .../register_mfs_json_message_components.cpp | 35 - .../register_mfs_json_message_components.hpp | 6 - ...gister_mfs_json_tox_message_components.cpp | 10 +- ...gister_mfs_json_tox_message_components.hpp | 4 +- src/json/message_components.hpp | 27 - src/main_screen.cpp | 19 +- src/main_screen.hpp | 6 +- 21 files changed, 32 insertions(+), 1708 deletions(-) create mode 160000 external/solanaceae_message_serializer delete mode 100644 src/fragment_store/README.md delete mode 100644 src/fragment_store/internal_mfs_contexts.cpp delete mode 100644 src/fragment_store/internal_mfs_contexts.hpp delete mode 100644 src/fragment_store/message_fragment_store.cpp delete mode 100644 src/fragment_store/message_fragment_store.hpp delete mode 100644 src/fragment_store/message_serializer.cpp delete mode 100644 src/fragment_store/message_serializer.hpp delete mode 100644 src/fragment_store/messages_meta_components.hpp delete mode 100644 src/fragment_store/messages_meta_components_id.inl delete mode 100644 src/fragment_store/register_mfs_json_message_components.cpp delete mode 100644 src/fragment_store/register_mfs_json_message_components.hpp delete mode 100644 src/json/message_components.hpp diff --git a/.gitmodules b/.gitmodules index 4d17134..bdf1441 100644 --- a/.gitmodules +++ b/.gitmodules @@ -23,3 +23,6 @@ [submodule "external/solanaceae_object_store"] path = external/solanaceae_object_store url = https://github.com/Green-Sky/solanaceae_object_store.git +[submodule "external/solanaceae_message_serializer"] + path = external/solanaceae_message_serializer + url = https://github.com/Green-Sky/solanaceae_message_serializer.git diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 5019deb..27a2242 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(./entt) add_subdirectory(./solanaceae_util) add_subdirectory(./solanaceae_contact) add_subdirectory(./solanaceae_message3) +add_subdirectory(./solanaceae_message_serializer) add_subdirectory(./solanaceae_plugin) diff --git a/external/solanaceae_message_serializer b/external/solanaceae_message_serializer new file mode 160000 index 0000000..1409485 --- /dev/null +++ b/external/solanaceae_message_serializer @@ -0,0 +1 @@ +Subproject commit 1409485ef1ee4a2bcf38d7f4631f33e8646d8718 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 91722ad..073a9b9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,41 +2,14 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) ######################################## -add_library(message_fragment_store - ./json/message_components.hpp # TODO: move - ./json/tox_message_components.hpp # TODO: move - - ./fragment_store/message_serializer.hpp - ./fragment_store/message_serializer.cpp - ./fragment_store/messages_meta_components.hpp - ./fragment_store/messages_meta_components_id.inl - ./fragment_store/internal_mfs_contexts.hpp - ./fragment_store/internal_mfs_contexts.cpp - ./fragment_store/message_fragment_store.hpp - ./fragment_store/message_fragment_store.cpp - - ./fragment_store/register_mfs_json_message_components.hpp - ./fragment_store/register_mfs_json_message_components.cpp - ./fragment_store/register_mfs_json_tox_message_components.hpp - ./fragment_store/register_mfs_json_tox_message_components.cpp -) -target_compile_features(message_fragment_store PRIVATE cxx_std_20) -target_link_libraries(message_fragment_store PUBLIC - solanaceae_object_store - solanaceae_message3 - solanaceae_tox_messages # TODO: move -) - -######################################## - -add_executable(convert_message_object_store +add_executable(convert_message_object_store EXCLUDE_FROM_ALL fragment_store/convert_frag_to_obj.cpp ) target_link_libraries(convert_message_object_store PUBLIC solanaceae_object_store solanaceae_object_store_backend_filesystem - message_fragment_store + solanaceae_message_fragment_store ) ######################################## @@ -44,6 +17,13 @@ add_executable(tomato ./main.cpp ./icon.rc + + # TODO: mfs leftovers, need to move + ./json/tox_message_components.hpp # TODO: move + ./fragment_store/register_mfs_json_tox_message_components.hpp + ./fragment_store/register_mfs_json_tox_message_components.cpp + + ./screen.hpp ./start_screen.hpp ./start_screen.cpp @@ -115,6 +95,7 @@ target_link_libraries(tomato PUBLIC solanaceae_util solanaceae_contact solanaceae_message3 + solanaceae_message_serializer solanaceae_plugin @@ -123,8 +104,6 @@ target_link_libraries(tomato PUBLIC solanaceae_tox_messages solanaceae_object_store - solanaceae_object_store_backend_filesystem - message_fragment_store SDL3::SDL3 diff --git a/src/fragment_store/README.md b/src/fragment_store/README.md deleted file mode 100644 index 0eba57b..0000000 --- a/src/fragment_store/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# Fragment Store - -Fragments are are pieces of information split into Metadata and Data. -They can be stored seperated or together. -They can be used as a Transport protocol/logic too. - -# Store types - -### Object Store - -Fragment files are stored with the first 2 hex chars as sub folders: -eg: -`objects/` (object store root) - - `5f/` (first 2hex subfolder) - - `4fffffff` (the fragment file without the first 2 hexchars) - -### Split Object Store - -Same as Object Store, but medadata and data stored in seperate files. -Metadata files have the `.meta` suffix. They also have a filetype specific suffix, like `.json`, `.msgpack` etc. - -### Memory Store - -Just keeps the Fragments in memory. - -# File formats - -Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted. - -### Text Json - -Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes). -Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised. - -A Metadata json object can have arbitrary keys, some are predefined: -- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any -- `FragComp::DataCompressionType` (uint) Compression type of the data, if any - -## Binary file headers - -### Split Metadata - -msgpack array: - -- `[0]`: file magic string `SOLMET` (6 bytes) -- `[1]`: uint8 encryption type (`0x00` is none) -- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd) -- `[3]`: binary metadata (optionally compressed and encrypted) - -note that the encryption and compression are for the metadata only. -The metadata itself contains encryption and compression info about the data. - -### Split Data - -All the metadata is in the metadata file. (like encryption and compression) -This is mostly to allow direct storage for files in the Fragment store without excessive duplication. -Keep in mind to not use the actual file name as the data/meta file name. - -### Single fragment - -Note: this format is unused for now - -file magic bytes `SOLFIL` (6 bytes) - -1 byte encryption type (`0x00` is none) - -1 byte compression type (`0x00` is none) - -...metadata here... - -...data here... - -## Compression types - -- `0x00` none -- `0x01` zstd (without dict) diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index b3cee8c..337edc7 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "./message_fragment_store.hpp" +#include #include diff --git a/src/fragment_store/internal_mfs_contexts.cpp b/src/fragment_store/internal_mfs_contexts.cpp deleted file mode 100644 index cadb952..0000000 --- a/src/fragment_store/internal_mfs_contexts.cpp +++ /dev/null @@ -1,149 +0,0 @@ -#include "./internal_mfs_contexts.hpp" - -#include "./message_fragment_store.hpp" - -#include -#include -#include - -#include - -static bool isLess(const std::vector& lhs, const std::vector& rhs) { - size_t i = 0; - for (; i < lhs.size() && i < rhs.size(); i++) { - if (lhs[i] < rhs[i]) { - return true; - } else if (lhs[i] > rhs[i]) { - return false; - } - // else continue - } - - // here we have equality of common lenths - - // we define smaller arrays to be less - return lhs.size() < rhs.size(); -} - -bool Message::Contexts::ContactFragments::insert(ObjectHandle frag) { - if (sorted_frags.contains(frag)) { - return false; - } - - // both sorted arrays are sorted ascending - // so for insertion we search for the last index that is <= and insert after it - // or we search for the first > (or end) and insert before it <--- - // since equal fragments are UB, we can assume they are only > or < - - size_t begin_index {0}; - { // begin - const auto pos = std::find_if( - sorted_begin.cbegin(), - sorted_begin.cend(), - [frag](const Object a) -> bool { - const auto begin_a = frag.registry()->get(a).begin; - const auto begin_frag = frag.get().begin; - if (begin_a > begin_frag) { - return true; - } else if (begin_a < begin_frag) { - return false; - } else { - // equal ts, we need to fall back to id (id can not be equal) - return isLess(frag.get().v, frag.registry()->get(a).v); - } - } - ); - - begin_index = std::distance(sorted_begin.cbegin(), pos); - - // we need to insert before pos (end is valid here) - sorted_begin.insert(pos, frag); - } - - size_t end_index {0}; - { // end - const auto pos = std::find_if_not( - sorted_end.cbegin(), - sorted_end.cend(), - [frag](const Object a) -> bool { - const auto end_a = frag.registry()->get(a).end; - const auto end_frag = frag.get().end; - if (end_a > end_frag) { - return true; - } else if (end_a < end_frag) { - return false; - } else { - // equal ts, we need to fall back to id (id can not be equal) - return isLess(frag.get().v, frag.registry()->get(a).v); - } - } - ); - - end_index = std::distance(sorted_end.cbegin(), pos); - - // we need to insert before pos (end is valid here) - sorted_end.insert(pos, frag); - } - - sorted_frags.emplace(frag, InternalEntry{begin_index, end_index}); - - // now adjust all indicies of fragments coming after the insert position - for (size_t i = begin_index + 1; i < sorted_begin.size(); i++) { - sorted_frags.at(sorted_begin[i]).i_b = i; - } - for (size_t i = end_index + 1; i < sorted_end.size(); i++) { - sorted_frags.at(sorted_end[i]).i_e = i; - } - - return true; -} - -bool Message::Contexts::ContactFragments::erase(Object frag) { - auto frags_it = sorted_frags.find(frag); - if (frags_it == sorted_frags.end()) { - return false; - } - - assert(sorted_begin.size() == sorted_end.size()); - assert(sorted_begin.size() > frags_it->second.i_b); - - sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b); - sorted_end.erase(sorted_end.begin() + frags_it->second.i_e); - - sorted_frags.erase(frags_it); - - return true; -} - -Object Message::Contexts::ContactFragments::prev(Object frag) const { - // uses range begin to go back in time - - auto it = sorted_frags.find(frag); - if (it == sorted_frags.end()) { - return entt::null; - } - - const auto src_i = it->second.i_b; - if (src_i > 0) { - return sorted_begin[src_i-1]; - } - - return entt::null; -} - -Object Message::Contexts::ContactFragments::next(Object frag) const { - // uses range end to go forward in time - - auto it = sorted_frags.find(frag); - if (it == sorted_frags.end()) { - return entt::null; - } - - const auto src_i = it->second.i_e; - if (src_i+1 < sorted_end.size()) { - return sorted_end[src_i+1]; - } - - return entt::null; -} - diff --git a/src/fragment_store/internal_mfs_contexts.hpp b/src/fragment_store/internal_mfs_contexts.hpp deleted file mode 100644 index 4283de8..0000000 --- a/src/fragment_store/internal_mfs_contexts.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include - -#include -#include - -// everything assumes a single object registry (and unique objects) - -namespace Message::Contexts { - - // ctx - struct OpenFragments { - // only contains fragments with <1024 messages and <2h tsrage (or whatever) - entt::dense_set open_frags; - }; - - // all message fragments of this contact - struct ContactFragments final { - // kept up-to-date by events - struct InternalEntry { - // indecies into the sorted arrays - size_t i_b; - size_t i_e; - }; - entt::dense_map sorted_frags; - - // add 2 sorted contact lists for both range begin and end - // TODO: adding and removing becomes expensive with enough frags, consider splitting or heap - std::vector sorted_begin; - std::vector sorted_end; - - // api - // return true if it was actually inserted - bool insert(ObjectHandle frag); - bool erase(Object frag); - // update? (just erase() + insert()) - - // uses range begin to go back in time - Object prev(Object frag) const; - // uses range end to go forward in time - Object next(Object frag) const; - }; - - // all LOADED message fragments - // TODO: merge into ContactFragments (and pull in openfrags) - struct LoadedContactFragments final { - // kept up-to-date by events - entt::dense_set loaded_frags; - }; - -} // Message::Contexts - diff --git a/src/fragment_store/message_fragment_store.cpp b/src/fragment_store/message_fragment_store.cpp deleted file mode 100644 index 4042b1d..0000000 --- a/src/fragment_store/message_fragment_store.cpp +++ /dev/null @@ -1,929 +0,0 @@ -#include "./message_fragment_store.hpp" - -#include "./internal_mfs_contexts.hpp" - -#include - -#include "../json/message_components.hpp" -#include "messages_meta_components.hpp" -#include "nlohmann/json_fwd.hpp" -#include "solanaceae/util/span.hpp" - -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -// https://youtu.be/CU2exyhYPfA - -// everything assumes a single object registry (and unique objects) - -namespace ObjectStore::Components { - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesVersion, v) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id) - - namespace Ephemeral { - // does not contain any messges - // (recheck on frag update) - struct MessagesEmptyTag {}; - - // cache the contact for faster lookups - struct MessagesContactEntity { - Contact3 e {entt::null}; - }; - } -} // ObjectStore::Component - -static nlohmann::json loadFromStorageNJ(ObjectHandle oh) { - assert(oh.all_of()); - auto* backend = oh.get().ptr; - assert(backend != nullptr); - - std::vector tmp_buffer; - std::function cb = [&tmp_buffer](const ByteSpan buffer) { - tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); - }; - if (!backend->read(oh, cb)) { - std::cerr << "failed to read obj '" << bin2hex(oh.get().v) << "'\n"; - return false; - } - - const auto obj_version = oh.get().v; - - if (obj_version == 1) { - return nlohmann::json::parse(tmp_buffer, nullptr, false); - } else if (obj_version == 2) { - return nlohmann::json::from_msgpack(tmp_buffer, true, false); - } else { - assert(false); - return {}; - } -} - -void MessageFragmentStore::handleMessage(const Message3Handle& m) { - if (_fs_ignore_event) { - // message event because of us loading a fragment, ignore - // TODO: this barely makes a difference - return; - } - - if (!static_cast(m)) { - return; // huh? - } - - if (!m.all_of()) { - return; // we only handle msg with ts - } - - _potentially_dirty_contacts.emplace(m.registry()->ctx().get()); // always mark dirty here - if (m.any_of()) { - // not an actual message, but we probalby need to check and see if we need to load fragments - //std::cout << "MFS: new or updated curser\n"; - return; - } - - // TODO: this is bad, we need a non persistence tag instead - if (!m.any_of()) { - // skip everything else for now - return; - } - - // TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag) - if (!m.all_of()) { - std::cout << "MFS: new msg missing Object\n"; - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); - } - - auto& fid_open = m.registry()->ctx().get().open_frags; - - const auto msg_ts = m.get().ts; - // missing fuid - // find closesed non-sealed off fragment - - Object fragment_id{entt::null}; - - // first search for fragment where the ts falls into the range - for (const auto& fid : fid_open) { - auto fh = _os.objectHandle(fid); - assert(static_cast(fh)); - - // assuming ts range exists - auto& fts_comp = fh.get(); - - if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) { - fragment_id = fid; - // TODO: check conditions for open here - // TODO: mark msg (and frag?) dirty - } - } - - // if it did not fit into an existing fragment, we next look for fragments that could be extended - if (!_os._reg.valid(fragment_id)) { - for (const auto& fid : fid_open) { - auto fh = _os.objectHandle(fid); - assert(static_cast(fh)); - - // assuming ts range exists - auto& fts_comp = fh.get(); - - const int64_t frag_range = int64_t(fts_comp.end) - int64_t(fts_comp.begin); - constexpr static int64_t max_frag_ts_extent {1000*60*60}; - //constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing - const int64_t possible_extention = max_frag_ts_extent - frag_range; - - // which direction - if ((fts_comp.begin - possible_extention) <= msg_ts && fts_comp.begin > msg_ts) { - fragment_id = fid; - - std::cout << "MFS: extended begin from " << fts_comp.begin << " to " << msg_ts << "\n"; - - // assuming ts range exists - fts_comp.begin = msg_ts; // extend into the past - - if (m.registry()->ctx().contains()) { - // should be the case - m.registry()->ctx().get().erase(fh); - m.registry()->ctx().get().insert(fh); - } - - - // TODO: check conditions for open here - // TODO: mark msg (and frag?) dirty - } else if ((fts_comp.end + possible_extention) >= msg_ts && fts_comp.end < msg_ts) { - fragment_id = fid; - - std::cout << "MFS: extended end from " << fts_comp.end << " to " << msg_ts << "\n"; - - // assuming ts range exists - fts_comp.end = msg_ts; // extend into the future - - if (m.registry()->ctx().contains()) { - // should be the case - m.registry()->ctx().get().erase(fh); - m.registry()->ctx().get().insert(fh); - } - - // TODO: check conditions for open here - // TODO: mark msg (and frag?) dirty - } - } - } - - // if its still not found, we need a new fragment - if (!_os.registry().valid(fragment_id)) { - const auto new_uuid = _session_uuid_gen(); - _fs_ignore_event = true; - auto fh = _sb.newObject(ByteSpan{new_uuid}); - _fs_ignore_event = false; - if (!static_cast(fh)) { - std::cout << "MFS error: failed to create new object for message\n"; - return; - } - - fragment_id = fh; - - fh.emplace_or_replace().comp = Compression::ZSTD; - fh.emplace_or_replace().comp = Compression::ZSTD; - fh.emplace_or_replace(); // default is current - - auto& new_ts_range = fh.emplace_or_replace(); - new_ts_range.begin = msg_ts; - new_ts_range.end = msg_ts; - - { - const auto msg_reg_contact = m.registry()->ctx().get(); - if (_cr.all_of(msg_reg_contact)) { - fh.emplace(_cr.get(msg_reg_contact).data); - } else { - // ? rage quit? - } - } - - // contact frag - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); - } - m.registry()->ctx().get().insert(fh); - - // loaded contact frag - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); - } - m.registry()->ctx().get().loaded_frags.emplace(fh); - - fid_open.emplace(fragment_id); - - std::cout << "MFS: created new fragment " << bin2hex(fh.get().v) << "\n"; - - _fs_ignore_event = true; - _os.throwEventConstruct(fh); - _fs_ignore_event = false; - } - - // if this is still empty, something is very wrong and we exit here - if (!_os.registry().valid(fragment_id)) { - std::cout << "MFS error: failed to find/create fragment for message\n"; - return; - } - - m.emplace_or_replace(fragment_id); - - // in this case we know the fragment needs an update - for (const auto& it : _frag_save_queue) { - if (it.id == fragment_id) { - // already in queue - return; // done - } - } - _frag_save_queue.push_back({Message::getTimeMS(), {_os.registry(), fragment_id}, m.registry()}); - return; // done - } - - const auto msg_fh = _os.objectHandle(m.get().o); - if (!static_cast(msg_fh)) { - std::cerr << "MFS error: fid in message is invalid\n"; - return; // TODO: properly handle this case - } - - if (!m.registry()->ctx().contains()) { - m.registry()->ctx().emplace(); - } - - auto& fid_open = m.registry()->ctx().get().open_frags; - - if (fid_open.contains(msg_fh)) { - // TODO: dedup events - // TODO: cooldown per fragsave - _frag_save_queue.push_back({Message::getTimeMS(), msg_fh, m.registry()}); - return; - } - - // TODO: save updates to old fragments, but writing them to a new fragment that would overwrite on merge - // new fragment?, since we dont write to others fragments? - - - // on new message: assign fuid - // on new and update: mark as fragment dirty -} - -// assumes not loaded frag -// need update from frag -void MessageFragmentStore::loadFragment(Message3Registry& reg, ObjectHandle fh) { - std::cout << "MFS: loadFragment\n"; - // version HAS to be set, or we just fail - if (!fh.all_of()) { - std::cerr << "MFS error: nope, object without version, cant load\n"; - return; - } - - nlohmann::json j; - const auto obj_version = fh.get().v; - if (obj_version == 1 || obj_version == 2) { - j = loadFromStorageNJ(fh); // also handles version and json/msgpack - } else { - std::cerr << "MFS error: nope, object with unknown version, cant load\n"; - return; - } - - if (!j.is_array()) { - // wrong data - fh.emplace_or_replace(); - return; - } - - if (j.size() == 0) { - // empty array - fh.emplace_or_replace(); - return; - } - - // TODO: this should probably never be the case, since we already know here that it is a msg frag - if (!reg.ctx().contains()) { - reg.ctx().emplace(); - } - reg.ctx().get().insert(fh); - - // mark loaded - if (!reg.ctx().contains()) { - reg.ctx().emplace(); - } - reg.ctx().get().loaded_frags.emplace(fh); - - size_t messages_new_or_updated {0}; - for (const auto& j_entry : j) { - auto new_real_msg = Message3Handle{reg, reg.create()}; - // load into staging reg - for (const auto& [k, v] : j_entry.items()) { - //std::cout << "K:" << k << " V:" << v.dump() << "\n"; - const auto type_id = entt::hashed_string(k.data(), k.size()); - const auto deserl_fn_it = _sc._deserl_json.find(type_id); - if (deserl_fn_it != _sc._deserl_json.cend()) { - try { - if (!deserl_fn_it->second(_sc, new_real_msg, v)) { - std::cerr << "MFS error: failed deserializing '" << k << "'\n"; - } - } catch(...) { - std::cerr << "MFS error: failed deserializing (threw) '" << k << "'\n"; - } - } else { - std::cerr << "MFS warning: missing deserializer for meta key '" << k << "'\n"; - } - } - - new_real_msg.emplace_or_replace(fh); - - // dup check (hacky, specific to protocols) - Message3 dup_msg {entt::null}; - { - // get comparator from contact - if (reg.ctx().contains()) { - const auto c = reg.ctx().get(); - if (_cr.all_of(c)) { - auto& comp = _cr.get(c).comp; - // walking EVERY existing message OOF - // this needs optimizing - for (const Message3 other_msg : reg.view()) { - if (other_msg == new_real_msg) { - continue; // skip self - } - - if (comp({reg, other_msg}, new_real_msg)) { - // dup - dup_msg = other_msg; - break; - } - } - } - } - } - - if (reg.valid(dup_msg)) { - // -> merge with preexisting (needs to be order independent) - // -> throw update - reg.destroy(new_real_msg); - //messages_new_or_updated++; // TODO: how do i know on merging, if data was useful - //_rmm.throwEventUpdate(reg, new_real_msg); - } else { - if (!new_real_msg.all_of()) { - // does not have needed components to be stand alone - reg.destroy(new_real_msg); - std::cerr << "MFS warning: message with missing basic compoments\n"; - continue; - } - - messages_new_or_updated++; - // -> throw create - _rmm.throwEventConstruct(reg, new_real_msg); - } - } - - if (messages_new_or_updated == 0) { - // useless frag - // TODO: unload? - fh.emplace_or_replace(); - } -} - -bool MessageFragmentStore::syncFragToStorage(ObjectHandle fh, Message3Registry& reg) { - auto& ftsrange = fh.get_or_emplace(Message::getTimeMS(), Message::getTimeMS()); - - auto j = nlohmann::json::array(); - - // TODO: does every message have ts? - auto msg_view = reg.view(); - // we also assume all messages have an associated object - for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) { - const Message3 m = *it; - - if (!reg.all_of(m)) { - continue; - } - - // filter: require msg for now - // this will be removed in the future - if (!reg.any_of(m)) { - continue; - } - - if (_frag_save_queue.front().id != reg.get(m).o) { - continue; // not ours - } - - { // potentially adjust tsrange (some external processes can change timestamps) - const auto msg_ts = msg_view.get(m).ts; - if (ftsrange.begin > msg_ts) { - ftsrange.begin = msg_ts; - } else if (ftsrange.end < msg_ts) { - ftsrange.end = msg_ts; - } - } - - auto& j_entry = j.emplace_back(nlohmann::json::object()); - - for (const auto& [type_id, storage] : reg.storage()) { - if (!storage.contains(m)) { - continue; - } - - //std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n"; - - // use type_id to find serializer - auto s_cb_it = _sc._serl_json.find(type_id); - if (s_cb_it == _sc._serl_json.end()) { - // could not find serializer, not saving - //std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n"; - continue; - } - - try { - s_cb_it->second(_sc, {reg, m}, j_entry[storage.type().name()]); - } catch (...) { - std::cerr << "MFS error: failed to serialize " << storage.type().name() << "(" << type_id << ")\n"; - } - } - } - - // we cant skip if array is empty (in theory it will not be empty later on) - - std::vector data_to_save; - const auto obj_version = fh.get_or_emplace().v; - if (obj_version == 1) { - auto j_dump = j.dump(2, ' ', true); - data_to_save = std::vector(j_dump.cbegin(), j_dump.cend()); - } else if (obj_version == 2) { - data_to_save = nlohmann::json::to_msgpack(j); - } else { - std::cerr << "MFS error: unknown object version\n"; - assert(false); - } - assert(fh.all_of()); - auto* backend = fh.get().ptr; - if (backend->write(fh, {reinterpret_cast(data_to_save.data()), data_to_save.size()})) { - // TODO: make this better, should this be called on fail? should this be called before sync? (prob not) - _fs_ignore_event = true; - _os.throwEventUpdate(fh); - _fs_ignore_event = false; - - //std::cout << "MFS: dumped " << j_dump << "\n"; - // succ - return true; - } - - // TODO: error - return false; -} - -MessageFragmentStore::MessageFragmentStore( - Contact3Registry& cr, - RegistryMessageModel& rmm, - ObjectStore2& os, - StorageBackendI& sb -) : _cr(cr), _rmm(rmm), _os(os), _sb(sb), _sc{_cr, {}, {}} { - _rmm.subscribe(this, RegistryMessageModel_Event::message_construct); - _rmm.subscribe(this, RegistryMessageModel_Event::message_updated); - _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); - - auto& sjc = _os.registry().ctx().get>(); - sjc.registerSerializer(); - sjc.registerDeSerializer(); - sjc.registerSerializer(); - sjc.registerDeSerializer(); - sjc.registerSerializer(); - sjc.registerDeSerializer(); - - // old frag names - sjc.registerSerializer(sjc.component_get_json); - sjc.registerDeSerializer(sjc.component_emplace_or_replace_json); - sjc.registerSerializer(sjc.component_get_json); - sjc.registerDeSerializer(sjc.component_emplace_or_replace_json); - - _os.subscribe(this, ObjectStore_Event::object_construct); - _os.subscribe(this, ObjectStore_Event::object_update); -} - -MessageFragmentStore::~MessageFragmentStore(void) { - while (!_frag_save_queue.empty()) { - auto fh = _frag_save_queue.front().id; - auto* reg = _frag_save_queue.front().reg; - assert(reg != nullptr); - syncFragToStorage(fh, *reg); - _frag_save_queue.pop_front(); // pop unconditionally - } -} - -MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) { - return _sc; -} - -// checks range against all cursers in msgreg -static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message3Registry& msg_reg) { - // 1D collision checks: - // - for range vs range: - // r1 rhs >= r0 lhs AND r1 lhs <= r0 rhs - // - for range vs point: - // p >= r0 lhs AND p <= r0 rhs - // NOTE: directions for us are reversed (begin has larger values as end) - - auto c_b_view = msg_reg.view(); - c_b_view.use(); - for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) { - // p and r1 rhs can be seen as the same - // but first we need to know if a curser begin is a point or a range - - // TODO: margin? - auto ts_begin = ts_begin_comp.ts; - auto ts_end = ts_begin_comp.ts; // simplyfy code by making a single begin curser act as an infinitly small range - if (msg_reg.valid(vcb.curser_end) && msg_reg.all_of(vcb.curser_end)) { - // TODO: respect curser end's begin? - // TODO: remember which ends we checked and check remaining - ts_end = msg_reg.get(vcb.curser_end).ts; - - // sanity check curser order - if (ts_end > ts_begin) { - std::cerr << "MFS warning: begin curser and end curser of view swapped!!\n"; - std::swap(ts_begin, ts_end); - } - } - - // perform both checks here - if (ts_begin < range_end || ts_end > range_begin) { - continue; - } - - // range hits a view - return true; - } - - return false; -} - -float MessageFragmentStore::tick(float) { - const auto ts_now = Message::getTimeMS(); - // sync dirty fragments here - if (!_frag_save_queue.empty()) { - // wait 10sec before saving - if (_frag_save_queue.front().ts_since_dirty + 10*1000 <= ts_now) { - auto fh = _frag_save_queue.front().id; - auto* reg = _frag_save_queue.front().reg; - assert(reg != nullptr); - if (syncFragToStorage(fh, *reg)) { - _frag_save_queue.pop_front(); - } - } - } - - // load needed fragments here - - // last check event frags - // only checks if it collides with ranges, not adjacent - // bc ~range~ msgreg will be marked dirty and checked next tick - const bool had_events = !_event_check_queue.empty(); - for (size_t i = 0; i < 10 && !_event_check_queue.empty(); i++) { - std::cout << "MFS: event check\n"; - auto fh = _event_check_queue.front().fid; - auto c = _event_check_queue.front().c; - _event_check_queue.pop_front(); - - if (!static_cast(fh)) { - return 0.05f; - } - - if (!fh.all_of()) { - return 0.05f; - } - - if (!fh.all_of()) { - // missing version, adding - fh.emplace(); - } - const auto object_version = fh.get().v; - // TODO: move this early version check somewhere else - if (object_version != 1 && object_version != 2) { - std::cerr << "MFS: object with version mismatch\n"; - return 0.05f; - } - - // get ts range of frag and collide with all curser(s/ranges) - const auto& frag_range = fh.get(); - - auto* msg_reg = _rmm.get(c); - if (msg_reg == nullptr) { - return 0.05f; - } - - if (rangeVisible(frag_range.begin, frag_range.end, !msg_reg)) { - loadFragment(*msg_reg, fh); - _potentially_dirty_contacts.emplace(c); - return 0.05f; // only one but soon again - } - } - if (had_events) { - std::cout << "MFS: event check none\n"; - return 0.05f; // only check events, even if non where hit - } - - if (!_potentially_dirty_contacts.empty()) { - //std::cout << "MFS: pdc\n"; - // here we check if any view of said contact needs frag loading - // only once per tick tho - - // TODO: this makes order depend on internal order and is not fair - auto it = _potentially_dirty_contacts.cbegin(); - - auto* msg_reg = _rmm.get(*it); - - // first do collision check agains every contact associated fragment - // that is not already loaded !! - if (msg_reg->ctx().contains()) { - const auto& cf = msg_reg->ctx().get(); - if (!cf.sorted_frags.empty()) { - if (!msg_reg->ctx().contains()) { - msg_reg->ctx().emplace(); - } - const auto& loaded_frags = msg_reg->ctx().get().loaded_frags; - - for (const auto& [fid, si] : msg_reg->ctx().get().sorted_frags) { - if (loaded_frags.contains(fid)) { - continue; - } - - auto fh = _os.objectHandle(fid); - - if (!static_cast(fh)) { - std::cerr << "MFS error: frag is invalid\n"; - // WHAT - msg_reg->ctx().get().erase(fid); - return 0.05f; - } - - if (!fh.all_of()) { - std::cerr << "MFS error: frag has no range\n"; - // ???? - msg_reg->ctx().get().erase(fid); - return 0.05f; - } - - if (fh.all_of()) { - continue; // skip known empty - } - - // get ts range of frag and collide with all curser(s/ranges) - const auto& [range_begin, range_end] = fh.get(); - - if (rangeVisible(range_begin, range_end, *msg_reg)) { - std::cout << "MFS: frag hit by vis range\n"; - loadFragment(*msg_reg, fh); - return 0.05f; - } - } - // no new visible fragment - //std::cout << "MFS: no new frag directly visible\n"; - - // now, finally, check for adjecent fragments that need to be loaded - // we do this by finding the outermost fragment in a rage, and extend it by one - - // TODO: rewrite using some bounding range tree to perform collision checks !!! - // (this is now performing better, but still) - - - // for each view - auto c_b_view = msg_reg->view(); - c_b_view.use(); - for (const auto& [_, ts_begin_comp, vcb] : c_b_view.each()) { - // aka "scroll down" - { // find newest(-ish) frag in range - // or in reverse frag end <= range begin - - - // lower bound of frag end and range begin - const auto right = std::lower_bound( - cf.sorted_end.crbegin(), - cf.sorted_end.crend(), - ts_begin_comp.ts, - [&](const Object element, const auto& value) -> bool { - return _os.registry().get(element).end >= value; - } - ); - - Object next_frag{entt::null}; - if (right != cf.sorted_end.crend()) { - next_frag = cf.next(*right); - } - // we checked earlier that cf is not empty - if (!_os.registry().valid(next_frag)) { - // fall back to closest, cf is not empty - next_frag = cf.sorted_end.front(); - } - - // a single adjacent frag is often not enough - // only ok bc next is cheap - for (size_t i = 0; i < 5 && _os.registry().valid(next_frag); next_frag = cf.next(next_frag)) { - auto fh = _os.objectHandle(next_frag); - if (fh.any_of()) { - continue; // skip known empty - } - - if (!loaded_frags.contains(next_frag)) { - std::cout << "MFS: next frag of range\n"; - loadFragment(*msg_reg, fh); - return 0.05f; - } - - i++; - } - } - - // curser end - if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of(vcb.curser_end)) { - continue; - } - const auto ts_end = msg_reg->get(vcb.curser_end).ts; - - // aka "scroll up" - { // find oldest(-ish) frag in range - // frag begin >= range end - - // lower bound of frag begin and range end - const auto left = std::lower_bound( - cf.sorted_begin.cbegin(), - cf.sorted_begin.cend(), - ts_end, - [&](const Object element, const auto& value) -> bool { - return _os.registry().get(element).begin < value; - } - ); - - Object prev_frag{entt::null}; - if (left != cf.sorted_begin.cend()) { - prev_frag = cf.prev(*left); - } - // we checked earlier that cf is not empty - if (!_os.registry().valid(prev_frag)) { - // fall back to closest, cf is not empty - prev_frag = cf.sorted_begin.back(); - } - - // a single adjacent frag is often not enough - // only ok bc next is cheap - for (size_t i = 0; i < 5 && _os.registry().valid(prev_frag); prev_frag = cf.prev(prev_frag)) { - auto fh = _os.objectHandle(prev_frag); - if (fh.any_of()) { - continue; // skip known empty - } - - if (!loaded_frags.contains(prev_frag)) { - std::cout << "MFS: prev frag of range\n"; - loadFragment(*msg_reg, fh); - return 0.05f; - } - - i++; - } - } - } - } - } else { - // contact has no fragments, skip - } - - _potentially_dirty_contacts.erase(it); - - return 0.05f; - } - - - return 1000.f*60.f*60.f; -} - -bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) { - handleMessage(e.e); - return false; -} - -bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) { - handleMessage(e.e); - return false; -} - -// TODO: handle deletes? diff between unload? - -bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectConstruct& e) { - if (_fs_ignore_event) { - return false; // skip self - } - - if (!e.e.all_of()) { - return false; // not for us - } - if (!e.e.all_of()) { - // missing version, adding - // version check is later - e.e.emplace(); - } - - // TODO: are we sure it is a *new* fragment? - - Contact3 frag_contact = entt::null; - { // get contact - const auto& frag_contact_id = e.e.get().id; - // TODO: id lookup table, this is very inefficent - for (const auto& [c_it, id_it] : _cr.view().each()) { - if (frag_contact_id == id_it.data) { - frag_contact = c_it; - break; - } - } - if (!_cr.valid(frag_contact)) { - // unkown contact - return false; - } - e.e.emplace_or_replace(frag_contact); - } - - // create if not exist - auto* msg_reg = _rmm.get(frag_contact); - if (msg_reg == nullptr) { - // msg reg not created yet - // TODO: this is an erroious path - return false; - } - - if (!msg_reg->ctx().contains()) { - msg_reg->ctx().emplace(); - } - msg_reg->ctx().get().erase(e.e); // TODO: can this happen? update - msg_reg->ctx().get().insert(e.e); - - _event_check_queue.push_back(ECQueueEntry{e.e, frag_contact}); - - return false; -} - -bool MessageFragmentStore::onEvent(const ObjectStore::Events::ObjectUpdate& e) { - if (_fs_ignore_event) { - return false; // skip self - } - - if (!e.e.all_of()) { - return false; // not for us - } - - // since its an update, we might have it associated, or not - // its also possible it was tagged as empty - e.e.remove(); - - Contact3 frag_contact = entt::null; - { // get contact - // probably cached already - if (e.e.all_of()) { - frag_contact = e.e.get().e; - } - - if (!_cr.valid(frag_contact)) { - const auto& frag_contact_id = e.e.get().id; - // TODO: id lookup table, this is very inefficent - for (const auto& [c_it, id_it] : _cr.view().each()) { - if (frag_contact_id == id_it.data) { - frag_contact = c_it; - break; - } - } - if (!_cr.valid(frag_contact)) { - // unkown contact - return false; - } - e.e.emplace_or_replace(frag_contact); - } - } - - // create if not exist - auto* msg_reg = _rmm.get(frag_contact); - if (msg_reg == nullptr) { - // msg reg not created yet - // TODO: this is an erroious path - return false; - } - - if (!msg_reg->ctx().contains()) { - msg_reg->ctx().emplace(); - } - msg_reg->ctx().get().erase(e.e); // TODO: check/update/fragment update - msg_reg->ctx().get().insert(e.e); - - // TODO: actually load it - //_event_check_queue.push_back(ECQueueEntry{e.e, frag_contact}); - - return false; -} - diff --git a/src/fragment_store/message_fragment_store.hpp b/src/fragment_store/message_fragment_store.hpp deleted file mode 100644 index c5f9f7e..0000000 --- a/src/fragment_store/message_fragment_store.hpp +++ /dev/null @@ -1,107 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include "./message_serializer.hpp" - -#include "./messages_meta_components.hpp" - -#include -#include - -#include -#include - -#include -#include -#include - -namespace Message::Components { - - // unused, consumes too much memory (highly compressable) - //using FUID = FragComp::ID; - - struct Obj { - // message fragment's object - Object o {entt::null}; - }; - - // TODO: add adjacency range comp or inside curser - - // TODO: unused - // mfs will only load a limited number of fragments per tick (1), - // so this tag will be set if we loaded a fragment and - // every tick we check all cursers for this tag and continue - // and remove once no fragment could be loaded anymore - // (internal) - struct TagCurserUnsatisfied {}; - -} // Message::Components - -// handles fragments for messages -// on new message: assign fuid -// on new and update: mark as fragment dirty -// on delete: mark as fragment dirty? -class MessageFragmentStore : public RegistryMessageModelEventI, public ObjectStoreEventI { - protected: - Contact3Registry& _cr; - RegistryMessageModel& _rmm; - ObjectStore2& _os; - StorageBackendI& _sb; - bool _fs_ignore_event {false}; - - UUIDGenerator_128_128 _session_uuid_gen; - - // for message components only - MessageSerializerCallbacks _sc; - - void handleMessage(const Message3Handle& m); - - void loadFragment(Message3Registry& reg, ObjectHandle oh); - - bool syncFragToStorage(ObjectHandle oh, Message3Registry& reg); - - struct SaveQueueEntry final { - uint64_t ts_since_dirty{0}; - //std::vector id; - ObjectHandle id; - Message3Registry* reg{nullptr}; - }; - std::deque _frag_save_queue; - - struct ECQueueEntry final { - ObjectHandle fid; - Contact3 c; - }; - std::deque _event_check_queue; - - // range changed or fragment loaded. - // we only load a limited number of fragments at once, - // so we need to keep them dirty until nothing was loaded. - entt::dense_set _potentially_dirty_contacts; - - public: - MessageFragmentStore( - Contact3Registry& cr, - RegistryMessageModel& rmm, - ObjectStore2& os, - StorageBackendI& sb - ); - virtual ~MessageFragmentStore(void); - - MessageSerializerCallbacks& getMSC(void); - - float tick(float time_delta); - - protected: // rmm - bool onEvent(const Message::Events::MessageConstruct& e) override; - bool onEvent(const Message::Events::MessageUpdated& e) override; - - protected: // fs - bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override; - bool onEvent(const ObjectStore::Events::ObjectUpdate& e) override; -}; - diff --git a/src/fragment_store/message_serializer.cpp b/src/fragment_store/message_serializer.cpp deleted file mode 100644 index 7b86c5a..0000000 --- a/src/fragment_store/message_serializer.cpp +++ /dev/null @@ -1,121 +0,0 @@ -#include "./message_serializer.hpp" - -#include -#include -#include - -#include - -#include - -static Contact3 findContactByID(Contact3Registry& cr, const std::vector& id) { - // TODO: id lookup table, this is very inefficent - for (const auto& [c_it, id_it] : cr.view().each()) { - if (id == id_it.data) { - return c_it; - } - } - - return entt::null; -} - -template<> -bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) { - const Contact3 c = h.get().c; - if (!msc.cr.valid(c)) { - // while this is invalid registry state, it is valid serialization - j = nullptr; - std::cerr << "MSC warning: encountered invalid contact\n"; - return true; - } - - if (!msc.cr.all_of(c)) { - // unlucky, this contact is purely ephemeral - j = nullptr; - std::cerr << "MSC warning: encountered contact without ID\n"; - return true; - } - - j = nlohmann::json::binary(msc.cr.get(c).data); - - return true; -} - -template<> -bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) { - if (j.is_null()) { - std::cerr << "MSC warning: encountered null contact\n"; - h.emplace_or_replace(); - return true; - } - - std::vector id; - if (j.is_binary()) { - id = j.get_binary(); - } else { - j["bytes"].get_to(id); - } - - Contact3 other_c = findContactByID(msc.cr, id); - if (!msc.cr.valid(other_c)) { - // create sparse contact with id only - other_c = msc.cr.create(); - msc.cr.emplace_or_replace(other_c, id); - } - - h.emplace_or_replace(other_c); - - // TODO: should we return false if the contact is unknown?? - return true; -} - -template<> -bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) { - const Contact3 c = h.get().c; - if (!msc.cr.valid(c)) { - // while this is invalid registry state, it is valid serialization - j = nullptr; - std::cerr << "MSC warning: encountered invalid contact\n"; - return true; - } - - if (!msc.cr.all_of(c)) { - // unlucky, this contact is purely ephemeral - j = nullptr; - std::cerr << "MSC warning: encountered contact without ID\n"; - return true; - } - - j = nlohmann::json::binary(msc.cr.get(c).data); - - return true; -} - -template<> -bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) { - if (j.is_null()) { - std::cerr << "MSC warning: encountered null contact\n"; - h.emplace_or_replace(); - return true; - } - - std::vector id; - if (j.is_binary()) { - id = j.get_binary(); - } else { - j["bytes"].get_to(id); - } - - Contact3 other_c = findContactByID(msc.cr, id); - if (!msc.cr.valid(other_c)) { - // create sparse contact with id only - other_c = msc.cr.create(); - msc.cr.emplace_or_replace(other_c, id); - } - - h.emplace_or_replace(other_c); - - // TODO: should we return false if the contact is unknown?? - return true; -} - diff --git a/src/fragment_store/message_serializer.hpp b/src/fragment_store/message_serializer.hpp deleted file mode 100644 index 70d6fcd..0000000 --- a/src/fragment_store/message_serializer.hpp +++ /dev/null @@ -1,85 +0,0 @@ -#pragma once - -#include -#include - -#include - -#include - -struct MessageSerializerCallbacks { - using Registry = Message3Registry; - using Handle = Message3Handle; - - Contact3Registry& cr; - - // nlohmann - // json/msgpack - using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out); - entt::dense_map _serl_json; - - using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in); - entt::dense_map _deserl_json; - - template - static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) { - if (h.template all_of()) { - if constexpr (!std::is_empty_v) { - j = h.template get(); - } - return true; - } - - return false; - } - - template - static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) { - if constexpr (std::is_empty_v) { - h.template emplace_or_replace(); // assert empty json? - } else { - h.template emplace_or_replace(static_cast(j)); - } - return true; - } - - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { - _serl_json[type_info.hash()] = fn; - } - - template - void registerSerializerJson( - serialize_json_fn fn = component_get_json, - const entt::type_info& type_info = entt::type_id() - ) { - registerSerializerJson(fn, type_info); - } - - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { - _deserl_json[type_info.hash()] = fn; - } - - template - void registerDeSerializerJson( - deserialize_json_fn fn = component_emplace_or_replace_json, - const entt::type_info& type_info = entt::type_id() - ) { - registerDeSerializerJson(fn, type_info); - } -}; - -// fwd -namespace Message::Components { -struct ContactFrom; -struct ContactTo; -} - -// make specializations known -template<> -bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j); -template<> -bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j); -template<> -bool MessageSerializerCallbacks::component_get_json(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j); -template<> -bool MessageSerializerCallbacks::component_emplace_or_replace_json(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j); diff --git a/src/fragment_store/messages_meta_components.hpp b/src/fragment_store/messages_meta_components.hpp deleted file mode 100644 index 6d95727..0000000 --- a/src/fragment_store/messages_meta_components.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include - -namespace ObjectStore::Components { - struct MessagesVersion { - // messages Object version - // 1 -> text_json - // 2 -> msgpack - uint16_t v {2}; - }; - - struct MessagesTSRange { - // timestamp range within the fragment - uint64_t begin {0}; // newer msg -> higher number - uint64_t end {0}; - }; - - struct MessagesContact { - std::vector id; - }; - - // TODO: add src contact (self id) - -} // ObjectStore::Components - -// old -namespace Fragment::Components { - struct MessagesTSRange : public ObjComp::MessagesTSRange {}; - struct MessagesContact : public ObjComp::MessagesContact {}; -} // Fragment::Components - -#include "./messages_meta_components_id.inl" - diff --git a/src/fragment_store/messages_meta_components_id.inl b/src/fragment_store/messages_meta_components_id.inl deleted file mode 100644 index 4713637..0000000 --- a/src/fragment_store/messages_meta_components_id.inl +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include "./messages_meta_components.hpp" - -#include - -// TODO: move more central -#define DEFINE_COMP_ID(x) \ -template<> \ -constexpr entt::id_type entt::type_hash::value() noexcept { \ - using namespace entt::literals; \ - return #x##_hs; \ -} \ -template<> \ -constexpr std::string_view entt::type_name::value() noexcept { \ - return #x; \ -} - -// cross compiler stable ids - -DEFINE_COMP_ID(ObjComp::MessagesVersion) -DEFINE_COMP_ID(ObjComp::MessagesTSRange) -DEFINE_COMP_ID(ObjComp::MessagesContact) - -// old stuff -//DEFINE_COMP_ID(FragComp::MessagesTSRange) -//DEFINE_COMP_ID(FragComp::MessagesContact) - -#undef DEFINE_COMP_ID - - diff --git a/src/fragment_store/register_mfs_json_message_components.cpp b/src/fragment_store/register_mfs_json_message_components.cpp deleted file mode 100644 index ed6bda6..0000000 --- a/src/fragment_store/register_mfs_json_message_components.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "./register_mfs_json_message_components.hpp" - -#include "./message_serializer.hpp" -#include "../json/message_components.hpp" - -void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) { - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); - - // files - //_sc.registerSerializerJson() - //_sc.registerSerializerJson(); - //_sc.registerDeSerializerJson(); - //_sc.registerSerializerJson(); - //_sc.registerDeSerializerJson(); - //_sc.registerSerializerJson(); - //_sc.registerDeSerializerJson(); -} - diff --git a/src/fragment_store/register_mfs_json_message_components.hpp b/src/fragment_store/register_mfs_json_message_components.hpp deleted file mode 100644 index 5efef28..0000000 --- a/src/fragment_store/register_mfs_json_message_components.hpp +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -#include "./message_serializer.hpp" - -void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc); - diff --git a/src/fragment_store/register_mfs_json_tox_message_components.cpp b/src/fragment_store/register_mfs_json_tox_message_components.cpp index 83b7dad..fbeb979 100644 --- a/src/fragment_store/register_mfs_json_tox_message_components.cpp +++ b/src/fragment_store/register_mfs_json_tox_message_components.cpp @@ -1,10 +1,10 @@ -#include "./register_mfs_json_message_components.hpp" +#include "./register_mfs_json_tox_message_components.hpp" -#include "./message_serializer.hpp" #include "../json/tox_message_components.hpp" +#include "solanaceae/message3/message_serializer.hpp" -void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) { - msc.registerSerializerJson(); - msc.registerDeSerializerJson(); +void registerMFSJsonToxMessageComponents(MessageSerializerNJ& msnj) { + msnj.registerSerializer(); + msnj.registerDeserializer(); } diff --git a/src/fragment_store/register_mfs_json_tox_message_components.hpp b/src/fragment_store/register_mfs_json_tox_message_components.hpp index fdf86bc..ea801df 100644 --- a/src/fragment_store/register_mfs_json_tox_message_components.hpp +++ b/src/fragment_store/register_mfs_json_tox_message_components.hpp @@ -1,6 +1,6 @@ #pragma once -#include "./message_serializer.hpp" +#include -void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc); +void registerMFSJsonToxMessageComponents(MessageSerializerNJ& msnj); diff --git a/src/json/message_components.hpp b/src/json/message_components.hpp deleted file mode 100644 index c272e6c..0000000 --- a/src/json/message_components.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -#include - -#include - -namespace Message::Components { - - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts) - // TODO: SyncedBy - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text) - - namespace Transfer { - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size) - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list) - } // Transfer - -} // Message::Components - diff --git a/src/main_screen.cpp b/src/main_screen.cpp index c588f35..9623381 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -1,7 +1,8 @@ #include "./main_screen.hpp" -#include "./fragment_store/register_mfs_json_message_components.hpp" +#include #include "./fragment_store/register_mfs_json_tox_message_components.hpp" +#include "solanaceae/message3/message_serializer.hpp" #include @@ -15,9 +16,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::string save_password, std::vector plugins) : renderer(renderer_), rmm(cr), + msnj{cr, {}, {}}, mts(rmm), - mfsb(os, "test2_message_store/"), - mfs(cr, rmm, os, mfsb), tc(save_path, save_password), tpi(tc.getTox()), ad(tc), @@ -38,8 +38,9 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri tdch(tpi) { tel.subscribeAll(tc); - registerMFSJsonMessageComponents(mfs.getMSC()); - registerMFSJsonToxMessageComponents(mfs.getMSC()); + + registerMessageComponents(msnj); + registerMFSJsonToxMessageComponents(msnj); conf.set("tox", "save_file_path", save_path); @@ -61,6 +62,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri g_provideInstance("ConfigModelI", "host", &conf); g_provideInstance("Contact3Registry", "1", "host", &cr); g_provideInstance("RegistryMessageModel", "host", &rmm); + g_provideInstance("MessageSerializerNJ", "host", &msnj); g_provideInstance("ToxI", "host", &tc); g_provideInstance("ToxPrivateI", "host", &tpi); @@ -83,8 +85,6 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri } conf.dump(); - - mfsb.scanAsync(); // HACK: after plugins and tox contacts got loaded } MainScreen::~MainScreen(void) { @@ -426,7 +426,6 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { tdch.tick(time_delta); // compute - const float mfs_interval = mfs.tick(time_delta); mts.iterate(); // compute (after mfs) _min_tick_interval = std::min( @@ -439,10 +438,6 @@ Screen* MainScreen::tick(float time_delta, bool& quit) { _min_tick_interval, fo_interval ); - _min_tick_interval = std::min( - _min_tick_interval, - mfs_interval - ); //std::cout << "MS: min tick interval: " << _min_tick_interval << "\n"; diff --git a/src/main_screen.hpp b/src/main_screen.hpp index 5cbb608..fbda690 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -3,12 +3,11 @@ #include "./screen.hpp" #include -#include #include #include #include #include -#include "./fragment_store/message_fragment_store.hpp" +#include #include #include #include "./tox_private_impl.hpp" @@ -51,9 +50,8 @@ struct MainScreen final : public Screen { SimpleConfigModel conf; Contact3Registry cr; RegistryMessageModel rmm; + MessageSerializerNJ msnj; MessageTimeSort mts; - backend::FilesystemStorage mfsb; // message fsb // TODO: make configurable - MessageFragmentStore mfs; ToxEventLogger tel{std::cout}; ToxClient tc; From 0030487613801a2e0eba1d126fe3a2aa9230dbd9 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 14 Apr 2024 14:16:53 +0200 Subject: [PATCH 97/98] move converter tool and pics out --- external/solanaceae_object_store | 2 +- src/CMakeLists.txt | 11 -- src/fragment_store/convert_frag_to_obj.cpp | 164 --------------------- src/fragment_store/fs_binary_msgpack1.png | Bin 37255 -> 0 bytes src/fragment_store/fs_binary_msgpack2.png | Bin 46969 -> 0 bytes 5 files changed, 1 insertion(+), 176 deletions(-) delete mode 100644 src/fragment_store/convert_frag_to_obj.cpp delete mode 100644 src/fragment_store/fs_binary_msgpack1.png delete mode 100644 src/fragment_store/fs_binary_msgpack2.png diff --git a/external/solanaceae_object_store b/external/solanaceae_object_store index 4695579..e26959c 160000 --- a/external/solanaceae_object_store +++ b/external/solanaceae_object_store @@ -1 +1 @@ -Subproject commit 46955795b034ed4b058958bba620bd8965ce91f7 +Subproject commit e26959c380bc76e8c01a517019c4c0a569156b1d diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 073a9b9..103d743 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,17 +2,6 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) ######################################## -add_executable(convert_message_object_store EXCLUDE_FROM_ALL - fragment_store/convert_frag_to_obj.cpp -) - -target_link_libraries(convert_message_object_store PUBLIC - solanaceae_object_store - solanaceae_object_store_backend_filesystem - solanaceae_message_fragment_store -) - -######################################## add_executable(tomato ./main.cpp ./icon.rc diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp deleted file mode 100644 index 337edc7..0000000 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ /dev/null @@ -1,164 +0,0 @@ -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include - -#include - -int main(int argc, const char** argv) { - if (argc != 3) { - std::cerr << "wrong paramter count, do " << argv[0] << " \n"; - return 1; - } - - if (!std::filesystem::is_directory(argv[1])) { - std::cerr << "input folder is no folder\n"; - } - - std::filesystem::create_directories(argv[2]); - - // we are going to use 2 different OS for convineance, but could be done with 1 too - ObjectStore2 os_src; - ObjectStore2 os_dst; - - backend::FilesystemStorage fsb_src(os_src, argv[1]); - backend::FilesystemStorage fsb_dst(os_dst, argv[2]); - - Contact3Registry cr; // dummy - RegistryMessageModel rmm(cr); // dummy - // they only exist for the serializers (for now) - // TODO: version - MessageFragmentStore mfs_src(cr, rmm, os_src, fsb_src); - MessageFragmentStore mfs_dst(cr, rmm, os_dst, fsb_dst); - - // add message fragment store too (adds meta?) - - // hookup events - struct EventListener : public ObjectStoreEventI { - ObjectStore2& _os_src; - backend::FilesystemStorage& _fsb_src; - - ObjectStore2& _os_dst; - backend::FilesystemStorage& _fsb_dst; - - EventListener( - ObjectStore2& os_src, - backend::FilesystemStorage& fsb_src, - ObjectStore2& os_dst, - backend::FilesystemStorage& fsb_dst - ) : - _os_src(os_src), - _fsb_src(fsb_src), - _os_dst(os_dst), - _fsb_dst(fsb_dst) - { - _os_src.subscribe(this, ObjectStore_Event::object_construct); - _os_src.subscribe(this, ObjectStore_Event::object_update); - } - - protected: // os - bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override { - assert(e.e.all_of()); - assert(e.e.all_of()); - - // !! we read the obj first, so we can discard empty objects - // technically we could just copy the file, but meh - // read src and write dst data - std::vector tmp_buffer; - std::function cb = [&tmp_buffer](const ByteSpan buffer) { - tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); - }; - if (!_fsb_src.read(e.e, cb)) { - std::cerr << "failed to read obj '" << bin2hex(e.e.get().v) << "'\n"; - return false; - } - - if (tmp_buffer.empty()) { - std::cerr << "discarded empty obj '" << bin2hex(e.e.get().v) << "'\n"; - return false; - } - { // try getting lucky and see if its an empty json - const auto j = nlohmann::json::parse(tmp_buffer, nullptr, false); - if (j.is_array() && j.empty()) { - std::cerr << "discarded empty json array obj '" << bin2hex(e.e.get().v) << "'\n"; - return false; - } - } - - // WARNING: manual and hardcoded - // manually upconvert message json to msgpack (v1 to v2) - if (true && e.e.get().v == 1) { - // TODO: error handling - const auto j = nlohmann::json::parse(tmp_buffer); - - if (true) { - e.e.replace(uint16_t(2)); - - // overwrite og - tmp_buffer = nlohmann::json::to_msgpack(j); - } - } - - // we dont copy meta file type, it will be the same for all "new" objects - auto oh = _fsb_dst.newObject(ByteSpan{e.e.get().v}); - - if (!static_cast(oh)) { - // already exists - return false; - } - - { // sync meta - // some hardcoded ehpemeral (besides mft/id) - oh.emplace_or_replace(e.e.get_or_emplace()); - oh.emplace_or_replace(e.e.get_or_emplace()); - - // serializable - for (const auto& [type, fn] : _os_src.registry().ctx().get>()._serl) { - //if (!e.e.registry()->storage(type)->contains(e.e)) { - //continue; - //} - - // this is hacky but we serialize and then deserialize the component - // raw copy might be better in the future - nlohmann::json tmp_j; - if (fn(e.e, tmp_j)) { - _os_dst.registry().ctx().get>()._deserl.at(type)(oh, tmp_j); - } - } - } - - static_cast(_fsb_dst).write(oh, ByteSpan{tmp_buffer}); - - //assert(std::filesystem::file_size(e.e.get().path) == std::filesystem::file_size(oh.get().path)); - - return false; - } - - bool onEvent(const ObjectStore::Events::ObjectUpdate&) override { - std::cerr << "Update called\n"; - assert(false); - return false; - } - } el { - os_src, - fsb_src, - os_dst, - fsb_dst, - }; - - // perform scan (which triggers events) - fsb_dst.scanAsync(); // fill with existing? - fsb_src.scanAsync(); // the scan - - // done - return 0; -} - diff --git a/src/fragment_store/fs_binary_msgpack1.png b/src/fragment_store/fs_binary_msgpack1.png deleted file mode 100644 index b536b203ca5e1f586564c93e732fb464f798f3f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37255 zcmdSBWk6K#w>CTy(jn4~lt@W;OG}r4faFMbgMgGEIS5D%QqnneBT6%%bT>nH2n_IU ze*bfx^PJ~B=XuZ9_tPGB-h1EoTI*WZx)xEI>WX;Sl-M8;2v1q*l{N^3stN+3e8)rq z_B{Oz=?8&=LCUXWbbV&`7yJWgegz8MiPSI+);M6zL1MUjMf(I@6C7929?5uFX`9Jr zehuR<;2++YAfgWw{4&a-F_yx=|CBOaIE2jh&$_K_`X+*fd6^ael>iw9g#$`oO+dm4 zaT>j9Jx@wGqwzGvQ=~(RG{xH6%gYAr4fYyA+@1_p=85=U!S|rBB|iT&-M~PJN5Ig6 z(b%+UfGr4whD$*UY^49tccTJZB}yxwhuzAI$t&j<7ow_};ZLf{`B%xm=u!}C*JT6e zxSYU`{FdO?lYa_S9iNJ+qI7)<8K8(*KmM9N7-3=$ed%GEI;yV&eTj9e^vU5faK&Gp zA-+WXUOpNNkk+&vaCa3KYad+&R*Vsb`8|9rq>sk_dsm#-EHMSDsTUU)>3p-MVyU*q zsh)laA`}OV2297_C$HINEMhc848X0s4yJaxE1tz}f*&E0Ccxc<-crz>(}QUPB@mus zxj4tK)x=S8wi|cf(uhT2G2+5WNf$^@azq=_)7ZsOzp-qtVC!eS1HL0Ruc>QyyvI^G4CV`OO>Gr?t9f7mSk=X?s zh2kQgY3l}%Sh<7^?L)6DJmw)Tq9fQYhv>l9{dDH*<2oW0unx9ag47+Le;^(43o)sZ zH7cX z!)k$jAmz}1+ddt{b)KowE0L2viZUVt=KII8sQB~u^44UpN6NUNu-c*k=*}C^GXYO8 zmsQ=v=|gBHC5<)>4hW5af|j@YVZM#`We? zdFH2M-1K3@@mVTrYUHJutr6MR0>_?0Yn~DbJJ74eK5lmR}k;UretHRq9w2R%72M6FtCl7s=7;($n z>+@{nI_oenwB-;)cF^5H%i*2b{cccfY-|+Qesly*$4MwSW{QlntobnRGK(sMe<|>I zqwD6w;uGEV;^L^8N=XTxKzLd@OPZ^h76nWcmcl=lY~Cehp)9C5?!74*@k8*bwGni z)+W>UYN%CUtjCO7GW)c8GHP27F^CxX;7Hf6@kDk&y+fSWc^LTS?{|<#-2@tHY73rI zCT7=vQePn?q$DRUcWl=@mr4t>16CXje|z8CS@P>WPZw5Hl1z{<_zoWtJw{%g;N(TU zbw~Ev?tFpE!otASnS-^%t4*)rdF@tlKSWxLLyx8;Vn}@ThxybIf9Canew{Xpwfc@0 zjJ7s>^gTAaUxQst$KBVrqQ!?p@kEC&!oT+gDM|Lg)tlYvps*0Xlx2qQ@X=&Nnz4$u6i+5F=8|B_JICfe&P?+x= z*RPS!qKL&%h!Eg9dJf8vNo;EDU(v z_-G9e$-z8{stmj`cNTUYC?ZhE>fv;V0lKa&psZ)VW(thPzC&3}Evz~vw@514^Y#5ae#2oXLI+It6X?S zz!6zH6kV7lW#U@1d+wX6-hNfp@a8#ZTK(yEYO%9p`!g7#Q(BU4%aJsPBBSYjiBHep znDkRVGW|>+)4S~m-D2C~8LARR=y28t2bYVRO_JfS-l^?X)xDXCaCqEfN#)zB#u$Nz_QV7czD$&C4fp^o1}vln(sr#7+;%~Accwm$ zN+3?y7pj$Ouptz3s>|zFb25F903S+^2=WZr*_(4>r@akgL}0+fn*rPjqR$1}$~8~% z=Kn&Go8G2ju-Z9ShRXs6FjJ1QNFTt!H<4|IDv$-v6fJ-nw4>&2ACO>FmCK5@Y+oW{ zTM25&G6mJtpGX67MyVF)&{w%=a7mmR>eI?3_d@MLf<&x&3g7~C^A{QltlDcj#l>$- zXw7tAsYIVx|GIjHIBm2@pF>LwYfW1r8R?s9{rqJ}+l1ATc!UXNvpHCzUD+1QL;wO2 zIRn?j$<%sYH25=~u<;5er+-z^`=Z{)+5g$u>FFz?lb$TntC{m^hmTdsLjxl{h7JHM zZT}smOFbqwwoXiup??7;!g)(!8@K&MACMQAba^->36YT_|Q?2X^1}t2KSLQ zJK&^;%9%g@TH&$FTC9-d)q2t$sj&Bo+nU*Q)6{(U;PAbi-hKOAOpy5{1x1@Ky#2*e zfWzBrR@w|si^qXm$2UclB4Ac7+ZwIDtsYp#7fQ$+5il0Wzz`~cBiiSjTGUwDV7w4< zMpQnSgt6=gr31g zTFOT!&&Jh&z}m0@;BltKBn?ShE70YFw;BC*R;)k>Y||7jKIEvZ3>;3m|*s$fhN_7#b$LLYZRxlY7?e8n|d{ zUY>E@EJOzbMiwkSH*r~Se`*~T2<7*yZuvCW+e5O|4g2mIgL4vQL#Be4rTUHQ>BD4G zC{tMYv0xDp{iJ-scLej>t{3VIZP~owb&mvb%bF)F!5; zp30u6E^^Pl4riRWEZQLiM)$Gy-`BrQ|7a7Fxb*%<3TKIPVTz{Mc&4Azjmp!2#|44j zh2sKi7XaA_D-Tw~q_&e=Dpe>7%s%MTcy3fGCIjlPL0t@#E1d&YlZ&jd*ktn17`*2%S2!R+XVh!CBT;ffq2%|)-rWc zzI-XdCmolj`&ZWXcj(Bii5krCZ~@kgEdLv@ZLFiFo?d}yU)5K!w7R%)*}IuoH>d)R z<^D;RyMC8Q%l=b#(U!V-#UojJ1cCC^CT6}2)6_vzSmzX16SwjKE?m&N6yI{<&I=V3 zM9THH@m&{+CC1=}n_@hzgxk6jz3S#YhfX}DQzTIUX|7h|)QaJy_^Ci$LXg*w+cBn? zxEC40STA4y17K>?j3qwrMOX^qML0quwnzXqL|mp2kX1n}~6 zm`drJOaoK$P3owO&y4Yhgt8C(@+c)L_{W_?gQW&T5SvU_ z?g8?5_Ihpv`?s0n1XK6K2Q^txRKi<;$Q8HY`pY}>69b7mbJPp zWbt=@U&B{5_jZE99kk&f)-f9ol8i!*ixb5EPD2=eeRopWY&B_WErVPPDN3lP^KnGA ztO`7{5_F<2b^g3$NdE2y^nTF2cQS{MfE=x&JL&b0A_D=o);I_y!F_?j-j0tKV@vrb z*nv4(W&QRu9La$M;SrXwYLQR#?LX;~5wq6m@XJ|LtD9)cLt2F>75$w)`aB>ng3A>s@FVsx5o zd{u|fob&_TuYTz9kybE;Ws3dryil4-lhb%As|^g~GzA)1xBP+M4ldB07x%Z+JST>+ zTrq|WM~E}OU6s?Dt4olgZgg{GfXyYxvA9y0nRZQJ25E zZRwN3awrKKe#+j>3ZWRxbL=p;AAukM{DC^5RkAvxYqV56% z6AJBE;iftfCheJdztbNrR|ZRx{=T2D^ldjj4tP63oAUQ3SnHq2epk!Z6Kwc!wx09y zN8g5<@b)39L^rJd`;}3*EX5=mJZ&hgKYPexX1W*)oW9Ck>Yic34^n_q+8rUU=pNoA zO(gRw-AKuEzuA6>8Xlm=>0vp%;&7FjcE$7Nso1H2SSg?3=?lR3jpBK#+|3O4&*!ch z)_G#dcHOKYO|0%TtJ?@IGQzJzR4_~_T>Eygf7Wi3Fp~)M*c%!jHSmNEEpiA?re6G2lf=0et8ZESn2fZ5<5les&K8&LePx%PZj zuw`EZi0zJ)4F{2+(0GTmCshR=^=EOv{mGypf)&-u#>rC%2lz1SJ!pKLmls|Zcl+e! zQtUSEu9d?ouw2(pTA0%GuN4j#%C1X`wzJGHPEridhwBZ?N;q_<`f0V7uR`_TOs#0? z7~tWAGN%<(k06bjvIy$3ewq)d@1y5)CAoWjFdyO~Zgp3=3T3X#i(Bvq%kLMR?zLGi zYaA9`%BF|!!Nu(#+YH9~z>|o!?RSc=-qIHu%I$PArVw*ZBZqqBDODP?CcV8ta%E`< z@E}=*g@$3f%S4)-|Kit;m`hs>1@81yH;%lYUi~R)naZXE)*QX!de4a=qtRcKeq@o} zrBBaZ+7PK7W~#o+ysA6d6SqXLI~`uPT&U}k!%h))doar=hH-!J@XFBBI$p3XUM7Y^ z+JR8RcDOpMdE-;gCdK5g;%!!f=RvLOOyo=k?zV}=>kL!#)~_qcD-@TeZ@|^Ln{rO6 zCL#n|<4{2a$=j1uH>_|bSZ-exulU{AiXBcM%*N6tXTQ!k#l~N?1~G!MD?(3qrB((e zuuU6p7lq@CueynaAL5IjT(BSMW>v%n*WtV@s^zG0%Fn;+1Yv5DizGC?wxfeeAz{56 z^IR3({>E-LZwQC^!rbh2Xu%N>pv}2m92?|qmHcngT8K( zX>H@JgL4C|-g9;{YC>oV|HyiY%tct(3DkQ9PmB#YCvMhN}5nKEwt8 z+n($_Uy7tBmr1RA31QHN+^}lbB$q;TAfBMvyE#||tP89q`Y7NdUZ0e0FZ}JZhr?>2 zspE4j%S~uYoW-lJ>tBJv^R_unKL(C$aLx3qP%Fqz_9!Fr11@wJE!)BImGYm-%HK9Y zu2T*pGg+nW6;~!{rWG`3U%W^&7Z=@aM_Q};4a!gbxdUW!OHTDrf zvmYJ3f-K$UXhjC4D%WQzV388@{+glB|7I+>uiVjQ@A{ja%)0>GljAkY42ktC7^zqJ ztpvVN17*nOcNPUjjW%AvfP9Y&6!&j9|BU{$Fn*J@R}TZz;|w?6FI&|rJnZ6=*j2Bd zSaqc836Gd3wZ}-fCfU|i+v_apt8x5&$YqO}2?2Fj7|UuIILThylQ z3)Ri&TXJLGPp%psgAB^+6~1*K*Z)pyW+UC`c|}bl!8C4hL4LpMUN+kW(CLUQc^YS} z%V^ac=qu=Uc@exJgPT&`%PjQnK_{vm1ED6eZe|u7tTlCHn-xkXaWM4EK%4QEg-n?7 zeeHz~ujtCFmW1;C?t?j&AxZd=9U7VdEHW14CTRUOO?o;1?Ss zNh&9SALn|bKjN9XIq38UMoHzroHSi-$xY^4%Yimsyv@6sttT{YIusTOZAkW4FE)+^ zNk_y5`s4;60w89}PB2w3kx%%SPFau+(i5{@$kN)l-N~hhG90c$mV^Cx5t>*|(~|3` z6~fUgWGDaXY2a-s7U;=4Yaej}THMM6sr;|lmYtp(R@dtR2%!_$)70S{FDm(Zuf1q^ zb1xUh$--2Ts0#>FmFW7YmeL^Z6r`-CcqY<#-O?yI&4O*~^t}9%6>G#w zga13+q$f_GAy+&wpfbVps_;BYB3xuy6htV?Wj5wAh?mjeFtZxKo;oP++w1h>5B-kg z@^z+>a{lqLkRM#oVOl!t-Bg81&u0jPXa}ul`3wVp&$6<3^ZCZMtp%{Q&zN&hP*j~zw`0-2b+<2s$!MgxxGpzObJ&?>^>|EzX?V7QB zcR26didnK2TV3&0#Wc24Q?OytZ-u5Cz?=h{lkre-CYb1Mf#IV+I~KRkN%SMUXKH-? z*}s>@p@7E8x9Eg@j_A+J)$B=|hdp<}&Uk_`on%FCPwddb5^TrVRbJTrE#xjPqM#Fz z6cMywZ-_JNgx7{W7ISeMT*u~XV|;+Wgo5H<&{p3Yu^AlF_>(g3^7b>i&=1rR#QUw4 zM1;4a^V&2WKAK{lqE$FboRG->4pXN;eJP&XyGdxL(c_|rY>+1E5JO#4H@({t%iCze z5l%>n4_?QEoqRrtyhaiSQIV~FIz^TRrttNWKNY4+U-6YD8^D-{!n;H$z)G$%BS&Q> zS)2BFIM$j3Hi>wRTum1*KTd(Gd~+teAl%|*Y$z+`ZiiXOyu_(j(W zTM#xLY9bBdtmLb!dl~t{bcdW$xLpk!z;%@g*}wK0U>o2}43FOdolVcdy*YDxU7O^h zqsOA+zf+Ru$?x!0&BcXePS}u%=#?vspUYKKL>Ry4+nMu!>iF(LJLqeJC2rJTJ;jv$ zYj~zXZU4rKSTsoUcxczq&~C~|Q-k!2vnc3flV>3XahzRYt&tX^%fVEJ8=o#Budl=z z&jBM9Z6n#cEM9NOA9{bBYkzEVnP4+@H2Q-g8QGrSn)=Hz=3Nj$8V(b8Vn6ZBLYge2 zhOmy~NZiObg@Kf9xW{fMqbZ%VgjCdW_xiMwYxQ(n6nu{3b^SrZ59)G$3(<=eioZ!s z-$ak5!`0!H>c7K6U7fi0W{SH8Gp88a)nV6PC0awH`IL!vP?Mr$KA2&rSS?(Ig019{ zo~c*FE#*_?E$~UnaT-INkE0ySumc`$Dpf}_o{M4@SI(9hK0dBrdPvx> zL~gJqv#JJ_VwC#4&nx`fT^o&S-DL0x-kZf=IE%K<25PN2Q zx(^7V$g%kNRs{)F`#_^;*r-VL+#WPMCQ5?Ua;f5p4gg8+%0yyhWm}v@YVtEq{SXDU zJWZ()eAxcWFm7x7%Y&w!gw-#$R|$QY9!c}m5Sa0y(b3zFLvZQ$ROB6jl>>(FIAs(-mMlHrD@(mr0OV7}-STtg*RM>-debHu@IF#(@(&9zI zt)ik*^prewj83>JzAg7(30zI&_hAjSObIU+I}2UU(oU+F*wQsaqX(bUHU4vJ#uB4< zA=^^?!zTG3F^O$|Zbx#czwOa@-`AbzAnDkD#|--Ry-y~7V>RB3T~cumeQ5FdMR3e3 zz$5CnlU%*RSO6n?Gcv6nmn9N({r-d7ThCx=gEth5)`Oe-<%Ist1R$m!>HE&EPz<1O zU;3N5-F|Qk0syd3gzhzu*!N5Vo2PNPFv79x%?Mn48IU~(6lkZ)R#cFo&E$GD+D_>1z@ z8`dTwoUO3QjazpEVezl=4C%q?Nun$tYxJZ$Is0`&P`O8ankuT0=@{bV+8q9qu2oau zku>0zd!-(~RSN%^xQ{M|LUaBohZm+=Ve-K%XJyga#@qP2rvAFW{0R(zBJQhi+}y(q ziZMUN5AF%V<9K=u{pa|dZta}199JCe=eBy_%|7}`Pg>s#Fz{%Pskm@Ewr)*Ff?c4Q z)*9QP>jsdyy-Zb{fPGn>s*XJ0zom7YAcSmgT9ygf2uYKg0a*}EFJ&+o#Gyh;@RPQ%cBZG-UPacN%sCHGJbA3t43q+nWKgXA9U8mLyP%r+a>+R?t!k!XV=lgbi zZcEJKZQ`C{FYu_i&$Bxb?K)N+XQI{CbCp_VRA;Xo^oZY^2};)PyX9nyJg+k(OC6kU zxBBJG!=CIh{|#sSpIN7pH~TJ2y3yHzKY8T3W^87At< zs2Tdl=>CkF?UTFLl{v1}W3A=v1w*ypa|D3|ms~R*C!8aOjK()%Q}hrR#x#A*G1aP` zw_WTjV`f}GjaJO~H$B`gq_Ra@6to%yc0S>gFQ*y$ z-^GV7Uj?A;>>;~DRPm{4#~asT);oJqr-i!{97OwP!tH+EH*)WwqFklsRY?@fU=MRW z4IuXfe*wv9F1<(~|6g%xpL5NM9Z^+4h#5HYhtfE!kkP>H-sMB;jKViW_Cl|cnv?3< zh+DwWf#Gn$P|iWOPYKLQorQwHve6s55G<8(DI|l2XHi;a^tEQx}-uT9AEm3ppEmHnsSk6FK7pNMDs4U+ynbxn#+ zUO6tD3n6P}c6`Ega}r5?=gtEGJ6hSKO1-4w{^%!LiH~Q2V|um+eZ4SIUpNP+ckc>b z-@rcEXOmK?MCSVZu)KfaZY<{SGs}#{+op_u$1wc9BLOMF%*yAGN zx$HfL*Ag1rHL3|06drrsPu12?_BwIaOK9#YwRXNB+wj}LsZ#GnI#%T~v}(4Qz=&ck zdYC(xnz=vk;)a8U7twO%wv}W=*QS+CFBPKLYR77vO94vKRBp0${pgPJ(R2({TvwMj z6j99vd}vw9QIr5nYo9%0v2oc3_M>M?LnLw1bPmn^L4}wVO>8dxa}*!T6Nk7lE_i+Q zU+w3*x3J6G$b)qALfbaTDy#_9GP2XjY4(!M$lz^{OO-r9#CoJ*mZ~@H+v$FuNbO$C zGbC&fSw$NA*2ZhZ?z-hl?!Cp58mpAQ6-p}>OY7sD16;p)R`5c&c@MASVEk`YlwL)`$SIjFS>PDnU)w={oz=ltIcNDzp!>j6QalV6su>g$6N5VMhrZLR|8 zvJH`jk*?g1RiVRA59YKs6kmHAYoT>T@{55XD?NxWnW~`rMJz?^h@wtz59rWF&4zB_lf%*NF6Y zbvDMMuFb2+1DkkR27b-9(KU@g4!)yi80&KJN8^RSc-3QoZ?eG$w|^%+5z319@u6+# zaU%3mV)%6^LLUh(x1LnbB4WbGz9k3LVxgxv^Np#4oUdaqz<5fopA?Of&i zn>pjSl!lHkVIo}gjP%}*qWi^BZ782I$-ue%6zyNH!-dPL;}KYb7Q7=DMAql@Y*)#; zpXvu1sOXKik!H(4E}MrK1C{P!7DIm5rgcxiI$e)0uTMYuI!1&yyaM@7ZokO#a#3V= zgWIeSf2fmA1j=u})*=(;_te@bBMUFmj|M8DUU8y94GL4~^|y62*!KPI(e5s}#Ckpr zJB`2EYME2mjLG+8fHE9UnY`C!zHsILp1U33nR_6668e_SfzX!$kCbf8WZbdr^H+Tg zkZ|=s?13RekdiXdFcxvc5O0xg_oZ(s<_=tXzjSS{(;er{Rz>Q&U0StBO~|Xs&)@SE zn4Waxfr7K+rdFa%4G@Q-tuK}0Na_B3+HPNqaud1RIheF2IbJc83o#IWAZ`4m>2u!;SiuJRlQa=1F{Qthhs@KJ=HfHC=$DJjQkAoCXPy z-4)$Yo**TNuC;Z+4teg_^-n0k(cR9(mF-A_q(lBy*x2MWt`}Y!0I2#d6(f0NWh_q=OP6a@jEmw5Y_yT-Mp0IB+y03sfgsb)EaM#Yb(u*&oL)+dU?;YpTm=o}p#UVljLKjCF^09TJd5YO z?wT0zM^h51{S=|=?{{CVD=~`9x4dpqt}tmo020%owYaA+uOKnV`Yh^F5=E(tF$a_h z7ex8zn4|nX3gR%T*Qw2$cTIi3N?uYQk%4@7TE8+B{*l0?$Ul%0&7L)dwOPKN@71VB zz1b55^ofHCK=JS{Bp;M@+Jo|mY>%iBgqkA!?(r|ZUN~j=qi&W@su$i8x z0TmZ?? zH!TIM5BCYx#_#b{YHy!iyfuz;rvB6XYSR{~d~ZM}@E^;VhgoX6wWPan&Sg_LHxx?@ z6%=grfZPX`JbiSLsA;Nk^vX02k~LavsMHRNqHS9fMvp20f6dlechh$ckP|**X^c;F z49q779ZBFbIIV;r;@ry>pja$Y88%j>88O#)km*gJ$+||z2@Ohv=wNfoxK%q5z%jk{ zp;IeYIFjvn&otcjAsKSOa_7%F$0xFbL@ug;Mm7Jb*W>=S4YU|(lbXJBP}KLN2Yu_@ zu%~otGCxh>G|qJ1a1#k?5PGC{*dCeWp%okPe_51TOTV&6W5Qnw(`^iLcls;G#+GhB z=x-=x{V>o=?>qB)KL09RsuU$XFH4Oy9OK=+QrG|(TmOTCE`a_m$pe7|NI6Yw-8NKq zjLT^hbrmtbRoxSsZXLDW5bJg-J?xNkX*Ys>AZ zJC+msYHO^)2jVbkh0m8y#Osp7(z(VdwZNA=(WxEAp$iv9M;90hne7ARQ~LkHUkAi2Sex$mLLpE9 zRArhIYoAJa{n*u5?r!^x-w1z0f5WcrhURhAHOokTki`(gK+R;oE+PiT9j82Y)*ngu zQPeV5(P!rR3^LJFzEG}^c+jm9q3aqg_t_26WG8mR84s@+v`PG#)k<;Ks#SqpO%O8axhIdy|f z>NAu76t+uhh9Z{IopfG^@C*oxRSLG@1f-3tk@m@d_y)^o@w%(h5rsz0Bvn79g4Y(l z@pzI*GBB=E-p8BYX+S*Oz&A3n6z-LoAl^^1>uyzz4;`qNDlPxgVgDFJ|A@a0Ut`Om ztb=b-H5uSgdV0%=?k1nXUe~E1T28rJDU~M|8m~wHDhA+z_cBd*eS!o|s*SZn$wVbM zYehBamgb-9^ewUQ4xaIdjck-d7;;bvq=g>1nz0^p;*&@wwj-XwK-|fLn>2iQp{^4C zq`HZXm0B-%zMd-Aq)0)4(jD#B9~D~Up5E##P5z3czEuvp$pJ^P?rD$-tQSmIGmq#g z|7Vaap$x4Ez+=^oMJ()3-lkJI!%{cab^hZxN@z+`JG+udfqnn z0Mx!zeb-ONWa^DJ$YoI;+dYy!nO6-hROr!Fd<3%NFlv3jF_1joU?dh-6o(I@R|3M4 zNxj2jX7u(y*8TVz1D{r0DbRi_J^X)#B@oD{i?9MjNHn#3R3ubk_lEq_w+R1Bp?Vfi zevKR+M|tpNCL~DHq`_5bhTw03I?L16LoGFlUAVhY5pvVTIB|5X(wZ`irzL(F^{v>- zVrr2NwnRtwm9iC5IWT~+WdSL%ogiverFp7=8n2>EzU0j${bf3p=1@j154is{~^Pww}So`tlvOrZqpTNb5a2ubN-o$z{_<-C+83AD(deMr~M zU@1_`GTK#c{y$w@G+Ki&gw*t7t|(pR7?gpAV`p>l0b%LZlyK^VtvlCF$yDQ4m-7Own{| zfSorgO6G|LpK(0(tH1&2noDqo&+3uNFs!Zggj_D7e{g|NIGAoJmpkd4NLd1F=Y447 zx%P942DQ86AH2po9{o^uq6I>TCqaN$zl2CkIRyup*7Cy#5qCBCe7dN9m~o!S4W-1* zV0qHa_v1Zo#gS*haE}05=oXe?kcgR5AOY_!7ZPz!Zc8z;&9w6cvP~1Tu<-H8_4bHx zt?^sG;eE=+H$$|K?-G7sPVdX!yq~1)*&MZkkcD+{8hKAWEPQSVKp!HRLzO@CKEXN8xnVXFd~MO940=Hn4v3?fv3L5@1{ z-kCgA>^L;7m7au?_9Po48PI+D&Rnm)SS=K<`v;>m;d8vfJ{^C)d*GWIH3Uu)j&LWr zeT#{H3czvz{)74VSyZm>aGm04C5*JSA2eFqV}(y;dwP1}s72h)wzMW(tNW~MRfX=- zc*J}F^gqtN!q0&#_meEi@YpP}LD#9%GY98xx_sRkD@L{Q)uEZ72F~O5m1p+MJJDsA z>(^gR+SBd4<@;$k-Z+s>^X&Y9PWQ(1(9!t76Uf7%u+z8aIY)!A1hv<%sYN}uMswu& zY=!{QxtO=Q=c{j@|7U0t`-VmjiyHy}%E)USYQX$O3OU%=a#E=N#lK!V`HXb$-X8kL zE{g5qoSNA8LJggE_xZpccgiN;J+YhyJs$Vd5qVm`NtE6EX`v6*UO;d}ufVT-2s&GN z{d*o;o=l0OpHzVC3vPe?YdUsp4FG74wDJ4cS-tio27T+v{qdN-vOWPBEI7_KO4A~IW`#^;5wr`@88J)=+}~2%C5f zjongZX$Inc#k*iD%dqXz;7ru>4;NKskxd(07O!RwhT78s5LOSUx+vk}Fi4eziL}iz zZu+9Q_%D)xk>fKG*#@#Ei7Z|2kH2XsL3R8j^Fu}bt=96;?qmqt5(Q(jLim=e44x<@ zJc>|sw96_bIR167OT6<%A!^?XWA_U*&;ia?%znb5JDsLDa#WP3^8U^9&>&FMOPdC+ z-X&fMy7pgm9qwK^;72>8ty0)>7V*-{SMPB8hjUgQ`1WX~%q)h1XjEe29T6S-oH_d* zSN^%v3_6Yw>u8l6mj-ewc zr^B4wUoawGv76wd<5djvt9rgF3i4p|>O>6e1(h0lHmFv&^BX;zf-#86xcLbVMzwN*-SJAo0(C{&~=f=`a((DS+pN=3~{u1cybXIRTBsI#9{`l z?nqv9_@yK!WRM_F!D%WNy+(loVzMpX)VS5g5q`9xi8&w9{gFHe$m~z^ZBB>{8GK>8 zaHCbh0_Qs?S4=yve(MP00De3ouOmrPRg>=lu~xE>`~Y_)2$p1ig17-^?c&P|A$U~x zG-sZzJBSccQdXu*Bj;h1J?(x-P%kx?s7HeddhT+S9%1Y>-D*8ZiZE_^t8Sj^cmmL~ z!h4_Xu;>5&6-In{kZQR7r^Ju^pJ+^P`+tGY&u|zAJReDSy(q&ytg$}b)O~I6K&Xj`hjTdUXA z%HnX8HPl_kL;wtG2NM);3melXq!3u9@hqgDzy_C6X&jDn?WaQKJ~@VY?Uw%VuYE+X zutfnq$8z7?eQ@y*8421WsWlZhuhqLPHuztO>~AchVk?qHeN$OdA#0f?cPmzqLEHd) zqQuaKA6X%@D&`LOcgq64+F2dfHFTQ2Z^gJy%^Td1M%m? zGpEt0)cmh&(H(vW_u1A6FoO+Fj_iU^-=UqmSoZgLa7CJIU7`I1WhXzMP1~s_y%P@P zr{Prxsr^f=@IW_&e6}I>+F{g?pIQ}zC-m4~ns`6^Ae%I*9`$&R9vKo|EnrQX&2Dxn z4Q*%V+o=v%K&6~8`U$qZ zKE|6?C#Ds|RK?A8_Bw{#Bsf>f!GW!DPO-bASI1Sax z%G0~WaTKZe)A<)>-sQ7{Z4)aoxvEnAcCV8Qosm7+em`HXq}z%BB`Evt-%SMs+PExe z6t9h-vZQ3->OJIPYO&}46e*Y&s@t})M6`vyT-7!9mIy_(QcLESv%xOu!$Yua5LZJAXyy>Y0{_$G(W$cJZ`*zJ-=Ne!?)!g43Zt#4++40G!7+d^ ziOX2f+4+~pKjjP5TT~1J+Whu9+hN(Ojim=>(SSz_I1x7!AYP8-uG&sPAl!;TpK&UK zJhocN+n?@U%0_sX>S%|GI9B_cUu0n!@v49OE5&BQ&4=LRW3qQw^pZ|K>m3+LL0->l z`1){9x;`b{7q^P;^|ymB@d||>NrM6D$aRAG4qoTRgj>cdMS*E<_lzee^i^ zLFWeudDL>|UXsY1FH-PRA4t?JZ^-3d)K<4Fp=3%-GYNF|wp}ycqZx%WzNsSHJ#=Tf z;yG&&C^m_Ay8Y=ig6GBc!hBt<706c70UcGW>aIvNVcMgqs2id5A(umBuhB(nnN=up zI!MFg+V~?-mr20%-q|E=MJjoScSRoajC0PYk=GJQbNq*yDiihTS^xg#RquMy**{`h z)n7z)p~jjk^6Y*O!v06CEHob12#cQdVVyVVG)wj&^g^gten`Ha6~#FpVM!j_cjFmG z{{R>D(yQWiCenCqPF{`%3d#Eyp+cudWXO;s@$#3W%91Rli;2yC*v%j2t4CATtM|Yx zW@!lK0AF3q|GD=aMgV3V>;1&L@uGks+L;QaS6)==kgmb01+!T3&__f+?;eAGEB%{5 zuUuoV=@Mfud%v#)9}Ps@7@*e8Dde#Dk4En;S8viM#=@2Hmcx~wX1aNk|BcHBGKvR1 zH%LEyQI!gUFr^e1;QcO&{oF%AL)+aQwj;gF36Li9i_QPkzgH(hB9jI|=oSA7SA`LG z#;zRK0IRv7;i0@;l6Q)ALoUtG{XmtAc?3U_o(J@9)y}SXEeS@ndu_Fkd&*PA4fN&? z*q&0JWcuwsN8IM0!1#Jf5Ogn~jpCgny_%O0*waKR19_^S-b$Gdoh)c^&=teT#Fu7@ z`bvTO7Ztb{?r+0w$Y>JBM8#79RWW(bp3@q$%Y|_tfidylJqV*Tn(%jYrv@|-h-S+t zWiqAHm&N@K>N9lz^#jzF$VHRjr9fgp*jGsndcN78-r^9L+?n5{gac9sh@Qb5lUbzW z6*d2sLl_oA^2mf!wKHSek)*&WE`UI^KvC=wkum7@nEDt3H;hn)LTsm z!X|Aa7)iHGTd+wa(H>m*ex`Oez?sI3T%XE>>{2#pr z|Hpq;`2Xp}I%T2{hDWHQt(_~1HcE5-pBpOvo0qh3=XRW#|EwDcLU%k{A1sjrZm}FL z`L712C^b50&P9pw!%@WFv3uaYARkq(RG{jhO(WmR7$z${0<(aVN-+ zx$4E{<{|u;b#IGgGW&I`_WO$x%kxW0u$5__1Jy0??Vh<1Iw$Cqg{&2csX4vvJ! zIqfM934OCkS~yn%C}A>tqOJ28Md6cx8WD}|alzf@m_1M`!zw&Nb-KCJS|Hl7ou;qH z9a0n|Rf|Wxe?fjs-fushgYxo|wU3@NulHNad%>?shB(!qTdp!RDh2MYpu9&hYTXI+ zyyMw+xw(6X4ZmzZ%)NxK+M2$h$0LeJ&|jbN5bi=3b(c^bY{%|On)~usoc!Uj9oOAn z2>UxWd3Ux8qc34-%~3aX6j&{Pl_k?Y$%nq0d_q49y%wSYWdONc3mER+q(@q^OHMV3 z808-iHT>R=PcITm?hVWn$itx6ez#*o-5C`TKV8qcSD zTI3)hpq!=MEqmqlSw8fo?1lp#!WWT%FH}#*wAWXxC!LNt@S9J(;TE!BzmHTp;%<6* zT&Tv2m7@89R88qyVTn*S_6o;%EZ>Vct!$c(h;Z~j^w6Bg+TO~E^7w==VvN3%F;b1~ zq@d%;ebj8qn)>{WPb2#@>KkV{T&zU~gLaD^(15TteR{9nmonV;iOGJPmk;dL!ad;| zUovM>nt9n~N2I++o|h5}4KVbiU#X!z?{9yx_xJmq>-=@NbS~y~eCMraJ@>lrd#wkJEAaPD9Pz?} z+KV7yqSax0{pz*yUm^m!p=-G|Td3WaeOfI_BGGB<0HHB#E267wbD4K;Fb-v;m4&fv zD66P2z(Oj+EGS0qs2nGCb<@1jkxo`P!?qxt;m8*9vEVxt>Ke&P!XM4JRY(1qQ?6C$ z^~D|euYsvfT4cieew&x9Zv<3CViq~e?Lg;_<@0=N!vacK z8P|VrfE4F@Qo+wrbuD=$ODSOcp&inb$ee-h+lFi?W!AZS@V*?4255rdK_0fo7zd#c zHPMzAJm`H)BqfjM6L`(()nw9&Y0c2w$g7Zr54qBBLGA;)ZN6%4oTr%gsj6D*_E&u6 z)rwr0Qtd-NwNC}|)uVE9ovyCYx0GFuOB`nOg63Ap<3Br?bd?_pSrm1nsrTkLa2}|Q z4&@YLQVj>*T=^6_(rIsj-OT~rZM%5|vQ5s=MZoIBYL44L7;{xo{ctSaa+e?1=x8Yb zuj;X%FkFyUVr2-oq)3Hyke{?aLm2PODV)fp-J$8;-pk^3vz^I>uYhS&2p!!<%O+;pxrSA zny85E>hTrUW0>$88mv~8xf?nc^o^u1GkaGCZ>?2eo|5Ehjf-a+bICHk5x6fvwCEg@ zUm{GUxsEAVRDwC-LFVcTCxSW?p#=bLjbZ=VsN}fnQk7=?`U&<*IY}2VT(&>VPaqMR zohnyH$^Z@I?jbANwHea;K1xRu&6z?bDdT1dLqv)(($Ms|MA7q>-A_uD8-kVdW6`aA zRI9TgEu6l?>vJi3jyrl^zvV#+SrZh?qZFuQOk)$n*o!@yi`8uQSD3NrfCn2O8_oxG^A~EasZgPe*G{r#hoyJBq_gl^G7~5 zo&ooaT825^abhnoRGIMDjdpe*81N0~I>}D83nKSwGxJELT0Pe)6Z_6KOn7tfiCSXnw3U^W6%`+Hb31K}m0OL%A%Ge6o)$Uj4cFAv3>}(0ojByNnaG6~f&G4< ze?_5dXSeIQi6y|;b0M$;olV>LoOAz4nn&I;8nWmQ zf+1J4h6$HCe*btC9~1@Q;}k`|-8Q(DKIU$rB&>DFx!E+l~k?oQe9LOc^ABl3Lm!VC19 zF`zvi2Dj6DFO@bdG_-(>ilr32i<6V?&K z>x!)WeD+Lk46KK{iLOp|=zc56B)IOWM~o3^_4GG;b_J z*R-@WPLnP{_q`=zc|p4Y6}6JmQh7N!Z_r37HZBfP<8|5;M&*UsJ3c->JZ#F^y0}|i zU0p*~=CnE}N$a&&IeEU($vf#eVia?}mm#84Y}#8|vkx88^;q9q?(^~jrEZ-&fOhpd z7$E>x>SAB-f<#K3SnTBFq+@j*fk1Q&)gt%v?Wk!?zmw-pNbPiN@N0D~f?fqbn&N~J zx}1ZrYQ6(!@OnSVNCiDbJJ;Y6sX4h>VXsaz=z36N1066!d{(jG$$Rt7UGgl|4l4pc2V|LA{;lf~Sm`8h_j!t?X&&PPN7rE@x ziE=t{V6y@_tcyeuCM0oxxr4i57l13J?$#Ii9kZ>&8^_J;@sS#OxOT<{pd&*|UdKge zFz6bzE3(2dUL?U834?xrbjQgVesS8kJ5;k-h3 zOifAD)YLAMUZouvHnrym;BoWvINV_HW6>@vC@XUvDKct)PtAPqUM#Q8L`zFcv00z4 z&$(0g{2J;4Bnp9U91+*95h|USVITP+=yb=snz7=y7A(q0{mrzHl653^q#~;I{oTjfc!6ui??>)rllQb7R*-Bz=V}L1%~jf-uot&|gbeLu16BOxN_Q zi{FP!nE2XW`vXQsMzYZ?a1mIi9YUUM%p|Y7`l^g120c9^Hc#3wa!z z9W{#xyX}14ZiyPNbRodUpKx1*fC6*SHc9k;)JdPnxl$q?yE^Fago=ZqL$Wb!Ru&eV zx_XYiPZw6@y;hXKaFuC%jb4_Vcr_~F1(u18#w`n%JO&Br_zxefq4OOgnd43o?Q=}U z&N*F)YRK7`b)UU&B^TT62jK0SFq~U>;2d;x#^rrZly($c%RMyq|s>clt)Gk%rGHVr1tGPB_Q`F@5cW6=F|S`qj4O>VE#=87=^ zyt=0N>GOlq+VgcA9}8q`r!j1Vh0buY#_M&X-NBlMn3R^*Xeb4z@P3wjmUVRLF&NfX zg4?bG*(#5Pw#h}syQpvzVsam~hXs$P$vop|?m9Or;yu^D6;wMi0%AcZSyQXV zYBd?(Q`{6(@&BGn&5>p+FX@Ta(h;>}%Q*Mmp`(uF%K4;J$;7e^_VM$>Ioj&C$V6Fv zK`+-x>(q z`fN?vgR0YFgHjH9w}A*M?Zk(g_U^W9^%qRYy_UQQcp1@x#-Bi?G`yM85_eO6Af9E&I2J>jceDBlpVKhgOJE9*jwTZ_h#?I zx&EaOp#L8D!FtG9^`~}wv%yK=vuX3Y=_&(|@e6T!kXfL8dCQyE5TB?Kz9-iNdh+#g zuBrg*0qUSpy>~RS`6GtKsQ~csWt?c)vR#J=7NZ)3_!)v8b%_kBdl_j}TkNzCFWmAX z_^)+`?##^s)(Nd=3^aAA1)XoSl=w}8pBIi&p}NL3g`oP!f$FXA407N@Xg-OaJFQeU zRokbN7sxQ)iKDqldCFzTHo_|U)yMhTfosO?X_@m~V&<)t*2n&e8wnldx33yJYT7#k zo)~PfgD(V}t)QdLsdeQ#kF&ev6OakwbDF!)b>wBjwE8$;3PmRGMdTdO!QqRt2!cjB z67C>)9TxTp)VFL6kJ}>^Ok9kec8+EnzoRY%Aufpfygw>Iha6~5PP++hBIjh0Q`K=0 zOPm}Bl5gGub!vp@t0gWvIy%4zMiR^-_~7MNCB9Bn)6QVS%98pj#_sOwVz~v@>!J3m z_tdoLoH*RpRA94?@u(I&qO=k+ChLU`(Zi_5QP^}*R5f}0NA%dSm1Yu=35tnsA7ovjLG$qgluzBAtX`g-X7N#{|kkHvW`n|S$^rx)uZ zFqfe`CWwuW*M7xeO{s%I*6e67+hR~>EbDzM|6yo!~vatDc4xYej}cOq-t z^mVbknv<2B{2IjBZlcEqonO^fJua9f-YMFHAbLPV^mYgGy@ldb!NH-8dxDwZsGh9G79e19|2Ok^rmHamSg)DY(^S;esbP3a>lE30)>Cs=n$2qftzTo(*hXJW(;{CwIqg>;PMq%N`#^S^LXI6W zr1&AoDNKWnS&s|Q`&C4t-muI1J`YJB8-?=xGpg0Zm+ z&T#MGU~pjISp5r>z~*pcyr~*%wA4ngK>xMkQLDUngcmz>8qDFW`(nKE$OW7>0K#ex zO7gu<`jUhlmp*jZ|BM2n`ZPXgcC+7^gtxJ2VU_#SIzH$7A62GPY0oo9wV31*hO0eX z6qr*I@e>mg&RQgeIk>qQ%54krY$mG8!3`T)=+3{EZ4Hi)P>D%C(Q_p$BXo#_goK-$ z8$iT`-7Vmakx$^GtLOmF0xfbYu)h>Md;l~$BcE2!Qg3d#WcZJ0EYT;qZetm;zM`0D z7D2{h-A6USIyv166tS#H6!tn{EgjpLrM)-;Su01o=X{hJ=%+?2b7I}W5ajKfII^~=;FV%y!xf`R8dm0;TnL*bmX%Y@2O0uBC}qB0uvtx zQ1K2;ti*kv{a>yHD}mJT+3 zyf2W8Dw5YLii(QjZECV|bDufgx{^V7z(!|EDD@TN(Wge)CkC42RLCE)lU7Kxnt%_V zX?dKL^xT&#ai*Kt2d6Uv4CmH>M5?DBoqnsH_A6#)}COux*pr!`o zt2JBxwQ~u*&L5oN*gnh*PMFTCS@rpD%%j2$rkQ?Tvx+|E?i-widG@b{@kii83tXWL zK=;|XaX9^mxAvGs-|IEaJ-0KfmLqipT#lvcuzGc&Bet)%aPnyP4KADOrpM6w#Q6=U zR}!S{cc;zL$VoQj_Mwd>d8srK*qpSkk@4iDyJ6(jnU#7A)M}Io znxRdxey?~~oi}TgdnfdSl(Ce$we@qyVTOu>wZ|Knb8wdZ%&~y25^iB~Lo~3y6qA$` zYrnNKGq?<0r;2-z9@dbLHrgf`W#FEpI(@fpoNohg920{FSo=^^^SIwEE^b?m1^^o3 z%C(|rzCl@3>g#t+rO~iJLnEys!}1O>Fl_IaU!DpaJh=9}*|%w0&DToBS4B?HvTqJJ zcdUmt>?-CWBw88JME=7QFDKj1y@p-Y{X^G%*~5|{OZk3M_bATcrjLY^5eioN5_YX_ z6^$(B`tb1ZI6FIQ%7THZq;nlL2wmT9^}Udhm5q;!1FtHQY*RXV9?yRcmLeeCL_|r{ z>M}4tC!h8(>sxk|mX_v+fSf2w!|rVB!)(!q$fi&QwX#PY=N~%Ar(B5F3 zfQ5J`WXH8nRI*Ivp|x4|qLss{Gew$N14cUk&Gu?8y_`W>_Fy7jJ%uh~B*o2}9flkc z6uOGiBSj*|>=v!E7mKOcT9Cpxw2Y7#qs zBCEt=C{MGbjb#KN7?4@Vu=~uySMu`mw5^{xe$Vho;vdxF-ip4zFHwGjToktf{W{GBq_F zuW$mh>iPNkrAwEfQ0TJb$xH~lwT(@;F$}p;%y)1kWE|kKw(T)Y9B&rO{nUX}bJB^? zL2A=Ca93ev4HIZ(Rxix6mA|i z6eS;mxcHo8y4gg{SuJIQrie2dqC%|W(Oj{#1BWIjMNp)MnYlS7xTEv-9LtbzOHI3L zb*V*8LvC_6^>ZENJj{c+sEm1@$Fmk&6-P1 zG8YkP8aE0QARsaqdg1)Uu9`A#Zf?(?KX-6&07Mpe1;EA4t=#i+GtH^HloL#+`mjF6 z{KMjF5qOAeolI#{;S~eiXMqh-weVL%vb4q~x&?;Ot-~{+U4q?Wbm(2WpvmF@(|g6o zNHMIpqO{;K1Pt5C(o%~t?AoS4e*iRF#Px)+3x~wkghP3tom(XDCgg6jbq&0s zT0KnPWvbkFBtS`axCM6P$pQTwy~QNhKE&;|m7Vo*TMthTGTo9#s}kBB8-|5kagLQl zk?ZpY2kP3?=;mqnnVGGgf+Xywf$|BXO9U5((rpd0-Tc$E$5H#4QP+A@Indh1C@h6n3vP^>S^Z!H-DN7y zrl!Re{5m=ip7y{1iHEur0JFWwsu)7&Rd3nt+RBAEfFLmSbsgK$O~(9Fm2qg|)@M?g zDWILBE#XmMF9mp0PXrT7s$il}0-EE%0PY#;PG1ytXp*tlNZDi85&S6-W~1dKxDKZm zD(O`F3cWtmK%AUh)sns+)wZ|`uNMz2%4Wv(BYeT{6ShJKIz|oP5B$t)->>ou&YImz z2e#y{E;$SAjm&|zP%Q#L;43X|{ zkT6`qe>3fQMjvn(E<09~mhu+eC7=~fkjt2lxir;!ChERnk5ZDK%s%Pk%SODrG0aQ9 z|3QOqAUt%(8Gc1yhuep|Nlba2ri&nbZV1DmRrya*NzvF9t+&A2a+sEG1%$U0yA-pV zDppUGU5K8rx2>zUIGQf{S?RUrUsd#`lb}0o<0ADrDDtiA|$?aLC$-j9uXWk^D^=gsDkt1U|Rvil;rxz;~LQvZZzrC0}F09dS z#4nyg8c-a$<-=NbrMs4G0#-a?>Fnz2I@=TmhN#~MPvN9iL+6dLk;zF~{Z8>W-4CQV zg#Au#J}-IGG##@B$9cetO9ggoYhkh?=^IyJm)Ei$IgeMD5a9ljwuFG)vTG{KEG;}A z|MlwD$?0ieU?4UfhyZWjz6E+J=`wZivI$$`o1P&V797-Jzt4vM@C#SU$zB=@l};&u zSRH>bb-yu@mY48SVaVFUYrTZGfb8~G@u!`gow&HTg@px>QGi##-GMD=In?N9)|Y&W zD-hDWrZ>H>{7_x$xJ0ETfa!hxX|9My=oB>A81ZJy=pBZaYwR;)bMtH!qzikOcUKDs!1y{LRMT)a*+?iM5KAl`Gz$H{SMlx!4k2d+F3UB41j~QC=?U zU&p6{uKH-Z;JHCYni0AO1|UsibWdndvBMr;3GQ>j2|@=d0{Pj3)?b3G-}{G$het-t zSie!1pf;ZnlK0gsXlf$*`;|2{v-$0lCiY8)P=8THJ!qO*hf5+#?G83O+87s~5HGa+ zFLQm-tUvsb-nSeP5#h)`J~~RJt4<;tm*9WLtM;Sb4z6A{=--RCP%5pW5)}?KbISR| zk4!aone%7gEJjuN`$O8K0utRX$TEGh!`cWfAB3jUbX`+ZRncS$@pzMBTPq`R={|kJ zv=K+_w{PE`F%p{xs42gIugN3ZHFtho9GmVzFZnmc(G)5%tb8)Zm2T1Vw==uBVu740 zv0|oPArTn-%K0YLfsZ%0q6y7r`@aO1H`i7jHihW(R;x2j1rJ?BS6*2Di=ZT=cLwtmSD6Ya>uMdLI|aB8L@hH^U`+lxoV# zwfhHP#iysI!>D)<_V=%g?HqeDMvjy_nE3ekNPP^0tOx)wgX_K71+WdLb9VT{G&?iM zajgcMTQxP?&?Vwu&_-^rIzIFPh9&d!6{80NTe<#Srz9OT!mFS9#b&%K-xB|UYDC>J zA2Qfd>-zXyY;8TD`M+EVefPHxGr(;k{fyX|2dW8Y*T;PUI)-Tj`k|+&*-EkG~MGN47qD{@lsY(GWHg5D);O8;Ta%*ie`iD8E=l z8qfLM$8nOepyS`g3EaQ6QgwI2{aqM;8bS=#=z9HVTbM1Z!8*zG$Cn2PnLzj8qIAgw z7iD!^fLvDI35w4oK->=~M18W_WKQ$L0DYNjFoH@s{!47)z_TG^_*2@#{5(6Cew%2W zZQ!fN0I^g23Skxg!==_}s0PZjGriWpX&psPNeRu9Q&Y?6+(}MW3{?Z@(*dgI!$Uta@=oq913~Ge%B6zR#_OH;i3@GADArEDvSi9kYGgLzF9U&H2`Y?M-sj zlN<5DC!w*zF4OG=2IBsV`{P@-d#_Ew5YJLJUq_Sxjh;i+8Ne7IFVDBfQ297JInCP} zC|<1w!XhTE=T^mP9w*Xs>l^0b+6b``;2>vqJnDj2XBs@UDFZ1B>Qyjl0`S9p<|j+y z@}XXq#^s8pdvz543xPV~sf=}kjpenz9!Xu~Xcw$(0X+@e>BXgm_?%M4%IkWkZhBoz& zRLDH{t}7T@iD^KAYiMYo;^hITpm)>z(`E~zj3XO~a{6qewtyPAa%Zbr1bai!=l={ZT zwDt5R!K;>*mac9d&xwuqZoK&=UyS5AqSCS;+1VxAOst;~D{3T`j3w5nXw%V7u0!K4 zecedka&bTspRm^n)CI0Z7mP?a`2IG9!Lvo>e)6Vh!+|veo>x#Q#rEZUaxA}Z~ zn`KTO{g^j4e8j-f>2fBju;?ed($~8^2+i8xzt;2_aDc2JgKiF|7R>8ex~P z;H>IMa&qs=ilfuMmX#0}7iG+L8=nVvj|Wf<^LsOB$dg#3A{TcH_g zpDC%R1Ox=240h$Ia8c#+@4hR9Tv4YBw}0kOPaf+(!xc9(GXrK9GqbFf7L-6VHD_Pm zh)LLT-|wGOqczH_w`1CYGYGVTd)08J?Y6=5D-gK2a=j0*BC)IfluC1qlX3DQeSF4)n=!@?3#J?Dm6mGxEfy2kaOiC5W?Ay1}Kuh4KOb+likqrUO!kjlk471 zrN9Y1!{lq@yCXc2YWKvzw&uM@^|Sbz?Ix>dP#0~pf6bL@hA)vx2`C-UQ=Xmp1;0m=+fvE{kuqmtTXt5BYYu> zQ9*w@tB3C_Vq6tH-RNZl*zFi%1Qs9mFm@CJlEh+haoa<2uloyH8tQB4u@A&3HDDFznS#(f0mO2aeNCH%3D_cU4jLLC7c&V*k|pfUPYg5 zG^mBw{yIBi4owU!2@dldf%Za%{P-9B4O6Y{c4MMThCqIV5Ch$L=>3EMB>eg0Tphhe zoS<|SOK+{WrvM@(qJb_kFL>B?@$%48e$;RW0*o}eKvzQ9OwNw%#H4uQh3sY;-;pry zb#Gzqt_eF4=y&6`HVtc$=Yw0NulICzX;4+u>+vIE0gK>Ki`AqqGQ6W#;S%KQ9h?sZ znC(ydb|V>zZyz(>wvlJc>hmk*wKsBY`eG<9o;onFys}cPA^hizsD5gMl`Ke96>yO2 zlNZl22NZ69)7djQdF5~g4^7mU=ToSMlqbY(fyoz|lhSEUb zDEU_|fiIi?0V;re$;s30`GYmK_HsQxXK&AS+MWrak3l=@LDVEKlY=J#rvQpm&=wqM zMJWK7m|&0O+nPx7xxw)7zN@Rsi4BhSC9b?(3!5E$Sf29P}dL zX?m!bb0;-`&P?`57vadS^WsH>T!x2-hYBWbfg^tyk90*vwo=>I!GtSI_k%2&bOL11J^` zAK#cGb`C6~FqlujI25_~lY9vs>KD8x_^~8vap#Uf`Y}yOJ3G0-YM`;upG{1*xm9pv8j2*#pa+qVN0PW4+x0zGybec=TA3UHFJ~Qvky5rMDJ~_1^4~Febh+#JQZWB(ClN09sWIb+liU{sg7gpE z^a6i@6PL{iNcUo8WhIzrF|S_L)Y9^BcL$@#d2?cXVq&AJMAc!L_jtHE^!&%++82>h zrK{FV)Wa}IZ7DI~mF8-@lpD{rsJ$Z*ZIsef`(hPcB1ayxNgA){-Ha1WHTDrng!rJg71;gN>|hZEXQ7;OWT;7TvL@aAfP+SUIe=R?mq<-BN_C!@hMK z%j0G8%ZvL1GJAF@c1QlbswaENaXOBTIToFsWEz?`W6zx%)$}O$l?X%FKMY?l!_Tj+ z^>KCmO!xz&CjdLP&vVIdjwy7P;(V+5YiS-^wYOWCo5#h+udc4f$H#-&y1l(!rFoOJAOEW30twb+Oz0EZui5+N6MRp? z#TB&?ZJ2^fUq#Et=gp&cJ$pX+5aIg$H4U4cb1U?MDw;p7gs{@AUV%4n-}~99#V9~{ z!KX@iJ$%=dMpb3Xbd*h43BIaz!*2(DUN2Z0BRTiF<9{-)Qt-!!@zsT``_qR z@(DH>_#hjk_NswqS=LcbWo~NpOGzWq*)8Te_>Zq`d}KFoJ%``h()t??g<%K;5AytH z;^gswy7*EDroF+gM?Fa;;W%iw+};SS<9<-CH$Yh}?)o+2`(mE+R}nM4&&CHp1dy-U zIb=uW_?zF!1dImVTN%P4WYN4$ttNXM@lfQ{u<8}{`NjsGFi3Rtm4DKH+8#X{0q&8^ zJQmb$2$ST&fTvkYS;9Aa4wPL)Vjx1YK_NI+)} z?i0M(oD#M7Cw?ULLBkzCL9h@d(A8^CH!Ahq=1F{HMJX}m25pEBcz+hg{6_NKKDr;0 zeAngaUBd|q-TP6RmCozgDmfhy)B?_yKyf+V0S5)lLgi!Xd&zo0VF%`k^Kq01Y1}2G z%fd5ns}oD`wjBPJXm?Dco?5O$Z8@hfdgp z?dql=)APH>AO`_PYLFrA1M>dxkN^*_7!C)tQV4rZS#`BgeCPeDzL^lcvy_jER+BXl zA2!eXzTXb2%><+#q$_Kg3ivlPqJt}{Zi@twwi3+1&Q8y*^x!~!N@Kt}@03DB9$nWypz2t={=Gjpj_ z#?8Q!QfSKTlQ3?-7Hq|SWO98z`h^nH9UJ+$EdM8S9e)yX^x50YZ0A!(8PRAkhNkzy zc@Pi~ivgDODki2wc>-cjZU?Nm!+WO3ur_%p1L#&9s%96w?d%w;MC(V-A0VO;vMZ;z zrMklI7I(DBwQ;)h-ihqjd(unzSGrAhS$_2a0*LS6oSumPjw!ILfOYtatZ{7Ujn}C4 zk5I`7s3zkY3mLACLg%@%MZAQ9odDTjKKv*c z?dr;a=0dpC>N3rrX#2Eowb)TB(m}uQ-crd<-0_8Z?Q|+%~@VSha-Q$(H?p?4hNU*S0l8EX$fr=ov{C4@PEDs*7CIPnxaL^1!l#MB(s8|K8f01Og>!x|H@G$`DrSa3v#7{R# z?O39jJlU=IA#C^VNoVu{t9teu;J}~MQB{STbkigkSTQW*8F;n3hE->OP;|jV{zIf` zyBnVVLRcFpA|ACx(ybg}3mjTa2O0MVXG3=M`7WsvWw`OZf6BZp{f{(6;$?s&*6}j% z^iI7cupWhX>hTzln-yGo9qiNm1ymjWx?cmTmusaAuq0`N0sy@rlQsyJ5iuXQ4>N3j zPpwsItyAM!zHOqWrpCrg>a{ck9Am(NT?&W=@F%z{gn(qLVG|wYH+g_)Q-9AWJrU(I zWd2twF?pE5`J^SU1BQixrpesBxg?>wJnhxWqek?%|1)YMXG<<*6!>=l&vFWk)W;x< z)Hkmk`T0g# zFrs^rGqwX*_D)Yv2UP3yXcN(s2&bM1-CvbZ(Zi>d@VQ^e=$(y50SGdBP;Y^zGR6r#nclR(`pDX8b3dO(TJtExHyoL!HnVJ z;RX@BQ98CFD#;p(2+VKqaFYv$i2n{)4Q9CxRcRk*^WX7ve9{Yaq6*nObbXzD=n^1O4PKKwPho?mk@YOqro(tn{kadQ3VaI)n#uyI8i?6{gprNOi_n!e6yiXmOBkA_ke7YP{hI}4#$NL0Vv2Nx zHGL^S`p(MAs;oRcJKoKHySSg1Uu|6FIxWln809%8EYxOGQ&Ax~m%*mHp0Cy_LoC79 zAo~x!S52kocje|~$4tj?W^2MW!V?$g!*+=nUyVklQZ_3+oeyhM+CS7A+5U8!HLL1b z8HV;^FH){n@{aX1@n6tHJuAC}>8))5ctpVD#~B-1kC&gBT68Q9cw8~*Cy$lCT8<>0 z#gxpYUF<3*GgJYIB^ESyD^e6^T29B?_-C*&03_#fEDGvuygmD z$;#a&gTqkB%5Ev)ZngluTOD9nS6dMu3B!Ijr^J;;iTn%8`myvdS`jz6B_otS&4lb$ z2`QZ6NdejN-vFBIy)J#n+2f76OSTOcD)mS&TD#>qJX)6s9~6;OK!zbo}#er z*lzZwcd8#Y9hyVumEQvD9ItQInfpZ2?~^HVy*pxJWoOR^0ui7@^Yd-MD-;U0s8O8* zJ!iv4VvH46%#v)|H|jSr2S1S6$Lf1tjBrO+m3}KrWZ`!Th+85mKvidF%jC%4Onr0a zPk8mUFp9wE<{s$n{n%+tMM;TDt5#MjDl17OeC#i}&pVHR*JW!-z_!TShTo@1Oa4$H zLs=`RX&aBbXaVR@ne#FSm+${uhzcEloAE*9%G@Nsu(A9Wg%8#{(Ev3ao%0@X0%{TO z1j2|;V_-xBmROt;ibOnmXVIrWp^kVmls-){b9Py}nA94w^O z6qn{E_v{HAW4jHPAx4_D*RT)wd%o6iYZF+4t+WmQT9~R@29TAFk9&GdwVRk{tE@;( zYygINrt)sXsqPx-&Yu9!2EXGPI+`y*Tml=x+XzaAe2P@2Ke4V~f49vf@YyLl>7xP* zuqsu5`9sGZI{{V!y=aZqneSxMZ0JYHC0^ov55L$Qk?3myJ0LN^D3>!xtqPd7gk`n# zCiR)6(_S2VPN=#rDX2Oby(ScsS~sQSI)zQnVTFZ-Fj$`^ zQH+2&ZX>(=g$UqSR^GzcRsY_gVo*FBr1p~GpMj9#)s<0TUI9e=nf({h&q<=);{iWq zIKi)8G$;z}BgScTD$O0-UqtL6n&s*|-rEaKDO1|py(8m{aJEMfImu%)k}ecEKQ*P# z7#S=ZfLYglGGgF^p9Lb`li9nLD5uKpO%esU<0e)@{$kiOyokw5VlYld)zj_&=2f{d z_$`vP-1+o?D`WAv7tdcjF!V8(^0hwU`4pR*rDzzT-~PZqRl`7zf)i0MReni8YgmuNWdgH^3k>KMzO8$G+_@A$6J7aozbY= zMogU=)L&dr@5(uwZT)-vbIbDR>{9UA3^qmYW+6zQ9-1@LVmxcFPxq9+OK82m7M=D` z=7|U&9YlBP49jBYA=!W6O^A{vu0y~>+0&rI*vU)t*uuOyH~gjIa+Owt=fx}ls0s8sHkE#-x} z-m7xyH?B?#S3F`N*noKrDuNhUClhK`?v62ENUA&PC>|{SVO(=`Tzhztn{V@=sn`_M zFfK1Y@;)n5)Ya0eeD(buFyD(wi8mMl2g)n1D#*_0>fFd5&J-|a;O{j_Q>nS#e{mUc zYLC#t*XS~Nffzd))}_|hf8MK{H_RJWdKp|ry^WSGSbs?w~4uruxSARiz%{B z9fj)|^%E3{B@S#4iN>DVtfyk2-Mxv8d^IHVVpJoiG9QCZF#RbV;*M>&m&F0yXZ@^jM{zvs|pv3llCqC6Vz#``M-Z zb;&yzf3myTwQivuY1HBm>s|XP3zhoJUqMy}H1!rhgC?j+R%y;nI16Dl0cExuI${D$ z7n(;VCO#G4{(^NyHZeT^$;5nKc^5Q&5xl9fp^f2O;`i}vM~3J3Bnn@eCZc=}+iY`Y z?D*1g)eTqPM6d|+FAps!wE({WanT=`c2agmq8eeVL%q^Yy_YBiH+grr;VVmx!Yf@J!@->?MWyv>d-`mQ+jV zSm6_4?;1VHG4DB2E<~LIeiCL`ggduPo9R7m{rZ!!?l|}!P z$7~7RpZ=rz0=gP}n?QSZR*C5ZR8iBL{$aBM0j>kMI0>WNUm^sU4%X}bN^;S5IxFgl z#T48tM@Wl?;+wABw{uM@0CJf7btDh)X zg!nRQcF|y`eE&rkuZh7WR4gGSjJmYCy&Q&3O$=GoJr-eaoPWD^u*{Yjay#KNN7c&8 zN+778I@+G~`(VlAG1C}K!mQeOyTQR)-x+(KyHWsQy#mYDY4HSxglV6<>EVPuA-(8I;>xZ=gKMKv3vVL@b z3tzB45~ZnxuG~_2H$F1_yB1oPGM6Bpe4VB*rAMC(3=H34R0Ao*J59~JNB^Y5KGqt& z(_f0J%q!amOja!mZ18&sdxSQ|mbk@fK)S&%4)=cDeI^>X*_yH%yT1ICS2Vn)<{L_n zO=$Q;{ny+F7{Vy(;|syG0P8QM1Yol6B$0gnqfFM*7wmL%Y5A1%EmT3>#pV2bwqxf^aw!sl=;}X7eX6dU$SP9QsNeKW3?!i zDZTvpJ->s3ULHWJvYa(w%F%j)1#mr@9nP~2Uh_TjJSx~9Zap$j1%j=@UF2n6{C%fP zR^sPcA4{2>yTq<>BSBMgR8|ueTdE=ZYxU&IXc@P=@BK2k+IaQt%(o|bpDY7oNIV13 zTb`%A$Yy@U>oFg=YVBq(x0_VR1UqOQ(WSZ&gHXTvc7pq2Q7;ny?DkCJ{gmgkcg--n z>N4{qzpdrKsu%pUuidf*WbXgWyQ=xTL+8}QsZ{V$&a`GPh29C`#cWbIi`7VsyOA;G ze1b_$D3OT?-aa1`4j@8!-RR0zydW-jFOZ%lkCJ>W&J0|pg412P@I}xHrvGN(Yx;D^ z!6!Y;yQ@R7L+LxtCRt)SIz9|DgsyYZs((+h>sXTseBfCcwJs(!D-Y4qU1EXu&FsB0 zv`4GK{ts?={tOq6p=k}6kWV?~Mrjx~Qk?NuMCf+CExTWDnj>aVJ@;M{AI%z6a>V_G zoFeZscX2wm4RHbOF}m=p(2_aXDLQiYRWR#cg*+%Sb+WkWEReZw-4BJ%V5luGv!lnT z(1-t>tfKE?J$0p(VM|X$67oYcsH@7gkj4vg$f_w{s0dPumb>hkO)vc@C1AipGq)Zn zaiFA}Vxn!|e0Q$u`=0Zk^WX2@aqqaB0oi0Ed#%0pTF*1*eC7(#P*bENyG;fHfhd)p zKhpw%uGoV>m+xI82Htu9uJIgrAvRM{ddref#`{GYG`bdGUM6J^P6( z2*eIjdM2ymj$di;i#FBuIrl3YfnL75qDOsAGul+a@l~tCr&~eXsr<(>!~tmyMOH$W6nI~6 zNu4W80euGDFEns0rj`qq@?uRaKd%WSn(FxEJJkKnpSn40UhhXgodd1e#+C6R+ImA2 zo)s}V;rgrzJJDDP+xJa}($yZD+Hc^8+!qcmfk5fgg~oLd>n3ty^$+5c4NfL3S@vEy zN56LK9f{Lb!-CrV-jzeQl4)Z6*<{N0a@x+C%ta5_pV$r+)A^ka`AIpX+u=>;g7nY2 zHp`e%X|*K3w@gQ5-8)ZQvR7yGr;Ed1*GekbYBgzd3G@{ov8EpZt%BI^xs7||R39xB z%u~dM1#A!Kn?78t-|l%cBo1q}wK`v7txm3(s&qaZl6uR3w#52?AebZRP0V7zwav5mJ{~UDmfytga z?PP{KhAJ5$fCFx5|HYqYzIlToJZP68k1x3b`?};3t;g_9dt0ylNRN+^@;<%lyKBaZ z1aiiS_u+n>k*_;(VDpN-eTlN_bbN)efP7MsccX<{+!t8OAZoRduAS=0B|~~D{3M8) z7oT9a^xMT$4kM5ky#+ql9a*J)G5sVSw&IFmTO3EvL432W0@nBNL^kcClJABQ z)$3n@xVuAg+T0|6wE;XySJ%mtpX$C^s!*t|xPK(VeErBJaM*1f0wu}MSZ(#pnyGdW zYkL&lkMCa}mK1ZjBXgg>-16y#?1suc+Jt}FqC-~O z>pE}fwj8mO z6T7lV2P{9pwd1&QUBX4x#BA!-hqk`Lvem=E*NL0z#^$$x9xl3>1znyHJV3V+b74m8 zjvA`QniB-^jYNO1GLMGb)T>cr3imEL``>GfL4Z<2OfJ||Y^@`CHPgyOE&Hx#6{J#X zsk&)3*f*!i#}>OL_3*~U7l9-F3;XEbXCRR9&G$_Iocn)}3;*lfnxaH}hfVGphcCpP zHSSUcu&6m&-u?OW`tOfgU&+iYcuy^-sy6=A0tBZ5$vo1%2x}Kddg=eorQf|J`uh{0 zJ`HLh@cn(h!!#4*GC;kQe^4kLKg=D#r^A2 z)ZcJ8e%?zU-b|cM-s0k7uJFZo{V_kh_(33@pXtlwus3hsza%@@(p&C|r?)MCPSqf) zJ1?X^V!rh4jrRVO?x1C&)qO?j<328@K}4iC+S*END{o}+c+Ue<&l%d%&C`V*j8-q-m$?~qp!AoixO3BS$Y zNyKu{F010Qc3eQ0<*zPeuTC>kh=BFdSL9dFfozD*Ed%b)^t+1jpyX_JWf$T#)<7iInXmmMh(L!Q|;fRaM#Cb-f{Z(@u@@DFtT4byaq-)fT zZ9OISP!(Qoqxl79k^9?{QS+;cg~N|+YC#S2PyrbqoJdi}ZgYB<5SVKM!xkl1uZB8O zBL{&V(%5Tx2@Q|V!A*V90_G;a3mCuU)UJ!6t7hE$RS^+5ZLfUanl7*uPYzP2=Pchr zU;w`H&6zUcz#g=~fU#<#lzs1!a}@+)2$Uvj9u15yD_(=%(wRNT&g;CSlPF#&o>qEj z-xd2cpWqZ{aerHIbvZyDwO~)!ZJ2BREvWpl!?+SG&4-oW%5~Y`y59!9zS%;rl1V8$YIhhFK|5v0&0$|z zT;GDT8nAHalCsenLFs+CrN5C+cosR&fnnx2RP~9{nkz5<9_fJM-$O}ys=`2hs$ZTO zgZa^>YGWlM<1wVF6p(()88j-K5DbCIqgeEfY)6T$li!-Zygo4B)neuOEPp#`CtGh; zI8Si4__t+b8M4$AA&q)d-nBcmRFbqK*@CzCGL0@9i$C;T`r$?i$~qqpks6vmZpRo` z4O4c1p*DA!3RBA>6=R$Gtdd;n(bEB|<u}5B)eTPIW}`_tKWJ+$alJzyw;yq=?r$ zn)Nml&}sj~h&RNg#AaVfg_m~jwyx*p|A0D>1_BRXZ&ah|_)oi4#o<9HRsAIRy>KWzGsxZ3d(Lh&AOUnwGj`5Gm%lg>u{5i02FF)#H zm~JG${Ym;0o_f4;Z_03@MkKG>YJGy=WU~zsQN{C~nv;`LxRAJFLaJ*0rxSEdV8-&* z+YkenDVUk{h!gw>=u%4ttvZ9y@m?r0%}`S;#&nm@8EX8&pwEKMZJEI8ov8>V0=8R% zEklcgk0?;3WwT3wnw)sF1oN~jKI}WK36II>F=&bsr^eCd@;wl({3a@iI*NXPR#CdD z5W1Ue*2FXRp^IE;ujfATasDueL%}$6zD#=gkg!vHuigZh^>yp6%VF@1Two#oon6w$ z9*7-|ie+?{Ir~DHo2KL(M?=Yhq*Y&GrCeWarMyD$sRAJCRiZ$%pFihA8PX1<<47u} z_<9!5T8U^)1PkKm)4I!Df@3Kp=#hFi?_RrFT-goI;^Ajex4_?T`dI;sVz9y}i$9GV zbxRlmu}nVFF)~F`#dMdJf*alnNlEz>(rOr?>Z0RsPuN1$)k$WvzZOWe$Io?U@|W-| zsqm*Qdx9a_vjg$H;$5sxCf0_6n|bO!HhbHQti zcctQJhE9odjViFg*Z%7a6l5N^JL8&05d?-FS~p5UXt7^ zr4;TV8NMO%;AQ>IT2q^&1DaXq36zOI;~gj?e}lWZ#w+uf9O%fU8k@{KBJUWDX#z<{ zJGeU0>pbkdr80ompn4%82bh54Hgs?@HSbkI>&0ZWeY#)R!^o@80&K*81z9Jd}J4=Qr4xNEq5!Nk=>P}sGDeYI8Uy4TQ0}+ z5a-7qkj=Btp*T`N&uy0~z+nQefukF%Fvy{Vd>r&V{! zO|~M0&d58*NhZn5wqPBnovj1rcFot}m}zM%-(-A_OdOn{dB4Q`tspQ~2m3`8{&l&x zz=27wBH-jl_wSM3_N&)K<@?MAK9s!uvn_4urM}_eR(OX(CQ)$4yLV|Iq2*3>bxVSD ziTEsyp35S5KDJw(|5C?Y>4RVp$KKLSSpEZ3>u{#DqpgtFnPjRE!$5H08h_R-O#n{7 zYz)I^+K7FhB=}Y>rB|9zIPD2pM;i1dFhfvIY_lctj45@vUt`06nxbI#rdHA5{i6pIVg76-gruZZqLhT&$*M{uWdvrJ(>MAJCN z!CWXYlXpqD_$7B3WNaUU3zH1uWoS%&%=Ys1NX)Obu`Hn_5tk8BFPP|%v^>H~tM~xn z6~<$lSWz(HR0)Q!*74s2K^3p_p8QVbS>HCnZ~G4_V4JA4+X$bgdD>KA4$h-&){&0_ z?*A#yqz0m;<#{c2bd|$wIe5V{r1C1<&itjCW>%;`$#9uSUgOgJrjl2mosOP$dc_7V zmo$x6OIAHBvO(#Nx(!(V;$_C%5|!R`uzMFC4pZ6?#OA!wx48feb3S!YT)uv#KA zZhHH%@P3T;NRO2LjJsRR($yu1S$^7yh_h?ybqA{8 z4DZ!9-#0ckG?N~KWNvXAImtPpOpn6|trKUv`YosDd#7(8NGcIOi?r>D9m~?Sr}|-T zO~+&ZN^4-UfYQ}PAU=W(R@jAvgv!%~N=zN3T!HLgp7!y>hr;&A$YL6~YhW>tC?P6x z1_8SEG}i|v5w>D)Io`XUO*DzsF;@eL;-H1-dso15OERhP+i&;sTTcA{(0qFOyX)** zKe0Dqd|`nXEK(V!1_JS#NAq6b1AmCq!K5z`eR#k?6ykWFK=(#)(Zn_82;e*rV?Csu#x z{|<=X^h8hFyZI^4fVdvQz*?sdnzj)Yd$g$#5aY(6?Qed!-+m# ztzErOI3k(FWm;ud4?+dRzqE-&kjVowpc;xY*)w7MR>sx7yLiq+UKkaiHm=AYK7j1jvki%tlJ>``lyzn+mh#>s&>^Al9 z_@!Umy#_~C5A>fHyGeU5gLiz};`0`FFH_$cXLjFfpPF>`x&TRODdM_s4<@BIx3;6g zGCx}7KCq_aTEC5k262Q5yL}dH2dK0|I4rdruj>Biya+W(4 zpBw9aYRBNnWv(ew^#%tvxdUWIEet{}FfJ^c;27|cGMAzPL(9k0mPO{0_*>;mp*yv$ zMEj(GqR$fXV;+)aiy! zW4lGLS%!K;{_eW7lyNQGk7XAStb!p>>bE#|QK)%EGlxTs7 zJiKeVh3UcYOIaM6b*|MS^BTxddIBI2oWTm$yL2;G%sl;E6-YIl8`Y&(x%g?PE$!@7 znN$tN9BALXp<;@WlBr(2$8$A>Ss;%e#Un+5v;A;($DvliPHM(;u@~(Bb>PKHNw=7t z-aWjI&v{a;dTQwsdavTDp>Zr7Q{77|5U7DE#^4GA@>ZY$?FW@`9w9{M-Qrx?4?aDP z*|JX8V$l1)>7!sV=c;;mPPVb;Oy<&eL>z^;n}4y^7s|9EW1v{#FN=jd31SV zzBjyx0mMd3j0~3NPUZR#|8NK`U{tRPEH)Q#xkCxSMz~u(JtYD3l#bBeo4g=4dI~Bk zw(Q#{BQ>YjH`V>NuWkk&uVo8@Ks0avh4MGo+?ojPzjk(9wyI4mz6sqcG_fB)q{bp8 zI)42E_r~om4;e;{Vc>+7`7d8MmP`LFAx)z zv&Qf&Jd%Xh0>I6`!mOs7FDTUKn4TLXsUs=A)WxdPC2uAtrVb)}+PUz#DuG=3LRk|%@>=5mfq;+wOs#Z;%jnO}Zw0igi_$riK#XaP{J2a_X)httzd3zqK=Z*h=#%WXEdN(Y1d(W-Zt&fDH2<(ONo8qxUF z8;A9QLqeGO6lQA`cpG;oo44_XrK`0Qpg1zoFl2UVsYA&GJK^gOSVx^@;=cZnO#ygb zXS0!F_G<{_n)m>@L>?O2+E!Yc^05jU^`2=LdBvlS99CnxFp&LzK-RJYH0)yci`+NT zThDlpYru*hReD)n8*guzN}?Ov4{v8D7ZhDfP92MXvWzh&?k$yzP1t2~!h>K@9xiIw;~Mmh2(C!4>msMP zB4JIpUfbtultGzsJ(qeKT}RP&jR~q;o-MkF4D9=!vktmDOI{22*UIi@p6$|1LkzFT zB}XJU&3Ll9V86(r6?H>;mcI)}@ah<8Vx5Bdi1586H9upcQHck?@Qj&lvrQzLU#RHky=&RGz7bM|v{98Fi^IS1&uj!F z^^dpUPlF-RrqzET)Vzv_vh{@0^-K%j>i71qXw8i7P~K5XcBd_SqEpyU`UniexV*c; zIqFW%1XZ$aE(KEeh#ZsLGF} zS|_+KB{!LwzEGV~HP*K9u6L08jUPoCZLE<;w5OK+(XJt&9Mf0>68 zUaN#%>Tbk zS$~+KSjFxF`j!7yL3sFV+l z2X0ZZ5&Ox=(b~+8>~(6A?n>_sTW$rT(hyd#ncR)c=NA9ZvfXsuXKh2QlSX0EXD2bE z6et2gj}_?}rs%dcKHRBfbTa8^a4Kpvx24s^Q`BJc8VSF-%+^xYMj`PH1rU^KadwY$ z;R;ZySd|y0%vaIQo(X7N|0d1#A>zT;g_hX-=_z4SLvKccPneRe8Gv;13S+-mme&_po3{k%99M;(n8m^kvX0Z;U@b|Gp=l9-1H6rxT{Mrv z4x&J@?r$%EN(vlM4=|Aa!7{ggOSZgnHh_(~21ATFEqRp-8B`ooduY*}y17XT;n(_w zqfoe6+uKDcyt;b9O84ffpP}~Qk|GB6n?9QABI!8FGVjPmX9mz zE29=SkMC?r9rOS*#_VC=wcXb?uh$Ttm$D@J6*p~cBKw8axcKC~`~Koy&DJ7sS^-6= zg@85yt^KWIJ|`?bFNUespdVLWn#g+w->Pk`%koFHk=qyOI^L4T$;k^}ZStzC z{_EV^xDveO%C6J!FOh{`U1*CKRVD3ioBjpNuBeYkH%P79-s}s;lcw zmu#Z8Fh$;95OdmlWeDPE)K&722D)S~@Ac%S4e)3P%D+rAAI$|@cwePAd^Kf&s<~gU z=khM#_LJ;eC#b3Xj&}i`k=-WP1-ZjFr3&2R3GW`b-FaLJ^Nd~h;OW@>yi~^MoWi^P zf~%>K=-o}C&9dEB+x{UbT*zk2!bdlz|HMq6&ECF-X4L_}0uF*Y7{y=7;3Bxh)vcN- z6pItxZ~it(bD{XR^p(xxcY?x@Apr2^x7=H$6L@m|{vvF|5${G>Ja`wT624e28mVH? zpWU9U4S7;?g+1nAlJoNW;R5ZfyfW`E<4>h`XK9-i=o0KO#OCiKxUzNc^~kt6wz?^C z5DWJMr5`uar7wxS`FtDT{iJ!Kqn|+2jwf5>E$=SvJAD3cwOuCSJ@MT&CP)d>0$b5; zHID>kem+b`m%=2;x(Rk!=K0ro+-6YHODjxSmF~Bnbo+S%8r%aOE{SO4O7LUwDsj@c z3lvTRC!?pJ2))Oh#%JO^b*#fG6Y=9pR#7HI)r0rxP?weYxaep{N3bK$Dj=raIyt%m zqX?n<%&TD++?}vIV%HYWaW^e*iA&K3(rP*20aGEYnvH1h2tfVp;g?#>FMmfiIOvXV z3l8NU`4Z{yfIsIfbOgMKx~ouA|u4j*K|K&zbxatnP~Rx5dRY>RsYEpcJv!8 zHdMU_0^ObKStW_tYj6(`E@1s9yqB^!Ef1AcvDU!}n33M6PoEB&D=O-f0-{ZFeS6|Z z;1PvRtJkILhyk~B=|W8QIG$y@6!ePBmu&;?1(27Pi+T~oe`r5tZ~M#cLR@}CP&>EY zu7AVMjm~a@^C|8kbn}e!L=HPCI%K#ANkk8osTvx7HA(?5{>EB(H31 zJ~6%#fU$Vl*ln2e92+9Tg>pZ4VM6?L@Mx+Pb}xmi1il@#8%BBxU&DPKV{+Z*T_T(vYxiKoBt@`!3Ht&fb!xxR-xJ?k=sBi>$Nm zKhn#?YXbjUdI624i+?L1W+ygvB#uhj8m_l?HFAXS=o^R1Z%-E!J!@)ihc?v^8l<%=@^t2Na?^^T&B|j{)VFTu&un}@HGBKk5ysReTR9_du1wSsp;iI*SoCLn zk*pb%h=#^#NjbgQz{pmE$=D%7y|uqpr_?$;c-}trTE?^4Xp7$3VKHb;*01dD`;G{{ zZzc}m>J?klypFP1x@QTqKtU(0vWxb6^)P}EZDhmJk!nAhy~H}EoQW@v!=b=iB0XT}*&Ra68WO`XfR3j+D{)lbI>6rO6yZgFXF zHf|m}2lWZ(KgW34b%i;+EQC&Mt6K0I_7Yl^+6HnteO@i*U8Z*+vuCiWogX=Pb#~O0(H}5p)t$%^o^H0 zOnwH%U4k@7T#SUMGb54L%_w!a zMwv$iEcc$=`w*W>gE;$jmQx>pS(H>5aznwFZMT%Aj7>Aga4P(0WKJnl|KTKg|_NqNgeReuRTB8Wi*GWzPPw_aF=n%Omg zDpLL@NQeuCx;T;wUW2>_B!0nhQ7%V$O2HLp;BKyviZ(Xg!6nXx!m;m7IgLvBlVo%= zwC&znn(@mRrmqef3giVrCzgZeTIb}a$>>wW1wh7szG0(+r;(=Y^b`lDTps@jnPsyd zdDKDCL};a5}@&%)fe)Q9a>z+9`7#$C*p4!@VemBSuh;~THCi|Lv)F|3* zi6!2sAu<@!9l~G;AJgl_j=l??1{e3h_rKGr^I;z1r;f5h4{1>hW4om6l#0PaB|Zv_Oige(dS~q4To8@XTDB!}Lze8*k(#8-Zw|y6S%m2LHI@I%_e08kzF2Vb6d0c(^}!QOYziZ!>Ac)sp5`Z((k#%8ELru{sb5YmPd>Kuo@R>K-fNg<$;7*OG*JB6y zAD^~~a@p1ZMnDaD#|z7lO#rv!t@wjoU7h%RHd zib^2^r;UL(hCVle4|U}k$6S@qUayafnFZCZ(ipf9JwW(%2uwFNReXHK2iyv)7K@~g;H%wYD=X%?q@k1UR35UW6c*Vp-Q>n5{c5MkPYLI+RE3g zt-xI2WCgT#OV?Og)Ym`D{aX4-w&FT~-0txxomQ_in7w#W1C#fnbcnpN61E%cs4QP- zJ*|>krIfg+0frE6Vt>p*re_r#R&lWi)T0mjg7S$Zp+0mP=yY^1I&p4bz>0!5nXnpnH8faTwa@c;JRxF83K zxd&T+klui4k%@z~k)3#9N@y~iKXDX{`CZsUmQt?dgn`UXcf#;x zlDc7DIZMZn__ z{(dfnK24IWnw;l}m+79Kp2qUYIJ1GFoX>KBWxPQ3;OaR1CDSz8E<8yGY`=7?I_vh% z%?s7h`7G22%K6cnGe+4+3>c&N6T0KhSpA57?JUuV*i^_>`O(O!l3{&fQ!rPH%Ih(C zrum|D9XrDBPBI3IoX2F252C9dKE$lR7x>giHh9eO9&D%9TlFJ$odq?z-wET+U%>F4 z9}^MzLMSLO6FZ19pjCcR6Si@yC{|(vga6-H3;65%s35H__A% zZ@k|#BY@!)+n{!AwGfg&bom~v%}?SBtmp_8U&3$i7N$hG4l&sD$IlN)_&L3%a#c78 z8aYoF(Q1o(AIWZQ-Q+M#Dgk|3@^EX-34L*0660-gWH{klzSLO$>!vEu8NhD&FQo-Q z)1!nk0l%qNPN-ffPa`K3p{KfJ%k2{V8E!6h zRX-bnrQfR-j9JRS!v>Be^P~KNititc`4kg-Y>9%{KfO%8kY}7*KJx++ap~pHc_v!q zm!aP)JN5p|ks(dxl-U>sOt6sRl+<>>{iDIec`Kpdaj0b)#?C^CIx`-epg1gJySYg#~pd5mOSRe@{wo%cxH10 za@B4x&BPN}pIR#s7b@P631S%_>Qoqo<=CgYeWL`~Rt_ zm>Yh|Q-V2pE_meC_#HP)MJ$iosq#Hy;p8ld)lM&t-L4#qihoJ-!gy{N^CL3Cu+hto zE^+^SNnzF3xRyLkH{AsW38hQK4J33H;#{!BhL%rE5kHR`uyS2RhF{R3afNcr3jF0S zT>&c*T&^nCQ&ak{vxfD&)F9lx)X+9{ARJLT?0X$Zx1>{QsOgd6Xp*BwqT_^Hkg{(X zwA%TRVc5HNn-+Ha54nm$X2Wbo(;^n9v$Zzx7RL%#~7W_EK&=TqTUhP5Db( z)k!JBoX={-UrG`6HSV5k-YyO<_g62O_oWm^%%7n$)^%Fp#!nasiohh55jR6;w0M@@ zoL`iDoo^JB4{HgghHh6Y88@QAeS_ghI`cPC$z>T4-B<KYE`Naen+&)xP8`E&xV3`4bhQSzDM0BF4QlON!d#S0+K$O#!n-Nx$6aMg#V_~<^O%p!x(x)st4C8nC!QbegWK(_* zsnd}52#vMxjOFQYS`MT|CSKQ-A4M5xwU^)c;c%q=X-z`6Qy$xL+jxaWtZ40$CyREr zQ?z-$&UGi%I2*Y-;34?aM_4hNh4D1>eh&$HOU_rjli=Jr$BhKNp<(lW=1=W~&_o={ z0ZKm(sZPP9`>O13JqxH7FWi36gA3rpKp#N&(fuKUm;_c3{8ImgD;2M7Ei={ldb>8@ z3=H6moKQ#eX<$`I3cM|S80{!KGTAIXz8TpM*nOlq=E%hn8EOo?#5u(k24?c80)CJ4 zdfm^V0J;GTXQwxJ|0pkeqJpzSA+O2RjOKkxwmjU|3B`SDNOSznQ|mu@2a#^!HbKkp z$j5!YYmXEZjU@bM{^^iY z-Fy-0l^H79$fulWHkwOBDRud4{g(%*`bmig);5ME3;E2;`TiR#|CjwYtCV zRvm5up_diM>SpAyoZSo0cxZ8c&U%JiXdJ?Q?EC?>PP1G_*&%ESm813M6PGPOR4QR@NGALF{i4H&1~RcRr5-~#*$)VdA$VIl9(kuW9=rvtkU#D{DcbRT;w67{X=PuZ z;$D?UTTB#p)#2o_Y06oRkF}K(O8?Eqo0`+g#p6Oz{sXny-ms$e%H1!R#=gOv*#(qc zZ~f|Fg_IR$zIAX7zB)8`9ln|u>3S)B3;UBW7c8;_$;$pkR`QnOg}>&0fr>u^V8gs- z)jinl+;SVilnw|E7l{+(*}>`NZIWE~XP>vJ_#u6^Lvknem;1XFMOlc=w0B`@^`)fX zU}47&6A`)KH&fyD!iYaUnVwS}gyQT%#2%ReEdL+>w@l7AEc&(?S}yNh7!V}ON3PYF zU^i~OBGYVlc`y#~{U60AS$%HijCcBp=;Mq>+4}u2j*CyG%9q0I4~;!6kb-=t-t&+A4|6H+WU=#KkLWN$A4*(srbG4_ruE%I_uKdeUru2ClFUNz|rHX7R%(93a5_8|otx~qp)(>7LZr$C{1wqrv$01@l zkLWPVNxXC_c3-E9L)(KPj#E(D_lykg!~UHz+~P&oUUTWjdZkbUnRTWjzuyBNE2hTs zsO}kysa;^k@TC}gf4keNT`>Mo*&SVEz27(pA3u20MJ`;a;uBWx8574dt$wb-%(nMG zu?k}hocJsoD?ukO!tn!{R-(vqNDDa9aW0P>AtD zR645ObH^#3h$_9XD#BLjEld8!oiuY5?8ruq$-p1lcgq?y|Bx$!(`2nnLKu+85^sr` zo15cb{aErvv2%J2C#+oh0WAojvr6J$z=3tSTL^7dUo|Yr!T|b=CDwpABx$HKrD$lc zlxJ6xVsGXP0v&v{U}I}ccjiqpipJ0qdR*ip>ebIig{deAI(e87h_0CpL(ovf30wO0 zP2#O-zTfXtm%Sp_>%6fOhsHprWBJ6zjJ&07xfH8pq!~zYoII@@=qq>&`%{FcsG9}B zxD@)GMm$#ys!gm>3*&8-HoTq17>l-cv#~=-hll}xF=PRGJoOGU2an(|2L4Te%YyJ! zDgpaQz4mGzAv)fH(P3@=8mFFoO*v*m`Of%3VKYD>Y-KLZZly@S;MN{|2s#3$qLu{P z2rD%@`Vi-=7eO;M%&UbFX5xmQ^d&E90G@XOm`LkToi(d%s}13hx~;#$ceea2g{z|c z=H*l->hSkW zW?Mx+T@F{%_`W`xtuSSC#=N5ZLshvbzdKh=x5LD_x66d>@>96amm>A@N=5>(0r4rJ z!F?4gpm32kvX=yh@j-t}e2HRv%*FrMYTNO{|Fx;6l(lN**B+F7bbol1jY-XS+kw0M zbf>{jKXJ8qim<1ks2`G-XY6vB`WY`!dgA_PuU`eF?iCoP*ws&s!4HkUSJYRn=WAWz zQ+rxk7W+_RGpPura#e8e(kBufz!~eY?{b8EPr`428nra6H`kNg+n#}>c{lU?g!inR z`Z}RLwsulH#H~1PcNFY#H{m}sxB}1s_uF;n>O}n`#o(`ApACO~>dO1D+klZ49xMQ; z=*ZD6L(|t@b8H?1P4ZG!rM+Pdr{lB(spu;pP)j=h^22rHU*VP5@!*=^(&s6V?!ODr zL_w1ov<`N16nCJH^B-|K9EU;FdqFm{G%BH`w`d;5Jl0PV;gTzdrQe<RP(VFtf35%LbF5J2;npmA1ZQNhM+WAL;dex;x9k`&1isdApIbKfzy(`?%X-L_4Yjsi?A%kvYNeu2qI-2~QhB^Y zMf3}QL{V||&GOqZ)IX{(C)zU}mhr{rt*c>}IfsxTgT8fQfR=ECw-PS~;`zh zZ9TFS#`iSQd+xs1%H(5UFO7H@P|7~de&BSx6Ia`~c(%IVv+QQ)6IoIP-Bc|&C-CEd z<>kTx@ZA^S_SuM-BT4HgL_QCzS3bokS2%_UUW|F=RAevvT{?~oRYo~UyEopx-PcbC z>OvREk&UlsB?Xc-oXCW}1RmUMN+^Ih1yfi^!o8)Gw5~l6|I)13(($Jzn!xnq&68Hd zthCOm^T65G6eSyuYyy0$Po|E#x%=`&l(a}1md}DRj)A5$M4tfAeBQm4x?(tv7o>%O0Y=PIAVG^^N=WOQ82* zLN@icMXXCD={nV*}N9&lUg*?kXnM{3)&yvfgB5WVANL02fQu7L3<%_#ESvIoc z9sjdr-Ct53rTh?o(?s;kX)%9WL~+{ZDra|mb*jaLRoSoD#^__x$*!~0G-ljgh+^M=s0S2&Z1L z5lodxB5HKggX2B*y#0X`V4}}>#j|0LCCq-)hGuGAy^R8-@P*vVP_ll7eOu=j{cUsd zv`eo!sX|rdR97nUc%Lz8gUlwrwQRlO7k)U9{$SvhxK(%zZ_o!cxdF7GH~LjCkJ;G5 zQMbbi;))OfM?1?Xmi7@y65CC$m*%B-;kA}4|Gk&(#S>J?IYbEHoC$+H1&&=UkmA{ zv>NqV3SlUK9j_ki1v1y4ukg7R_~@281SzIzwGkA2i`8ppl)=KEi&qcPvz`i0yKH@S z*IC5xx~`8uy!efjSbGfgTA*Qvh7QZmWK|43zr6bc;7*dY2X8$C zCQH+R%-1(Pw4q_S#|r3eU$XfCMs@hdPXwI;?2(lG)@NWcyesz~DQ_1_(ktG7e{}eW zj3D$xc*`LU=WLuR1UPJ0-&k65E!BVFIIhD1J3oMa#Ki0yL`fWWHyNwcu+>a&m#0FO ztQZHF6xD`2mO3!1$96;7tF^^eyoPp+cJpmB_ed^O45fd2n$^7wz%2#M8*!Hk<0734d+AH3*ZdaYi6i*j|$J5gyJ-OLAEO629V%_!Qz{(?SXBszQ~Xi z-z9`ORNR9aSkmpaukfs zROF>PyiDQ7q4@=Bbydwe9m@aONCH^Eel<84GRpOAQ`RRHKr0#q?$>4L48TPZrel`* zWa4&p?W)a}AJNv0R;|{XHSuqhc$x%jn2GnsBcm_4iK8ODP3>L$|FF#xb0CO0rjG$c zx7OHudMDrz2tO)N94K;$&s=zFd|#+-ZY4LP;e9XXJTd0~i?O!=tEy|eMz^A*v~+`l zv>@H3DBU0_otp+}kdQ6`K^kc!q&p>~k(Tc6ZqD5J__gvNTb#Ts%>STEW=mvZ*=aK0 zY!v;zgFb9j(|S1fvFsrgzm2BKx&q~9nFK9*pU#Ftt&WM1UQfIW?{VY&H&E5stjX1b z?+T~6+E1BwpD2GJ<64>)>@m8kL6E*K=N>nq(mMKWrbnBAn9Zer5$^m%`wLt zlOs*aS$L2cX0vE_JwOZCc`dJ^2JXe{$Xom!Tj}sMj%nI4#FF>enTaJ^Mjo}F+UuaqFdlO#ZJ~|e*p8wp zN7(<8t|v|m&v{aHPJTPO{7Z0)`AZu7oPEWh=0@7WAO`%L)p0NfnWANJ&FvRLL2mJz z4|aG+NKc?Xww;HBMWaiH)4hy}^}_?=!|wWBa;mJUT%8L=bn-kV*RbYr^Cm*Y_tjl< zn{OiXq+4A|?C)B4P_6P>ohSfGRdHdBZmGY`A|%x-Yg8_&5P)jRqwt5H_iCDZ)9t)rp``bcUv)azp!iSN!HZ^JH29|L=*HlRL~1p_nzXIK4)? z(Vvru?CWZ!4u{0WucqO1?qm(Tg#H1-LNaMv@%c>hg#W+cSg%`7cBkn^waOxcAdtOQ z@kClNl4G?YHo5`*{|oD_UqVLub<~h(zocJ{oWLL>v{f#i*iLNtIs5mMuAa3 zh=pH)L{$uhSdM%fLDIAIo!uJokDJvt+ui$#VC(maqZ()9|A!8RK+2ls?(a`VOqm%98_(z!HOZpBohSUiahazI}F3$Ww3h=9}ESBxoAg1VzAD%?QaHNX%@J)#4Dp}wakn-p=+@(bQ3F~_99o|S@ojY?SL zts8q)afV)64BxXs*Qu#X zk&l))(T^Hg1=MwBT^B=j;;!pOXge-Ocww z)Rg$^t6`Ec*KB3zVS7@?#P%Ps)Qy_dSk!64!9t0uACL>8R~(}~B+xHSRpajh=(DvT zg_@Xp`)}xe4>o#rUpx)qfUz`e#E(mKCzQ(9vV@mG#+#_4-H~*Lg9O_$D?57?=}8-) zXxfW}?K*Rd5e<2fJEp!XSywmWY|p9@TvA;pHoV;^knO_+2qm4gS2*$p`(nV@DMJ-; z4DBarZM6D78>LHtgiGsZ{?Ei5P`xmQ(VR)94{a3>93F97z%=97l*>B(x`}9xod*== zvzRlm>};!ME+?d(erLW?kD%V#?^-$==5T|5HGAnoH`RiVb%L+*o#@ycZTkbVz#8; zc)mV4=8H_)WudQT3EYL*2_nnEy@zdM^jXLWHNx&_HljG*3R+bDRR$$*D;{xYCoB7z z5-RtkPR0N&6D?2u1GM4H&iq=F^k1SZql@2*azv79ppw9x8`e>_<;v^-&JhkCF;!@- zLi+L8tqS9g`ul9;b6-|b(+i-lauiUUxju-(Of=>^AU~gly0$QJhRja;kYq@*QRe3! z2}Tt>7K;yxhJa|6=K0EtjyZ(ggfhL?JL@93Tk#evbfWo)i*&B}->EuE_q&UHkezL z!DcSb$=X74(u$*2g)&~JqsOOj``cn+IIVDj2vU6gCs=(e?;JKKf zOGTO8Wt7b0m62_c$ho!7Hm-PmJMUHd^ zxFGhd&TkzKK-vMpBRy&x7ED=@Pn4znhb5!-b$(+#=1eP}O^~bTr?*honylKMMQd%0 zoRGFG*OpK+l%8C(=(2b)RZ*4)omeRUd8r>8jaHrP(mQTg0bPEr`+}ogT6y2y7FiLb zvJ8i9=jdmuTk!4Lfohr}=lAgAQN(pC3bNvD&_NK$g4%OFKDTJIy;W)Bf^;k_R=3gV zPAHS-0yz#!!BX?;V28EKlJcd7{0W|8eMH$V<0ik!W&i+;*VCC4p1c=%7| z7$W2z_xXmSfcWX|SZ3vi|26&v#qFd=B3tTjKm>f;o4$Rsu7W}@RE4xOuQ>Q^bM?l; z)HY#O$5~#|%pU3zmHkU*vS)roZJ^S&gGKPLFw%(#xP+{eAcO36&!j z=j2!#-q~abB;HrXm-ghFUTB?CRXD0w1vv^12!9n;Wf@CrLSa0mq<2Fdf--=NAoT4S z04S5PQClT3Ve?+UK5mWdXEayj&mJdx;n2P8mvPwh9ra2X9aGu2$DVu z*mtDWdz@?WyQD4tGf*3`)5X#O&Vx~rZp~*?ojoPm*_gx)nbJ(v^1Nt1FN~IknpbP& zqLq3fmPqerM+^j<=QoW6XP+O7?t-Z!aEB4StBBTEdWJ|lH?F%dxhjtz<;LQ>ZSWWp z=-t^T@|`t)^O`v*75b4o!U+qDg~5lHG+9a{c-e2p7rMuHG^EtAyYUop6VCLpp%%`M z;_2A=`T1obv=v}io^&LM6)gi;?hV^2fv=$qKnpy$Mc1`fq2G#> zAgvJ}N(Zg!!l0kXsCb*T1)G`!f_mQA9sh-yaZ3Fw@nfDXvECYoVM84gw@5#c@3e|+ zwf_nS>KlMp6Oo44Q0Y(^Ml1szg!DJ2qkT-J*`b!?X+d1vde7Igr-XsQ&qm;036dZ99P`}Lj-2-@~;NDoOI$mK0o zkrM+?rMYz}JEBvS9HlwzVw$se6vj|m0Qw2iQ57zBZftdGUn=Q*9KW%L3ei$w==w)J^UZ&N zT1G?ggaZtRp+poN?Il2oXaztfHDRT^LzVEs0_unBofJ>VWgtpF?y7In+RKH(rl5UL zeL&9(z#?^c+;Ufa!@GZQQsBn10t7d`1GdTsGuSk0HtV86h^`2{%?0V->++mZ*8~5E z(p0E19o$4b&D7f0)zj}dV?s)9F$%-yRBpQwF7wvBW`s=;Vk#qKv}JPyl#l-Y;e)N$ z-1DEXKTY!8MR!*>UBOjv2PDv}Zh2xL!Z2{&whJiS-*@<{LK(sCZf_q0WWzc;U2_a_ z9?w>A^3+I3NpF8`5kjIf&ky7s0j(uGCjqMKuz& ztN=d$OeID_aw|_E*Bb$D{0_(uexHK`gT8cohX70S|6*Ce-v4i}|I4Fl_J#4NGz7E- zI*f|EaK2t3{ded4L@1&q;PnDwpU`Bwr(&AVbrW**C46rFikJ?_*$<^lY4f~$_1|Ii z|J&-!`@y zuUj3*t$!}UZS3sNqbsFN3lEgver)*LQ3KC04D-^_tP+V= zL46FtX>#qaYOM)5s1qUb_O^9sJY9BX+H*4H%!g&}!8P#wm1T`-pH4A)DueV%Lu4$n zzy5zL7;)+S-{Oh#J~&isRsX4J`4U4To~eE>z$OXLXIw~naJ`Hlg3+~E?s z&Hyvss6n7k*C~^JClKDeRM4H{6(bV#&6Cb3Pnwc?<5hfGX5MQ5Z_b z0toU$akkPnW^fJ3ly_|GernQo#NfhQ1r`&2%V9j~yZdp(9bNy&{U*6LYTlgoXOjX^ z_Sx)De`akUe-X+bUbO>Vw43%nINat`j0a25X^*71)MR}i(iyKJz|LF~$HTMV9!;i| zYWiCS=6LFa+TV{@=X>zg<*CTp@jdSh2@Z%;!x0S7sSez$)tBk9<0hS z$)^L^rWUh^4BxI@4gGI|bSnJ(@qaom^vc!?bIVQQuU;A5F2x2ew56R5)w`|r`ItTY zw7|UVoOW`d4Zqg$PW{Hbh5WhkTWOy+lXJgSn6+M~7beIXryaj-V|U-h)O!T6Pukz# z&oO=G-i%8Rf6$6(V5NWoq$Gb+Wn`5UCD9*=f=B`=-OTos0x#cGu;qOS_Z!eTdiNe;~4{VZj^P zb#Vg9Nlj^#D0tM(KkR8A8dm@-A9eAsaO& zDP}X$yGYQ-Tzh5rAZyHE<&1jOD6haxNXr2!l3?A}gv`%-(=h}l7!2&w9t{*Um*w`G zPe6D_QCe1(6)nCEIa#baP!u5Rrkgu#E4IhEa7a^pTTE(QI$u$=)F2^8zGuHy->jc& zb&T(9Nc$N{_6I=mw1KUVJqw8)bPr2 zk(FA>9Nc0FdW10>kyN{5&YQj&ms@xu-(8eZ@TDg9pOfSj=XigljVR)o{^^i>!EF?i z&GGpjxzS!u@C*F}87y9|%-e?;NuH!mefM|{7{miOSUS8&VxLn5^U;bCJ4i&;KqZEf zvKO$MVriAPsdBA$q0bZztELlN?JP@Ra<9>~fzzTNW$j=>r;73VbD zYSB2&gRjzKIg^(k@i}|5he$bPXRcMJbRi~Vl7xC-hZeGm;5Q5EZ}1M;2=$53H-`C_ zs$mLnICveKeUdEF5?a(#W0&pabx!$OCIWZvvSWdqqzq32FUr4W(`zf_uaqKs%)L;H z{~^Yk(I`jE@9*0zOtB@(xJ*57x@$QE@*PM}p&`mV`m?bN+*gqG6Dc!A_}7PE^r~XN z_EpaO^Lq&7Ux9d`&>VWPbK)27Bg&=kN&5fVjslv?3c8xstdo@Xd`ji+?V<4tcDN9L^;e}+I*|Gv{mD+ zMAIsS;OQ1k9D8D#5ejlS0SxJz9t^V21vj#SBuO42`Q%T7vZIh)E z|D|q6ZK6TFNfZZ?Sn6#5(eicYpaR(_X)VEsTCM2Z{^;v>L3WB`G4z5*>BoGH6?O>+ z7TJBqG5wAt{q7qV&t|@1SV3xnCB1B%2OsO9iG-z?=wq1yw%sOxMuM zdXd<7e-PuOWktid;&9?q7PhTzMQp>EgW|@}lYwCh_@%sl*3+~uSA3p03$v7&W8>P5 z)!MDnX%ys(g8bf*MjaB4T@%w^Hhs>cIC%V;*b|s#J|y}6yYH}= zHv6(JuG*qKnqsM~Pbs@1R#QV@;*&}Tc}};3*6|!$nSlEPYDJw-rH@!n>o-`V*AGQ5 znKV)?Yr_x9rAT>?(x-Rmi8!4C-GHLB>Pqz({|ky;p@0{V=h?E=>sc@=MN%r<{C%0V z4S9z&ms#M#*ulB}Ox=E|eJW|O)ot>VJpDw5Rh0Cst;h4ZuvRSskun;x{$^>|h3xjj z2=gHM9S^n)*>GrZVckHu$|Of#VSk=qF6%kNZ^zBkv>Ry2KN$TvQk7(ZUOBGXMw!^X-gL#(NT7~A5%-z+kzJ-G6 z(zGd-H81-PHQimdnku0~`tdWW70bU1sf-l1C?85=ox2_Mhx04vJBJSS2C5F%6{8MA+B1`KDbkH& z4hS@TNmfI{>dD*ZaQ5uG4xihUQlz_#W0E->1f$%Y#eui&q=sDA_~@dJ>*ovyOEU{} z6%3E&3yu_8ouAV89x0C1qACg&xDFF2((p(HMQP2)PiR^oh6+>35v%8tk~S{x|-1s!EH5T7hrl;wN7E-2*%fxrdPJd#Y}b zWT**xz&H@$X~7NQ!U1W-QqP)A)kuG!+CmQp^ViqrA>n-Y_bj41R$~1cFj5827V z4os6CQ0Y%>30cQaIh>DO!ZNndrYW3j%{m$u4s|cetkr()62$u5_A}|*=*B z#9uEurfGeCD`9!E25p?JyMTkNVFLSHtGdj;*Aot;&S6BVsVgi7f#W)ZisXjVT^<|R z6aEt@A-oqt0LO^DKc`ICVP3@-xH{+&KELJ1o3S7Ryf5e1{MsE3>+vX-vYa#rYrLGC z6*qTJNRGbo_}Sa<#hJp_XHMTT`1&7`GH-v(1qT^H?j>G~&H{1@Y>!dmd9s?1?!mHl zCf<0!LWi2Fr-b=Rd)Rfw}-a z9J^#pYM@!3lH_4YkiYC?;7{BR;QZMbo_0&@FL|r z#OZodhdJ5(l`?%Z7~AlKsdkGMWOc#i6_1E%YSBS?a;A%Ipr=1E z-ijO>F^@gYsqEo)I$7cFgMA6F3-9Uem(biS%)-r0>#PEN7pJt}LQ%S@puOY5@Cf$L5qzu;*pk_~BBiJ*bU74+mQ` zg_tYcIqvz`wsCxP^1FGX-gx$fhRYpnB2LSEsjBlIf>)y3H}pky3U{p)yjc`1(L99> zTFO}~O{>4))%=qarnW<5#aDagZ19M z6C&oe6-#s&eS0@M&SG@VarWDoJ>;hx#JG+04&q>D8Dyz%6z;Hro%i zauWp^D1Z=c>kv3$u~{3S*k5M$d<&xN0CbN%pM|9;SzOL(`>63o&&JAaQGc|+m%+Ho z3tDz7UyOs8-Pa{cCyi6wE6y&)8U<>vYsTs=`%c6KFXVyrf&rH3#lp+`5D&L`GAq1Y z_l~xOeAOgY=Ivi8{7)P9!Yfrj$BM+UWPW)-0$ex*DLXTG;g;U7FN0f=XcEauJ+Ewv z-7g*YM=KkYx5kS_i3Qx$t~yh$0=aCL0;gI2bWQu&G_~+qeg4E$5DF5Sk3Lt zWg8Ii;{tsH{KqNx)9HoJ8kU%VUOuMvKJfbTR2+qL_Il@i+oz|Jt#@-~%BCYQ<>n~6 zOvo!4nVVz2>B*C&y#*F&EjoQcj+CsrabIFG%+4oVBW)=++hN=bAxzU^IeTAaib|v_ z>8Sd0E~OWwD13k38cMDHen2{TtSJ&{svfoDR!&Tv&~SBlvKw@IbJ4Z{#cRBYQ#(lV zZxsaOcthx~r#(Nt(G)Ww2MeW%m2M$*gd<)R(?6&S_}<(`zD;$t@1BGty~kF!0G_I$CTS)uime(FBeKvuJg~D z5Cq7zUDluO&zv@JI=6xIy8;`}f*UbXuD6VhPp5&sTVTFDV2;w+97?$gl=MJErH|EUY)-Opg z!1XZY_HuFskrVbpIO1ran_a*X0#V=BPQ}!93)6i{yN0!~p)8H}rz}^;&%!d#JjL*~ zM2tbF!A+$M6PDF~Q*1fSQ*6+QWdmE&rTqdLiEam-Z|PRP>0IZX9c2qL?$n>&TJ7HW zEavR;-8LFpv>{qO@pyd@f2`)y)ULLVj(8!$%0XFfz9R+wGN;)!@A|T5TkrMLMA3U1 z33<~(Kb3{&QN!LW+v=K^3QZTg{UGeX(N}S>{u%HP+lPVPr}cfL&oavyakq&W7ZK6G z6xo2N>^D!hnrzpVq3M2c9KS5Vp+A1<0 zx^A&>dt5o4%P-cx-~9{>eD!8wZANwVTT=HUc~cXUh{mh##v_ucVF__@e?Gkn*2cXS zl+2={qN@vU6s8=fdfgxJ+)jJsv-m>8G{#JoI4@ar*u9UG0&RJdDtz{RrbZUuC<-&} zO}mVU6*r{j{g(OG9O-i)=rPTFyPHxJeY?NuK|nnJ9{=^Ub8~jR$y5tb%xQ-7z+`wn zzfI4fO0Co;U;7bxrKR(+NXh1xd58I3TplqNh|T!dDZ$E%lS!%`DDQMd$7b=&}P>K{xce z-4L-y^ojeO;O24E)3La)8zw3$E#XWwT1u@bpCb9L+}!#&5^KMF&9ru1b-8sLpvOS zv$5aQT5pr8QTcux`Jqxbk7Y=g6Q|>0HKU?~yQQJ-wods>lABJ2K!qfDZCvTdb$&#& zXtj9qY%{fOiu&R3yk^VOvSJxk0xEIU6%_XoluL5`@i;i#+UJ?0KmfEA>PArMh@z0S zc=PZ#UJ)s)FdSuelc|v=T9!g?bPg#cunlC5uY^RL{Sbt^zac%HT0ynFjGSE_(brJg zpbttFf0|+ADmwsp3te9k&9|5q3O)pOxoH=rGaXLA$29un4{hL0CKq+Tj~Sb5VWK*kfECEjXSdiAccMtBbj5!W%Gu;#%Rx!$Q z)@-K?dHr00v@_MoLmTr*5zrt)6C~GTDe_1G^_3pZ^f3^VCfPYXcdK~UDW`|@j?2O; zxsI-1k-ZXI5H`u{=nkD?Mnr*9oz*Ud;O4;#uHs4ij=7Z|yfcN)a8FH>GR--%zux)d zP0WpOSlOW(MMN3l?$&mm8rzjmgf|)Z?ZkOD&>;#-K8uZ#%O1D7TZV)>r5EYakSF&( zh@Ck+_SZix}s#w^eivtf=b6&$yx!IstZOw?9ET(8c z7I9m}a0aY8R~b9(Xkg&m&8CyM%12ZWGlL=g#rx?}fUl!j-O!r9{uB zl5v%Tn*%ndFO7z>q=_HICpfn!Ru79jN@OuEbYpTYPb78@>VmZutBGgsdw@EZK1qI@ zP=^NSGGaJF^Dom-h0#-0ePBcieI9=6!yC0tYy2o_HLdcMa6xyYwd!hKhYY08LC6w8 zpg#r1vg6pj`?D5>sA!R#ZcgoYc!HH&g^L7XoYtJ;swCL#wstrkQN4^u%OFUT=96+I zJ~J1rZCE%qoK4^>E) zD$hg`oZ?J5e-X<(Vab1eaRZxCrbkzY{oTEsvznNL7!%}g@GGg4j4qOzr}W#H_N?jO z)7d3>{H{! zu8eTNBx^xhP&GqP@(bb9aBV*D=U$}Ao;T;VCWlceS7Mg~Syj3|t5In^yDWn8T}@G$ zd5b3?*`l3^L$WI(#6)6DEGBwBK7rkFHF4!}$Y!b3Sl0J;7WP1s zb8;-xH*TDJftnhG+~w74eN^Mbtj(U?Do$%-G~@i{5X(x5x2{WZzb2`ZvL143H@{(D zrh~hL*ZOQWhdpxh%N+K`9KmdB>M+@uk{QyMa*8BWA-~)o5xI;vnZYZA-vXHjQ|G0~ zpxY2rc#*`p{K{ABJrj3Rm+c*LZw4qgfok|k55HMRBa;5td5-5c?D|s{Q&qnlpX+Yg z(`VlcNc!w=Pj#nGo}trp8KdG zR|bqPon@o~s(rqGb(J5n<6=Y3BW>dLDsIO~Q;%(>m2%JWDgUxReFy2Sn!af*P?*mX zw@Q4r^6i*4UC4^a6{lHRwm*JAb#Fog8`PZulH%CEI997$HUCUuZF8z}_vtC;)$y*V z&T7!irrs~oWQTMOjW*x0agKpl39FdrtA`@tu}KcB?Vl&wAzOc=gSJsPKx4}TLk|MQVyKphv4ISU-p~ohpfUKWi}}Eb>5F+KDhswARSq^ zEK)|GQ6cdq*tqL8FT|b%PT89it6Rs3qwEw9b;q5lHxKV28UXW2i8cGdEp)pwp%1EX z!snAflj5cnM$ElYomy=-xxA@!OU(|Op^T_#XkD(m^{O83qi>BnccyDIv$Mr#Ja4ZL zj_msRRd2TXZ=`0NSCjHa$EKzjRuk>QrohjaO;xoVVl-_#pO)CI$~Z3s(kxva3KC(j z0*5WSa>hl{^mo6&W&Wz`$?h!k`zag$gOiifQ~Hs45Ja{&KNx;f;1l``>`W5~B1Fc? z&Aoejx6I59XC+ttx94?>t_=q<%C;lQ+-9)-Z^C7_{?uS3PdOzCu|JU(&@m&KJJ$7zoEJ!#drZLXnOsF^FRalPj+Shij8gwnR1!1!m<`x#nvfwNS%vw45waUtKT z-Z$u=i6AESa$Du)?4GLLKA63|yrs!h$o+ctk?6*!Cux!z zjcjr>yZPwPKx~?&rOv1l5I3Foy>FnROgW-M`VmPbzEoJ76`r8WFpFsIeYP&^Nk>Tu z6qlKAlb!&qUzj18=T$MWpob=@$C++5$Y3m|hYKt;g{4)^?!XOyGPXYaB~I2E?s@g% zZKFpLaOU4HrTO0^Bnn)o^(1mDTGnpkaywdRR#==D4c!Mh6NnHez{!_5AFi0B*A@IzfKh^>KHc;AR8SXV|^G9IHVH1nmY1 ztEFz8o06-mRl(R$BF-*w)GzVoqQbKL?C5Fhb*{pzcR#AAy*QVf15=#z5;Pr@WdQ%J zf&O;6(c*LM&PD&h#}kCh?{_p!q0pGl&d!0+m|ml0@N5nY zx9yw_6a@tZ2Lqz$o!~Sr43nk>5Jz*CX(Za$C`*VviR%2Ba(kY_`#A+P+ZLSh^>eD) z&V3d)3)>EoenDD`;h!yauXA}bSE2TO<1?M>jdDuLgAPx=*a4V(Ry_FoH!bM#*eDv= zE?8RZ;ArHVYIhr+jwUHD^&_kLW0kfz9>*DvOKAUTRrT3IFo*%ICXTizD=&B2-c=HR z0~yV8aH@|+RK-N?`=YJs+OZ0Y{e`xJ!<1W15U$sZ`*Sdc^DPk0TU)kV8RrcZbEOz#ZWk zbT@-SQK4@^?iT|g@Kt_2G|^?wninU+Z3>2e@lxV(*ofZMrWK0o;WrFvR2Ic=+(p39 zdGKfFLhum5+jK>;I4++s`HKbcIwJq06*&AD4h!y&jg5G%r4>8omP9tQ@5lhN_t@Fn z!bgY}5C_!W-rlEqFjr0}p7C#V&~fN~=S@P)jQ(*FAgMH#cOjM+Pl;ciNEg^ktWP^O+aIHN!a&9{BgydxQVE}Q2c(tx?PG(MX5PS&MaDsyjxqxEX zwZ^l!ka)is(a4Hh+$&*CHy9`uBHfpC^|`B-=L~pSesx)|(1o#Ro_# zK`o0FqN$htLau_)efMqiF-4d+fg~OQmjTy|T2^5-`>h20Q}RVGHSYhcVO9Uqs6ScS zFk`Fq{stjr75_C_d;Pt4IhzFv?cvKoI6C=pMjoLC+FLml`{28!jIL}A%iqsdlU)m^ zY`b6mt~pL?!b_L;WIE61^@j`xYeV&*=o1~Y`%?83U5mXtRSMVc^TcpG&;4Jt&jP0q zMA9oW?3RX*PyQ^HaFX-hLD z^4Ue|AjA6*In)BzDXwdic-ez|&!JK(Zkqj(WToFEG;bV7l)o*3l^prugZr{6EXS?s zFhcjc{_gwMy5!0uBM+!*kVB z04zlgmC>6hTbomuS**(I!?~J7H`2(?G_w;t8=b>SW%j3yINZeXJd&b;Y<};P36#Ur zN5d4u{S@ZMzZcaWK!)Qh_$~9!#|)Ufy{wtk>jfl_E+8onPx~J zNjSK(H$3;~`I#yv|Kt##wmg0~Qg+gh+q$<_b_%9~0^jBn&kpf5+yzmzq4+DOn(7;4 z>!qm&BoBrcG(XKUiS;tN*|^$1PD70#y<0nrFJ=1)35YH$2UG`#*~G!|K`9e5ToYe7 z?mNi3%K3X_*pNH#eR=85p!Qwv;=-}I!T5Ixy_lH6?G1nOE)k*2)97tF%W-mGU=?l} zwZ6=PHug*9!KORJte@VV==sv{k>Zs{ngayn7q#7l*5Q@Ou zUe_USa2N*e$A(Y(9W#Lj)dML1xuy^i_$?s#S*S8w>?W!hlAjr3$8^XtDzL*GvqFWY zhe(Ghn)cyIay3_n&8_w5?bH_8+C#_OAfMU}RdR9Y8w_~s)6#?`GCx6oH@Gv)m%>^% z*FtY$Lr0TwZ}6pQNP?TkZkcI!EOTsg^ZM< zd-+t$adwv*ENGz(oZy!QmC|F|L(Y?c@{4k$lq*F|Rch4x3B07Ir>9vBCqA4GGu#co z{-^>+DL`OlYKmmoP!+9QdOPgyaIDeqGM2`lk7sLX$TK3(UU>eid(gEbM!l#%$F=q( zK#{15MCZ*F>X7qo2z1Du_a4Lr`SFxN>05A|VXOrQ(HrX*`lA-rYs@#!6Gxo4^X6sv zaz5DFBiNFa=L-rdy*G-dBO_M)n_96a-%$_}yS&Wq!->E_v^k&?Pbeut&M%YItLh;q zV7HXmE=#Q7`qbq;cjvfKgVWHNOM~i_bZO5TdM*!tl^@=#NYF)*N6H6?S z;j4c9^d7$O^0AO<^K6IQXx@jUqY=NRd|sCO$7nfyRlv3XQodHcdQ5Sy?IlM1;}E1Q_b;>pyiG7=S9k;Os2( zxtl;)b6_}MWi@3L_ra9WvU6`*5W9g5U1>u&s0F{ReG}j_HHnqML5$d%kO4QMx zD@v}9>ayic2BAf^e01)nO-(~%c)T&f%E6(6=1}rGe`~5*5!p{9hsbebn#FnbcQ5O% z?Cz)c$BJ*b|Twy!_OiA2XF@mHt|?2>R;8lhe6lAscm$o(Nj6qPMs0 zx8lRiv-kRQp*|j@-XA^lHix*&0w>2swL3tf7f}=Btta(!(I`;)ZM-h_f$U3~Tm+~P zU}CQC?7-6Mw79`(V3$uH3r(5fu-tMPIs1b5TJ$x>i5u+kHx(PTefG05`9HN=6@AHY z*Qq&f*nh#H6y2f4zB#mCN*L>v;Q64rOn;opOTL;U2%Hi+)EYYP(UEeEKkKZ!cg(ey zl|>D6Ks1?Xl9G}>buVYs*K5Vc$5T!=QS}SR#Re4<(XW>8mSfJ5$85E~e?8b1>(FRF z(O)%IZ+6|j{WZAbB9G;?AvfmxST+w=`Nn>cg32}v;W4X*x;oZAx7>EFQZNs^MI#NV zbH;*Ttv0v3F9(>tG13VfF$nY=vf?FvBop}EWj6@7ZpT%fMy0Im>`&p~;8Mbjhn`kz z%JA^;7&tj`eM04H*nKm5$OX<@z^zm3PP7bUo#|jyd~NGAe^k^oItf3u5-j;eg4=;i z>w3enTc}|2$NE>bOes}uIBbm{#4)7YmiHpY4!rY=CKjK(c7;cE_V)1fE0(fMf4D)P zm&`GR_41u!_wg?(8t=KR@^WRcoimAtlj20mw$}rHERI3Dj;myj8nIID>FX~JJZY$( zfJZ8Idgz2P}bN)L%H>~6=#Nntz+p=K8C)cm%@7lDP53z&e)IG_QfBRVragjYm!+AoKq<_w6 zwjq|Q3piG6ReP%ovv08wQpJ(OG{V2P*Ggk^;wqBN^hiyh*${z1 z@%2<23EmBa(&*7Ei}$>bE8nz5&04q8<0HL*AY(1y?@H=>-6eEcwU3VSBY~-(M}#GL zBJc|e4l|0ZVmDnsGuDcTiGh%H@w_1|O*Kc!q@Nap#T$LPBNoX;QUyzT(?;>x71L2n zPl~7TlWsq46XOZQVg7AllKaF~1M=pZe02A67jP!jZbu`vnhYvaOcAa4tYS8{wn~y_ z1_sD86*?=nnm+KQJPrtWy*MSA>&wG)-H4C_1OAb}-0}|N@+Z>1Cqvjml*wXwa9%%@ z#i-Mu36Fq~Exo#~@SW#y5+TQzhwFSuh7R-&Is6vsF4WyD`t@>ZtEFtBh|4-FhRx%t z`-vIhp4cYV3+?T%VC!&xUYFbbf-MCNd`c>^KN<FNm7==DMaMw@nuvBO4FE_uQ{?ciHwSR5$fXQ+`lthH^43+y+jg@J&JDr?$ zRIibto0}#uQI5^1UK_G4O0vGN+5eXPQg*tdcE`aAy$D6oX=r`?{&vFBV$tYq(whaG zCZ}mM!)WX!a?{0$GbF3T6P?p+BR|3^Ha51OT`}~A>w}p%L+`i7DL`F2#t#MWb2ivp z-V{4Y1e2j}g4vDkGl~Aq@|N&X>%0L&;br#FYGRG-sd2NK2l0;t`rREykncb*EKagi zH9DQBVOT6bua_^QU4`$wxz4L_pBA_kZXil@>8{%LsqP4zKJ^>=7@TV$`4pxB;rnZ>7&N&JyOpl$UmMgq1iz^aT2xlexl_%S#+N$-zN_IYMT zn}C2IGB=kNL}Hakr>DF?*6cIu7pe0`@D_$SAfpq2yC_KuAIxG(fXZj zOmW?*_SWiqCbM%J?Ao{~D7QWO<7yq>iqX_9(9EW~AY8$lirGeI&UT85QDx=jZ8&%RvO`UZUmGxS>F zSBrZ@2ArwZ5$7AfW)g{>TvjrMZEv2`W4ASYV;bv!@BDZ+$*~(AQn^RHQz^yH#>p8g zj=i(~mI@AzS*0$mv{VwT{gltAKf#e;#QZL0nls=@gYlR@H#7)^C)iE42}Fh#hrOP)cyMZ$E`@ok|MiGNw$!+tR)G9ELk!tTWIY2 znx#?*6|xRV_GFu}4P|GtWEsoYC)+S~W9R#xI_KQyzR$Vu-{buC^N)W#X69o)pKE!& zp4atuD-R9T(D3ES&u6uP=kEq3z1vLP<@zG52x5%l?yT6Y%Jy*_P!0J2S_p(XHM-L` z2q?4c`ODF3h1S=dbS5#TEOO_}HqA^29gx}bIlD-3q=vKWAtEuV{*i2(bD9^Jj$%ajvIu0-QbWt&zr!QAm2E=Jv4;5y?~8Hi+14Km)IC! zH_F-;r5G5ZUuuKpvkT#HNd{+MtKkZ_$r=5Ro^od#8@wZgY%T zHqQk0>1c=cJ=!fHC7YGYACtMHfay(ME)>wVPM&01cc$DA_~UQ@o>eKU2z zhMvGM<{A9T+k48U-1MeaeCoJ*>1yIIkYFcM>vjWzw;b$FPhh)MN(E=c_6~{X%(<9+ zv_1W!cZV+=uJeG-CLAV(L&9~zc$FSgd)azpwtHnxno9wDhe?J(-sWrK+Oua={4uwY zmOCVLeZfy&uxyE-V@?PB_l%q=(nW>52T!+3O`^t6IcEuXv+*1z>f^2b+%apo;3jx{ zuYH)=O`eCjf49-;l zserf*1tnxDv%5I3S9aFIx$I7l>rWvyMC@0$W*hJZ^zNABtH@kilnQLuTLq#+Z=V){ zuzc5Bfr^l*`d-?nnfmnwJ3D(uWu*xiI-Ojb?atmgfu64(4GV3y8@0o;9ELo33yR6f z8c>*>MbNio-5un9P&~qQ%8i~QCPmzR{F$xR32^egtq51KEtciVeP&(sP2itYp<{Atq*r&8vvKtzfN$!UyP_{j79H%QxbnHbI2Wa%$Fp zBe~-4D0R%X7T)?RakG28f6cB}Jb*7ZW$S8(jZ&Q5en&@@&LbO}LiA>eAT;(sH<|i- zDK##X_ptb>J|%$poH}6beoyzPDu`;XOghK1_Zg3l$SSytBOdPrKUVMjVg*`Vdp|Pqv7`WaY+i$Ki~2o4Z59Ip2uXp>fDd3-9i~f;IU#H02_2 z9T+OtuLtMdE58#fY-%j(*Yt|B$en0xROa}=YY-4iC&Myrk;{t_)jUnq{`}LhZKJz| zc*>f$8#m3Ck~WHV{LQ|*G%l1Xl|I)zi8(aEMyQW5O;sFzj-tn24(d?87O7eA#1j0I zrHBVqtD{qw8wbU2N@VH8ci1Om;l=i0I6J zO|M9Vn8IofNtVXW&%QgD4$9Gh++-|?i_rc2P|C2d)wklAL!g!V24 zTzt&+WNuJK2~?}6+gB3P0xoc@89_}g`5W> zRVhPUz-;CMzTCmafqt&&b?3;`h@^p1Q__nG1OCj;EMuV1=igD3MFU4+1xWvQ81=n@ z(}4+a7%CHW#|3n2O0{4_YB=_TL~?vtTVx3z7IvJYtJev3;fcy=Ja2%9EO5ue0|mwd z)C&#sOdgqE_8t!D#WfO{Gi^KTsGQ0!9`^xCgU-f!w%J_1str7{#BolA67s}|qb`9L zuCO)%^o}Ra~ zw6wmmDVo2h@$rg<=4C&vXGp zImD%_M!sz3!;f%Wb=P1Pf!*5}`%j@xUEOp?0hUaQh zh|wh`xk<{5y@eKfQyZ6xeF_LyGMyF9&^^%H4cK_SIdBM4pLoa7{2IUQ!wE_oc-t*I zyN~aEA5H`(C};wuC>0j8#*AlXM-QgMa7Q3d%z#-%mYUQ-i=aPbU-0tbk}X)$;+?Ta z0a5ztODm#hCEq!*QQxIXwV&-ecz#A>#|3g5R~oVGe2JAP`5rx;eOL!+jyUGJq8i4F%6M@;GFyMYv?^11iG*x5HBL8aDyK zeIEP*KwPOql=#69v8JLo&&_m=pXn}5Ft?vBXj`OkRTy9Qc{nR+#?kt%d(Q2S=5SxZ zg4-kd^dUIcFd>WW?#Qr!$iWy}exJ2RG-u;e4+*%fM>voCy7^;v;t_~lyaxuuG;SQ5rYI2lMkUNta{CnMdls$9QZQ{FPqjH^mAgOyCuGB0k3e? z)8PY%IGK3Nq>B)1{6J4_pf=0K+k=EY7oUeDLOPR*jQ%y=MeacX?tmp2Z`0^fkPin& zn6*-MYJJ9pFXgLdw%+UgegF7(@1pc`4O&1+3|b2r-_6f!B+_lrq5{5LEg+!g$Ep{c zn$Lfu;W=mdZyH$AV#EaNl_V}%>=Xzh&P8h74^Q8608KAGT7dX~u z4_zxD2ti?KuMnRbBU9yCh6Z~CEOca$N35R=hDZkZ_IhL#ZtdP6_T?$q#u)|l2p5}( zygpYMa80O6#+)=gnIv+Luee<{E?AMN?$|=KdgFpx<8)58ij`#PE%Vi_l?8_Y38AWN z(&p+>xN8B^q)EogR+R=6+DhonwS4?IkAnbY)q!%COmJEG^daM67H+K!23@STJtLd4q4IiCY(vL*ckd=oTqhH@k8WC0hkILwb4IhMg? zmZr`|W4`J}@IW6Gli6z7yFWZUEG9263{2sj&c^#|6*R@Dh$C9z%0?k zo_APge}b$3UE9*?)L)2q4~y%Xc2Fxe)W4-X`Zzvu1NDYb*5}tHe5Y&8=tyw+hGu>U zT3)#5&7L2(J8Rb&I`hFQQt{j*ps~0K@>p+;#BsCXwy(^}ii{=l=n;k8+{$xy=PJ$H z7MG{>_~OF#xm|nl{cC@dCJZ@x0-6sll~|&^MxTC)q@ng@wb0Z69Ri{;h`YPn+sVXCUw8tba7fO87nT^IXL%f7Unzwhk`0QCo{_^xnk!`6s z8lHqN^^Sn)%E7tcBjso29JS9We}eu|!5vaG|2Y;cf0(@Mu`@`Wxq;I}kt(uP{b0_+ z^d(37KEmrhKFh&1(J?FHAaOeIk?Ya5nhSOv?CpoIe9oi0e!kr89BwF{~C5I!m zQyWhXFI&C5DZs^tnnrwDP?V8~G@|qvW!Sq^j)0dVK?Oy@xSk7-=}_fVx+M z(Hr2xIgWR)=k{Ru7UOKphnuml?a-UUHx+Q)7V-XZb7k_V5mgg)F^-AqTn-<9j6R`W zyiCTC2eBk-;61~y(Nh&ONWT`i7GF%;KTC7+EO-vye96MaG<+BBj` zSZg}i{65kL;6uo~R1B^7TSs*Ns0EFnq=c_CdrX_VY_GM%&UCLl46awId0edOOGb6$ z0;7|WM-(nCs3*e&l4;B*zD4$yZ%rMR4WwW&n{u+Poo8ht^-C!U*SDpGj)r+<-am7x zl%uI37p8}_^!zI4+~T%AtbhVF*GA;Zid^G{Xl<@E?bLf?KHxfSt%`(N1iv@_V|Ped zo7S~+VMZ*?|3N+_E=3UWYWU%CF+Fa^1uPS9j_k3)ots{lr}8tLMH0)xYd+i|zY_4I zX#K9o3L0^>IH$4v%R_@SvAgzki;HwkvZKx4vnBQ0Z#~GGhO{by^2$UADW#2mETrj^ z@Y>c>w?I~RxMM#|>u7gB_ye(kh+v4sG>V8l*V+t^rUI7;(I$&h?9>bRn_bcC0=Mz`piHfQApnm|p7KNl8Ex~wkDu9$G6;z{6P zZT6gt;looY@WtA%Hl%&KhKBL=+T{C~iafF%=7O@+Va>()S^WBTXDnyY_}k=~LZflH z*5vsj%6K8SeaHH7;sS^t1M|wSjj7MJ2GfS!!lbuN)=aP0S1sgLt^d2Q`uKeiJ7aX% z{sdx@Y@9W_Bu@5Jn)_BnAH_1M{llWO@^CW1ydjkI`3))@gHmX+376>LzC7uKK)UMk zrz37hop67L=gSkt9zaK9ElSm5@IO4M{>ra=I%xyIqC0{qFa0u7d}jN|UY+pbwMwH` z*HAK@VtFVi*|)?5m}FL>sV2P1^#{k#`eU&aZBL=fCV#9^Y^XL-y5jFFyPqY<8(VsT z%B33_KpU~^RTmVhp?Px4yRqa!&70|>Zk&Ai{^NTrHk~J?kS{}%wwgEs4E~q;0SIf? zL}SFsKADTiUIm92i9)7&u_Z^Dy_U6J^n+7{=d#NLSDY9@etSEj`!Fnj@cQ}wqs~gO z#m+_n4yQ{}9l73NZ|-J+{~L|BZx&S)3_+oQM(QCO)Jx?ikHDjtu}a>7Wj4`Mv!mlL zBL`l&R&q%=upo+ag4nPBW!D!~)=JZbZ{&X~x=}{)aKo}dN8}_yKBY?7%7%XN*j$+- zK_XqYV0ct1j?vQRDko|xMFu*K(8W>h36`}@XR<^h@YwPWsksf!$=k5-+iVMld>z{PXiy775XO3-t5p2C zq|*`J@j^mEFE%-{NP0Oc;JU^d0pkQZWiS9DcKPzstf!yb2C=pgI{<% zd3rqJJEoCs>YdMM)*(5?o;X&q z&#&^*$mfR*alDAp9UCvO{ye-H%p77Si9})%IA3CRZjOmD`qL+jB`~}$#e$hCXRp0= zC0i4vBx>Q0|Lnd(XEHkKe^%fOvZQu(tSZIs0%v70rK&v2A*9 zwKROAvNF}k7LmLw_gWmq3kpc@60Ln6+L}Q=6PE>&^*5INd%IWFCm?Hpwc6ysQ+?;j z1Al*S6XepXAf237k5&CIDk$VgN6FYn?M>4efp~?3pj&@ZNmJ9YSTDWbp7gI&>E{#| zV}4H@l+-Idg5Z6DIvn%1u}qYU+8*h<<;=>Fv~PIxpS>N!t3U7#AE>T7f&o*|#W9di z0V+|j9`}9eA_q}~A3ka?79ajCK|&oEChsrd!_0Qy|8Oo(D3AjCcQ;eQsbvaX6MyP> z>$2G&7uG9Mzh}$2)@ajBv{w3*njvt@Y<-LCAD&bFGQ-k|GBJhl5Glny_+SY?Glo=| zcX302ZceL0F_N+an7W`49(Zg3nnfo4GvFkUvd1{2CO8kFHd5eDo%A@+-7LWE!^+n~ zRt+Nq$nKaLEnFM%MKQDEG-Tu0NWKZ`nLh>5IvyN@?(AD*Vw6;GdGLCKPS)4p#Hqh( z=lc68myu0%r>e7IBsh7o?|%rRTQ`nFeRqf`OoiLrkFV6+Z!=T8FO`JF zU4}Zpc2uI?6VXT(hV3Sm&FM|)eX4sZ>egZX)!xs^;<*02bWGy3zR|1OKO-n!sw|-+ z6p>=|Ac_7Ek*2YnT6^>u_HwK>%Tm_ct1;!8QbGJEuOwu{qP(Z+5Du;?v15(mRsvj- z+)E;@x5lAij~INoqGqRqS5HaFSuAGI$w z|M>ACG}P6@0|TUjlVJ_@PbD8jIx+RWJG?t~?j=*}=jrYYLApQ);G%f>s~B|_CEk=Q zs-s4vbiy4@HcoeSY`bnhKNph%l|8emQmPb&2s6ptNclYIPlt48!t+>W z`wSc(Supl7J6mmVaF90+(*j18E`pKB!f)~7ssNn$0Z8c=SyAlvWlAvL<%TbO^y}M7 zQ#T-`($y2*Nex@8N`QEBvRbRow)A8oZ3odp^POP~y-s>zmW@J!G80eWP&I@A>nmkV4zl0@$74WZ2;(1?AkSsomNjUXmifU zzVqw(;i=+Sr{4lEN9tq-97e((m9=H+#usu!SDIS+Qm(Sf5YTLVQfC-SsV_s8ye<{6Wf(*T}f-}L}$MakdQ({oA>v8 zu8m4vSvwkYG}Y>w&yeADuy-BOGhK@a3$Rw>c)|ttIU`WFL4bX7EC66~SJe~Y# zEiHXclJ}Ecj?oij#sr{BN^bm=34@XgWJ6$iwDs-AHRCVuN2aSzt52047xd9%P!qqF>ZNmvgS#8bEj(c$Lk_2f6uQq6T zGG19bQlwG2^4S4bgv=e@uFa|Ip_T&qs1soFmXNAP@G=s&(4f6Ldg8?Y?`cR#(&P>(av7%en^o@ZKVz$|>vFh%kw z$Hx!%aNpZrFY@2>E(Z$(Ciiz1>4}FBOH?7U%V_N)9JeV+goYe$tK1s1_X+@vc&Z>@ z4)$&GX8d>Y#*5+|=0pkejbF)YlznQ{Yi<#3D{@Ie&bI289_dPxK~b-8|HwJV7Z2l| za=TN5o-rv=`OvWmt9(bgqg3PT-rFLj+QAqMpfL9K_QXU*&*U29zmAS(1d5mQ=F)>% zA~6P_lVQw4WVNwndbRL$(q_TIb^(T>ok{B&V@m^1RLC)1!0yFtxe9h~)t=9Gb3pg6 z`y4Z28$4%&Ps<)k_@QT?a(A7XuRyy=%aCb}<4ZmZpbIB@Mqp>cS_@UADR~vNvkguL zpQ)4U4|PsD%Nfh!s%VMgX@<+#TzeMd5YsxnDhBqW^wlYGH@f-6yL@X~^46c0EBSxo=@H6%$&74o0e< zZ1PNSK!@Q@YeUiLQT8|KgFlNb3=4-Q3nv7xB`j4~Hp1g!_~Z}?t*AZ6iu27Ko~D=MSC z7hhj7UCi(}9(o4KK^Op3pY*fCuwadJB%-&>S8|5&aOSZE+t%Q!QK9=se5!3p9gEv~ z)bgJi_VtT}l;*_0Rpd1x%g`c3x%$6p9|jj^z>X3=FlHo>c3ivRh1ER~=n(WRk))ov zr?Fya>@<}58Fy}$Wp`>ss`LbDyQ~>Xaxm$)1RB-XziAUnhg-=}sJr#?IB8z2 ziOD<+N6tROFOUc?pvadPfp*zx^1sP^)71XeVL@DpN+^J$@&09I!OE7sSUJRfYkPTK zLfVSMzE=nRC2gS`eLIu%Igu+9_hGl1Z{|s3Z##R5X6?T49hsCM4yK{5^J)IVG6$59 zp5Xs#1Q}k_%wbdV?r{tPC>ev?F!f%BwIn2!?UxheB*yTSCkBK{9reL{Pbay>i6jPy zc0{}$2>Oa*BAql~)m&%|2<<;f1s%OVfo%{-h_$if{}X&ueC#!Hsdr$&@^|1lkkZTc z=XfF{_4aM^p&zS0nVKfmzU8!ZbkM|GMsb&8sA9s5|#U`W9bg z-~e!%yP@U=G~4`pO;0MZ#0P!|+@j_Q;q6aVm3LwLUnKzhys*O)zIdc^m7;g@e8yh- z?hxUg!jy~hAR}b#?w_2TvuFRU5jbHS^>A-_vgx#%%(%W>EZM~|LMv1Pe-G6s>;D{E zAy*|#&~IfBN#XWiQHDgafL-su*w_IeTb z=_9mF5dwK&;>7DlEk5|fpPPHLm>X;6ysO<7D;<#zi zx2VRfPM8wHbF|v~FKgpGBD8v&kSnqCefki+T3MNUgX?g`FU4IKJZmwmm!uV=Qh z7)y-4@n=}LW6^2C-F{5+V=MxHrrGJn7+ACbrndoZ7S19kXJ>JLtl_1UtGheUSU0*? zxX3&raIa%AI2_Ir46$|f_3@oO3#Okifs|V>Edlcyc=BjX82r5cj>uP=*?6W6cPQmx zXazOkUkQHgQCN5`$rEqD?^*tLCFoo4KXAgvTfyR@%SwVN7xanr`Jpud{F7$o;30XR zxszH$dy5v5_3S4e_6GYV(6}8$0)}T-hCgPm?mIz`K(ZQtj(hOLv&u%Snq7@l}0 z{A0NTWaigk9f+8^zb44Qi56#nO^ZP~PyU*ChCKQBYnmCL%PRj^8=lJlkLha&*qi|_ m4E#KL`D;1E|A&`a51=sQk)+AmD{mox5;Ya*jZez=gZ>Xv91VN` From 5728432b760a4cdb7f327d4d301d65d94692df1b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sun, 14 Apr 2024 14:27:25 +0200 Subject: [PATCH 98/98] everything put into their respective repos --- external/CMakeLists.txt | 9 --------- external/solanaceae_tox | 2 +- src/CMakeLists.txt | 7 ------- .../register_mfs_json_tox_message_components.cpp | 10 ---------- .../register_mfs_json_tox_message_components.hpp | 6 ------ src/json/tox_message_components.hpp | 16 ---------------- src/main_screen.cpp | 5 ++--- 7 files changed, 3 insertions(+), 52 deletions(-) delete mode 100644 src/fragment_store/register_mfs_json_tox_message_components.cpp delete mode 100644 src/fragment_store/register_mfs_json_tox_message_components.hpp delete mode 100644 src/json/tox_message_components.hpp diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 27a2242..f4415fd 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -22,12 +22,3 @@ add_subdirectory(./stb) add_subdirectory(./libwebp) add_subdirectory(./qoi) -if (NOT TARGET nlohmann_json::nlohmann_json) - FetchContent_Declare(json - URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz - URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d - EXCLUDE_FROM_ALL - ) - FetchContent_MakeAvailable(json) -endif() - diff --git a/external/solanaceae_tox b/external/solanaceae_tox index ce81ef7..d5c1bf0 160000 --- a/external/solanaceae_tox +++ b/external/solanaceae_tox @@ -1 +1 @@ -Subproject commit ce81ef7cf7cea2fe2091912c9eafe787cbba6100 +Subproject commit d5c1bf07db96143939d47e3c49cbc4fef308b3d3 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 103d743..2d37118 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,13 +6,6 @@ add_executable(tomato ./main.cpp ./icon.rc - - # TODO: mfs leftovers, need to move - ./json/tox_message_components.hpp # TODO: move - ./fragment_store/register_mfs_json_tox_message_components.hpp - ./fragment_store/register_mfs_json_tox_message_components.cpp - - ./screen.hpp ./start_screen.hpp ./start_screen.cpp diff --git a/src/fragment_store/register_mfs_json_tox_message_components.cpp b/src/fragment_store/register_mfs_json_tox_message_components.cpp deleted file mode 100644 index fbeb979..0000000 --- a/src/fragment_store/register_mfs_json_tox_message_components.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "./register_mfs_json_tox_message_components.hpp" - -#include "../json/tox_message_components.hpp" -#include "solanaceae/message3/message_serializer.hpp" - -void registerMFSJsonToxMessageComponents(MessageSerializerNJ& msnj) { - msnj.registerSerializer(); - msnj.registerDeserializer(); -} - diff --git a/src/fragment_store/register_mfs_json_tox_message_components.hpp b/src/fragment_store/register_mfs_json_tox_message_components.hpp deleted file mode 100644 index ea801df..0000000 --- a/src/fragment_store/register_mfs_json_tox_message_components.hpp +++ /dev/null @@ -1,6 +0,0 @@ -#pragma once - -#include - -void registerMFSJsonToxMessageComponents(MessageSerializerNJ& msnj); - diff --git a/src/json/tox_message_components.hpp b/src/json/tox_message_components.hpp deleted file mode 100644 index 09ebbdc..0000000 --- a/src/json/tox_message_components.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include - -#include - -#include - -namespace Message::Components { - - // TODO: friend msg id, does not have the same qualities - NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id) - // TODO: transfer stuff, needs content rewrite - -} // Message::Components - diff --git a/src/main_screen.cpp b/src/main_screen.cpp index 9623381..79f27dc 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -1,8 +1,7 @@ #include "./main_screen.hpp" #include -#include "./fragment_store/register_mfs_json_tox_message_components.hpp" -#include "solanaceae/message3/message_serializer.hpp" +#include #include @@ -40,7 +39,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri tel.subscribeAll(tc); registerMessageComponents(msnj); - registerMFSJsonToxMessageComponents(msnj); + registerToxMessageComponents(msnj); conf.set("tox", "save_file_path", save_path);