diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 375f95ba..4bc6c630 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR) add_library(fragment_store ./fragment_store/fragment_store_i.hpp + ./fragment_store/types.hpp + ./fragment_store/meta_components.hpp + ./fragment_store/meta_components_id.inl + ./fragment_store/serializer.hpp + ./fragment_store/serializer.cpp ./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.cpp ) @@ -106,6 +111,8 @@ target_link_libraries(tomato PUBLIC solanaceae_tox_contacts solanaceae_tox_messages + fragment_store + SDL3::SDL3 imgui diff --git a/src/fragment_store/fragment_store.cpp b/src/fragment_store/fragment_store.cpp index a13431bd..be4c3938 100644 --- a/src/fragment_store/fragment_store.cpp +++ b/src/fragment_store/fragment_store.cpp @@ -51,7 +51,7 @@ FragmentStore::FragmentStore( registerSerializers(); } -entt::basic_handle> FragmentStore::fragmentHandle(FragmentID fid) { +FragmentStore::FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) { return {_reg, fid}; } @@ -107,7 +107,7 @@ FragmentID FragmentStore::newFragmentMemoryOwned( const auto new_frag = _reg.create(); - _reg.emplace(new_frag, id); + _reg.emplace(new_frag, id); // TODO: memory comp _reg.emplace>>(new_frag) = std::move(new_data); @@ -149,12 +149,12 @@ FragmentID FragmentStore::newFragmentFile( const auto new_frag = _reg.create(); - _reg.emplace(new_frag, id); + _reg.emplace(new_frag, id); // file (info) comp - _reg.emplace(new_frag, fragment_file_path.generic_u8string()); + _reg.emplace(new_frag, fragment_file_path.generic_u8string()); - _reg.emplace(new_frag, mft); + _reg.emplace(new_frag, mft); // meta needs to be synced to file std::function empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; }; @@ -177,7 +177,7 @@ FragmentID FragmentStore::getFragmentByID( ) { // TODO: accelerate // maybe keep it sorted and binary search? hash table lookup? - for (const auto& [frag, id_comp] : _reg.view().each()) { + for (const auto& [frag, id_comp] : _reg.view().each()) { if (id == id_comp.v) { return frag; } @@ -206,7 +206,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { + if (!_reg.all_of(fid)) { // not a file fragment? return false; } @@ -214,20 +214,20 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - meta_type = _reg.get(fid).type; + if (_reg.all_of(fid)) { + meta_type = _reg.get(fid).type; } Encryption meta_enc = Encryption::NONE; // TODO: better defaults Compression meta_comp = Compression::NONE; // TODO: better defaults if (meta_type != MetaFileType::TEXT_JSON) { - if (_reg.all_of(fid)) { - meta_enc = _reg.get(fid).enc; + if (_reg.all_of(fid)) { + meta_enc = _reg.get(fid).enc; } - if (_reg.all_of(fid)) { - meta_comp = _reg.get(fid).comp; + if (_reg.all_of(fid)) { + meta_comp = _reg.get(fid).comp; } } else { // we cant have encryption or compression @@ -236,15 +236,15 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - _reg.emplace_or_replace(fid, Encryption::NONE); + _reg.emplace_or_replace(fid, Encryption::NONE); //} //if (_reg.all_of(fid)) { - _reg.emplace_or_replace(fid, Compression::NONE); + _reg.emplace_or_replace(fid, Compression::NONE); //} } std::ofstream meta_file{ - _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type), + _reg.get(fid).path + ".meta" + metaFileTypeSuffix(meta_type), std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text }; @@ -253,7 +253,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid).path, + _reg.get(fid).path, std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text }; @@ -320,19 +320,32 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function(fid)) { - _reg.remove(fid); + if (_reg.all_of(fid)) { + _reg.remove(fid); } return true; } +bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) { + std::function fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { + uint64_t i = 0; + for (; i+read < data_size && i < buffer_size; i++) { + request_buffer[i] = data[i+read]; + } + read += i; + + return i; + }; + return syncToStorage(fid, fn_cb); +} + static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) { if (comp == nullptr) { return false; } - auto& r_comp = *reinterpret_cast(comp); + auto& r_comp = *reinterpret_cast(comp); out = static_cast>(r_comp.enc); @@ -344,7 +357,7 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { return false; } - auto& r_comp = *reinterpret_cast(comp); + auto& r_comp = *reinterpret_cast(comp); out = static_cast>(r_comp.comp); @@ -352,8 +365,8 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { } void FragmentStore::registerSerializers(void) { - _sc.registerSerializerJson(serl_json_data_enc_type); - _sc.registerSerializerJson(serl_json_data_comp_type); + _sc.registerSerializerJson(serl_json_data_enc_type); + _sc.registerSerializerJson(serl_json_data_comp_type); std::cout << "registered serl text json cbs:\n"; for (const auto& [type_id, _] : _sc._serl_json) { diff --git a/src/fragment_store/fragment_store.hpp b/src/fragment_store/fragment_store.hpp index 90cff43b..6dda33de 100644 --- a/src/fragment_store/fragment_store.hpp +++ b/src/fragment_store/fragment_store.hpp @@ -2,101 +2,24 @@ #include "./fragment_store_i.hpp" -#include +#include "./types.hpp" +#include "./meta_components.hpp" + +#include "./serializer.hpp" + #include #include -#include #include -#include #include #include #include #include -enum class Encryption : uint8_t { - NONE = 0x00, -}; -enum class Compression : uint8_t { - NONE = 0x00, -}; -enum class MetaFileType : uint8_t { - TEXT_JSON, - //BINARY_ARB, - BINARY_MSGPACK, -}; - -namespace Components { - - // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) - struct ID { - std::vector v; - }; - - struct DataEncryptionType { - Encryption enc {Encryption::NONE}; - }; - - struct DataCompressionType { - Compression comp {Compression::NONE}; - }; - - - // meta that is not written to (meta-)file - namespace Ephemeral { - - // excluded from file meta - struct FilePath { - // contains store path, if any - std::string path; - }; - - // TODO: seperate into remote and local? - // (remote meaning eg. the file on disk was changed by another program) - struct DirtyTag {}; - - - // type as comp - struct MetaFileType { - ::MetaFileType type {::MetaFileType::TEXT_JSON}; - }; - - struct MetaEncryptionType { - Encryption enc {Encryption::NONE}; - }; - - struct MetaCompressionType { - Compression comp {Compression::NONE}; - }; - - } // Ephemeral - -} // Components - -struct SerializerCallbacks { - // nlohmann - // json/msgpack - using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); - entt::dense_map _serl_json; - - using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in); - entt::dense_map _deserl_json; - - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { - _serl_json[type_info.hash()] = fn; - } - template - void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerSerializerJson(fn, type_info); } - - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { - _deserl_json[type_info.hash()] = fn; - } - template - void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerDeSerializerJson(fn, type_info); } -}; - struct FragmentStore : public FragmentStoreI { + using FragmentHandle = entt::basic_handle>; + entt::basic_registry _reg; std::minstd_rand _rng{std::random_device{}()}; @@ -113,10 +36,11 @@ struct FragmentStore : public FragmentStoreI { FragmentStore(std::array session_uuid_namespace); // HACK: get access to the reg - entt::basic_handle> fragmentHandle(FragmentID fid); + FragmentHandle fragmentHandle(FragmentID fid); // TODO: make the frags ref counted + // TODO: check for exising std::vector generateNewUID(std::array& uuid_namespace); std::vector generateNewUID(void); @@ -154,21 +78,20 @@ struct FragmentStore : public FragmentStoreI { ); // remove fragment? + // unload? - // syncs fragment to file - + // ========== sync fragment to storage ========== using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. bool syncToStorage(FragmentID fid, std::function& data_cb); - - // unload frags? - // if frags are file backed, we can close the file if not needed + bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size); // fragment discovery? private: void registerSerializers(void); // internal comps // internal actual backends + // TODO: seperate out bool syncToMemory(FragmentID fid, std::function& data_cb); bool syncToFile(FragmentID fid, std::function& data_cb); }; diff --git a/src/fragment_store/meta_components.hpp b/src/fragment_store/meta_components.hpp new file mode 100644 index 00000000..a8ec20e5 --- /dev/null +++ b/src/fragment_store/meta_components.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include "./types.hpp" + +#include +#include +#include + +namespace Fragment::Components { + + // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) + struct ID { + std::vector v; + }; + + struct DataEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct DataCompressionType { + Compression comp {Compression::NONE}; + }; + + + // meta that is not written to (meta-)file + namespace Ephemeral { + + // excluded from file meta + struct FilePath { + // contains store path, if any + std::string path; + }; + + // TODO: seperate into remote and local? + // (remote meaning eg. the file on disk was changed by another program) + struct DirtyTag {}; + + + // type as comp + struct MetaFileType { + ::MetaFileType type {::MetaFileType::TEXT_JSON}; + }; + + struct MetaEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct MetaCompressionType { + Compression comp {Compression::NONE}; + }; + + } // Ephemeral + +} // Components + +// shortened to save bytes (until I find a way to save by ID in msgpack) +namespace FragComp = Fragment::Components; + +#include "./meta_components_id.inl" + diff --git a/src/fragment_store/meta_components_id.inl b/src/fragment_store/meta_components_id.inl new file mode 100644 index 00000000..c912cce7 --- /dev/null +++ b/src/fragment_store/meta_components_id.inl @@ -0,0 +1,26 @@ +#pragma once + +#include "./meta_components.hpp" + +#include + +// TODO: move more central +#define DEFINE_COMP_ID(x) \ +template<> \ +constexpr entt::id_type entt::type_hash::value() noexcept { \ + using namespace entt::literals; \ + return #x##_hs; \ +} \ +template<> \ +constexpr std::string_view entt::type_name::value() noexcept { \ + return #x; \ +} + +// cross compiler stable ids + +DEFINE_COMP_ID(FragComp::DataEncryptionType) +DEFINE_COMP_ID(FragComp::DataCompressionType) + +#undef DEFINE_COMP_ID + + diff --git a/src/fragment_store/serializer.cpp b/src/fragment_store/serializer.cpp new file mode 100644 index 00000000..1f3bfe41 --- /dev/null +++ b/src/fragment_store/serializer.cpp @@ -0,0 +1,10 @@ +#include "./serializer.hpp" + +void SerializerCallbacks::registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) { + _serl_json[type_info.hash()] = fn; +} + +void SerializerCallbacks::registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) { + _deserl_json[type_info.hash()] = fn; +} + diff --git a/src/fragment_store/serializer.hpp b/src/fragment_store/serializer.hpp new file mode 100644 index 00000000..a09feec7 --- /dev/null +++ b/src/fragment_store/serializer.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include + +struct SerializerCallbacks { + // nlohmann + // json/msgpack + using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); + entt::dense_map _serl_json; + + using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in); + entt::dense_map _deserl_json; + + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info); + template + void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerSerializerJson(fn, type_info); } + + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info); + template + void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id()) { registerDeSerializerJson(fn, type_info); } +}; + diff --git a/src/fragment_store/test_fragstore.cpp b/src/fragment_store/test_fragstore.cpp index 9c021650..581f64ac 100644 --- a/src/fragment_store/test_fragstore.cpp +++ b/src/fragment_store/test_fragstore.cpp @@ -43,8 +43,8 @@ int main(void) { { auto frag0h = fs.fragmentHandle(frag0); - frag0h.emplace_or_replace(); - frag0h.emplace_or_replace(); + frag0h.emplace_or_replace(); + frag0h.emplace_or_replace(); frag0h.emplace_or_replace(); std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { @@ -62,8 +62,8 @@ int main(void) { { auto frag1h = fs.fragmentHandle(frag1); - frag1h.emplace_or_replace(); - frag1h.emplace_or_replace(); + frag1h.emplace_or_replace(); + frag1h.emplace_or_replace(); std::function fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { static constexpr std::string_view text = "This is some random data"; @@ -78,6 +78,15 @@ int main(void) { fs.syncToStorage(frag1, fn_cb); } + { + auto frag2h = fs.fragmentHandle(frag2); + + frag2h.emplace_or_replace(); + frag2h.emplace_or_replace(); + + static constexpr std::string_view text = "This is more random data"; + fs.syncToStorage(frag2, reinterpret_cast(text.data()), text.size()); + } return 0; } diff --git a/src/fragment_store/types.hpp b/src/fragment_store/types.hpp new file mode 100644 index 00000000..cbb64c51 --- /dev/null +++ b/src/fragment_store/types.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +enum class Encryption : uint8_t { + NONE = 0x00, +}; +enum class Compression : uint8_t { + NONE = 0x00, +}; +enum class MetaFileType : uint8_t { + TEXT_JSON, + //BINARY_ARB, + BINARY_MSGPACK, +}; + diff --git a/src/main_screen.cpp b/src/main_screen.cpp index e84a2c91..9db04a37 100644 --- a/src/main_screen.cpp +++ b/src/main_screen.cpp @@ -1,4 +1,5 @@ #include "./main_screen.hpp" +#include "fragment_store/fragment_store.hpp" #include @@ -49,6 +50,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n"; { // setup plugin instances + // TODO: make interface useful + g_provideInstance("FragmentStoreI", "host", &fs); + g_provideInstance("FragmentStore", "host", &fs); + g_provideInstance("ConfigModelI", "host", &conf); g_provideInstance("Contact3Registry", "1", "host", &cr); g_provideInstance("RegistryMessageModel", "host", &rmm); diff --git a/src/main_screen.hpp b/src/main_screen.hpp index 27f7d7bd..aabcb51d 100644 --- a/src/main_screen.hpp +++ b/src/main_screen.hpp @@ -2,6 +2,7 @@ #include "./screen.hpp" +#include "./fragment_store/fragment_store.hpp" #include #include #include @@ -43,6 +44,8 @@ extern "C" { struct MainScreen final : public Screen { SDL_Renderer* renderer; + FragmentStore fs; + SimpleConfigModel conf; Contact3Registry cr; RegistryMessageModel rmm;