Tha_14/tomato-testing
1
0
Fork 0
forked from Green-Sky/tomato
Code Pull Requests Actions Packages Releases Activity

further serializer refactoring

Browse Source
This commit is contained in:
Green Sky 2024-02-15 19:06:56 +01:00
parent f6e55851cc
commit 73d1d65142
No known key found for this signature in database
8 changed files with 78 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/CMakeLists.txt
View File

@ -6,7 +6,6 @@ add_library(fragment_store
./fragment_store/meta_components.hpp ./fragment_store/meta_components.hpp
./fragment_store/meta_components_id.inl ./fragment_store/meta_components_id.inl
./fragment_store/serializer.hpp ./fragment_store/serializer.hpp
./fragment_store/serializer.cpp
./fragment_store/fragment_store.hpp ./fragment_store/fragment_store.hpp
./fragment_store/fragment_store.cpp ./fragment_store/fragment_store.cpp
) )

50
src/fragment_store/fragment_store.cpp
View File

@ -291,7 +291,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead //if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]); //s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
//} else if (meta_type == MetaFileType::TEXT_JSON) { //} else if (meta_type == MetaFileType::TEXT_JSON) {
s_cb_it->second(storage.value(fid), meta_data[storage.type().name()]); s_cb_it->second({_reg, fid}, meta_data[storage.type().name()]);
//} //}
} }
@ -521,52 +521,44 @@ void FragmentStore::scanStoragePathAsync(std::string path) {
scanStoragePath(path); // TODO: make async and post result scanStoragePath(path); // TODO: make async and post result
} }
static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) { static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) {
if (comp == nullptr) { out = static_cast<std::underlying_type_t<Encryption>>(
return false; fh.get<FragComp::DataEncryptionType>().enc
} );
auto& r_comp = *reinterpret_cast<FragComp::DataEncryptionType*>(comp);
out = static_cast<std::underlying_type_t<Encryption>>(r_comp.enc);
return true; return true;
} }
static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) { static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) {
// TODO: this is ugly in multiple places
try {
fh.emplace_or_replace<FragComp::DataEncryptionType>( fh.emplace_or_replace<FragComp::DataEncryptionType>(
static_cast<Encryption>( static_cast<Encryption>(
static_cast<std::underlying_type_t<Encryption>>(in) static_cast<std::underlying_type_t<Encryption>>(in)
) )
); );
} catch(...) {
return false;
}
return true; return true;
} }
static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) { static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) {
if (comp == nullptr) { out = static_cast<std::underlying_type_t<Compression>>(
return false; fh.get<FragComp::DataCompressionType>().comp
} );
auto& r_comp = *reinterpret_cast<FragComp::DataCompressionType*>(comp);
out = static_cast<std::underlying_type_t<Compression>>(r_comp.comp);
return true; return true;
} }
static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) {
fh.emplace_or_replace<FragComp::DataCompressionType>(
static_cast<Compression>(
static_cast<std::underlying_type_t<Compression>>(in)
)
);
return true;
}
// TODO: dserl comp type
void FragmentStore::registerSerializers(void) { void FragmentStore::registerSerializers(void) {
_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type); _sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type); _sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type); _sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
std::cout << "registered serl text json cbs:\n";
for (const auto& [type_id, _] : _sc._serl_json) {
std::cout << " " << type_id << "\n";
}
} }

5
src/fragment_store/fragment_store.hpp
View File

@ -17,9 +17,6 @@
#include <cstdint> #include <cstdint>
#include <random> #include <random>
// fwd
struct SerializerCallbacks;
struct FragmentStore : public FragmentStoreI { struct FragmentStore : public FragmentStoreI {
FragmentRegistry _reg; FragmentRegistry _reg;
@ -31,7 +28,7 @@ struct FragmentStore : public FragmentStoreI {
uint64_t _memory_budget {10u*1024u*1024u}; uint64_t _memory_budget {10u*1024u*1024u};
uint64_t _memory_usage {0u}; uint64_t _memory_usage {0u};
SerializerCallbacks _sc; SerializerCallbacks<FragmentID> _sc;
FragmentStore(void); FragmentStore(void);
FragmentStore(std::array<uint8_t, 8> session_uuid_namespace); FragmentStore(std::array<uint8_t, 8> session_uuid_namespace);

30
src/fragment_store/message_fragment_store.cpp
View File

@ -214,37 +214,37 @@ MessageFragmentStore::MessageFragmentStore(
_rmm.subscribe(this, RegistryMessageModel_Event::message_updated); _rmm.subscribe(this, RegistryMessageModel_Event::message_updated);
_rmm.subscribe(this, RegistryMessageModel_Event::message_destroy); _rmm.subscribe(this, RegistryMessageModel_Event::message_destroy);
_fs._sc.registerSerializerJson<FragComp::MessagesTSRange>(serl_json_msg_ts_range); _fs._sc.registerSerializerJson<FragComp::MessagesTSRange>();
_fs._sc.registerDeSerializerJson<FragComp::MessagesTSRange>(); _fs._sc.registerDeSerializerJson<FragComp::MessagesTSRange>();
_fs._sc.registerSerializerJson<FragComp::MessagesContact>(serl_json_msg_c_id); _fs._sc.registerSerializerJson<FragComp::MessagesContact>();
_fs._sc.registerDeSerializerJson<FragComp::MessagesContact>(); _fs._sc.registerDeSerializerJson<FragComp::MessagesContact>();
_sc.registerSerializerJson<Message::Components::Timestamp>(serl_json_default<Message::Components::Timestamp>); _sc.registerSerializerJson<Message::Components::Timestamp>();
_sc.registerDeSerializerJson<Message::Components::Timestamp>(); _sc.registerDeSerializerJson<Message::Components::Timestamp>();
_sc.registerSerializerJson<Message::Components::TimestampProcessed>(serl_json_default<Message::Components::TimestampProcessed>); _sc.registerSerializerJson<Message::Components::TimestampProcessed>();
_sc.registerDeSerializerJson<Message::Components::TimestampProcessed>(); _sc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
_sc.registerSerializerJson<Message::Components::TimestampWritten>(serl_json_default<Message::Components::TimestampWritten>); _sc.registerSerializerJson<Message::Components::TimestampWritten>();
_sc.registerDeSerializerJson<Message::Components::TimestampWritten>(); _sc.registerDeSerializerJson<Message::Components::TimestampWritten>();
_sc.registerSerializerJson<Message::Components::ContactFrom>(serl_json_default<Message::Components::ContactFrom>); _sc.registerSerializerJson<Message::Components::ContactFrom>();
_sc.registerDeSerializerJson<Message::Components::ContactFrom>(); _sc.registerDeSerializerJson<Message::Components::ContactFrom>();
_sc.registerSerializerJson<Message::Components::ContactTo>(serl_json_default<Message::Components::ContactTo>); _sc.registerSerializerJson<Message::Components::ContactTo>();
_sc.registerDeSerializerJson<Message::Components::ContactTo>(); _sc.registerDeSerializerJson<Message::Components::ContactTo>();
_sc.registerSerializerJson<Message::Components::TagUnread>(serl_json_default<Message::Components::TagUnread>); _sc.registerSerializerJson<Message::Components::TagUnread>();
_sc.registerDeSerializerJson<Message::Components::TagUnread>(); _sc.registerDeSerializerJson<Message::Components::TagUnread>();
_sc.registerSerializerJson<Message::Components::Read>(serl_json_default<Message::Components::Read>); _sc.registerSerializerJson<Message::Components::Read>();
_sc.registerDeSerializerJson<Message::Components::Read>(); _sc.registerDeSerializerJson<Message::Components::Read>();
_sc.registerSerializerJson<Message::Components::MessageText>(serl_json_default<Message::Components::MessageText>); _sc.registerSerializerJson<Message::Components::MessageText>();
_sc.registerDeSerializerJson<Message::Components::MessageText>(); _sc.registerDeSerializerJson<Message::Components::MessageText>();
_sc.registerSerializerJson<Message::Components::TagMessageIsAction>(serl_json_default<Message::Components::TagMessageIsAction>); _sc.registerSerializerJson<Message::Components::TagMessageIsAction>();
_sc.registerDeSerializerJson<Message::Components::TagMessageIsAction>(); _sc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
// files // files
//_sc.registerSerializerJson<Message::Components::Transfer::FileID>() //_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>(serl_json_default<Message::Components::Transfer::FileInfo>); _sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>(); _sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>(serl_json_default<Message::Components::Transfer::FileInfoLocal>); _sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>(); _sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>(serl_json_default<Message::Components::Transfer::TagHaveAll>); _sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>(); _sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
_fs.scanStoragePath("test_message_store/"); _fs.scanStoragePath("test_message_store/");
@ -292,7 +292,7 @@ float MessageFragmentStore::tick(float time_delta) {
continue; continue;
} }
s_cb_it->second(storage.value(m), j_entry[storage.type().name()]); s_cb_it->second({*reg, m}, j_entry[storage.type().name()]);
} }
} }

2
src/fragment_store/message_fragment_store.hpp
View File

@ -36,7 +36,7 @@ namespace Fragment::Components {
}; };
} // Fragment::Components } // Fragment::Components
struct MessageSerializerCallbacks : public SerializerCallbacks { struct MessageSerializerCallbacks : public SerializerCallbacks<Message3> {
// TODO: add contact and message reg, so entities can be looked up and be converted to fragment uids OR persistent ids // TODO: add contact and message reg, so entities can be looked up and be converted to fragment uids OR persistent ids
}; };

10
src/fragment_store/serializer.cpp
View File

@ -1,10 +0,0 @@
#include "./serializer.hpp"
void SerializerCallbacks::registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
_serl_json[type_info.hash()] = fn;
}
void SerializerCallbacks::registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
_deserl_json[type_info.hash()] = fn;
}

51
src/fragment_store/serializer.hpp
View File

@ -6,32 +6,24 @@
#include <nlohmann/json_fwd.hpp> #include <nlohmann/json_fwd.hpp>
#include "./fragment_store_i.hpp" template<typename EntityType = entt::entity>
struct SerializerCallbacks { struct SerializerCallbacks {
using Registry = entt::basic_registry<EntityType>;
using Handle = entt::basic_handle<Registry>;
// nlohmann // nlohmann
// json/msgpack // json/msgpack
using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out); using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out);
entt::dense_map<entt::id_type, serialize_json_fn> _serl_json; entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
using deserialize_json_fn = bool(*)(FragmentHandle fh, const nlohmann::json& in); using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in);
entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json; entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
template<typename T> template<typename T>
static bool component_emplace_or_replace_json(FragmentHandle fh, const nlohmann::json& j) { static bool component_get_json(const Handle h, nlohmann::json& j) {
if constexpr (std::is_empty_v<T>) { if (h.template all_of<T>()) {
fh.emplace_or_replace<T>(); // assert empty json?
} else {
fh.emplace_or_replace<T>(static_cast<T>(j));
}
return true;
}
template<typename T>
static bool component_get_json(const FragmentHandle fh, nlohmann::json& j) {
if (fh.all_of<T>()) {
if constexpr (!std::is_empty_v<T>) { if constexpr (!std::is_empty_v<T>) {
j = fh.get<T>(); j = h.template get<T>();
} }
return true; return true;
} }
@ -39,12 +31,31 @@ struct SerializerCallbacks {
return false; return false;
} }
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info); template<typename T>
static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
if constexpr (std::is_empty_v<T>) {
h.template emplace_or_replace<T>(); // assert empty json?
} else {
h.template emplace_or_replace<T>(static_cast<T>(j));
}
return true;
}
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
_serl_json[type_info.hash()] = fn;
}
template<typename CompType> template<typename CompType>
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerSerializerJson(fn, type_info); } void registerSerializerJson(
serialize_json_fn fn = component_get_json<CompType>,
const entt::type_info& type_info = entt::type_id<CompType>()
) {
registerSerializerJson(fn, type_info);
}
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info); void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
_deserl_json[type_info.hash()] = fn;
}
template<typename CompType> template<typename CompType>
void registerDeSerializerJson( void registerDeSerializerJson(

20
src/fragment_store/test_fragstore.cpp
View File

@ -11,28 +11,16 @@ namespace Components {
uint64_t begin {0}; uint64_t begin {0};
uint64_t end {1000}; uint64_t end {1000};
}; };
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end)
} // Components } // Components
static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) {
if (comp == nullptr) {
return false;
}
out = nlohmann::json::object();
auto& r_comp = *reinterpret_cast<Components::MessagesTimestampRange*>(comp);
out["begin"] = r_comp.begin;
out["end"] = r_comp.end;
return true;
}
int main(void) { int main(void) {
FragmentStore fs; FragmentStore fs;
fs._default_store_path = "test_store/"; fs._default_store_path = "test_store/";
fs._sc.registerSerializerJson<Components::MessagesTimestampRange>(serl_json_msg_ts_range); fs._sc.registerSerializerJson<Components::MessagesTimestampRange>();
fs._sc.registerDeSerializerJson<Components::MessagesTimestampRange>();
const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9}); const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9});