Tha_14/tomato-testing
1
0
Fork 0
forked from Green-Sky/tomato
Code Pull Requests Actions Packages Releases Activity

further serializer refactoring

Browse Source
This commit is contained in:
Green Sky 2024-02-15 19:06:56 +01:00
parent f6e55851cc
commit 73d1d65142
No known key found for this signature in database
8 changed files with 78 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/CMakeLists.txt
View File

@ -6,7 +6,6 @@ add_library(fragment_store
./fragment_store/meta_components.hpp
./fragment_store/meta_components_id.inl
./fragment_store/serializer.hpp
./fragment_store/serializer.cpp
./fragment_store/fragment_store.hpp
./fragment_store/fragment_store.cpp
)

60
src/fragment_store/fragment_store.cpp
View File

@ -291,7 +291,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
//} else if (meta_type == MetaFileType::TEXT_JSON) {
s_cb_it->second(storage.value(fid), meta_data[storage.type().name()]);
s_cb_it->second({_reg, fid}, meta_data[storage.type().name()]);
//}
}
@ -521,52 +521,44 @@ void FragmentStore::scanStoragePathAsync(std::string path) {
scanStoragePath(path); // TODO: make async and post result
}
static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) {
if (comp == nullptr) {
return false;
}
auto& r_comp = *reinterpret_cast<FragComp::DataEncryptionType*>(comp);
out = static_cast<std::underlying_type_t<Encryption>>(r_comp.enc);
static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) {
out = static_cast<std::underlying_type_t<Encryption>>(
fh.get<FragComp::DataEncryptionType>().enc
);
return true;
}
static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) {
// TODO: this is ugly in multiple places
try {
fh.emplace_or_replace<FragComp::DataEncryptionType>(
static_cast<Encryption>(
static_cast<std::underlying_type_t<Encryption>>(in)
)
);
} catch(...) {
return false;
}
fh.emplace_or_replace<FragComp::DataEncryptionType>(
static_cast<Encryption>(
static_cast<std::underlying_type_t<Encryption>>(in)
)
);
return true;
}
static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) {
if (comp == nullptr) {
return false;
}
auto& r_comp = *reinterpret_cast<FragComp::DataCompressionType*>(comp);
out = static_cast<std::underlying_type_t<Compression>>(r_comp.comp);
static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) {
out = static_cast<std::underlying_type_t<Compression>>(
fh.get<FragComp::DataCompressionType>().comp
);
return true;
}
static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) {
fh.emplace_or_replace<FragComp::DataCompressionType>(
static_cast<Compression>(
static_cast<std::underlying_type_t<Compression>>(in)
)
);
return true;
}
// TODO: dserl comp type
void FragmentStore::registerSerializers(void) {
_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
std::cout << "registered serl text json cbs:\n";
for (const auto& [type_id, _] : _sc._serl_json) {
std::cout << " " << type_id << "\n";
}
_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
}

5
src/fragment_store/fragment_store.hpp
View File

@ -17,9 +17,6 @@
#include <cstdint>
#include <random>
// fwd
struct SerializerCallbacks;
struct FragmentStore : public FragmentStoreI {
FragmentRegistry _reg;
@ -31,7 +28,7 @@ struct FragmentStore : public FragmentStoreI {
uint64_t _memory_budget {10u*1024u*1024u};
uint64_t _memory_usage {0u};
SerializerCallbacks _sc;
SerializerCallbacks<FragmentID> _sc;
FragmentStore(void);
FragmentStore(std::array<uint8_t, 8> session_uuid_namespace);

30
src/fragment_store/message_fragment_store.cpp
View File

@ -214,37 +214,37 @@ MessageFragmentStore::MessageFragmentStore(
_rmm.subscribe(this, RegistryMessageModel_Event::message_updated);
_rmm.subscribe(this, RegistryMessageModel_Event::message_destroy);
_fs._sc.registerSerializerJson<FragComp::MessagesTSRange>(serl_json_msg_ts_range);
_fs._sc.registerSerializerJson<FragComp::MessagesTSRange>();
_fs._sc.registerDeSerializerJson<FragComp::MessagesTSRange>();
_fs._sc.registerSerializerJson<FragComp::MessagesContact>(serl_json_msg_c_id);
_fs._sc.registerSerializerJson<FragComp::MessagesContact>();
_fs._sc.registerDeSerializerJson<FragComp::MessagesContact>();
_sc.registerSerializerJson<Message::Components::Timestamp>(serl_json_default<Message::Components::Timestamp>);
_sc.registerSerializerJson<Message::Components::Timestamp>();
_sc.registerDeSerializerJson<Message::Components::Timestamp>();
_sc.registerSerializerJson<Message::Components::TimestampProcessed>(serl_json_default<Message::Components::TimestampProcessed>);
_sc.registerSerializerJson<Message::Components::TimestampProcessed>();
_sc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
_sc.registerSerializerJson<Message::Components::TimestampWritten>(serl_json_default<Message::Components::TimestampWritten>);
_sc.registerSerializerJson<Message::Components::TimestampWritten>();
_sc.registerDeSerializerJson<Message::Components::TimestampWritten>();
_sc.registerSerializerJson<Message::Components::ContactFrom>(serl_json_default<Message::Components::ContactFrom>);
_sc.registerSerializerJson<Message::Components::ContactFrom>();
_sc.registerDeSerializerJson<Message::Components::ContactFrom>();
_sc.registerSerializerJson<Message::Components::ContactTo>(serl_json_default<Message::Components::ContactTo>);
_sc.registerSerializerJson<Message::Components::ContactTo>();
_sc.registerDeSerializerJson<Message::Components::ContactTo>();
_sc.registerSerializerJson<Message::Components::TagUnread>(serl_json_default<Message::Components::TagUnread>);
_sc.registerSerializerJson<Message::Components::TagUnread>();
_sc.registerDeSerializerJson<Message::Components::TagUnread>();
_sc.registerSerializerJson<Message::Components::Read>(serl_json_default<Message::Components::Read>);
_sc.registerSerializerJson<Message::Components::Read>();
_sc.registerDeSerializerJson<Message::Components::Read>();
_sc.registerSerializerJson<Message::Components::MessageText>(serl_json_default<Message::Components::MessageText>);
_sc.registerSerializerJson<Message::Components::MessageText>();
_sc.registerDeSerializerJson<Message::Components::MessageText>();
_sc.registerSerializerJson<Message::Components::TagMessageIsAction>(serl_json_default<Message::Components::TagMessageIsAction>);
_sc.registerSerializerJson<Message::Components::TagMessageIsAction>();
_sc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
// files
//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>(serl_json_default<Message::Components::Transfer::FileInfo>);
_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>(serl_json_default<Message::Components::Transfer::FileInfoLocal>);
_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>(serl_json_default<Message::Components::Transfer::TagHaveAll>);
_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
_fs.scanStoragePath("test_message_store/");
@ -292,7 +292,7 @@ float MessageFragmentStore::tick(float time_delta) {
continue;
}
s_cb_it->second(storage.value(m), j_entry[storage.type().name()]);
s_cb_it->second({*reg, m}, j_entry[storage.type().name()]);
}
}

2
src/fragment_store/message_fragment_store.hpp
View File

@ -36,7 +36,7 @@ namespace Fragment::Components {
};
} // Fragment::Components
struct MessageSerializerCallbacks : public SerializerCallbacks {
struct MessageSerializerCallbacks : public SerializerCallbacks<Message3> {
// TODO: add contact and message reg, so entities can be looked up and be converted to fragment uids OR persistent ids
};

10
src/fragment_store/serializer.cpp
View File

@ -1,10 +0,0 @@
#include "./serializer.hpp"
void SerializerCallbacks::registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
_serl_json[type_info.hash()] = fn;
}
void SerializerCallbacks::registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
_deserl_json[type_info.hash()] = fn;
}

51
src/fragment_store/serializer.hpp
View File

@ -6,32 +6,24 @@
#include <nlohmann/json_fwd.hpp>
#include "./fragment_store_i.hpp"
template<typename EntityType = entt::entity>
struct SerializerCallbacks {
using Registry = entt::basic_registry<EntityType>;
using Handle = entt::basic_handle<Registry>;
// nlohmann
// json/msgpack
using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out);
using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out);
entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
using deserialize_json_fn = bool(*)(FragmentHandle fh, const nlohmann::json& in);
using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in);
entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
template<typename T>
static bool component_emplace_or_replace_json(FragmentHandle fh, const nlohmann::json& j) {
if constexpr (std::is_empty_v<T>) {
fh.emplace_or_replace<T>(); // assert empty json?
} else {
fh.emplace_or_replace<T>(static_cast<T>(j));
}
return true;
}
template<typename T>
static bool component_get_json(const FragmentHandle fh, nlohmann::json& j) {
if (fh.all_of<T>()) {
static bool component_get_json(const Handle h, nlohmann::json& j) {
if (h.template all_of<T>()) {
if constexpr (!std::is_empty_v<T>) {
j = fh.get<T>();
j = h.template get<T>();
}
return true;
}
@ -39,12 +31,31 @@ struct SerializerCallbacks {
return false;
}
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info);
template<typename T>
static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
if constexpr (std::is_empty_v<T>) {
h.template emplace_or_replace<T>(); // assert empty json?
} else {
h.template emplace_or_replace<T>(static_cast<T>(j));
}
return true;
}
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
_serl_json[type_info.hash()] = fn;
}
template<typename CompType>
void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerSerializerJson(fn, type_info); }
void registerSerializerJson(
serialize_json_fn fn = component_get_json<CompType>,
const entt::type_info& type_info = entt::type_id<CompType>()
) {
registerSerializerJson(fn, type_info);
}
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info);
void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
_deserl_json[type_info.hash()] = fn;
}
template<typename CompType>
void registerDeSerializerJson(

20
src/fragment_store/test_fragstore.cpp
View File

@ -11,28 +11,16 @@ namespace Components {
uint64_t begin {0};
uint64_t end {1000};
};
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end)
} // Components
static bool serl_json_msg_ts_range(void* comp, nlohmann::json& out) {
if (comp == nullptr) {
return false;
}
out = nlohmann::json::object();
auto& r_comp = *reinterpret_cast<Components::MessagesTimestampRange*>(comp);
out["begin"] = r_comp.begin;
out["end"] = r_comp.end;
return true;
}
int main(void) {
FragmentStore fs;
fs._default_store_path = "test_store/";
fs._sc.registerSerializerJson<Components::MessagesTimestampRange>(serl_json_msg_ts_range);
fs._sc.registerSerializerJson<Components::MessagesTimestampRange>();
fs._sc.registerDeSerializerJson<Components::MessagesTimestampRange>();
const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9});