Tha_14/tomato-testing
1
0
Fork 0
forked from Green-Sky/tomato
Code Pull Requests Actions Packages Releases Activity

refactor the serializer again

Browse Source
This commit is contained in:
Green Sky 2024-04-10 22:27:01 +02:00
parent 3796841961
commit 10b689ca95
No known key found for this signature in database
7 changed files with 136 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/CMakeLists.txt
View File

@ -10,15 +10,16 @@ add_library(fragment_store
./fragment_store/types.hpp
./fragment_store/meta_components.hpp
./fragment_store/meta_components_id.inl
./fragment_store/serializer.hpp
./fragment_store/file2_stack.hpp
./fragment_store/file2_stack.cpp
#old
./fragment_store/serializer.hpp
./fragment_store/fragment_store_i.hpp
./fragment_store/fragment_store_i.cpp
./fragment_store/fragment_store.hpp
./fragment_store/fragment_store.cpp
#new
./fragment_store/serializer_json.hpp
./fragment_store/object_store.hpp
./fragment_store/object_store.cpp
./fragment_store/backends/filesystem_storage.hpp

63
src/fragment_store/backends/filesystem_storage.cpp
View File

@ -1,6 +1,7 @@
#include "./filesystem_storage.hpp"
#include "../meta_components.hpp"
#include "../serializer_json.hpp"
#include <solanaceae/util/utils.hpp>
@ -38,60 +39,10 @@ static ByteSpan spanFromRead(const std::variant<ByteSpan, std::vector<uint8_t>>&
}
}
// TODO: move somewhere else
static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataEncryptionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Encryption>>(
oh.get<ObjComp::DataEncryptionType>().enc
);
return true;
}
static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) {
oh.emplace_or_replace<ObjComp::DataEncryptionType>(
static_cast<Encryption>(
static_cast<std::underlying_type_t<Encryption>>(in)
)
);
return true;
}
static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataCompressionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Compression>>(
oh.get<ObjComp::DataCompressionType>().comp
);
return true;
}
static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) {
oh.emplace_or_replace<ObjComp::DataCompressionType>(
static_cast<Compression>(
static_cast<std::underlying_type_t<Compression>>(in)
)
);
return true;
}
namespace backend {
FilesystemStorage::FilesystemStorage(ObjectStore2& os, std::string_view storage_path) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path) {
_sc.registerSerializerJson<ObjComp::DataEncryptionType>(serl_json_data_enc_type);
_sc.registerDeSerializerJson<ObjComp::DataEncryptionType>(deserl_json_data_enc_type);
_sc.registerSerializerJson<ObjComp::DataCompressionType>(serl_json_data_comp_type);
_sc.registerDeSerializerJson<ObjComp::DataCompressionType>(deserl_json_data_comp_type);
// old stuff
_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
}
FilesystemStorage::~FilesystemStorage(void) {
@ -227,6 +178,8 @@ bool FilesystemStorage::write(Object o, std::function<write_to_storage_fetch_dat
nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed
// metadata file
auto& sjc = _os.registry().ctx().get<SerializerJsonCallbacks<Object>>();
// TODO: refactor extract to OS
for (const auto& [type_id, storage] : reg.storage()) {
if (!storage.contains(o)) {
@ -236,8 +189,8 @@ bool FilesystemStorage::write(Object o, std::function<write_to_storage_fetch_dat
//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
// use type_id to find serializer
auto s_cb_it = _sc._serl_json.find(type_id);
if (s_cb_it == _sc._serl_json.end()) {
auto s_cb_it = sjc._serl.find(type_id);
if (s_cb_it == sjc._serl.end()) {
// could not find serializer, not saving
continue;
}
@ -527,6 +480,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
}
}
auto& sjc = _os.registry().ctx().get<SerializerJsonCallbacks<Object>>();
std::vector<Object> scanned_objs;
// step 3: parse meta and insert into reg of non preexising
// main thread
@ -637,8 +592,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
for (const auto& [k, v] : j.items()) {
// type id from string hash
const auto type_id = entt::hashed_string(k.data(), k.size());
const auto deserl_fn_it = _sc._deserl_json.find(type_id);
if (deserl_fn_it != _sc._deserl_json.cend()) {
const auto deserl_fn_it = sjc._deserl.find(type_id);
if (deserl_fn_it != sjc._deserl.cend()) {
// TODO: check return value
deserl_fn_it->second(oh, v);
} else {

5
src/fragment_store/backends/filesystem_storage.hpp
View File

@ -26,11 +26,6 @@ struct FilesystemStorage : public StorageBackendI {
private:
size_t scanPath(std::string_view path);
void scanPathAsync(std::string path);
public: // TODO: private?
// this thing needs to change and be facilitated over the OS
// but the json serializer are specific to the backend
SerializerCallbacks<Object> _sc;
};
} // backend

5
src/fragment_store/convert_frag_to_obj.cpp
View File

@ -1,6 +1,7 @@
#include "./object_store.hpp"
#include "./backends/filesystem_storage.hpp"
#include "./meta_components.hpp"
#include "./serializer_json.hpp"
#include <solanaceae/util/utils.hpp>
@ -97,7 +98,7 @@ int main(int argc, const char** argv) {
oh.emplace_or_replace<ObjComp::Ephemeral::MetaCompressionType>(e.e.get_or_emplace<ObjComp::Ephemeral::MetaCompressionType>());
// serializable
for (const auto& [type, fn] : _fsb_src._sc._serl_json) {
for (const auto& [type, fn] : _os_src.registry().ctx().get<SerializerJsonCallbacks<Object>>()._serl) {
//if (!e.e.registry()->storage(type)->contains(e.e)) {
//continue;
//}
@ -106,7 +107,7 @@ int main(int argc, const char** argv) {
// raw copy might be better in the future
nlohmann::json tmp_j;
if (fn(e.e, tmp_j)) {
_fsb_dst._sc._deserl_json.at(type)(oh, tmp_j);
_os_dst.registry().ctx().get<SerializerJsonCallbacks<Object>>()._deserl.at(type)(oh, tmp_j);
}
}
}

55
src/fragment_store/object_store.cpp
View File

@ -2,10 +2,53 @@
#include "./meta_components.hpp"
#include "./serializer_json.hpp"
#include <nlohmann/json.hpp> // this sucks
#include <iostream>
// TODO: move somewhere else
static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataEncryptionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Encryption>>(
oh.get<ObjComp::DataEncryptionType>().enc
);
return true;
}
static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) {
oh.emplace_or_replace<ObjComp::DataEncryptionType>(
static_cast<Encryption>(
static_cast<std::underlying_type_t<Encryption>>(in)
)
);
return true;
}
static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataCompressionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Compression>>(
oh.get<ObjComp::DataCompressionType>().comp
);
return true;
}
static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) {
oh.emplace_or_replace<ObjComp::DataCompressionType>(
static_cast<Compression>(
static_cast<std::underlying_type_t<Compression>>(in)
)
);
return true;
}
StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) {
}
@ -23,6 +66,18 @@ bool StorageBackendI::write(Object o, const ByteSpan data) {
}
ObjectStore2::ObjectStore2(void) {
// HACK: set them up independently
auto& sjc = _reg.ctx().emplace<SerializerJsonCallbacks<Object>>();
sjc.registerSerializer<ObjComp::DataEncryptionType>(serl_json_data_enc_type);
sjc.registerDeSerializer<ObjComp::DataEncryptionType>(deserl_json_data_enc_type);
sjc.registerSerializer<ObjComp::DataCompressionType>(serl_json_data_comp_type);
sjc.registerDeSerializer<ObjComp::DataCompressionType>(deserl_json_data_comp_type);
// old stuff
sjc.registerSerializer<FragComp::DataEncryptionType>(serl_json_data_enc_type);
sjc.registerDeSerializer<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
sjc.registerSerializer<FragComp::DataCompressionType>(serl_json_data_comp_type);
sjc.registerDeSerializer<FragComp::DataCompressionType>(deserl_json_data_comp_type);
}
ObjectStore2::~ObjectStore2(void) {

2
src/fragment_store/object_store.hpp
View File

@ -3,8 +3,6 @@
#include <solanaceae/util/event_provider.hpp>
#include <solanaceae/util/span.hpp>
#include "./serializer.hpp" // TODO: get rid of the tight nljson integration
#include <entt/entity/registry.hpp>
#include <entt/entity/handle.hpp>

67
src/fragment_store/serializer_json.hpp Normal file
View File

@ -0,0 +1,67 @@
#pragma once
#include <entt/core/type_info.hpp>
#include <entt/container/dense_map.hpp>
#include <entt/entity/handle.hpp>
#include <nlohmann/json_fwd.hpp>
// nlohmann
template<typename EntityType = entt::entity>
struct SerializerJsonCallbacks {
using Registry = entt::basic_registry<EntityType>;
using Handle = entt::basic_handle<Registry>;
using serialize_fn = bool(*)(const Handle h, nlohmann::json& out);
entt::dense_map<entt::id_type, serialize_fn> _serl;
using deserialize_fn = bool(*)(Handle h, const nlohmann::json& in);
entt::dense_map<entt::id_type, deserialize_fn> _deserl;
template<typename T>
static bool component_get_json(const Handle h, nlohmann::json& j) {
if (h.template all_of<T>()) {
if constexpr (!std::is_empty_v<T>) {
j = h.template get<T>();
}
return true;
}
return false;
}
template<typename T>
static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
if constexpr (std::is_empty_v<T>) {
h.template emplace_or_replace<T>(); // assert empty json?
} else {
h.template emplace_or_replace<T>(static_cast<T>(j));
}
return true;
}
void registerSerializer(serialize_fn fn, const entt::type_info& type_info) {
_serl[type_info.hash()] = fn;
}
template<typename CompType>
void registerSerializer(
serialize_fn fn = component_get_json<CompType>,
const entt::type_info& type_info = entt::type_id<CompType>()
) {
registerSerializer(fn, type_info);
}
void registerDeSerializer(deserialize_fn fn, const entt::type_info& type_info) {
_deserl[type_info.hash()] = fn;
}
template<typename CompType>
void registerDeSerializer(
deserialize_fn fn = component_emplace_or_replace_json<CompType>,
const entt::type_info& type_info = entt::type_id<CompType>()
) {
registerDeSerializer(fn, type_info);
}
};