fixes missing comps and conversion tool is almost working (most of meta is transfered)

This commit is contained in:
Green Sky 2024-04-10 16:26:24 +02:00
parent 5c3b797a99
commit 854ed851b4
No known key found for this signature in database
4 changed files with 98 additions and 4 deletions

View File

@ -40,6 +40,10 @@ static ByteSpan spanFromRead(const std::variant<ByteSpan, std::vector<uint8_t>>&
// TODO: move somewhere else
static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataEncryptionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Encryption>>(
oh.get<ObjComp::DataEncryptionType>().enc
);
@ -56,6 +60,10 @@ static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in)
}
static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) {
if (!oh.all_of<ObjComp::DataCompressionType>()) {
return false;
}
out = static_cast<std::underlying_type_t<Compression>>(
oh.get<ObjComp::DataCompressionType>().comp
);
@ -524,6 +532,9 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
// main thread
// TODO: check timestamps of preexisting and reload? mark external/remote dirty?
for (const auto& it : file_obj_list) {
MetaFileType mft {MetaFileType::TEXT_JSON};
Encryption meta_enc {Encryption::NONE};
Compression meta_comp {Compression::NONE};
nlohmann::json j;
if (it.meta_ext == ".meta.msgpack") {
std::ifstream file(it.obj_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
@ -532,6 +543,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
continue;
}
mft = MetaFileType::BINARY_MSGPACK;
// file is a msgpack within a msgpack
std::vector<uint8_t> full_meta_data;
@ -558,13 +571,13 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
continue;
}
Encryption meta_enc = meta_header_j.at(1);
meta_enc = meta_header_j.at(1);
if (meta_enc != Encryption::NONE) {
std::cerr << "FS error: unknown encryption " << it.obj_path << "\n";
continue;
}
Compression meta_comp = meta_header_j.at(2);
meta_comp = meta_header_j.at(2);
if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) {
std::cerr << "FS error: unknown compression " << it.obj_path << "\n";
continue;
@ -598,6 +611,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
continue;
}
mft = MetaFileType::TEXT_JSON;
file >> j;
} else {
assert(false);
@ -613,6 +628,9 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
ObjectHandle oh{_os.registry(), _os.registry().create()};
oh.emplace<ObjComp::Ephemeral::Backend>(this);
oh.emplace<ObjComp::ID>(hex2bin(it.id_str));
oh.emplace<ObjComp::Ephemeral::MetaFileType>(mft);
oh.emplace<ObjComp::Ephemeral::MetaEncryptionType>(meta_enc);
oh.emplace<ObjComp::Ephemeral::MetaCompressionType>(meta_comp);
oh.emplace<ObjComp::Ephemeral::FilePath>(it.obj_path.generic_u8string());

View File

@ -27,7 +27,7 @@ struct FilesystemStorage : public StorageBackendI {
size_t scanPath(std::string_view path);
void scanPathAsync(std::string path);
private:
public: // TODO: private?
// this thing needs to change and be facilitated over the OS
// but the json serializer are specific to the backend
SerializerCallbacks<Object> _sc;

View File

@ -1,9 +1,14 @@
#include "./object_store.hpp"
#include "./backends/filesystem_storage.hpp"
#include "./meta_components.hpp"
#include <nlohmann/json.hpp>
#include <filesystem>
#include <iostream>
#include <cassert>
int main(int argc, const char** argv) {
if (argc != 3) {
std::cerr << "wrong paramter count, do " << argv[0] << " <input_folder> <output_folder>\n";
@ -26,6 +31,77 @@ int main(int argc, const char** argv) {
// add message fragment store too (adds meta?)
// hookup events
struct EventListener : public ObjectStoreEventI {
ObjectStore2& _os_src;
backend::FilesystemStorage& _fsb_src;
ObjectStore2& _os_dst;
backend::FilesystemStorage& _fsb_dst;
EventListener(
ObjectStore2& os_src,
backend::FilesystemStorage& fsb_src,
ObjectStore2& os_dst,
backend::FilesystemStorage& fsb_dst
) :
_os_src(os_src),
_fsb_src(fsb_src),
_os_dst(os_dst),
_fsb_dst(fsb_dst)
{
_os_src.subscribe(this, ObjectStore_Event::object_construct);
_os_src.subscribe(this, ObjectStore_Event::object_update);
}
protected: // os
bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override {
assert(e.e.all_of<ObjComp::Ephemeral::MetaFileType>());
assert(e.e.all_of<ObjComp::ID>());
auto oh = _fsb_dst.newObject(e.e.get<ObjComp::Ephemeral::MetaFileType>().type, ByteSpan{e.e.get<ObjComp::ID>().v});
if (!static_cast<bool>(oh)) {
// already exists
return false;
}
{ // sync meta
// some hardcoded ehpemeral (besides mft/id)
oh.emplace_or_replace<ObjComp::Ephemeral::MetaEncryptionType>(e.e.get_or_emplace<ObjComp::Ephemeral::MetaEncryptionType>());
oh.emplace_or_replace<ObjComp::Ephemeral::MetaCompressionType>(e.e.get_or_emplace<ObjComp::Ephemeral::MetaCompressionType>());
// serializable
for (const auto& [type, fn] : _fsb_src._sc._serl_json) {
//if (!e.e.registry()->storage(type)->contains(e.e)) {
//continue;
//}
// this is hacky but we serialize and then deserialize the component
// raw copy might be better in the future
nlohmann::json tmp_j;
if (fn(e.e, tmp_j)) {
_fsb_dst._sc._deserl_json.at(type)(oh, tmp_j);
}
}
}
// read src and write dst data
static_cast<StorageBackendI&>(_fsb_dst).write(oh, ByteSpan{});
return false;
}
bool onEvent(const ObjectStore::Events::ObjectUpdate&) override {
std::cerr << "Update called\n";
assert(false);
return false;
}
} el {
os_src,
fsb_src,
os_dst,
fsb_dst,
};
// perform scan (which triggers events)
fsb_dst.scanAsync(); // fill with existing?
fsb_src.scanAsync(); // the scan

View File

@ -29,7 +29,7 @@ struct StorageBackendI {
// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
virtual bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) = 0;
//virtual bool write(Object o, const uint8_t* data, const uint64_t data_size); // default impl
virtual bool write(Object o, const ByteSpan data); // default impl
bool write(Object o, const ByteSpan data);
// ========== read object from storage ==========
using read_from_storage_put_data_cb = void(const ByteSpan buffer);