diff --git a/src/fragment_store/backends/filesystem_storage.cpp b/src/fragment_store/backends/filesystem_storage.cpp index b5ae4209..4a13663e 100644 --- a/src/fragment_store/backends/filesystem_storage.cpp +++ b/src/fragment_store/backends/filesystem_storage.cpp @@ -42,13 +42,17 @@ static ByteSpan spanFromRead(const std::variant>& namespace backend { -FilesystemStorage::FilesystemStorage(ObjectStore2& os, std::string_view storage_path) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path) { +FilesystemStorage::FilesystemStorage( + ObjectStore2& os, + std::string_view storage_path, + MetaFileType mft_new +) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path), _mft_new(mft_new) { } FilesystemStorage::~FilesystemStorage(void) { } -ObjectHandle FilesystemStorage::newObject(MetaFileType mft, ByteSpan id) { +ObjectHandle FilesystemStorage::newObject(ByteSpan id) { { // first check if id is already used (TODO: solve the multi obj/backend problem) auto exising_oh = _os.getOneObjectByID(id); if (static_cast(exising_oh)) { @@ -85,7 +89,7 @@ ObjectHandle FilesystemStorage::newObject(MetaFileType mft, ByteSpan id) { oh.emplace(this); oh.emplace(std::vector{id}); oh.emplace(object_file_path.generic_u8string()); - oh.emplace(mft); + oh.emplace(_mft_new); // meta needs to be synced to file std::function empty_data_cb = [](auto*, auto) -> uint64_t { return 0; }; diff --git a/src/fragment_store/backends/filesystem_storage.hpp b/src/fragment_store/backends/filesystem_storage.hpp index 626f0008..b8ddafd1 100644 --- a/src/fragment_store/backends/filesystem_storage.hpp +++ b/src/fragment_store/backends/filesystem_storage.hpp @@ -3,17 +3,26 @@ #include "../types.hpp" #include "../object_store.hpp" +#include + namespace backend { struct FilesystemStorage : public StorageBackendI { - FilesystemStorage(ObjectStore2& os, std::string_view storage_path = "test_obj_store"); + FilesystemStorage( + ObjectStore2& os, + std::string_view storage_path = "test_obj_store", + MetaFileType mft_new = MetaFileType::BINARY_MSGPACK + ); ~FilesystemStorage(void); // TODO: fix the path for this specific fs? // for now we assume a single storage path per backend (there can be multiple per type) std::string _storage_path; - ObjectHandle newObject(MetaFileType mft, ByteSpan id); + // meta file type for new objects + MetaFileType _mft_new {MetaFileType::BINARY_MSGPACK}; + + ObjectHandle newObject(ByteSpan id) override; bool write(Object o, std::function& data_cb) override; bool read(Object o, std::function& data_cb) override; diff --git a/src/fragment_store/convert_frag_to_obj.cpp b/src/fragment_store/convert_frag_to_obj.cpp index 7d36bfa9..eccc9e9c 100644 --- a/src/fragment_store/convert_frag_to_obj.cpp +++ b/src/fragment_store/convert_frag_to_obj.cpp @@ -85,7 +85,8 @@ int main(int argc, const char** argv) { } } - auto oh = _fsb_dst.newObject(e.e.get().type, ByteSpan{e.e.get().v}); + // we dont copy meta file type, it will be the same for all "new" objects + auto oh = _fsb_dst.newObject(ByteSpan{e.e.get().v}); if (!static_cast(oh)) { // already exists diff --git a/src/fragment_store/object_store.cpp b/src/fragment_store/object_store.cpp index d5e2dae1..e459e945 100644 --- a/src/fragment_store/object_store.cpp +++ b/src/fragment_store/object_store.cpp @@ -52,6 +52,11 @@ static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) { } +ObjectHandle StorageBackendI::newObject(ByteSpan) { + //return {_os.registry(), entt::null}; + return {}; +} + bool StorageBackendI::write(Object o, const ByteSpan data) { std::function fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { uint64_t i = 0; diff --git a/src/fragment_store/object_store.hpp b/src/fragment_store/object_store.hpp index 03bac9ba..ebbb8f74 100644 --- a/src/fragment_store/object_store.hpp +++ b/src/fragment_store/object_store.hpp @@ -22,11 +22,13 @@ struct StorageBackendI { StorageBackendI(ObjectStore2& os); + // default impl fails, acting like a read only store + virtual ObjectHandle newObject(ByteSpan id); + // ========== write object to storage ========== using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. virtual bool write(Object o, std::function& data_cb) = 0; - //virtual bool write(Object o, const uint8_t* data, const uint64_t data_size); // default impl bool write(Object o, const ByteSpan data); // ========== read object from storage ==========