From e5b3546292c906da4680f838c235e3464476917b Mon Sep 17 00:00:00 2001 From: Green Sky Date: Sat, 18 Jan 2025 00:51:41 +0100 Subject: [PATCH] slight backend refactor, splitting it up --- .../backends/filesystem_storage.cpp | 16 ++++--- .../backends/filesystem_storage.hpp | 11 ++--- src/solanaceae/object_store/fwd.hpp | 4 +- .../object_store/meta_components.hpp | 11 +++-- .../object_store/meta_components_id.inl | 4 +- src/solanaceae/object_store/object_store.cpp | 22 +--------- src/solanaceae/object_store/object_store.hpp | 43 +++++++++++++------ 7 files changed, 60 insertions(+), 51 deletions(-) diff --git a/src/solanaceae/object_store/backends/filesystem_storage.cpp b/src/solanaceae/object_store/backends/filesystem_storage.cpp index d4506bb..a16c41b 100644 --- a/src/solanaceae/object_store/backends/filesystem_storage.cpp +++ b/src/solanaceae/object_store/backends/filesystem_storage.cpp @@ -33,13 +33,13 @@ FilesystemStorage::FilesystemStorage( ObjectStore2& os, std::string_view storage_path, MetaFileType mft_new -) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path), _mft_new(mft_new) { +) : _os(os), _storage_path(storage_path), _mft_new(mft_new) { } FilesystemStorage::~FilesystemStorage(void) { } -ObjectHandle FilesystemStorage::newObject(ByteSpan id) { +ObjectHandle FilesystemStorage::newObject(ByteSpan id, bool throw_construct) { { // first check if id is already used (TODO: solve the multi obj/backend problem) auto exising_oh = _os.getOneObjectByID(id); if (static_cast(exising_oh)) { @@ -74,7 +74,8 @@ ObjectHandle FilesystemStorage::newObject(ByteSpan id) { ObjectHandle oh{_os.registry(), _os.registry().create()}; - oh.emplace(this); + oh.emplace(this); + oh.emplace(this); oh.emplace(std::vector{id}); oh.emplace(object_file_path.generic_u8string()); oh.emplace(_mft_new); @@ -87,8 +88,10 @@ ObjectHandle FilesystemStorage::newObject(ByteSpan id) { return {}; } - // while new metadata might be created here, making sure the file could be created is more important - _os.throwEventConstruct(oh); + if (throw_construct) { + // while new metadata might be created here, making sure the file could be created is more important + _os.throwEventConstruct(oh); + } return oh; } @@ -579,7 +582,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) { // TODO: existing fragment file //newFragmentFile(); ObjectHandle oh{_os.registry(), _os.registry().create()}; - oh.emplace(this); + oh.emplace(this); + oh.emplace(this); oh.emplace(hex2bin(it.id_str)); oh.emplace(mft); oh.emplace(meta_enc); diff --git a/src/solanaceae/object_store/backends/filesystem_storage.hpp b/src/solanaceae/object_store/backends/filesystem_storage.hpp index 3c19bce..aeb3530 100644 --- a/src/solanaceae/object_store/backends/filesystem_storage.hpp +++ b/src/solanaceae/object_store/backends/filesystem_storage.hpp @@ -7,7 +7,11 @@ namespace Backends { -struct FilesystemStorage : public StorageBackendI { +// TODO: rename to atomic filesystem store? +// provides meta and atomic read/write on your filesystem +struct FilesystemStorage : public StorageBackendIMeta, public StorageBackendIAtomic { + ObjectStore2& _os; + FilesystemStorage( ObjectStore2& os, std::string_view storage_path = "test_obj_store", @@ -22,14 +26,11 @@ struct FilesystemStorage : public StorageBackendI { // meta file type for new objects MetaFileType _mft_new {MetaFileType::BINARY_MSGPACK}; - ObjectHandle newObject(ByteSpan id) override; + ObjectHandle newObject(ByteSpan id, bool throw_construct = true) override; bool write(Object o, std::function& data_cb) override; bool read(Object o, std::function& data_cb) override; - //// convenience function - //nlohmann::json loadFromStorageNJ(FragmentID fid); - void scanAsync(void); private: diff --git a/src/solanaceae/object_store/fwd.hpp b/src/solanaceae/object_store/fwd.hpp index 90ec072..d9e866c 100644 --- a/src/solanaceae/object_store/fwd.hpp +++ b/src/solanaceae/object_store/fwd.hpp @@ -8,7 +8,9 @@ using ObjectRegistry = entt::basic_registry; using ObjectHandle = entt::basic_handle; // fwd -struct StorageBackendI; +struct StorageBackendIMeta; +struct StorageBackendIAtomic; +struct StorageBackendIFile2; struct ObjectStore2; struct File2I; diff --git a/src/solanaceae/object_store/meta_components.hpp b/src/solanaceae/object_store/meta_components.hpp index 4ea5327..5306c61 100644 --- a/src/solanaceae/object_store/meta_components.hpp +++ b/src/solanaceae/object_store/meta_components.hpp @@ -39,9 +39,14 @@ namespace ObjectStore::Components { Compression comp {Compression::NONE}; }; - struct Backend { - // TODO: shared_ptr instead?? - StorageBackendI* ptr; + struct BackendMeta { + StorageBackendIMeta* ptr; + }; + struct BackendAtomic { + StorageBackendIAtomic* ptr; + }; + struct BackendFile2 { + StorageBackendIFile2* ptr; }; // excluded from file meta diff --git a/src/solanaceae/object_store/meta_components_id.inl b/src/solanaceae/object_store/meta_components_id.inl index 856f247..7b62852 100644 --- a/src/solanaceae/object_store/meta_components_id.inl +++ b/src/solanaceae/object_store/meta_components_id.inl @@ -25,7 +25,9 @@ DEFINE_COMP_ID(ObjComp::DataCompressionType) DEFINE_COMP_ID(ObjComp::Ephemeral::MetaFileType) DEFINE_COMP_ID(ObjComp::Ephemeral::MetaEncryptionType) DEFINE_COMP_ID(ObjComp::Ephemeral::MetaCompressionType) -DEFINE_COMP_ID(ObjComp::Ephemeral::Backend) +DEFINE_COMP_ID(ObjComp::Ephemeral::BackendMeta) +DEFINE_COMP_ID(ObjComp::Ephemeral::BackendAtomic) +DEFINE_COMP_ID(ObjComp::Ephemeral::BackendFile2) DEFINE_COMP_ID(ObjComp::Ephemeral::FilePath) DEFINE_COMP_ID(ObjComp::Ephemeral::DirtyTag) // ?? diff --git a/src/solanaceae/object_store/object_store.cpp b/src/solanaceae/object_store/object_store.cpp index 15ca98c..9316074 100644 --- a/src/solanaceae/object_store/object_store.cpp +++ b/src/solanaceae/object_store/object_store.cpp @@ -51,19 +51,7 @@ static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in return true; } -StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) { -} - -ObjectHandle StorageBackendI::newObject(ByteSpan) { - //return {_os.registry(), entt::null}; - return {}; -} - -bool StorageBackendI::write(Object, std::function&) { - return false; -} - -bool StorageBackendI::write(Object o, const ByteSpan data) { +bool StorageBackendIAtomic::write(Object o, const ByteSpan data) { std::function fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { uint64_t i = 0; for (; i+read < data.size && i < buffer_size; i++) { @@ -76,14 +64,6 @@ bool StorageBackendI::write(Object o, const ByteSpan data) { return write(o, fn_cb); } -bool StorageBackendI::read(Object, std::function&) { - return false; -} - -std::unique_ptr StorageBackendI::file2(Object o, FILE2_FLAGS flags) { - return nullptr; -} - ObjectStore2::ObjectStore2(void) { // HACK: set them up independently auto& sjc = _reg.ctx().emplace>(); diff --git a/src/solanaceae/object_store/object_store.hpp b/src/solanaceae/object_store/object_store.hpp index 026fca6..5e33985 100644 --- a/src/solanaceae/object_store/object_store.hpp +++ b/src/solanaceae/object_store/object_store.hpp @@ -8,24 +8,39 @@ #include #include -struct StorageBackendI { - // OR or OS ? - ObjectStore2& _os; +// the different backend components +// depending on what a backend (or composed backends) provide +// different component (pointers) might be provided - StorageBackendI(ObjectStore2& os); +// provides metadata storage(?) +struct StorageBackendIMeta { + virtual ~StorageBackendIMeta(void) = default; + virtual ObjectHandle newObject(ByteSpan id, bool throw_construct = true) = 0; +}; - // default impl fails, acting like a read only store - virtual ObjectHandle newObject(ByteSpan id); +// provides atomic-ish read write +// simplest backend strategy to implement data encryption for +// +// collides with file backend and implementers providing both need to +// orchestrate access. +struct StorageBackendIAtomic { + using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); + using read_from_storage_put_data_cb = void(const ByteSpan buffer); + + virtual ~StorageBackendIAtomic(void) = default; + + // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. // ========== write object to storage (atomic-ish) ========== - using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); - // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. - virtual bool write(Object o, std::function& data_cb); - bool write(Object o, const ByteSpan data); + virtual bool write(Object o, std::function& data_cb) = 0; + bool write(Object o, const ByteSpan data); // helper, calls cb variant internally // ========== read object from storage (atomic-ish) ========== - using read_from_storage_put_data_cb = void(const ByteSpan buffer); - virtual bool read(Object o, std::function& data_cb); + virtual bool read(Object o, std::function& data_cb) = 0; +}; + +struct StorageBackendIFile2 { + virtual ~StorageBackendIFile2(void) = default; // ========== File2 interop ========== enum FILE2_FLAGS : uint32_t { @@ -43,7 +58,7 @@ struct StorageBackendI { // TODO: stronger requirements // the backend might decide to not support writing using file2, if it's eg. zstd compressed // backends might only support a single file2 instance per object! - virtual std::unique_ptr file2(Object o, FILE2_FLAGS flags); // default does nothing + virtual std::unique_ptr file2(Object o, FILE2_FLAGS flags) = 0; }; namespace ObjectStore::Events { @@ -80,7 +95,7 @@ struct ObjectStoreEventI { using ObjectStoreEventProviderI = EventProviderI; struct ObjectStore2 : public ObjectStoreEventProviderI { - static constexpr const char* version {"3"}; + static constexpr const char* version {"4"}; ObjectRegistry _reg;