Green-Sky/solanaceae_object_store
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

slight backend refactor, splitting it up

Browse Source
This commit is contained in:
Green Sky 2025-01-18 00:51:41 +01:00
parent 18d2888e34
commit e5b3546292
No known key found for this signature in database
7 changed files with 60 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/solanaceae/object_store/backends/filesystem_storage.cpp
View File

@ -33,13 +33,13 @@ FilesystemStorage::FilesystemStorage(
ObjectStore2& os,
std::string_view storage_path,
MetaFileType mft_new
) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path), _mft_new(mft_new) {
) : _os(os), _storage_path(storage_path), _mft_new(mft_new) {
}
FilesystemStorage::~FilesystemStorage(void) {
}
ObjectHandle FilesystemStorage::newObject(ByteSpan id) {
ObjectHandle FilesystemStorage::newObject(ByteSpan id, bool throw_construct) {
{ // first check if id is already used (TODO: solve the multi obj/backend problem)
auto exising_oh = _os.getOneObjectByID(id);
if (static_cast<bool>(exising_oh)) {
@ -74,7 +74,8 @@ ObjectHandle FilesystemStorage::newObject(ByteSpan id) {
ObjectHandle oh{_os.registry(), _os.registry().create()};
oh.emplace<ObjComp::Ephemeral::Backend>(this);
oh.emplace<ObjComp::Ephemeral::BackendMeta>(this);
oh.emplace<ObjComp::Ephemeral::BackendAtomic>(this);
oh.emplace<ObjComp::ID>(std::vector<uint8_t>{id});
oh.emplace<ObjComp::Ephemeral::FilePath>(object_file_path.generic_u8string());
oh.emplace<ObjComp::Ephemeral::MetaFileType>(_mft_new);
@ -87,8 +88,10 @@ ObjectHandle FilesystemStorage::newObject(ByteSpan id) {
return {};
}
if (throw_construct) {
// while new metadata might be created here, making sure the file could be created is more important
_os.throwEventConstruct(oh);
}
return oh;
}
@ -579,7 +582,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
// TODO: existing fragment file
//newFragmentFile();
ObjectHandle oh{_os.registry(), _os.registry().create()};
oh.emplace<ObjComp::Ephemeral::Backend>(this);
oh.emplace<ObjComp::Ephemeral::BackendMeta>(this);
oh.emplace<ObjComp::Ephemeral::BackendAtomic>(this);
oh.emplace<ObjComp::ID>(hex2bin(it.id_str));
oh.emplace<ObjComp::Ephemeral::MetaFileType>(mft);
oh.emplace<ObjComp::Ephemeral::MetaEncryptionType>(meta_enc);

11
src/solanaceae/object_store/backends/filesystem_storage.hpp
View File

@ -7,7 +7,11 @@
namespace Backends {
struct FilesystemStorage : public StorageBackendI {
// TODO: rename to atomic filesystem store?
// provides meta and atomic read/write on your filesystem
struct FilesystemStorage : public StorageBackendIMeta, public StorageBackendIAtomic {
ObjectStore2& _os;
FilesystemStorage(
ObjectStore2& os,
std::string_view storage_path = "test_obj_store",
@ -22,14 +26,11 @@ struct FilesystemStorage : public StorageBackendI {
// meta file type for new objects
MetaFileType _mft_new {MetaFileType::BINARY_MSGPACK};
ObjectHandle newObject(ByteSpan id) override;
ObjectHandle newObject(ByteSpan id, bool throw_construct = true) override;
bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) override;
bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) override;
//// convenience function
//nlohmann::json loadFromStorageNJ(FragmentID fid);
void scanAsync(void);
private:

4
src/solanaceae/object_store/fwd.hpp
View File

@ -8,7 +8,9 @@ using ObjectRegistry = entt::basic_registry<Object>;
using ObjectHandle = entt::basic_handle<ObjectRegistry>;
// fwd
struct StorageBackendI;
struct StorageBackendIMeta;
struct StorageBackendIAtomic;
struct StorageBackendIFile2;
struct ObjectStore2;
struct File2I;

11
src/solanaceae/object_store/meta_components.hpp
View File

@ -39,9 +39,14 @@ namespace ObjectStore::Components {
Compression comp {Compression::NONE};
};
struct Backend {
// TODO: shared_ptr instead??
StorageBackendI* ptr;
struct BackendMeta {
StorageBackendIMeta* ptr;
};
struct BackendAtomic {
StorageBackendIAtomic* ptr;
};
struct BackendFile2 {
StorageBackendIFile2* ptr;
};
// excluded from file meta

4
src/solanaceae/object_store/meta_components_id.inl
View File

@ -25,7 +25,9 @@ DEFINE_COMP_ID(ObjComp::DataCompressionType)
DEFINE_COMP_ID(ObjComp::Ephemeral::MetaFileType)
DEFINE_COMP_ID(ObjComp::Ephemeral::MetaEncryptionType)
DEFINE_COMP_ID(ObjComp::Ephemeral::MetaCompressionType)
DEFINE_COMP_ID(ObjComp::Ephemeral::Backend)
DEFINE_COMP_ID(ObjComp::Ephemeral::BackendMeta)
DEFINE_COMP_ID(ObjComp::Ephemeral::BackendAtomic)
DEFINE_COMP_ID(ObjComp::Ephemeral::BackendFile2)
DEFINE_COMP_ID(ObjComp::Ephemeral::FilePath)
DEFINE_COMP_ID(ObjComp::Ephemeral::DirtyTag) // ??

22
src/solanaceae/object_store/object_store.cpp
View File

@ -51,19 +51,7 @@ static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in
return true;
}
StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) {
}
ObjectHandle StorageBackendI::newObject(ByteSpan) {
//return {_os.registry(), entt::null};
return {};
}
bool StorageBackendI::write(Object, std::function<write_to_storage_fetch_data_cb>&) {
return false;
}
bool StorageBackendI::write(Object o, const ByteSpan data) {
bool StorageBackendIAtomic::write(Object o, const ByteSpan data) {
std::function<write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
uint64_t i = 0;
for (; i+read < data.size && i < buffer_size; i++) {
@ -76,14 +64,6 @@ bool StorageBackendI::write(Object o, const ByteSpan data) {
return write(o, fn_cb);
}
bool StorageBackendI::read(Object, std::function<read_from_storage_put_data_cb>&) {
return false;
}
std::unique_ptr<File2I> StorageBackendI::file2(Object o, FILE2_FLAGS flags) {
return nullptr;
}
ObjectStore2::ObjectStore2(void) {
// HACK: set them up independently
auto& sjc = _reg.ctx().emplace<SerializerJsonCallbacks<Object>>();

43
src/solanaceae/object_store/object_store.hpp
View File

@ -8,24 +8,39 @@
#include <entt/entity/registry.hpp>
#include <entt/entity/handle.hpp>
struct StorageBackendI {
// OR or OS ?
ObjectStore2& _os;
// the different backend components
// depending on what a backend (or composed backends) provide
// different component (pointers) might be provided
StorageBackendI(ObjectStore2& os);
// provides metadata storage(?)
struct StorageBackendIMeta {
virtual ~StorageBackendIMeta(void) = default;
virtual ObjectHandle newObject(ByteSpan id, bool throw_construct = true) = 0;
};
// default impl fails, acting like a read only store
virtual ObjectHandle newObject(ByteSpan id);
// provides atomic-ish read write
// simplest backend strategy to implement data encryption for
//
// collides with file backend and implementers providing both need to
// orchestrate access.
struct StorageBackendIAtomic {
using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
using read_from_storage_put_data_cb = void(const ByteSpan buffer);
virtual ~StorageBackendIAtomic(void) = default;
// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
// ========== write object to storage (atomic-ish) ==========
using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
virtual bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb);
bool write(Object o, const ByteSpan data);
virtual bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) = 0;
bool write(Object o, const ByteSpan data); // helper, calls cb variant internally
// ========== read object from storage (atomic-ish) ==========
using read_from_storage_put_data_cb = void(const ByteSpan buffer);
virtual bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb);
virtual bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) = 0;
};
struct StorageBackendIFile2 {
virtual ~StorageBackendIFile2(void) = default;
// ========== File2 interop ==========
enum FILE2_FLAGS : uint32_t {
@ -43,7 +58,7 @@ struct StorageBackendI {
// TODO: stronger requirements
// the backend might decide to not support writing using file2, if it's eg. zstd compressed
// backends might only support a single file2 instance per object!
virtual std::unique_ptr<File2I> file2(Object o, FILE2_FLAGS flags); // default does nothing
virtual std::unique_ptr<File2I> file2(Object o, FILE2_FLAGS flags) = 0;
};
namespace ObjectStore::Events {
@ -80,7 +95,7 @@ struct ObjectStoreEventI {
using ObjectStoreEventProviderI = EventProviderI<ObjectStoreEventI>;
struct ObjectStore2 : public ObjectStoreEventProviderI {
static constexpr const char* version {"3"};
static constexpr const char* version {"4"};
ObjectRegistry _reg;