slight backend refactor, splitting it up

src/solanaceae/object_store/backends/filesystem_storage.cpp

@@ -33,13 +33,13 @@ FilesystemStorage::FilesystemStorage(
 	ObjectStore2& os,
 	std::string_view storage_path,
 	MetaFileType mft_new
-) : StorageBackendI::StorageBackendI(os), _storage_path(storage_path), _mft_new(mft_new) {
+) : _os(os), _storage_path(storage_path), _mft_new(mft_new) {
 }
 
 FilesystemStorage::~FilesystemStorage(void) {
 }
 
-ObjectHandle FilesystemStorage::newObject(ByteSpan id) {
+ObjectHandle FilesystemStorage::newObject(ByteSpan id, bool throw_construct) {
 	{ // first check if id is already used (TODO: solve the multi obj/backend problem)
 		auto exising_oh = _os.getOneObjectByID(id);
 		if (static_cast<bool>(exising_oh)) {
@@ -74,7 +74,8 @@ ObjectHandle FilesystemStorage::newObject(ByteSpan id) {
 
 	ObjectHandle oh{_os.registry(), _os.registry().create()};
 
-	oh.emplace<ObjComp::Ephemeral::Backend>(this);
+	oh.emplace<ObjComp::Ephemeral::BackendMeta>(this);
+	oh.emplace<ObjComp::Ephemeral::BackendAtomic>(this);
 	oh.emplace<ObjComp::ID>(std::vector<uint8_t>{id});
 	oh.emplace<ObjComp::Ephemeral::FilePath>(object_file_path.generic_u8string());
 	oh.emplace<ObjComp::Ephemeral::MetaFileType>(_mft_new);
@@ -87,8 +88,10 @@ ObjectHandle FilesystemStorage::newObject(ByteSpan id) {
 		return {};
 	}
 
-	// while new metadata might be created here, making sure the file could be created is more important
+	if (throw_construct) {
-	_os.throwEventConstruct(oh);
+		// while new metadata might be created here, making sure the file could be created is more important
+		_os.throwEventConstruct(oh);
+	}
 
 	return oh;
 }
@@ -579,7 +582,8 @@ size_t FilesystemStorage::scanPath(std::string_view path) {
 		// TODO: existing fragment file
 		//newFragmentFile();
 		ObjectHandle oh{_os.registry(), _os.registry().create()};
-		oh.emplace<ObjComp::Ephemeral::Backend>(this);
+		oh.emplace<ObjComp::Ephemeral::BackendMeta>(this);
+		oh.emplace<ObjComp::Ephemeral::BackendAtomic>(this);
 		oh.emplace<ObjComp::ID>(hex2bin(it.id_str));
 		oh.emplace<ObjComp::Ephemeral::MetaFileType>(mft);
 		oh.emplace<ObjComp::Ephemeral::MetaEncryptionType>(meta_enc);

src/solanaceae/object_store/backends/filesystem_storage.hpp

@@ -7,7 +7,11 @@
 
 namespace Backends {
 
-struct FilesystemStorage : public StorageBackendI {
+// TODO: rename to atomic filesystem store?
+// provides meta and atomic read/write on your filesystem
+struct FilesystemStorage : public StorageBackendIMeta, public StorageBackendIAtomic {
+	ObjectStore2& _os;
+
 	FilesystemStorage(
 		ObjectStore2& os,
 		std::string_view storage_path = "test_obj_store",
@@ -22,14 +26,11 @@ struct FilesystemStorage : public StorageBackendI {
 	// meta file type for new objects
 	MetaFileType _mft_new {MetaFileType::BINARY_MSGPACK};
 
-	ObjectHandle newObject(ByteSpan id) override;
+	ObjectHandle newObject(ByteSpan id, bool throw_construct = true) override;
 
 	bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) override;
 	bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) override;
 
-	//// convenience function
-	//nlohmann::json loadFromStorageNJ(FragmentID fid);
-
 	void scanAsync(void);
 
 	private:

src/solanaceae/object_store/fwd.hpp

@@ -8,7 +8,9 @@ using ObjectRegistry = entt::basic_registry<Object>;
 using ObjectHandle = entt::basic_handle<ObjectRegistry>;
 
 // fwd
-struct StorageBackendI;
+struct StorageBackendIMeta;
+struct StorageBackendIAtomic;
+struct StorageBackendIFile2;
 struct ObjectStore2;
 struct File2I;
 

src/solanaceae/object_store/meta_components.hpp

@@ -39,9 +39,14 @@ namespace ObjectStore::Components {
 			Compression comp {Compression::NONE};
 		};
 
-		struct Backend {
+		struct BackendMeta {
-			// TODO: shared_ptr instead??
+			StorageBackendIMeta* ptr;
-			StorageBackendI* ptr;
+		};
+		struct BackendAtomic {
+			StorageBackendIAtomic* ptr;
+		};
+		struct BackendFile2 {
+			StorageBackendIFile2* ptr;
 		};
 
 		// excluded from file meta

src/solanaceae/object_store/meta_components_id.inl

@@ -25,7 +25,9 @@ DEFINE_COMP_ID(ObjComp::DataCompressionType)
 DEFINE_COMP_ID(ObjComp::Ephemeral::MetaFileType)
 DEFINE_COMP_ID(ObjComp::Ephemeral::MetaEncryptionType)
 DEFINE_COMP_ID(ObjComp::Ephemeral::MetaCompressionType)
-DEFINE_COMP_ID(ObjComp::Ephemeral::Backend)
+DEFINE_COMP_ID(ObjComp::Ephemeral::BackendMeta)
+DEFINE_COMP_ID(ObjComp::Ephemeral::BackendAtomic)
+DEFINE_COMP_ID(ObjComp::Ephemeral::BackendFile2)
 DEFINE_COMP_ID(ObjComp::Ephemeral::FilePath)
 DEFINE_COMP_ID(ObjComp::Ephemeral::DirtyTag) // ??
 

src/solanaceae/object_store/object_store.cpp

@@ -51,19 +51,7 @@ static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in
 	return true;
 }
 
-StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) {
+bool StorageBackendIAtomic::write(Object o, const ByteSpan data) {
-}
-
-ObjectHandle StorageBackendI::newObject(ByteSpan) {
-	//return {_os.registry(), entt::null};
-	return {};
-}
-
-bool StorageBackendI::write(Object, std::function<write_to_storage_fetch_data_cb>&) {
-	return false;
-}
-
-bool StorageBackendI::write(Object o, const ByteSpan data) {
 	std::function<write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
 		uint64_t i = 0;
 		for (; i+read < data.size && i < buffer_size; i++) {
@@ -76,14 +64,6 @@ bool StorageBackendI::write(Object o, const ByteSpan data) {
 	return write(o, fn_cb);
 }
 
-bool StorageBackendI::read(Object, std::function<read_from_storage_put_data_cb>&) {
-	return false;
-}
-
-std::unique_ptr<File2I> StorageBackendI::file2(Object o, FILE2_FLAGS flags) {
-	return nullptr;
-}
-
 ObjectStore2::ObjectStore2(void) {
 	// HACK: set them up independently
 	auto& sjc = _reg.ctx().emplace<SerializerJsonCallbacks<Object>>();

src/solanaceae/object_store/object_store.hpp

@@ -8,24 +8,39 @@
 #include <entt/entity/registry.hpp>
 #include <entt/entity/handle.hpp>
 
-struct StorageBackendI {
+// the different backend components
-	// OR or OS ?
+// depending on what a backend (or composed backends) provide
-	ObjectStore2& _os;
+// different component (pointers) might be provided
 
-	StorageBackendI(ObjectStore2& os);
+// provides metadata storage(?)
+struct StorageBackendIMeta {
+	virtual ~StorageBackendIMeta(void) = default;
+	virtual ObjectHandle newObject(ByteSpan id, bool throw_construct = true) = 0;
+};
 
-	// default impl fails, acting like a read only store
+// provides atomic-ish read write
-	virtual ObjectHandle newObject(ByteSpan id);
+// simplest backend strategy to implement data encryption for
+//
+// collides with file backend and implementers providing both need to
+// orchestrate access.
+struct StorageBackendIAtomic {
+	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
+	using read_from_storage_put_data_cb = void(const ByteSpan buffer);
+
+	virtual ~StorageBackendIAtomic(void) = default;
+
+	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
 
 	// ========== write object to storage (atomic-ish) ==========
-	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
+	virtual bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) = 0;
-	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
+	bool write(Object o, const ByteSpan data); // helper, calls cb variant internally
-	virtual bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb);
-	bool write(Object o, const ByteSpan data);
 
 	// ========== read object from storage (atomic-ish) ==========
-	using read_from_storage_put_data_cb = void(const ByteSpan buffer);
+	virtual bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) = 0;
-	virtual bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb);
+};
+
+struct StorageBackendIFile2 {
+	virtual ~StorageBackendIFile2(void) = default;
 
 	// ========== File2 interop ==========
 	enum FILE2_FLAGS : uint32_t {
@@ -43,7 +58,7 @@ struct StorageBackendI {
 	// TODO: stronger requirements
 	// the backend might decide to not support writing using file2, if it's eg. zstd compressed
 	// backends might only support a single file2 instance per object!
-	virtual std::unique_ptr<File2I> file2(Object o, FILE2_FLAGS flags); // default does nothing
+	virtual std::unique_ptr<File2I> file2(Object o, FILE2_FLAGS flags) = 0;
 };
 
 namespace ObjectStore::Events {
@@ -80,7 +95,7 @@ struct ObjectStoreEventI {
 using ObjectStoreEventProviderI = EventProviderI<ObjectStoreEventI>;
 
 struct ObjectStore2 : public ObjectStoreEventProviderI {
-	static constexpr const char* version {"3"};
+	static constexpr const char* version {"4"};
 
 	ObjectRegistry _reg;