inital commit with ObjectStore2

some small hacks remain and still missing object deletion

.gitignore

@@ -0,0 +1,26 @@
+.vs/
+*.o
+*.swp
+~*
+*~
+.idea/
+cmake-build-debug/
+cmake-build-debugandtest/
+cmake-build-release/
+*.stackdump
+*.coredump
+compile_commands.json
+/build*
+/result*
+.clangd
+.cache
+
+.DS_Store
+.AppleDouble
+.LSOverride
+
+CMakeLists.txt.user*
+CMakeCache.txt
+
+*.tox
+imgui.ini

CMakeLists.txt

@@ -0,0 +1,72 @@
+cmake_minimum_required(VERSION 3.24 FATAL_ERROR)
+
+# cmake setup begin
+project(solanaceae_object_store)
+
+if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+	set(SOLANACEAE_OBJECT_STORE_STANDALONE ON)
+else()
+	set(SOLANACEAE_OBJECT_STORE_STANDALONE OFF)
+endif()
+message("II SOLANACEAE_OBJECT_STORE_STANDALONE " ${SOLANACEAE_OBJECT_STORE_STANDALONE})
+
+#option(SOLANACEAE_OBJECT_STORE_BUILD_PLUGINS "Build the solanaceae_object_store plugins" ${SOLANACEAE_OBJECT_STORE_STANDALONE})
+
+if (SOLANACEAE_OBJECT_STORE_STANDALONE)
+	set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+	# defaulting to debug mode, if not specified
+	if(NOT CMAKE_BUILD_TYPE)
+		set(CMAKE_BUILD_TYPE "Debug")
+	endif()
+
+	# setup my vim ycm :D
+	set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+	# more paths
+	set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
+	set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+	set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+endif()
+
+# external libs
+add_subdirectory(./external EXCLUDE_FROM_ALL) # before increasing warn levels, sad :(
+
+if (SOLANACEAE_OBJECT_STORE_STANDALONE)
+	set(CMAKE_CXX_EXTENSIONS OFF)
+
+	# bump up warning levels appropriately for clang, gcc & msvc
+	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
+		add_compile_options(
+			-Wall -Wextra # Reasonable and standard
+			-Wpedantic # Warn if non-standard C++ is used
+			-Wunused # Warn on anything being unused
+			#-Wconversion # Warn on type conversions that may lose data
+			#-Wsign-conversion # Warn on sign conversions
+			-Wshadow # Warn if a variable declaration shadows one from a parent context
+		)
+
+		if (NOT WIN32)
+			#link_libraries(-fsanitize=address)
+			#link_libraries(-fsanitize=address,undefined)
+			#link_libraries(-fsanitize-address-use-after-scope)
+			#link_libraries(-fsanitize=undefined)
+		endif()
+	elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL "MSVC")
+		if (CMAKE_CXX_FLAGS MATCHES "/W[0-4]")
+			string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+		else()
+			set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4")
+		endif()
+	endif()
+
+endif()
+
+# cmake setup end
+
+add_subdirectory(./src)
+
+#if (SOLANACEAE_OBJECT_STORE_BUILD_PLUGINS)
+	#add_subdirectory(./plugins)
+#endif()
+

external/CMakeLists.txt

@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 3.24 FATAL_ERROR)
+
+include(FetchContent)
+
+if (NOT TARGET solanaceae_util)
+	FetchContent_Declare(solanaceae_util
+		GIT_REPOSITORY https://github.com/Green-Sky/solanaceae_util.git
+		GIT_TAG master
+	)
+	FetchContent_MakeAvailable(solanaceae_util)
+endif()
+
+if (NOT TARGET EnTT::EnTT)
+	FetchContent_Declare(EnTT
+		GIT_REPOSITORY https://github.com/skypjack/entt.git
+		GIT_TAG v3.12.2
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(EnTT)
+endif()
+
+if (NOT TARGET nlohmann_json::nlohmann_json)
+	FetchContent_Declare(json
+		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(json)
+endif()
+
+#if (NOT TARGET solanaceae_plugin)
+	#FetchContent_Declare(solanaceae_plugin
+		#GIT_REPOSITORY https://github.com/Green-Sky/solanaceae_plugin.git
+		#GIT_TAG master
+	#)
+	#FetchContent_MakeAvailable(solanaceae_plugin)
+#endif()
+

src/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.9...3.24 FATAL_ERROR)
+
+project(solanaceae)
+
+add_library(solanaceae_object_store
+	./solanaceae/object_store/types.hpp
+	./solanaceae/object_store/meta_components.hpp
+	./solanaceae/object_store/meta_components_id.inl
+	./solanaceae/object_store/serializer_json.hpp
+	./solanaceae/object_store/object_store.hpp
+	./solanaceae/object_store/object_store.cpp
+)
+
+target_include_directories(solanaceae_object_store PUBLIC .)
+target_compile_features(solanaceae_object_store PUBLIC cxx_std_17)
+target_link_libraries(solanaceae_object_store PUBLIC
+	nlohmann_json::nlohmann_json
+	EnTT::EnTT
+	solanaceae_util
+)
+

src/solanaceae/object_store/meta_components.hpp

@@ -0,0 +1,77 @@
+#pragma once
+
+#include "./types.hpp"
+#include "./object_store.hpp"
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+namespace ObjectStore::Components {
+
+	// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
+	struct ID {
+		std::vector<uint8_t> v;
+	};
+
+	struct DataEncryptionType {
+		Encryption enc {Encryption::NONE};
+	};
+
+	struct DataCompressionType {
+		Compression comp {Compression::NONE};
+	};
+
+
+	// meta that is not written to (meta-)file
+	namespace Ephemeral {
+
+		// TODO: move, backend specific
+		struct MetaFileType {
+			::MetaFileType type {::MetaFileType::TEXT_JSON};
+		};
+
+		struct MetaEncryptionType {
+			Encryption enc {Encryption::NONE};
+		};
+
+		struct MetaCompressionType {
+			Compression comp {Compression::NONE};
+		};
+
+		struct Backend {
+			// TODO: shared_ptr instead??
+			StorageBackendI* ptr;
+		};
+
+		// excluded from file meta
+		// TODO: move to backend specific
+		struct FilePath {
+			// contains store path, if any
+			std::string path;
+		};
+
+		// TODO: seperate into remote and local?
+		// (remote meaning eg. the file on disk was changed by another program)
+		struct DirtyTag {};
+
+	} // Ephemeral
+
+} // Components
+
+// shortened to save bytes (until I find a way to save by ID in msgpack)
+namespace ObjComp = ObjectStore::Components;
+
+// old names from frag era
+namespace Fragment::Components {
+	//struct ID {};
+	//struct DataEncryptionType {};
+	//struct DataCompressionType {};
+	struct ID : public ObjComp::ID {};
+	struct DataEncryptionType : public ObjComp::DataEncryptionType {};
+	struct DataCompressionType : public ObjComp::DataCompressionType {};
+}
+namespace FragComp = Fragment::Components;
+
+#include "./meta_components_id.inl"
+

src/solanaceae/object_store/meta_components_id.inl

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "./meta_components.hpp"
+
+#include <entt/core/type_info.hpp>
+
+// TODO: move more central
+#define DEFINE_COMP_ID(x) \
+template<> \
+constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
+	using namespace entt::literals; \
+	return #x##_hs; \
+} \
+template<> \
+constexpr std::string_view entt::type_name<x>::value() noexcept { \
+	return #x; \
+}
+
+// cross compiler stable ids
+
+DEFINE_COMP_ID(ObjComp::DataEncryptionType)
+DEFINE_COMP_ID(ObjComp::DataCompressionType)
+
+// old stuff
+DEFINE_COMP_ID(FragComp::DataEncryptionType)
+DEFINE_COMP_ID(FragComp::DataCompressionType)
+
+#undef DEFINE_COMP_ID
+
+

src/solanaceae/object_store/object_store.cpp

@@ -0,0 +1,140 @@
+#include "./object_store.hpp"
+
+#include "./meta_components.hpp"
+
+#include "./serializer_json.hpp"
+
+#include <nlohmann/json.hpp> // this sucks
+
+#include <iostream>
+
+// TODO: move somewhere else
+static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) {
+	if (!oh.all_of<ObjComp::DataEncryptionType>()) {
+		return false;
+	}
+
+	out = static_cast<std::underlying_type_t<Encryption>>(
+		oh.get<ObjComp::DataEncryptionType>().enc
+	);
+	return true;
+}
+
+static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) {
+	oh.emplace_or_replace<ObjComp::DataEncryptionType>(
+		static_cast<Encryption>(
+			static_cast<std::underlying_type_t<Encryption>>(in)
+		)
+	);
+	return true;
+}
+
+static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) {
+	if (!oh.all_of<ObjComp::DataCompressionType>()) {
+		return false;
+	}
+
+	out = static_cast<std::underlying_type_t<Compression>>(
+		oh.get<ObjComp::DataCompressionType>().comp
+	);
+	return true;
+}
+
+static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) {
+	oh.emplace_or_replace<ObjComp::DataCompressionType>(
+		static_cast<Compression>(
+			static_cast<std::underlying_type_t<Compression>>(in)
+		)
+	);
+	return true;
+}
+
+StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) {
+}
+
+ObjectHandle StorageBackendI::newObject(ByteSpan) {
+	//return {_os.registry(), entt::null};
+	return {};
+}
+
+bool StorageBackendI::write(Object o, const ByteSpan data) {
+	std::function<write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+		uint64_t i = 0;
+		for (; i+read < data.size && i < buffer_size; i++) {
+			request_buffer[i] = data[i+read];
+		}
+		read += i;
+
+		return i;
+	};
+	return write(o, fn_cb);
+}
+
+ObjectStore2::ObjectStore2(void) {
+	// HACK: set them up independently
+	auto& sjc = _reg.ctx().emplace<SerializerJsonCallbacks<Object>>();
+	sjc.registerSerializer<ObjComp::DataEncryptionType>(serl_json_data_enc_type);
+	sjc.registerDeSerializer<ObjComp::DataEncryptionType>(deserl_json_data_enc_type);
+	sjc.registerSerializer<ObjComp::DataCompressionType>(serl_json_data_comp_type);
+	sjc.registerDeSerializer<ObjComp::DataCompressionType>(deserl_json_data_comp_type);
+
+	// old stuff
+	sjc.registerSerializer<FragComp::DataEncryptionType>(serl_json_data_enc_type);
+	sjc.registerDeSerializer<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
+	sjc.registerSerializer<FragComp::DataCompressionType>(serl_json_data_comp_type);
+	sjc.registerDeSerializer<FragComp::DataCompressionType>(deserl_json_data_comp_type);
+}
+
+ObjectStore2::~ObjectStore2(void) {
+}
+
+ObjectRegistry& ObjectStore2::registry(void) {
+	return _reg;
+}
+
+ObjectHandle ObjectStore2::objectHandle(const Object o) {
+	return {_reg, o};
+}
+
+ObjectHandle ObjectStore2::getOneObjectByID(const ByteSpan id) {
+	// TODO: accelerate
+	// maybe keep it sorted and binary search? hash table lookup?
+	for (const auto& [obj, id_comp] : _reg.view<ObjComp::ID>().each()) {
+		if (id == ByteSpan{id_comp.v}) {
+			return {_reg, obj};
+		}
+	}
+
+	return {_reg, entt::null};
+}
+
+void ObjectStore2::throwEventConstruct(const Object o) {
+	std::cout << "OS debug: event construct " << entt::to_integral(o) << "\n";
+	dispatch(
+		ObjectStore_Event::object_construct,
+		ObjectStore::Events::ObjectConstruct{
+			ObjectHandle{_reg, o}
+		}
+	);
+}
+
+void ObjectStore2::throwEventUpdate(const Object o) {
+	std::cout << "OS debug: event update " << entt::to_integral(o) << "\n";
+	dispatch(
+		ObjectStore_Event::object_update,
+		ObjectStore::Events::ObjectUpdate{
+			ObjectHandle{_reg, o}
+		}
+	);
+}
+
+void ObjectStore2::throwEventDestroy(const Object o) {
+	std::cout << "OS debug: event destroy " << entt::to_integral(o) << "\n";
+	dispatch(
+		ObjectStore_Event::object_destroy,
+		ObjectStore::Events::ObjectUpdate{
+			ObjectHandle{_reg, o}
+		}
+	);
+}
+

src/solanaceae/object_store/object_store.hpp

@@ -0,0 +1,95 @@
+#pragma once
+
+#include <solanaceae/util/event_provider.hpp>
+#include <solanaceae/util/span.hpp>
+
+#include <entt/entity/registry.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <cstdint>
+
+// internal id
+enum class Object : uint32_t {};
+using ObjectRegistry = entt::basic_registry<Object>;
+using ObjectHandle = entt::basic_handle<ObjectRegistry>;
+
+// fwd
+struct ObjectStore2;
+
+struct StorageBackendI {
+	// OR or OS ?
+	ObjectStore2& _os;
+
+	StorageBackendI(ObjectStore2& os);
+
+	// default impl fails, acting like a read only store
+	virtual ObjectHandle newObject(ByteSpan id);
+
+	// ========== write object to storage ==========
+	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
+	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
+	virtual bool write(Object o, std::function<write_to_storage_fetch_data_cb>& data_cb) = 0;
+	bool write(Object o, const ByteSpan data);
+
+	// ========== read object from storage ==========
+	using read_from_storage_put_data_cb = void(const ByteSpan buffer);
+	virtual bool read(Object o, std::function<read_from_storage_put_data_cb>& data_cb) = 0;
+
+};
+
+namespace ObjectStore::Events {
+
+	struct ObjectConstruct {
+		const ObjectHandle e;
+	};
+	struct ObjectUpdate {
+		const ObjectHandle e;
+	};
+	struct ObjectDestory {
+		const ObjectHandle e;
+	};
+
+} // ObjectStore::Events
+
+enum class ObjectStore_Event : uint16_t {
+	object_construct,
+	object_update,
+	object_destroy,
+
+	MAX
+};
+
+struct ObjectStoreEventI {
+	using enumType = ObjectStore_Event;
+
+	virtual ~ObjectStoreEventI(void) {}
+
+	virtual bool onEvent(const ObjectStore::Events::ObjectConstruct&) { return false; }
+	virtual bool onEvent(const ObjectStore::Events::ObjectUpdate&) { return false; }
+	virtual bool onEvent(const ObjectStore::Events::ObjectDestory&) { return false; }
+};
+using ObjectStoreEventProviderI = EventProviderI<ObjectStoreEventI>;
+
+struct ObjectStore2 : public ObjectStoreEventProviderI {
+	static constexpr const char* version {"2"};
+
+	ObjectRegistry _reg;
+
+	// TODO: default backend?
+
+	ObjectStore2(void);
+	virtual ~ObjectStore2(void);
+
+	ObjectRegistry& registry(void);
+	ObjectHandle objectHandle(const Object o);
+
+	// TODO: properly think about multiple objects witht he same id / different backends
+	ObjectHandle getOneObjectByID(const ByteSpan id);
+
+	// sync?
+
+	void throwEventConstruct(const Object o);
+	void throwEventUpdate(const Object o);
+	void throwEventDestroy(const Object o);
+};
+

src/solanaceae/object_store/serializer_json.hpp

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+// nlohmann
+template<typename EntityType = entt::entity>
+struct SerializerJsonCallbacks {
+	using Registry = entt::basic_registry<EntityType>;
+	using Handle = entt::basic_handle<Registry>;
+
+	using serialize_fn = bool(*)(const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_fn> _serl;
+
+	using deserialize_fn = bool(*)(Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_fn> _deserl;
+
+	template<typename T>
+	static bool component_get_json(const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializer(serialize_fn fn, const entt::type_info& type_info) {
+		_serl[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializer(
+		serialize_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializer(fn, type_info);
+	}
+
+	void registerDeSerializer(deserialize_fn fn, const entt::type_info& type_info) {
+		_deserl[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializer(
+		deserialize_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializer(fn, type_info);
+	}
+};
+

src/solanaceae/object_store/types.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+
+enum class Encryption : uint8_t {
+	NONE = 0x00,
+};
+enum class Compression : uint8_t {
+	NONE = 0x00,
+	ZSTD = 0x01,
+	// TODO: zstd without magic
+	// TODO: zstd meta dict
+	// TODO: zstd data(message) dict
+};
+enum class MetaFileType : uint8_t {
+	TEXT_JSON,
+	BINARY_MSGPACK, // msgpacked msgpack
+};
+