From f21dd1dbf055a3d1124b0c7833968b59e74bd4c7 Mon Sep 17 00:00:00 2001 From: Green Sky Date: Fri, 12 Apr 2024 01:00:15 +0200 Subject: [PATCH] inital commit with ObjectStore2 some small hacks remain and still missing object deletion --- .gitignore | 26 ++++ CMakeLists.txt | 72 +++++++++ external/CMakeLists.txt | 38 +++++ src/CMakeLists.txt | 21 +++ .../object_store/meta_components.hpp | 77 ++++++++++ .../object_store/meta_components_id.inl | 30 ++++ src/solanaceae/object_store/object_store.cpp | 140 ++++++++++++++++++ src/solanaceae/object_store/object_store.hpp | 95 ++++++++++++ .../object_store/serializer_json.hpp | 67 +++++++++ src/solanaceae/object_store/types.hpp | 19 +++ 10 files changed, 585 insertions(+) create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 external/CMakeLists.txt create mode 100644 src/CMakeLists.txt create mode 100644 src/solanaceae/object_store/meta_components.hpp create mode 100644 src/solanaceae/object_store/meta_components_id.inl create mode 100644 src/solanaceae/object_store/object_store.cpp create mode 100644 src/solanaceae/object_store/object_store.hpp create mode 100644 src/solanaceae/object_store/serializer_json.hpp create mode 100644 src/solanaceae/object_store/types.hpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56f48bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +.vs/ +*.o +*.swp +~* +*~ +.idea/ +cmake-build-debug/ +cmake-build-debugandtest/ +cmake-build-release/ +*.stackdump +*.coredump +compile_commands.json +/build* +/result* +.clangd +.cache + +.DS_Store +.AppleDouble +.LSOverride + +CMakeLists.txt.user* +CMakeCache.txt + +*.tox +imgui.ini diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d683aa3 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,72 @@ +cmake_minimum_required(VERSION 3.24 FATAL_ERROR) + +# cmake setup begin +project(solanaceae_object_store) + +if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(SOLANACEAE_OBJECT_STORE_STANDALONE ON) +else() + set(SOLANACEAE_OBJECT_STORE_STANDALONE OFF) +endif() +message("II SOLANACEAE_OBJECT_STORE_STANDALONE " ${SOLANACEAE_OBJECT_STORE_STANDALONE}) + +#option(SOLANACEAE_OBJECT_STORE_BUILD_PLUGINS "Build the solanaceae_object_store plugins" ${SOLANACEAE_OBJECT_STORE_STANDALONE}) + +if (SOLANACEAE_OBJECT_STORE_STANDALONE) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + + # defaulting to debug mode, if not specified + if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Debug") + endif() + + # setup my vim ycm :D + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + + # more paths + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") +endif() + +# external libs +add_subdirectory(./external EXCLUDE_FROM_ALL) # before increasing warn levels, sad :( + +if (SOLANACEAE_OBJECT_STORE_STANDALONE) + set(CMAKE_CXX_EXTENSIONS OFF) + + # bump up warning levels appropriately for clang, gcc & msvc + if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang") + add_compile_options( + -Wall -Wextra # Reasonable and standard + -Wpedantic # Warn if non-standard C++ is used + -Wunused # Warn on anything being unused + #-Wconversion # Warn on type conversions that may lose data + #-Wsign-conversion # Warn on sign conversions + -Wshadow # Warn if a variable declaration shadows one from a parent context + ) + + if (NOT WIN32) + #link_libraries(-fsanitize=address) + #link_libraries(-fsanitize=address,undefined) + #link_libraries(-fsanitize-address-use-after-scope) + #link_libraries(-fsanitize=undefined) + endif() + elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL "MSVC") + if (CMAKE_CXX_FLAGS MATCHES "/W[0-4]") + string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") + endif() + endif() + +endif() + +# cmake setup end + +add_subdirectory(./src) + +#if (SOLANACEAE_OBJECT_STORE_BUILD_PLUGINS) + #add_subdirectory(./plugins) +#endif() + diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt new file mode 100644 index 0000000..7ea89db --- /dev/null +++ b/external/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 3.24 FATAL_ERROR) + +include(FetchContent) + +if (NOT TARGET solanaceae_util) + FetchContent_Declare(solanaceae_util + GIT_REPOSITORY https://github.com/Green-Sky/solanaceae_util.git + GIT_TAG master + ) + FetchContent_MakeAvailable(solanaceae_util) +endif() + +if (NOT TARGET EnTT::EnTT) + FetchContent_Declare(EnTT + GIT_REPOSITORY https://github.com/skypjack/entt.git + GIT_TAG v3.12.2 + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(EnTT) +endif() + +if (NOT TARGET nlohmann_json::nlohmann_json) + FetchContent_Declare(json + URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz + URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d + EXCLUDE_FROM_ALL + ) + FetchContent_MakeAvailable(json) +endif() + +#if (NOT TARGET solanaceae_plugin) + #FetchContent_Declare(solanaceae_plugin + #GIT_REPOSITORY https://github.com/Green-Sky/solanaceae_plugin.git + #GIT_TAG master + #) + #FetchContent_MakeAvailable(solanaceae_plugin) +#endif() + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..727b60f --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.9...3.24 FATAL_ERROR) + +project(solanaceae) + +add_library(solanaceae_object_store + ./solanaceae/object_store/types.hpp + ./solanaceae/object_store/meta_components.hpp + ./solanaceae/object_store/meta_components_id.inl + ./solanaceae/object_store/serializer_json.hpp + ./solanaceae/object_store/object_store.hpp + ./solanaceae/object_store/object_store.cpp +) + +target_include_directories(solanaceae_object_store PUBLIC .) +target_compile_features(solanaceae_object_store PUBLIC cxx_std_17) +target_link_libraries(solanaceae_object_store PUBLIC + nlohmann_json::nlohmann_json + EnTT::EnTT + solanaceae_util +) + diff --git a/src/solanaceae/object_store/meta_components.hpp b/src/solanaceae/object_store/meta_components.hpp new file mode 100644 index 0000000..0363caa --- /dev/null +++ b/src/solanaceae/object_store/meta_components.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include "./types.hpp" +#include "./object_store.hpp" + +#include +#include +#include + +namespace ObjectStore::Components { + + // TODO: is this special and should this be saved to meta or not (its already in the file name on disk) + struct ID { + std::vector v; + }; + + struct DataEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct DataCompressionType { + Compression comp {Compression::NONE}; + }; + + + // meta that is not written to (meta-)file + namespace Ephemeral { + + // TODO: move, backend specific + struct MetaFileType { + ::MetaFileType type {::MetaFileType::TEXT_JSON}; + }; + + struct MetaEncryptionType { + Encryption enc {Encryption::NONE}; + }; + + struct MetaCompressionType { + Compression comp {Compression::NONE}; + }; + + struct Backend { + // TODO: shared_ptr instead?? + StorageBackendI* ptr; + }; + + // excluded from file meta + // TODO: move to backend specific + struct FilePath { + // contains store path, if any + std::string path; + }; + + // TODO: seperate into remote and local? + // (remote meaning eg. the file on disk was changed by another program) + struct DirtyTag {}; + + } // Ephemeral + +} // Components + +// shortened to save bytes (until I find a way to save by ID in msgpack) +namespace ObjComp = ObjectStore::Components; + +// old names from frag era +namespace Fragment::Components { + //struct ID {}; + //struct DataEncryptionType {}; + //struct DataCompressionType {}; + struct ID : public ObjComp::ID {}; + struct DataEncryptionType : public ObjComp::DataEncryptionType {}; + struct DataCompressionType : public ObjComp::DataCompressionType {}; +} +namespace FragComp = Fragment::Components; + +#include "./meta_components_id.inl" + diff --git a/src/solanaceae/object_store/meta_components_id.inl b/src/solanaceae/object_store/meta_components_id.inl new file mode 100644 index 0000000..d9f7de3 --- /dev/null +++ b/src/solanaceae/object_store/meta_components_id.inl @@ -0,0 +1,30 @@ +#pragma once + +#include "./meta_components.hpp" + +#include + +// TODO: move more central +#define DEFINE_COMP_ID(x) \ +template<> \ +constexpr entt::id_type entt::type_hash::value() noexcept { \ + using namespace entt::literals; \ + return #x##_hs; \ +} \ +template<> \ +constexpr std::string_view entt::type_name::value() noexcept { \ + return #x; \ +} + +// cross compiler stable ids + +DEFINE_COMP_ID(ObjComp::DataEncryptionType) +DEFINE_COMP_ID(ObjComp::DataCompressionType) + +// old stuff +DEFINE_COMP_ID(FragComp::DataEncryptionType) +DEFINE_COMP_ID(FragComp::DataCompressionType) + +#undef DEFINE_COMP_ID + + diff --git a/src/solanaceae/object_store/object_store.cpp b/src/solanaceae/object_store/object_store.cpp new file mode 100644 index 0000000..e459e94 --- /dev/null +++ b/src/solanaceae/object_store/object_store.cpp @@ -0,0 +1,140 @@ +#include "./object_store.hpp" + +#include "./meta_components.hpp" + +#include "./serializer_json.hpp" + +#include // this sucks + +#include + +// TODO: move somewhere else +static bool serl_json_data_enc_type(const ObjectHandle oh, nlohmann::json& out) { + if (!oh.all_of()) { + return false; + } + + out = static_cast>( + oh.get().enc + ); + return true; +} + +static bool deserl_json_data_enc_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + +static bool serl_json_data_comp_type(const ObjectHandle oh, nlohmann::json& out) { + if (!oh.all_of()) { + return false; + } + + out = static_cast>( + oh.get().comp + ); + return true; +} + +static bool deserl_json_data_comp_type(ObjectHandle oh, const nlohmann::json& in) { + oh.emplace_or_replace( + static_cast( + static_cast>(in) + ) + ); + return true; +} + +StorageBackendI::StorageBackendI(ObjectStore2& os) : _os(os) { +} + +ObjectHandle StorageBackendI::newObject(ByteSpan) { + //return {_os.registry(), entt::null}; + return {}; +} + +bool StorageBackendI::write(Object o, const ByteSpan data) { + std::function fn_cb = [read = 0ull, data](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t { + uint64_t i = 0; + for (; i+read < data.size && i < buffer_size; i++) { + request_buffer[i] = data[i+read]; + } + read += i; + + return i; + }; + return write(o, fn_cb); +} + +ObjectStore2::ObjectStore2(void) { + // HACK: set them up independently + auto& sjc = _reg.ctx().emplace>(); + sjc.registerSerializer(serl_json_data_enc_type); + sjc.registerDeSerializer(deserl_json_data_enc_type); + sjc.registerSerializer(serl_json_data_comp_type); + sjc.registerDeSerializer(deserl_json_data_comp_type); + + // old stuff + sjc.registerSerializer(serl_json_data_enc_type); + sjc.registerDeSerializer(deserl_json_data_enc_type); + sjc.registerSerializer(serl_json_data_comp_type); + sjc.registerDeSerializer(deserl_json_data_comp_type); +} + +ObjectStore2::~ObjectStore2(void) { +} + +ObjectRegistry& ObjectStore2::registry(void) { + return _reg; +} + +ObjectHandle ObjectStore2::objectHandle(const Object o) { + return {_reg, o}; +} + +ObjectHandle ObjectStore2::getOneObjectByID(const ByteSpan id) { + // TODO: accelerate + // maybe keep it sorted and binary search? hash table lookup? + for (const auto& [obj, id_comp] : _reg.view().each()) { + if (id == ByteSpan{id_comp.v}) { + return {_reg, obj}; + } + } + + return {_reg, entt::null}; +} + +void ObjectStore2::throwEventConstruct(const Object o) { + std::cout << "OS debug: event construct " << entt::to_integral(o) << "\n"; + dispatch( + ObjectStore_Event::object_construct, + ObjectStore::Events::ObjectConstruct{ + ObjectHandle{_reg, o} + } + ); +} + +void ObjectStore2::throwEventUpdate(const Object o) { + std::cout << "OS debug: event update " << entt::to_integral(o) << "\n"; + dispatch( + ObjectStore_Event::object_update, + ObjectStore::Events::ObjectUpdate{ + ObjectHandle{_reg, o} + } + ); +} + +void ObjectStore2::throwEventDestroy(const Object o) { + std::cout << "OS debug: event destroy " << entt::to_integral(o) << "\n"; + dispatch( + ObjectStore_Event::object_destroy, + ObjectStore::Events::ObjectUpdate{ + ObjectHandle{_reg, o} + } + ); +} + diff --git a/src/solanaceae/object_store/object_store.hpp b/src/solanaceae/object_store/object_store.hpp new file mode 100644 index 0000000..ebbb8f7 --- /dev/null +++ b/src/solanaceae/object_store/object_store.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include +#include + +#include +#include + +#include + +// internal id +enum class Object : uint32_t {}; +using ObjectRegistry = entt::basic_registry; +using ObjectHandle = entt::basic_handle; + +// fwd +struct ObjectStore2; + +struct StorageBackendI { + // OR or OS ? + ObjectStore2& _os; + + StorageBackendI(ObjectStore2& os); + + // default impl fails, acting like a read only store + virtual ObjectHandle newObject(ByteSpan id); + + // ========== write object to storage ========== + using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size); + // calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer. + virtual bool write(Object o, std::function& data_cb) = 0; + bool write(Object o, const ByteSpan data); + + // ========== read object from storage ========== + using read_from_storage_put_data_cb = void(const ByteSpan buffer); + virtual bool read(Object o, std::function& data_cb) = 0; + +}; + +namespace ObjectStore::Events { + + struct ObjectConstruct { + const ObjectHandle e; + }; + struct ObjectUpdate { + const ObjectHandle e; + }; + struct ObjectDestory { + const ObjectHandle e; + }; + +} // ObjectStore::Events + +enum class ObjectStore_Event : uint16_t { + object_construct, + object_update, + object_destroy, + + MAX +}; + +struct ObjectStoreEventI { + using enumType = ObjectStore_Event; + + virtual ~ObjectStoreEventI(void) {} + + virtual bool onEvent(const ObjectStore::Events::ObjectConstruct&) { return false; } + virtual bool onEvent(const ObjectStore::Events::ObjectUpdate&) { return false; } + virtual bool onEvent(const ObjectStore::Events::ObjectDestory&) { return false; } +}; +using ObjectStoreEventProviderI = EventProviderI; + +struct ObjectStore2 : public ObjectStoreEventProviderI { + static constexpr const char* version {"2"}; + + ObjectRegistry _reg; + + // TODO: default backend? + + ObjectStore2(void); + virtual ~ObjectStore2(void); + + ObjectRegistry& registry(void); + ObjectHandle objectHandle(const Object o); + + // TODO: properly think about multiple objects witht he same id / different backends + ObjectHandle getOneObjectByID(const ByteSpan id); + + // sync? + + void throwEventConstruct(const Object o); + void throwEventUpdate(const Object o); + void throwEventDestroy(const Object o); +}; + diff --git a/src/solanaceae/object_store/serializer_json.hpp b/src/solanaceae/object_store/serializer_json.hpp new file mode 100644 index 0000000..cd74540 --- /dev/null +++ b/src/solanaceae/object_store/serializer_json.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include + +#include + +// nlohmann +template +struct SerializerJsonCallbacks { + using Registry = entt::basic_registry; + using Handle = entt::basic_handle; + + using serialize_fn = bool(*)(const Handle h, nlohmann::json& out); + entt::dense_map _serl; + + using deserialize_fn = bool(*)(Handle h, const nlohmann::json& in); + entt::dense_map _deserl; + + template + static bool component_get_json(const Handle h, nlohmann::json& j) { + if (h.template all_of()) { + if constexpr (!std::is_empty_v) { + j = h.template get(); + } + return true; + } + + return false; + } + + template + static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) { + if constexpr (std::is_empty_v) { + h.template emplace_or_replace(); // assert empty json? + } else { + h.template emplace_or_replace(static_cast(j)); + } + return true; + } + + void registerSerializer(serialize_fn fn, const entt::type_info& type_info) { + _serl[type_info.hash()] = fn; + } + + template + void registerSerializer( + serialize_fn fn = component_get_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerSerializer(fn, type_info); + } + + void registerDeSerializer(deserialize_fn fn, const entt::type_info& type_info) { + _deserl[type_info.hash()] = fn; + } + + template + void registerDeSerializer( + deserialize_fn fn = component_emplace_or_replace_json, + const entt::type_info& type_info = entt::type_id() + ) { + registerDeSerializer(fn, type_info); + } +}; + diff --git a/src/solanaceae/object_store/types.hpp b/src/solanaceae/object_store/types.hpp new file mode 100644 index 0000000..97cfb14 --- /dev/null +++ b/src/solanaceae/object_store/types.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +enum class Encryption : uint8_t { + NONE = 0x00, +}; +enum class Compression : uint8_t { + NONE = 0x00, + ZSTD = 0x01, + // TODO: zstd without magic + // TODO: zstd meta dict + // TODO: zstd data(message) dict +}; +enum class MetaFileType : uint8_t { + TEXT_JSON, + BINARY_MSGPACK, // msgpacked msgpack +}; +