refactoring, add to mainscreen

src/CMakeLists.txt

@@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
 add_library(fragment_store
 	./fragment_store/fragment_store_i.hpp
+	./fragment_store/types.hpp
+	./fragment_store/meta_components.hpp
+	./fragment_store/meta_components_id.inl
+	./fragment_store/serializer.hpp
+	./fragment_store/serializer.cpp
 	./fragment_store/fragment_store.hpp
 	./fragment_store/fragment_store.cpp
 )
@@ -106,6 +111,8 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_tox_messages
 
+	fragment_store
+
 	SDL3::SDL3
 
 	imgui

src/fragment_store/fragment_store.cpp

@@ -51,7 +51,7 @@ FragmentStore::FragmentStore(
 	registerSerializers();
 }
 
-entt::basic_handle<entt::basic_registry<FragmentID>> FragmentStore::fragmentHandle(FragmentID fid) {
+FragmentStore::FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) {
 	return {_reg, fid};
 }
 
@@ -107,7 +107,7 @@ FragmentID FragmentStore::newFragmentMemoryOwned(
 
 	const auto new_frag = _reg.create();
 
-	_reg.emplace<Components::ID>(new_frag, id);
+	_reg.emplace<FragComp::ID>(new_frag, id);
 	// TODO: memory comp
 	_reg.emplace<std::unique_ptr<std::vector<uint8_t>>>(new_frag) = std::move(new_data);
 
@@ -149,12 +149,12 @@ FragmentID FragmentStore::newFragmentFile(
 
 	const auto new_frag = _reg.create();
 
-	_reg.emplace<Components::ID>(new_frag, id);
+	_reg.emplace<FragComp::ID>(new_frag, id);
 
 	// file (info) comp
-	_reg.emplace<Components::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
+	_reg.emplace<FragComp::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
 
-	_reg.emplace<Components::Ephemeral::MetaFileType>(new_frag, mft);
+	_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
 
 	// meta needs to be synced to file
 	std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
@@ -177,7 +177,7 @@ FragmentID FragmentStore::getFragmentByID(
 ) {
 	// TODO: accelerate
 	// maybe keep it sorted and binary search? hash table lookup?
-	for (const auto& [frag, id_comp] : _reg.view<Components::ID>().each()) {
+	for (const auto& [frag, id_comp] : _reg.view<FragComp::ID>().each()) {
 		if (id == id_comp.v) {
 			return frag;
 		}
@@ -206,7 +206,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
 		return false;
 	}
 
-	if (!_reg.all_of<Components::Ephemeral::FilePath>(fid)) {
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
 		// not a file fragment?
 		return false;
 	}
@@ -214,20 +214,20 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
 	// split object storage
 
 	MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults
-	if (_reg.all_of<Components::Ephemeral::MetaFileType>(fid)) {
+	if (_reg.all_of<FragComp::Ephemeral::MetaFileType>(fid)) {
-		meta_type = _reg.get<Components::Ephemeral::MetaFileType>(fid).type;
+		meta_type = _reg.get<FragComp::Ephemeral::MetaFileType>(fid).type;
 	}
 
 	Encryption meta_enc = Encryption::NONE; // TODO: better defaults
 	Compression meta_comp = Compression::NONE; // TODO: better defaults
 
 	if (meta_type != MetaFileType::TEXT_JSON) {
-		if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
+		if (_reg.all_of<FragComp::Ephemeral::MetaEncryptionType>(fid)) {
-			meta_enc = _reg.get<Components::Ephemeral::MetaEncryptionType>(fid).enc;
+			meta_enc = _reg.get<FragComp::Ephemeral::MetaEncryptionType>(fid).enc;
 		}
 
-		if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
+		if (_reg.all_of<FragComp::Ephemeral::MetaCompressionType>(fid)) {
-			meta_comp = _reg.get<Components::Ephemeral::MetaCompressionType>(fid).comp;
+			meta_comp = _reg.get<FragComp::Ephemeral::MetaCompressionType>(fid).comp;
 		}
 	} else {
 		// we cant have encryption or compression
@@ -236,15 +236,15 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
 
 		// TODO: forcing for testing
 		//if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
-			_reg.emplace_or_replace<Components::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
 		//}
 		//if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
-			_reg.emplace_or_replace<Components::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
 		//}
 	}
 
 	std::ofstream meta_file{
-		_reg.get<Components::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type),
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type),
 		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
 	};
 
@@ -253,7 +253,7 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
 	}
 
 	std::ofstream data_file{
-		_reg.get<Components::Ephemeral::FilePath>(fid).path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path,
 		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
 	};
 
@@ -320,19 +320,32 @@ bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage
 
 	// TODO: use temp files and move to old location
 
-	if (_reg.all_of<Components::Ephemeral::DirtyTag>(fid)) {
+	if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
-		_reg.remove<Components::Ephemeral::DirtyTag>(fid);
+		_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
 	}
 
 	return true;
 }
 
+bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) {
+	std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+		uint64_t i = 0;
+		for (; i+read < data_size && i < buffer_size; i++) {
+			request_buffer[i] = data[i+read];
+		}
+		read += i;
+
+		return i;
+	};
+	return syncToStorage(fid, fn_cb);
+}
+
 static bool serl_json_data_enc_type(void* comp, nlohmann::json& out) {
 	if (comp == nullptr) {
 		return false;
 	}
 
-	auto& r_comp = *reinterpret_cast<Components::DataEncryptionType*>(comp);
+	auto& r_comp = *reinterpret_cast<FragComp::DataEncryptionType*>(comp);
 
 	out = static_cast<std::underlying_type_t<Encryption>>(r_comp.enc);
 
@@ -344,7 +357,7 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) {
 		return false;
 	}
 
-	auto& r_comp = *reinterpret_cast<Components::DataCompressionType*>(comp);
+	auto& r_comp = *reinterpret_cast<FragComp::DataCompressionType*>(comp);
 
 	out = static_cast<std::underlying_type_t<Compression>>(r_comp.comp);
 
@@ -352,8 +365,8 @@ static bool serl_json_data_comp_type(void* comp, nlohmann::json& out) {
 }
 
 void FragmentStore::registerSerializers(void) {
-	_sc.registerSerializerJson<Components::DataEncryptionType>(serl_json_data_enc_type);
+	_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
-	_sc.registerSerializerJson<Components::DataCompressionType>(serl_json_data_comp_type);
+	_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
 
 	std::cout << "registered serl text json cbs:\n";
 	for (const auto& [type_id, _] : _sc._serl_json) {

src/fragment_store/fragment_store.hpp

@@ -2,101 +2,24 @@
 
 #include "./fragment_store_i.hpp"
 
-#include <entt/core/fwd.hpp>
+#include "./types.hpp"
+#include "./meta_components.hpp"
+
+#include "./serializer.hpp"
+
 #include <entt/core/type_info.hpp>
 #include <entt/entity/registry.hpp>
-#include <entt/container/dense_map.hpp>
 
 #include <nlohmann/json_fwd.hpp>
 
-#include <utility>
 #include <vector>
 #include <array>
 #include <cstdint>
 #include <random>
 
-enum class Encryption : uint8_t {
-	NONE = 0x00,
-};
-enum class Compression : uint8_t {
-	NONE = 0x00,
-};
-enum class MetaFileType : uint8_t {
-	TEXT_JSON,
-	//BINARY_ARB,
-	BINARY_MSGPACK,
-};
-
-namespace Components {
-
-	// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
-	struct ID {
-		std::vector<uint8_t> v;
-	};
-
-	struct DataEncryptionType {
-		Encryption enc {Encryption::NONE};
-	};
-
-	struct DataCompressionType {
-		Compression comp {Compression::NONE};
-	};
-
-
-	// meta that is not written to (meta-)file
-	namespace Ephemeral {
-
-		// excluded from file meta
-		struct FilePath {
-			// contains store path, if any
-			std::string path;
-		};
-
-		// TODO: seperate into remote and local?
-		// (remote meaning eg. the file on disk was changed by another program)
-		struct DirtyTag {};
-
-
-		// type as comp
-		struct MetaFileType {
-			::MetaFileType type {::MetaFileType::TEXT_JSON};
-		};
-
-		struct MetaEncryptionType {
-			Encryption enc {Encryption::NONE};
-		};
-
-		struct MetaCompressionType {
-			Compression comp {Compression::NONE};
-		};
-
-	} // Ephemeral
-
-} // Components
-
-struct SerializerCallbacks {
-	// nlohmann
-	// json/msgpack
-	using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out);
-	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
-
-	using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in);
-	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
-
-	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
-		_serl_json[type_info.hash()] = fn;
-	}
-	template<typename CompType>
-	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerSerializerJson(fn, type_info); }
-
-	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
-		_deserl_json[type_info.hash()] = fn;
-	}
-	template<typename CompType>
-	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerDeSerializerJson(fn, type_info); }
-};
-
 struct FragmentStore : public FragmentStoreI {
+	using FragmentHandle = entt::basic_handle<entt::basic_registry<FragmentID>>;
+
 	entt::basic_registry<FragmentID> _reg;
 
 	std::minstd_rand _rng{std::random_device{}()};
@@ -113,10 +36,11 @@ struct FragmentStore : public FragmentStoreI {
 	FragmentStore(std::array<uint8_t, 8> session_uuid_namespace);
 
 	// HACK: get access to the reg
-	entt::basic_handle<entt::basic_registry<FragmentID>> fragmentHandle(FragmentID fid);
+	FragmentHandle fragmentHandle(FragmentID fid);
 
 	// TODO: make the frags ref counted
 
+	// TODO: check for exising
 	std::vector<uint8_t> generateNewUID(std::array<uint8_t, 8>& uuid_namespace);
 	std::vector<uint8_t> generateNewUID(void);
 
@@ -154,21 +78,20 @@ struct FragmentStore : public FragmentStoreI {
 	);
 
 	// remove fragment?
+	// unload?
 
-	// syncs fragment to file
+	// ========== sync fragment to storage ==========
-
 	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
 	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
 	bool syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
-
+	bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size);
-	// unload frags?
-	// if frags are file backed, we can close the file if not needed
 
 	// fragment discovery?
 
 	private:
 		void registerSerializers(void); // internal comps
 		// internal actual backends
+		// TODO: seperate out
 		bool syncToMemory(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
 		bool syncToFile(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
 };

src/fragment_store/meta_components.hpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "./types.hpp"
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+namespace Fragment::Components {
+
+	// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
+	struct ID {
+		std::vector<uint8_t> v;
+	};
+
+	struct DataEncryptionType {
+		Encryption enc {Encryption::NONE};
+	};
+
+	struct DataCompressionType {
+		Compression comp {Compression::NONE};
+	};
+
+
+	// meta that is not written to (meta-)file
+	namespace Ephemeral {
+
+		// excluded from file meta
+		struct FilePath {
+			// contains store path, if any
+			std::string path;
+		};
+
+		// TODO: seperate into remote and local?
+		// (remote meaning eg. the file on disk was changed by another program)
+		struct DirtyTag {};
+
+
+		// type as comp
+		struct MetaFileType {
+			::MetaFileType type {::MetaFileType::TEXT_JSON};
+		};
+
+		struct MetaEncryptionType {
+			Encryption enc {Encryption::NONE};
+		};
+
+		struct MetaCompressionType {
+			Compression comp {Compression::NONE};
+		};
+
+	} // Ephemeral
+
+} // Components
+
+// shortened to save bytes (until I find a way to save by ID in msgpack)
+namespace FragComp = Fragment::Components;
+
+#include "./meta_components_id.inl"
+

src/fragment_store/meta_components_id.inl

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "./meta_components.hpp"
+
+#include <entt/core/type_info.hpp>
+
+// TODO: move more central
+#define DEFINE_COMP_ID(x) \
+template<> \
+constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
+	using namespace entt::literals; \
+	return #x##_hs; \
+} \
+template<> \
+constexpr std::string_view entt::type_name<x>::value() noexcept { \
+	return #x; \
+}
+
+// cross compiler stable ids
+
+DEFINE_COMP_ID(FragComp::DataEncryptionType)
+DEFINE_COMP_ID(FragComp::DataCompressionType)
+
+#undef DEFINE_COMP_ID
+
+

src/fragment_store/serializer.cpp

@@ -0,0 +1,10 @@
+#include "./serializer.hpp"
+
+void SerializerCallbacks::registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+	_serl_json[type_info.hash()] = fn;
+}
+
+void SerializerCallbacks::registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+	_deserl_json[type_info.hash()] = fn;
+}
+

src/fragment_store/serializer.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+struct SerializerCallbacks {
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(void* comp, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(void* comp, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info);
+	template<typename CompType>
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerSerializerJson(fn, type_info); }
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info);
+	template<typename CompType>
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info = entt::type_id<CompType>()) { registerDeSerializerJson(fn, type_info); }
+};
+

src/fragment_store/test_fragstore.cpp

@@ -43,8 +43,8 @@ int main(void) {
 	{
 		auto frag0h = fs.fragmentHandle(frag0);
 
-		frag0h.emplace_or_replace<Components::DataCompressionType>();
+		frag0h.emplace_or_replace<FragComp::DataCompressionType>();
-		frag0h.emplace_or_replace<Components::DataEncryptionType>();
+		frag0h.emplace_or_replace<FragComp::DataEncryptionType>();
 		frag0h.emplace_or_replace<Components::MessagesTimestampRange>();
 
 		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
@@ -62,8 +62,8 @@ int main(void) {
 	{
 		auto frag1h = fs.fragmentHandle(frag1);
 
-		frag1h.emplace_or_replace<Components::DataCompressionType>();
+		frag1h.emplace_or_replace<FragComp::DataCompressionType>();
-		frag1h.emplace_or_replace<Components::DataEncryptionType>();
+		frag1h.emplace_or_replace<FragComp::DataEncryptionType>();
 
 		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
 			static constexpr std::string_view text = "This is some random data";
@@ -78,6 +78,15 @@ int main(void) {
 		fs.syncToStorage(frag1, fn_cb);
 	}
 
+	{
+		auto frag2h = fs.fragmentHandle(frag2);
+
+		frag2h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag2h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		static constexpr std::string_view text = "This is more random data";
+		fs.syncToStorage(frag2, reinterpret_cast<const uint8_t*>(text.data()), text.size());
+	}
 	return 0;
 }
 

src/fragment_store/types.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <cstdint>
+
+enum class Encryption : uint8_t {
+	NONE = 0x00,
+};
+enum class Compression : uint8_t {
+	NONE = 0x00,
+};
+enum class MetaFileType : uint8_t {
+	TEXT_JSON,
+	//BINARY_ARB,
+	BINARY_MSGPACK,
+};
+

src/main_screen.cpp

@@ -1,4 +1,5 @@
 #include "./main_screen.hpp"
+#include "fragment_store/fragment_store.hpp"
 
 #include <solanaceae/contact/components.hpp>
 
@@ -49,6 +50,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";
 
 	{ // setup plugin instances
+		// TODO: make interface useful
+		g_provideInstance<FragmentStoreI>("FragmentStoreI", "host", &fs);
+		g_provideInstance<FragmentStore>("FragmentStore", "host", &fs);
+
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);

src/main_screen.hpp

@@ -2,6 +2,7 @@
 
 #include "./screen.hpp"
 
+#include "./fragment_store/fragment_store.hpp"
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
@@ -43,6 +44,8 @@ extern "C" {
 struct MainScreen final : public Screen {
 	SDL_Renderer* renderer;
 
+	FragmentStore fs;
+
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;