Merge pull request #8 from Green-Sky/fragment_store

ObjectStore backed MessageFragments

.gitmodules

@@ -23,3 +23,6 @@
 [submodule "external/solanaceae_object_store"]
 	path = external/solanaceae_object_store
 	url = https://github.com/Green-Sky/solanaceae_object_store.git
+[submodule "external/solanaceae_message_serializer"]
+	path = external/solanaceae_message_serializer
+	url = https://github.com/Green-Sky/solanaceae_message_serializer.git

external/CMakeLists.txt

@@ -5,6 +5,7 @@ add_subdirectory(./entt)
 add_subdirectory(./solanaceae_util)
 add_subdirectory(./solanaceae_contact)
 add_subdirectory(./solanaceae_message3)
+add_subdirectory(./solanaceae_message_serializer)
 
 add_subdirectory(./solanaceae_plugin)
 
@@ -21,12 +22,3 @@ add_subdirectory(./stb)
 add_subdirectory(./libwebp)
 add_subdirectory(./qoi)
 
-if (NOT TARGET nlohmann_json::nlohmann_json)
-	FetchContent_Declare(json
-		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
-		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
-		EXCLUDE_FROM_ALL
-	)
-	FetchContent_MakeAvailable(json)
-endif()
-

flake.lock

@@ -34,10 +34,28 @@
         "type": "github"
       }
     },
+    "nlohmann-json": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1701207391,
+        "narHash": "sha256-7F0Jon+1oWL7uqet5i1IgHX0fUw/+z0QwEcA3zs5xHg=",
+        "owner": "nlohmann",
+        "repo": "json",
+        "rev": "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlohmann",
+        "ref": "v3.11.3",
+        "repo": "json",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nlohmann-json": "nlohmann-json"
       }
     },
     "systems": {

flake.nix

@@ -6,9 +6,13 @@
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
     flake-utils.url = "github:numtide/flake-utils";
+    nlohmann-json = {
+      url = "github:nlohmann/json/v3.11.3"; # TODO: read version from file
+      flake = false;
+    };
   };
 
-  outputs = { self, nixpkgs, flake-utils }:
+  outputs = { self, nixpkgs, flake-utils, nlohmann-json }:
     flake-utils.lib.eachDefaultSystem (system:
     let
       pkgs = import nixpkgs { inherit system; };
@@ -61,9 +65,9 @@
           "-DTOMATO_ASAN=OFF"
           "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
 
-          "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
-          # do we really care less about the version? do we need a stable abi?
-          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}"
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${nlohmann-json}" # we care about the version
+          # TODO: use package instead
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # we dont care about the version (we use 1.4.x features)
         ];
 
         # TODO: replace with install command

src/CMakeLists.txt

@@ -2,45 +2,6 @@ cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
 ########################################
 
-add_library(message_fragment_store
-	./fragment_store/uuid_generator.hpp
-	./fragment_store/uuid_generator.cpp
-
-	./json/message_components.hpp # TODO: move
-	./json/tox_message_components.hpp # TODO: move
-
-	./fragment_store/message_serializer.hpp
-	./fragment_store/message_serializer.cpp
-	./fragment_store/messages_meta_components.hpp
-	./fragment_store/messages_meta_components_id.inl
-	./fragment_store/message_fragment_store.hpp
-	./fragment_store/message_fragment_store.cpp
-
-	./fragment_store/register_mfs_json_message_components.hpp
-	./fragment_store/register_mfs_json_message_components.cpp
-	./fragment_store/register_mfs_json_tox_message_components.hpp
-	./fragment_store/register_mfs_json_tox_message_components.cpp
-)
-target_compile_features(message_fragment_store PRIVATE cxx_std_20)
-target_link_libraries(message_fragment_store PUBLIC
-	solanaceae_object_store
-	solanaceae_message3
-	solanaceae_tox_messages # TODO: move
-)
-
-########################################
-
-add_executable(convert_message_object_store
-	fragment_store/convert_frag_to_obj.cpp
-)
-
-target_link_libraries(convert_message_object_store PUBLIC
-	solanaceae_object_store
-	solanaceae_object_store_backend_filesystem
-	message_fragment_store
-)
-
-########################################
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@@ -116,6 +77,7 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_util
 	solanaceae_contact
 	solanaceae_message3
+	solanaceae_message_serializer
 
 	solanaceae_plugin
 
@@ -124,8 +86,6 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_messages
 
 	solanaceae_object_store
-	solanaceae_object_store_backend_filesystem
-	message_fragment_store
 
 	SDL3::SDL3
 

src/chat_gui4.hpp

@@ -10,9 +10,6 @@
 #include "./file_selector.hpp"
 #include "./send_image_popup.hpp"
 
-// HACK: move to public msg api?
-#include "./fragment_store/message_fragment_store.hpp"
-
 #include <entt/container/dense_map.hpp>
 
 #include <cstdint>

src/fragment_store/README.md

@@ -1,76 +0,0 @@
-# Fragment Store
-
-Fragments are are pieces of information split into Metadata and Data.
-They can be stored seperated or together.
-They can be used as a Transport protocol/logic too.
-
-# Store types
-
-### Object Store
-
-Fragment files are stored with the first 2 hex chars as sub folders:
-eg:
-`objects/` (object store root)
-  - `5f/` (first 2hex subfolder)
-    - `4fffffff` (the fragment file without the first 2 hexchars)
-
-### Split Object Store
-
-Same as Object Store, but medadata and data stored in seperate files.
-Metadata files have the `.meta` suffix. They also have a filetype specific suffix, like `.json`, `.msgpack` etc.
-
-### Memory Store
-
-Just keeps the Fragments in memory.
-
-# File formats
-
-Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted.
-
-### Text Json
-
-Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
-Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
-
-A Metadata json object can have arbitrary keys, some are predefined:
-- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any
-- `FragComp::DataCompressionType` (uint) Compression type of the data, if any
-
-## Binary file headers
-
-### Split Metadata
-
-msgpack array:
-
-- `[0]`: file magic string `SOLMET` (6 bytes)
-- `[1]`: uint8 encryption type (`0x00` is none)
-- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd)
-- `[3]`: binary metadata (optionally compressed and encrypted)
-
-note that the encryption and compression are for the metadata only.
-The metadata itself contains encryption and compression info about the data.
-
-### Split Data
-
-All the metadata is in the metadata file. (like encryption and compression)
-This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
-Keep in mind to not use the actual file name as the data/meta file name.
-
-### Single fragment
-
-Note: this format is unused for now
-
-file magic bytes `SOLFIL` (6 bytes)
-
-1 byte encryption type (`0x00` is none)
-
-1 byte compression type (`0x00` is none)
-
-...metadata here...
-
-...data here...
-
-## Compression types
-
-- `0x00` none
-- `0x01` zstd (without dict)

src/fragment_store/convert_frag_to_obj.cpp

@@ -1,148 +0,0 @@
-#include <solanaceae/object_store/object_store.hpp>
-#include <solanaceae/object_store/backends/filesystem_storage.hpp>
-#include <solanaceae/object_store/meta_components.hpp>
-#include <solanaceae/object_store/serializer_json.hpp>
-#include "./message_fragment_store.hpp"
-
-#include <solanaceae/util/utils.hpp>
-
-#include <nlohmann/json.hpp>
-
-#include <filesystem>
-#include <iostream>
-
-#include <cassert>
-
-int main(int argc, const char** argv) {
-	if (argc != 3) {
-		std::cerr << "wrong paramter count, do " << argv[0] << " <input_folder> <output_folder>\n";
-		return 1;
-	}
-
-	if (!std::filesystem::is_directory(argv[1])) {
-		std::cerr << "input folder is no folder\n";
-	}
-
-	std::filesystem::create_directories(argv[2]);
-
-	// we are going to use 2 different OS for convineance, but could be done with 1 too
-	ObjectStore2 os_src;
-	ObjectStore2 os_dst;
-
-	backend::FilesystemStorage fsb_src(os_src, argv[1]);
-	backend::FilesystemStorage fsb_dst(os_dst, argv[2]);
-
-	Contact3Registry cr; // dummy
-	RegistryMessageModel rmm(cr); // dummy
-	// they only exist for the serializers (for now)
-	// TODO: version
-	MessageFragmentStore mfs_src(cr, rmm, os_src, fsb_src);
-	MessageFragmentStore mfs_dst(cr, rmm, os_dst, fsb_dst);
-
-	// add message fragment store too (adds meta?)
-
-	// hookup events
-	struct EventListener : public ObjectStoreEventI {
-		ObjectStore2& _os_src;
-		backend::FilesystemStorage& _fsb_src;
-
-		ObjectStore2& _os_dst;
-		backend::FilesystemStorage& _fsb_dst;
-
-		EventListener(
-			ObjectStore2& os_src,
-			backend::FilesystemStorage& fsb_src,
-			ObjectStore2& os_dst,
-			backend::FilesystemStorage& fsb_dst
-		) :
-			_os_src(os_src),
-			_fsb_src(fsb_src),
-			_os_dst(os_dst),
-			_fsb_dst(fsb_dst)
-		{
-			_os_src.subscribe(this, ObjectStore_Event::object_construct);
-			_os_src.subscribe(this, ObjectStore_Event::object_update);
-		}
-
-		protected: // os
-			bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override {
-				assert(e.e.all_of<ObjComp::Ephemeral::MetaFileType>());
-				assert(e.e.all_of<ObjComp::ID>());
-
-				// !! we read the obj first, so we can discard empty objects
-				// technically we could just copy the file, but meh
-				// read src and write dst data
-				std::vector<uint8_t> tmp_buffer;
-				std::function<StorageBackendI::read_from_storage_put_data_cb> cb = [&tmp_buffer](const ByteSpan buffer) {
-					tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend());
-				};
-				if (!_fsb_src.read(e.e, cb)) {
-					std::cerr << "failed to read obj '" << bin2hex(e.e.get<ObjComp::ID>().v) << "'\n";
-					return false;
-				}
-
-				if (tmp_buffer.empty()) {
-					std::cerr << "discarded empty obj '" << bin2hex(e.e.get<ObjComp::ID>().v) << "'\n";
-					return false;
-				}
-				{ // try getting lucky and see if its an empty json
-					const auto j = nlohmann::json::parse(tmp_buffer, nullptr, false);
-					if (j.is_array() && j.empty()) {
-						std::cerr << "discarded empty json array obj '" << bin2hex(e.e.get<ObjComp::ID>().v) << "'\n";
-						return false;
-					}
-				}
-
-				// we dont copy meta file type, it will be the same for all "new" objects
-				auto oh = _fsb_dst.newObject(ByteSpan{e.e.get<ObjComp::ID>().v});
-
-				if (!static_cast<bool>(oh)) {
-					// already exists
-					return false;
-				}
-
-				{ // sync meta
-					// some hardcoded ehpemeral (besides mft/id)
-					oh.emplace_or_replace<ObjComp::Ephemeral::MetaEncryptionType>(e.e.get_or_emplace<ObjComp::Ephemeral::MetaEncryptionType>());
-					oh.emplace_or_replace<ObjComp::Ephemeral::MetaCompressionType>(e.e.get_or_emplace<ObjComp::Ephemeral::MetaCompressionType>());
-
-					// serializable
-					for (const auto& [type, fn] : _os_src.registry().ctx().get<SerializerJsonCallbacks<Object>>()._serl) {
-						//if (!e.e.registry()->storage(type)->contains(e.e)) {
-							//continue;
-						//}
-
-						// this is hacky but we serialize and then deserialize the component
-						// raw copy might be better in the future
-						nlohmann::json tmp_j;
-						if (fn(e.e, tmp_j)) {
-							_os_dst.registry().ctx().get<SerializerJsonCallbacks<Object>>()._deserl.at(type)(oh, tmp_j);
-						}
-					}
-				}
-
-				static_cast<StorageBackendI&>(_fsb_dst).write(oh, ByteSpan{tmp_buffer});
-
-				return false;
-			}
-
-			bool onEvent(const ObjectStore::Events::ObjectUpdate&) override {
-				std::cerr << "Update called\n";
-				assert(false);
-				return false;
-			}
-	} el {
-		os_src,
-		fsb_src,
-		os_dst,
-		fsb_dst,
-	};
-
-	// perform scan (which triggers events)
-	fsb_dst.scanAsync(); // fill with existing?
-	fsb_src.scanAsync(); // the scan
-
-	// done
-	return 0;
-}
-

src/fragment_store/message_fragment_store.hpp

@@ -1,122 +0,0 @@
-#pragma once
-
-#include <solanaceae/object_store/object_store.hpp>
-#include <solanaceae/object_store/meta_components.hpp>
-
-#include "./uuid_generator.hpp"
-
-#include "./message_serializer.hpp"
-
-#include "./messages_meta_components.hpp"
-
-#include <entt/container/dense_map.hpp>
-#include <entt/container/dense_set.hpp>
-
-#include <solanaceae/contact/contact_model3.hpp>
-#include <solanaceae/message3/registry_message_model.hpp>
-
-#include <deque>
-#include <vector>
-#include <cstdint>
-
-namespace Message::Components {
-
-	// unused, consumes too much memory (highly compressable)
-	//using FUID = FragComp::ID;
-
-	struct Obj {
-		Object o {entt::null};
-	};
-
-	// points to the front/newer message
-	// together they define a range that is,
-	// eg the first(end) and last(begin) message being rendered
-	// MFS requires there to be atleast one other fragment after/before,
-	// if not loaded fragment with fitting tsrange(direction) available
-	// uses fragmentAfter/Before()
-	// they can exist standalone
-	// if they are a pair, the inside is filled first
-	// cursers require a timestamp ???
-	struct ViewCurserBegin {
-		Message3 curser_end{entt::null};
-	};
-	struct ViewCurserEnd {
-		Message3 curser_begin{entt::null};
-	};
-
-	// TODO: add adjacency range comp or inside curser
-
-	// TODO: unused
-	// mfs will only load a limited number of fragments per tick (1),
-	// so this tag will be set if we loaded a fragment and
-	// every tick we check all cursers for this tag and continue
-	// and remove once no fragment could be loaded anymore
-	// (internal)
-	struct TagCurserUnsatisfied {};
-
-} // Message::Components
-
-// handles fragments for messages
-// on new message: assign fuid
-// on new and update: mark as fragment dirty
-// on delete: mark as fragment dirty?
-class MessageFragmentStore : public RegistryMessageModelEventI, public ObjectStoreEventI {
-	protected:
-		Contact3Registry& _cr;
-		RegistryMessageModel& _rmm;
-		ObjectStore2& _os;
-		StorageBackendI& _sb;
-		bool _fs_ignore_event {false};
-
-		UUIDGenerator_128_128 _session_uuid_gen;
-
-		// for message components only
-		MessageSerializerCallbacks _sc;
-
-		void handleMessage(const Message3Handle& m);
-
-		void loadFragment(Message3Registry& reg, ObjectHandle oh);
-
-		bool syncFragToStorage(ObjectHandle oh, Message3Registry& reg);
-
-		struct SaveQueueEntry final {
-			uint64_t ts_since_dirty{0};
-			//std::vector<uint8_t> id;
-			ObjectHandle id;
-			Message3Registry* reg{nullptr};
-		};
-		std::deque<SaveQueueEntry> _fuid_save_queue;
-
-		struct ECQueueEntry final {
-			ObjectHandle fid;
-			Contact3 c;
-		};
-		std::deque<ECQueueEntry> _event_check_queue;
-
-		// range changed or fragment loaded.
-		// we only load a limited number of fragments at once,
-		// so we need to keep them dirty until nothing was loaded.
-		entt::dense_set<Contact3> _potentially_dirty_contacts;
-
-	public:
-		MessageFragmentStore(
-			Contact3Registry& cr,
-			RegistryMessageModel& rmm,
-			ObjectStore2& os,
-			StorageBackendI& sb
-		);
-		virtual ~MessageFragmentStore(void);
-
-		MessageSerializerCallbacks& getMSC(void);
-
-		float tick(float time_delta);
-
-	protected: // rmm
-		bool onEvent(const Message::Events::MessageConstruct& e) override;
-		bool onEvent(const Message::Events::MessageUpdated& e) override;
-
-	protected: // fs
-		bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override;
-		bool onEvent(const ObjectStore::Events::ObjectUpdate& e) override;
-};
-

src/fragment_store/message_serializer.cpp

@@ -1,107 +0,0 @@
-#include "./message_serializer.hpp"
-
-#include <solanaceae/message3/components.hpp>
-#include <solanaceae/contact/components.hpp>
-
-#include <nlohmann/json.hpp>
-
-#include <iostream>
-
-static Contact3 findContactByID(Contact3Registry& cr, const std::vector<uint8_t>& id) {
-	// TODO: id lookup table, this is very inefficent
-	for (const auto& [c_it, id_it] : cr.view<Contact::Components::ID>().each()) {
-		if (id == id_it.data) {
-			return c_it;
-		}
-	}
-
-	return entt::null;
-}
-
-template<>
-bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
-	const Contact3 c = h.get<Message::Components::ContactFrom>().c;
-	if (!msc.cr.valid(c)) {
-		// while this is invalid registry state, it is valid serialization
-		j = nullptr;
-		std::cerr << "MSC warning: encountered invalid contact\n";
-		return true;
-	}
-
-	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
-		// unlucky, this contact is purely ephemeral
-		j = nullptr;
-		std::cerr << "MSC warning: encountered contact without ID\n";
-		return true;
-	}
-
-	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
-
-	return true;
-}
-template<>
-bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
-	if (j.is_null()) {
-		std::cerr << "MSC warning: encountered null contact\n";
-		h.emplace_or_replace<Message::Components::ContactFrom>();
-		return true;
-	}
-
-	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
-
-	Contact3 other_c = findContactByID(msc.cr, id);
-	if (!msc.cr.valid(other_c)) {
-		// create sparse contact with id only
-		other_c = msc.cr.create();
-		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
-	}
-
-	h.emplace_or_replace<Message::Components::ContactFrom>(other_c);
-
-	// TODO: should we return false if the contact is unknown??
-	return true;
-}
-
-template<>
-bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
-	const Contact3 c = h.get<Message::Components::ContactTo>().c;
-	if (!msc.cr.valid(c)) {
-		// while this is invalid registry state, it is valid serialization
-		j = nullptr;
-		std::cerr << "MSC warning: encountered invalid contact\n";
-		return true;
-	}
-
-	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
-		// unlucky, this contact is purely ephemeral
-		j = nullptr;
-		std::cerr << "MSC warning: encountered contact without ID\n";
-		return true;
-	}
-
-	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
-
-	return true;
-}
-template<>
-bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
-	if (j.is_null()) {
-		std::cerr << "MSC warning: encountered null contact\n";
-		h.emplace_or_replace<Message::Components::ContactTo>();
-		return true;
-	}
-
-	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
-
-	Contact3 other_c = findContactByID(msc.cr, id);
-	if (!msc.cr.valid(other_c)) {
-		// create sparse contact with id only
-		other_c = msc.cr.create();
-		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
-	}
-
-	h.emplace_or_replace<Message::Components::ContactTo>(other_c);
-
-	// TODO: should we return false if the contact is unknown??
-	return true;
-}

src/fragment_store/message_serializer.hpp

@@ -1,85 +0,0 @@
-#pragma once
-
-#include <entt/core/type_info.hpp>
-#include <entt/container/dense_map.hpp>
-
-#include <solanaceae/message3/registry_message_model.hpp>
-
-#include <nlohmann/json_fwd.hpp>
-
-struct MessageSerializerCallbacks {
-	using Registry = Message3Registry;
-	using Handle = Message3Handle;
-
-	Contact3Registry& cr;
-
-	// nlohmann
-	// json/msgpack
-	using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out);
-	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
-
-	using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in);
-	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
-
-	template<typename T>
-	static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) {
-		if (h.template all_of<T>()) {
-			if constexpr (!std::is_empty_v<T>) {
-				j = h.template get<T>();
-			}
-			return true;
-		}
-
-		return false;
-	}
-
-	template<typename T>
-	static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) {
-		if constexpr (std::is_empty_v<T>) {
-			h.template emplace_or_replace<T>(); // assert empty json?
-		} else {
-			h.template emplace_or_replace<T>(static_cast<T>(j));
-		}
-		return true;
-	}
-
-	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
-		_serl_json[type_info.hash()] = fn;
-	}
-
-	template<typename CompType>
-	void registerSerializerJson(
-		serialize_json_fn fn = component_get_json<CompType>,
-		const entt::type_info& type_info = entt::type_id<CompType>()
-	) {
-		registerSerializerJson(fn, type_info);
-	}
-
-	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
-		_deserl_json[type_info.hash()] = fn;
-	}
-
-	template<typename CompType>
-	void registerDeSerializerJson(
-		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
-		const entt::type_info& type_info = entt::type_id<CompType>()
-	) {
-		registerDeSerializerJson(fn, type_info);
-	}
-};
-
-// fwd
-namespace Message::Components {
-struct ContactFrom;
-struct ContactTo;
-}
-
-// make specializations known
-template<>
-bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
-template<>
-bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
-template<>
-bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
-template<>
-bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);

src/fragment_store/messages_meta_components.hpp

@@ -1,33 +0,0 @@
-#pragma once
-
-#include <solanaceae/object_store/meta_components.hpp>
-
-namespace ObjectStore::Components {
-	struct MessagesVersion {
-		// messages Object version
-		// 1 -> text_json
-		uint16_t v {1};
-	};
-
-	struct MessagesTSRange {
-		// timestamp range within the fragment
-		uint64_t begin {0}; // newer msg -> higher number
-		uint64_t end {0};
-	};
-
-	struct MessagesContact {
-		std::vector<uint8_t> id;
-	};
-
-	// TODO: add src contact (self id)
-
-} // ObjectStore::Components
-
-// old
-namespace Fragment::Components {
-	struct MessagesTSRange : public ObjComp::MessagesTSRange {};
-	struct MessagesContact : public ObjComp::MessagesContact {};
-} // Fragment::Components
-
-#include "./messages_meta_components_id.inl"
-

src/fragment_store/messages_meta_components_id.inl

@@ -1,31 +0,0 @@
-#pragma once
-
-#include "./messages_meta_components.hpp"
-
-#include <entt/core/type_info.hpp>
-
-// TODO: move more central
-#define DEFINE_COMP_ID(x) \
-template<> \
-constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
-	using namespace entt::literals; \
-	return #x##_hs; \
-} \
-template<> \
-constexpr std::string_view entt::type_name<x>::value() noexcept { \
-	return #x; \
-}
-
-// cross compiler stable ids
-
-DEFINE_COMP_ID(ObjComp::MessagesVersion)
-DEFINE_COMP_ID(ObjComp::MessagesTSRange)
-DEFINE_COMP_ID(ObjComp::MessagesContact)
-
-// old stuff
-//DEFINE_COMP_ID(FragComp::MessagesTSRange)
-//DEFINE_COMP_ID(FragComp::MessagesContact)
-
-#undef DEFINE_COMP_ID
-
-

src/fragment_store/register_mfs_json_message_components.cpp

@@ -1,35 +0,0 @@
-#include "./register_mfs_json_message_components.hpp"
-
-#include "./message_serializer.hpp"
-#include "../json/message_components.hpp"
-
-void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) {
-	msc.registerSerializerJson<Message::Components::Timestamp>();
-	msc.registerDeSerializerJson<Message::Components::Timestamp>();
-	msc.registerSerializerJson<Message::Components::TimestampProcessed>();
-	msc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
-	msc.registerSerializerJson<Message::Components::TimestampWritten>();
-	msc.registerDeSerializerJson<Message::Components::TimestampWritten>();
-	msc.registerSerializerJson<Message::Components::ContactFrom>();
-	msc.registerDeSerializerJson<Message::Components::ContactFrom>();
-	msc.registerSerializerJson<Message::Components::ContactTo>();
-	msc.registerDeSerializerJson<Message::Components::ContactTo>();
-	msc.registerSerializerJson<Message::Components::TagUnread>();
-	msc.registerDeSerializerJson<Message::Components::TagUnread>();
-	msc.registerSerializerJson<Message::Components::Read>();
-	msc.registerDeSerializerJson<Message::Components::Read>();
-	msc.registerSerializerJson<Message::Components::MessageText>();
-	msc.registerDeSerializerJson<Message::Components::MessageText>();
-	msc.registerSerializerJson<Message::Components::TagMessageIsAction>();
-	msc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
-
-	// files
-	//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
-	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
-	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
-	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
-	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
-	//_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
-	//_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
-}
-

src/fragment_store/register_mfs_json_message_components.hpp

@@ -1,6 +0,0 @@
-#pragma once
-
-#include "./message_serializer.hpp"
-
-void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc);
-

src/fragment_store/register_mfs_json_tox_message_components.cpp

@@ -1,10 +0,0 @@
-#include "./register_mfs_json_message_components.hpp"
-
-#include "./message_serializer.hpp"
-#include "../json/tox_message_components.hpp"
-
-void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) {
-	msc.registerSerializerJson<Message::Components::ToxGroupMessageID>();
-	msc.registerDeSerializerJson<Message::Components::ToxGroupMessageID>();
-}
-

src/fragment_store/register_mfs_json_tox_message_components.hpp

@@ -1,6 +0,0 @@
-#pragma once
-
-#include "./message_serializer.hpp"
-
-void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc);
-

src/fragment_store/uuid_generator.cpp

@@ -1,68 +0,0 @@
-#include "./uuid_generator.hpp"
-
-UUIDGenerator_128_128::UUIDGenerator_128_128(void) {
-	{ // random namespace
-		const auto num0 = _rng();
-		const auto num1 = _rng();
-		const auto num2 = _rng();
-		const auto num3 = _rng();
-
-		_uuid_namespace[0+0] = (num0 >> 0) & 0xff;
-		_uuid_namespace[0+1] = (num0 >> 8) & 0xff;
-		_uuid_namespace[0+2] = (num0 >> 16) & 0xff;
-		_uuid_namespace[0+3] = (num0 >> 24) & 0xff;
-
-		_uuid_namespace[4+0] = (num1 >> 0) & 0xff;
-		_uuid_namespace[4+1] = (num1 >> 8) & 0xff;
-		_uuid_namespace[4+2] = (num1 >> 16) & 0xff;
-		_uuid_namespace[4+3] = (num1 >> 24) & 0xff;
-
-		_uuid_namespace[8+0] = (num2 >> 0) & 0xff;
-		_uuid_namespace[8+1] = (num2 >> 8) & 0xff;
-		_uuid_namespace[8+2] = (num2 >> 16) & 0xff;
-		_uuid_namespace[8+3] = (num2 >> 24) & 0xff;
-
-		_uuid_namespace[12+0] = (num3 >> 0) & 0xff;
-		_uuid_namespace[12+1] = (num3 >> 8) & 0xff;
-		_uuid_namespace[12+2] = (num3 >> 16) & 0xff;
-		_uuid_namespace[12+3] = (num3 >> 24) & 0xff;
-	}
-}
-
-UUIDGenerator_128_128::UUIDGenerator_128_128(const std::array<uint8_t, 16>& uuid_namespace) :
-	_uuid_namespace(uuid_namespace)
-{
-}
-
-std::vector<uint8_t> UUIDGenerator_128_128::operator()(void) {
-	std::vector<uint8_t> new_uid(_uuid_namespace.cbegin(), _uuid_namespace.cend());
-	new_uid.resize(new_uid.size() + 16);
-
-	const auto num0 = _rng();
-	const auto num1 = _rng();
-	const auto num2 = _rng();
-	const auto num3 = _rng();
-
-	new_uid[_uuid_namespace.size()+0] = (num0 >> 0) & 0xff;
-	new_uid[_uuid_namespace.size()+1] = (num0 >> 8) & 0xff;
-	new_uid[_uuid_namespace.size()+2] = (num0 >> 16) & 0xff;
-	new_uid[_uuid_namespace.size()+3] = (num0 >> 24) & 0xff;
-
-	new_uid[_uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff;
-	new_uid[_uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff;
-	new_uid[_uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff;
-	new_uid[_uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff;
-
-	new_uid[_uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff;
-	new_uid[_uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff;
-	new_uid[_uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff;
-	new_uid[_uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff;
-
-	new_uid[_uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff;
-	new_uid[_uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff;
-	new_uid[_uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff;
-	new_uid[_uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff;
-
-	return new_uid;
-}
-

src/fragment_store/uuid_generator.hpp

@@ -1,24 +0,0 @@
-#pragma once
-
-#include <vector>
-#include <array>
-#include <random>
-#include <cstdint>
-
-struct UUIDGeneratorI {
-	virtual std::vector<uint8_t> operator()(void) = 0;
-};
-
-// TODO: templates?
-struct UUIDGenerator_128_128 final : public UUIDGeneratorI {
-	private:
-		std::array<uint8_t, 16> _uuid_namespace;
-		std::minstd_rand _rng{std::random_device{}()};
-
-	public:
-		UUIDGenerator_128_128(void); // default randomly initializes namespace
-		UUIDGenerator_128_128(const std::array<uint8_t, 16>& uuid_namespace);
-
-		std::vector<uint8_t> operator()(void) override;
-};
-

src/json/message_components.hpp

@@ -1,27 +0,0 @@
-#pragma once
-
-#include <solanaceae/util/utils.hpp>
-
-#include <solanaceae/message3/components.hpp>
-
-#include <nlohmann/json.hpp>
-
-namespace Message::Components {
-
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts)
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts)
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts)
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c)
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c)
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts)
-	// TODO: SyncedBy
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text)
-
-	namespace Transfer {
-		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size)
-		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size)
-		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list)
-	} // Transfer
-
-} // Message::Components
-

src/json/tox_message_components.hpp

@@ -1,16 +0,0 @@
-#pragma once
-
-#include <solanaceae/util/utils.hpp>
-
-#include <solanaceae/tox_messages/components.hpp>
-
-#include <nlohmann/json.hpp>
-
-namespace Message::Components {
-
-	// TODO: friend msg id, does not have the same qualities
-	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id)
-	// TODO: transfer stuff, needs content rewrite
-
-} // Message::Components
-

src/main_screen.cpp

@@ -1,7 +1,7 @@
 #include "./main_screen.hpp"
 
-#include "./fragment_store/register_mfs_json_message_components.hpp"
-#include "./fragment_store/register_mfs_json_tox_message_components.hpp"
+#include <solanaceae/message3/nj/message_components_serializer.hpp>
+#include <solanaceae/tox_messages/nj/tox_message_components_serializer.hpp>
 
 #include <solanaceae/contact/components.hpp>
 
@@ -15,9 +15,8 @@
 MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::string save_password, std::vector<std::string> plugins) :
 	renderer(renderer_),
 	rmm(cr),
+	msnj{cr, {}, {}},
 	mts(rmm),
-	mfsb(os, "test2_message_store/"),
-	mfs(cr, rmm, os, mfsb),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
@@ -38,8 +37,9 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	tdch(tpi)
 {
 	tel.subscribeAll(tc);
-	registerMFSJsonMessageComponents(mfs.getMSC());
-	registerMFSJsonToxMessageComponents(mfs.getMSC());
+
+	registerMessageComponents(msnj);
+	registerToxMessageComponents(msnj);
 
 	conf.set("tox", "save_file_path", save_path);
 
@@ -61,6 +61,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
+		g_provideInstance<MessageSerializerNJ>("MessageSerializerNJ", "host", &msnj);
 
 		g_provideInstance<ToxI>("ToxI", "host", &tc);
 		g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
@@ -83,8 +84,6 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	}
 
 	conf.dump();
-
-	mfsb.scanAsync(); // HACK: after plugins and tox contacts got loaded
 }
 
 MainScreen::~MainScreen(void) {
@@ -426,7 +425,6 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 
 	tdch.tick(time_delta); // compute
 
-	const float mfs_interval = mfs.tick(time_delta);
 	mts.iterate(); // compute (after mfs)
 
 	_min_tick_interval = std::min<float>(
@@ -439,10 +437,6 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 		_min_tick_interval,
 		fo_interval
 	);
-	_min_tick_interval = std::min<float>(
-		_min_tick_interval,
-		mfs_interval
-	);
 
 	//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";
 

src/main_screen.hpp

@@ -3,12 +3,11 @@
 #include "./screen.hpp"
 
 #include <solanaceae/object_store/object_store.hpp>
-#include <solanaceae/object_store/backends/filesystem_storage.hpp>
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
-#include "./fragment_store/message_fragment_store.hpp"
+#include <solanaceae/message3/message_serializer.hpp>
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@@ -51,9 +50,8 @@ struct MainScreen final : public Screen {
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
+	MessageSerializerNJ msnj;
 	MessageTimeSort mts;
-	backend::FilesystemStorage mfsb; // message fsb // TODO: make configurable
-	MessageFragmentStore mfs;
 
 	ToxEventLogger tel{std::cout};
 	ToxClient tc;