Merge branch 'master' into fragment_store

(by disabling them, since the error case is already handled)

CMakeLists.txt

@@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			link_libraries(-fsanitize=address)
+			#link_libraries(-fsanitize=address)
-			#link_libraries(-fsanitize=address,undefined)
+			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			message("II enabled ASAN")
 		else()

external/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
 
 add_subdirectory(./entt)
 
@@ -21,3 +21,12 @@ add_subdirectory(./stb)
 add_subdirectory(./libwebp)
 add_subdirectory(./qoi)
 
+if (NOT TARGET nlohmann_json::nlohmann_json)
+	FetchContent_Declare(json
+		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(json)
+endif()
+

flake.nix

@@ -12,13 +12,15 @@
     flake-utils.lib.eachDefaultSystem (system:
     let
       pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
     in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
         pname = "tomato";
         version = "0.0.0";
 
         src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing
 
         nativeBuildInputs = with pkgs; [
           cmake
@@ -58,6 +60,10 @@
         cmakeFlags = [
           "-DTOMATO_ASAN=OFF"
           "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
+
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
+          # do we really care less about the version? do we need a stable abi?
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}"
         ];
 
         # TODO: replace with install command
@@ -66,7 +72,7 @@
           mv bin/tomato $out/bin
         '';
 
-        dontStrip = true;
+        dontStrip = true; # does nothing
 
         # copied from nixpkgs's SDL2 default.nix
         # SDL is weird in that instead of just dynamically linking with
@@ -93,6 +99,8 @@
           '';
       };
 
+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
       devShells.${system}.default = pkgs.mkShell {
         #inputsFrom = with pkgs; [ SDL2 ];
         buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?

src/CMakeLists.txt

@@ -1,5 +1,46 @@
 cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
+########################################
+
+add_library(message_fragment_store
+	./fragment_store/uuid_generator.hpp
+	./fragment_store/uuid_generator.cpp
+
+	./json/message_components.hpp # TODO: move
+	./json/tox_message_components.hpp # TODO: move
+
+	./fragment_store/message_serializer.hpp
+	./fragment_store/message_serializer.cpp
+	./fragment_store/messages_meta_components.hpp
+	./fragment_store/messages_meta_components_id.inl
+	./fragment_store/message_fragment_store.hpp
+	./fragment_store/message_fragment_store.cpp
+
+	./fragment_store/register_mfs_json_message_components.hpp
+	./fragment_store/register_mfs_json_message_components.cpp
+	./fragment_store/register_mfs_json_tox_message_components.hpp
+	./fragment_store/register_mfs_json_tox_message_components.cpp
+)
+target_compile_features(message_fragment_store PRIVATE cxx_std_20)
+target_link_libraries(message_fragment_store PUBLIC
+	solanaceae_object_store
+	solanaceae_message3
+	solanaceae_tox_messages # TODO: move
+)
+
+########################################
+
+add_executable(convert_message_object_store
+	fragment_store/convert_frag_to_obj.cpp
+)
+
+target_link_libraries(convert_message_object_store PUBLIC
+	solanaceae_object_store
+	solanaceae_object_store_backend_filesystem
+	message_fragment_store
+)
+
+########################################
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@@ -83,6 +124,8 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_messages
 
 	solanaceae_object_store
+	solanaceae_object_store_backend_filesystem
+	message_fragment_store
 
 	SDL3::SDL3
 

src/chat_gui4.cpp

@@ -47,6 +47,18 @@ namespace Components {
 
 } // Components
 
+namespace Context {
+
+	// TODO: move back to chat log window and keep per window instead of per contact
+	struct CGView {
+		// set to the ts of the newest rendered msg
+		Message3Handle begin{};
+		// set to the ts of the oldest rendered msg
+		Message3Handle end{};
+	};
+
+} // Context
+
 static constexpr float lerp(float a, float b, float t) {
 	return a + t * (b - a);
 }
@@ -269,28 +281,6 @@ float ChatGui4::render(float time_delta) {
 
 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);
 
-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@@ -303,6 +293,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
 
+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;
 
@@ -389,12 +382,26 @@ float ChatGui4::render(float time_delta) {
 								}
 
 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
+								if (ImGui::IsItemVisible()) {
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
 										// get time now
 										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
 										msg_reg.remove<Message::Components::TagUnread>(e);
 										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+
+										// we remove the unread tag here
+										_rmm.throwEventUpdate(msg_reg, e);
+									}
+
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
 								}
 
 								// highlight self
@@ -559,9 +566,90 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 
+						{ // update view cursers
+							if (!msg_reg.ctx().contains<Context::CGView>()) {
+								msg_reg.ctx().emplace<Context::CGView>();
+							}
+
+							auto& cg_view = msg_reg.ctx().get<Context::CGView>();
+
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view, we setup a view at current time, so the next frags are loaded
+								if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
+									// fix invalid state
+									if (static_cast<bool>(cg_view.begin)) {
+										cg_view.begin.destroy();
+										_rmm.throwEventDestroy(cg_view.begin);
+									}
+									if (static_cast<bool>(cg_view.end)) {
+										cg_view.end.destroy();
+										_rmm.throwEventDestroy(cg_view.end);
+									}
+
+									// create new
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									cg_view.end = {msg_reg, msg_reg.create()};
+
+									cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+									cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+									cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+									cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(cg_view.begin);
+									std::cout << "CG: created view FRONT end ts\n";
+									_rmm.throwEventConstruct(cg_view.end);
+								} // else? we do nothing?
+							} else {
+								bool begin_created {false};
+								if (!static_cast<bool>(cg_view.begin)) {
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								bool end_created {false};
+								if (!static_cast<bool>(cg_view.end)) {
+									cg_view.end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+								cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+								{
+									auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+										old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+										if (begin_created) {
+											std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.begin);
+										} else {
+											//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.begin);
+										}
+									}
+								}
+
+								{
+									auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+										old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+										if (end_created) {
+											std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.end);
+										} else {
+											//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.end);
+										}
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}
 
+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}

src/chat_gui4.hpp

@@ -10,6 +10,9 @@
 #include "./file_selector.hpp"
 #include "./send_image_popup.hpp"
 
+// HACK: move to public msg api?
+#include "./fragment_store/message_fragment_store.hpp"
+
 #include <entt/container/dense_map.hpp>
 
 #include <cstdint>
@@ -32,6 +35,7 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;
 
+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;
 
 	// TODO: per contact

src/fragment_store/README.md

@@ -25,41 +25,41 @@ Just keeps the Fragments in memory.
 
 # File formats
 
-Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
+Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted.
 
 ### Text Json
 
 Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
 Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
 
-A Metadata json object has the following keys:
+A Metadata json object can have arbitrary keys, some are predefined:
-- `enc` (uint) Encryption type of the data, if any
+- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any
-- `comp` (uint) Compression type of the data, if any
+- `FragComp::DataCompressionType` (uint) Compression type of the data, if any
-- `metadata` (obj) the 
 
 ## Binary file headers
 
 ### Split Metadata
 
-file magic bytes `SOLMET` (6 bytes)
+msgpack array:
 
-1 byte encryption type (`0x00` is none)
+- `[0]`: file magic string `SOLMET` (6 bytes)
-
+- `[1]`: uint8 encryption type (`0x00` is none)
-1 byte compression type (`0x00` is none)
+- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd)
-
+- `[3]`: binary metadata (optionally compressed and encrypted)
-...metadata here...
 
 note that the encryption and compression are for the metadata only.
 The metadata itself contains encryption and compression info about the data.
 
 ### Split Data
 
-(none) all the data is in the metadata file.
+All the metadata is in the metadata file. (like encryption and compression)
 This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
 Keep in mind to not use the actual file name as the data/meta file name.
 
 ### Single fragment
 
+Note: this format is unused for now
+
 file magic bytes `SOLFIL` (6 bytes)
 
 1 byte encryption type (`0x00` is none)
@@ -70,3 +70,7 @@ file magic bytes `SOLFIL` (6 bytes)
 
 ...data here...
 
+## Compression types
+
+- `0x00` none
+- `0x01` zstd (without dict)

src/fragment_store/convert_frag_to_obj.cpp

@@ -0,0 +1,148 @@
+#include <solanaceae/object_store/object_store.hpp>
+#include <solanaceae/object_store/backends/filesystem_storage.hpp>
+#include <solanaceae/object_store/meta_components.hpp>
+#include <solanaceae/object_store/serializer_json.hpp>
+#include "./message_fragment_store.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <filesystem>
+#include <iostream>
+
+#include <cassert>
+
+int main(int argc, const char** argv) {
+	if (argc != 3) {
+		std::cerr << "wrong paramter count, do " << argv[0] << " <input_folder> <output_folder>\n";
+		return 1;
+	}
+
+	if (!std::filesystem::is_directory(argv[1])) {
+		std::cerr << "input folder is no folder\n";
+	}
+
+	std::filesystem::create_directories(argv[2]);
+
+	// we are going to use 2 different OS for convineance, but could be done with 1 too
+	ObjectStore2 os_src;
+	ObjectStore2 os_dst;
+
+	backend::FilesystemStorage fsb_src(os_src, argv[1]);
+	backend::FilesystemStorage fsb_dst(os_dst, argv[2]);
+
+	Contact3Registry cr; // dummy
+	RegistryMessageModel rmm(cr); // dummy
+	// they only exist for the serializers (for now)
+	// TODO: version
+	MessageFragmentStore mfs_src(cr, rmm, os_src, fsb_src);
+	MessageFragmentStore mfs_dst(cr, rmm, os_dst, fsb_dst);
+
+	// add message fragment store too (adds meta?)
+
+	// hookup events
+	struct EventListener : public ObjectStoreEventI {
+		ObjectStore2& _os_src;
+		backend::FilesystemStorage& _fsb_src;
+
+		ObjectStore2& _os_dst;
+		backend::FilesystemStorage& _fsb_dst;
+
+		EventListener(
+			ObjectStore2& os_src,
+			backend::FilesystemStorage& fsb_src,
+			ObjectStore2& os_dst,
+			backend::FilesystemStorage& fsb_dst
+		) :
+			_os_src(os_src),
+			_fsb_src(fsb_src),
+			_os_dst(os_dst),
+			_fsb_dst(fsb_dst)
+		{
+			_os_src.subscribe(this, ObjectStore_Event::object_construct);
+			_os_src.subscribe(this, ObjectStore_Event::object_update);
+		}
+
+		protected: // os
+			bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override {
+				assert(e.e.all_of<ObjComp::Ephemeral::MetaFileType>());
+				assert(e.e.all_of<ObjComp::ID>());
+
+				// !! we read the obj first, so we can discard empty objects
+				// technically we could just copy the file, but meh
+				// read src and write dst data
+				std::vector<uint8_t> tmp_buffer;
+				std::function<StorageBackendI::read_from_storage_put_data_cb> cb = [&tmp_buffer](const ByteSpan buffer) {
+					tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend());
+				};
+				if (!_fsb_src.read(e.e, cb)) {
+					std::cerr << "failed to read obj '" << bin2hex(e.e.get<ObjComp::ID>().v) << "'\n";
+					return false;
+				}
+
+				if (tmp_buffer.empty()) {
+					std::cerr << "discarded empty obj '" << bin2hex(e.e.get<ObjComp::ID>().v) << "'\n";
+					return false;
+				}
+				{ // try getting lucky and see if its an empty json
+					const auto j = nlohmann::json::parse(tmp_buffer, nullptr, false);
+					if (j.is_array() && j.empty()) {
+						std::cerr << "discarded empty json array obj '" << bin2hex(e.e.get<ObjComp::ID>().v) << "'\n";
+						return false;
+					}
+				}
+
+				// we dont copy meta file type, it will be the same for all "new" objects
+				auto oh = _fsb_dst.newObject(ByteSpan{e.e.get<ObjComp::ID>().v});
+
+				if (!static_cast<bool>(oh)) {
+					// already exists
+					return false;
+				}
+
+				{ // sync meta
+					// some hardcoded ehpemeral (besides mft/id)
+					oh.emplace_or_replace<ObjComp::Ephemeral::MetaEncryptionType>(e.e.get_or_emplace<ObjComp::Ephemeral::MetaEncryptionType>());
+					oh.emplace_or_replace<ObjComp::Ephemeral::MetaCompressionType>(e.e.get_or_emplace<ObjComp::Ephemeral::MetaCompressionType>());
+
+					// serializable
+					for (const auto& [type, fn] : _os_src.registry().ctx().get<SerializerJsonCallbacks<Object>>()._serl) {
+						//if (!e.e.registry()->storage(type)->contains(e.e)) {
+							//continue;
+						//}
+
+						// this is hacky but we serialize and then deserialize the component
+						// raw copy might be better in the future
+						nlohmann::json tmp_j;
+						if (fn(e.e, tmp_j)) {
+							_os_dst.registry().ctx().get<SerializerJsonCallbacks<Object>>()._deserl.at(type)(oh, tmp_j);
+						}
+					}
+				}
+
+				static_cast<StorageBackendI&>(_fsb_dst).write(oh, ByteSpan{tmp_buffer});
+
+				return false;
+			}
+
+			bool onEvent(const ObjectStore::Events::ObjectUpdate&) override {
+				std::cerr << "Update called\n";
+				assert(false);
+				return false;
+			}
+	} el {
+		os_src,
+		fsb_src,
+		os_dst,
+		fsb_dst,
+	};
+
+	// perform scan (which triggers events)
+	fsb_dst.scanAsync(); // fill with existing?
+	fsb_src.scanAsync(); // the scan
+
+	// done
+	return 0;
+}
+

src/fragment_store/message_fragment_store.hpp

@@ -0,0 +1,122 @@
+#pragma once
+
+#include <solanaceae/object_store/object_store.hpp>
+#include <solanaceae/object_store/meta_components.hpp>
+
+#include "./uuid_generator.hpp"
+
+#include "./message_serializer.hpp"
+
+#include "./messages_meta_components.hpp"
+
+#include <entt/container/dense_map.hpp>
+#include <entt/container/dense_set.hpp>
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <deque>
+#include <vector>
+#include <cstdint>
+
+namespace Message::Components {
+
+	// unused, consumes too much memory (highly compressable)
+	//using FUID = FragComp::ID;
+
+	struct Obj {
+		Object o {entt::null};
+	};
+
+	// points to the front/newer message
+	// together they define a range that is,
+	// eg the first(end) and last(begin) message being rendered
+	// MFS requires there to be atleast one other fragment after/before,
+	// if not loaded fragment with fitting tsrange(direction) available
+	// uses fragmentAfter/Before()
+	// they can exist standalone
+	// if they are a pair, the inside is filled first
+	// cursers require a timestamp ???
+	struct ViewCurserBegin {
+		Message3 curser_end{entt::null};
+	};
+	struct ViewCurserEnd {
+		Message3 curser_begin{entt::null};
+	};
+
+	// TODO: add adjacency range comp or inside curser
+
+	// TODO: unused
+	// mfs will only load a limited number of fragments per tick (1),
+	// so this tag will be set if we loaded a fragment and
+	// every tick we check all cursers for this tag and continue
+	// and remove once no fragment could be loaded anymore
+	// (internal)
+	struct TagCurserUnsatisfied {};
+
+} // Message::Components
+
+// handles fragments for messages
+// on new message: assign fuid
+// on new and update: mark as fragment dirty
+// on delete: mark as fragment dirty?
+class MessageFragmentStore : public RegistryMessageModelEventI, public ObjectStoreEventI {
+	protected:
+		Contact3Registry& _cr;
+		RegistryMessageModel& _rmm;
+		ObjectStore2& _os;
+		StorageBackendI& _sb;
+		bool _fs_ignore_event {false};
+
+		UUIDGenerator_128_128 _session_uuid_gen;
+
+		// for message components only
+		MessageSerializerCallbacks _sc;
+
+		void handleMessage(const Message3Handle& m);
+
+		void loadFragment(Message3Registry& reg, ObjectHandle oh);
+
+		bool syncFragToStorage(ObjectHandle oh, Message3Registry& reg);
+
+		struct SaveQueueEntry final {
+			uint64_t ts_since_dirty{0};
+			//std::vector<uint8_t> id;
+			ObjectHandle id;
+			Message3Registry* reg{nullptr};
+		};
+		std::deque<SaveQueueEntry> _fuid_save_queue;
+
+		struct ECQueueEntry final {
+			ObjectHandle fid;
+			Contact3 c;
+		};
+		std::deque<ECQueueEntry> _event_check_queue;
+
+		// range changed or fragment loaded.
+		// we only load a limited number of fragments at once,
+		// so we need to keep them dirty until nothing was loaded.
+		entt::dense_set<Contact3> _potentially_dirty_contacts;
+
+	public:
+		MessageFragmentStore(
+			Contact3Registry& cr,
+			RegistryMessageModel& rmm,
+			ObjectStore2& os,
+			StorageBackendI& sb
+		);
+		virtual ~MessageFragmentStore(void);
+
+		MessageSerializerCallbacks& getMSC(void);
+
+		float tick(float time_delta);
+
+	protected: // rmm
+		bool onEvent(const Message::Events::MessageConstruct& e) override;
+		bool onEvent(const Message::Events::MessageUpdated& e) override;
+
+	protected: // fs
+		bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override;
+		bool onEvent(const ObjectStore::Events::ObjectUpdate& e) override;
+};
+

src/fragment_store/message_serializer.cpp

@@ -0,0 +1,107 @@
+#include "./message_serializer.hpp"
+
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/contact/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <iostream>
+
+static Contact3 findContactByID(Contact3Registry& cr, const std::vector<uint8_t>& id) {
+	// TODO: id lookup table, this is very inefficent
+	for (const auto& [c_it, id_it] : cr.view<Contact::Components::ID>().each()) {
+		if (id == id_it.data) {
+			return c_it;
+		}
+	}
+
+	return entt::null;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactFrom>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactFrom>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactFrom>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactTo>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactTo>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactTo>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}

src/fragment_store/message_serializer.hpp

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+struct MessageSerializerCallbacks {
+	using Registry = Message3Registry;
+	using Handle = Message3Handle;
+
+	Contact3Registry& cr;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+
+// fwd
+namespace Message::Components {
+struct ContactFrom;
+struct ContactTo;
+}
+
+// make specializations known
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);

src/fragment_store/messages_meta_components.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <solanaceae/object_store/meta_components.hpp>
+
+namespace ObjectStore::Components {
+	struct MessagesVersion {
+		// messages Object version
+		// 1 -> text_json
+		uint16_t v {1};
+	};
+
+	struct MessagesTSRange {
+		// timestamp range within the fragment
+		uint64_t begin {0}; // newer msg -> higher number
+		uint64_t end {0};
+	};
+
+	struct MessagesContact {
+		std::vector<uint8_t> id;
+	};
+
+	// TODO: add src contact (self id)
+
+} // ObjectStore::Components
+
+// old
+namespace Fragment::Components {
+	struct MessagesTSRange : public ObjComp::MessagesTSRange {};
+	struct MessagesContact : public ObjComp::MessagesContact {};
+} // Fragment::Components
+
+#include "./messages_meta_components_id.inl"
+

src/fragment_store/messages_meta_components_id.inl

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "./messages_meta_components.hpp"
+
+#include <entt/core/type_info.hpp>
+
+// TODO: move more central
+#define DEFINE_COMP_ID(x) \
+template<> \
+constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
+	using namespace entt::literals; \
+	return #x##_hs; \
+} \
+template<> \
+constexpr std::string_view entt::type_name<x>::value() noexcept { \
+	return #x; \
+}
+
+// cross compiler stable ids
+
+DEFINE_COMP_ID(ObjComp::MessagesVersion)
+DEFINE_COMP_ID(ObjComp::MessagesTSRange)
+DEFINE_COMP_ID(ObjComp::MessagesContact)
+
+// old stuff
+//DEFINE_COMP_ID(FragComp::MessagesTSRange)
+//DEFINE_COMP_ID(FragComp::MessagesContact)
+
+#undef DEFINE_COMP_ID
+
+

src/fragment_store/register_mfs_json_message_components.cpp

@@ -0,0 +1,35 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/message_components.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::Timestamp>();
+	msc.registerDeSerializerJson<Message::Components::Timestamp>();
+	msc.registerSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerDeSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerSerializerJson<Message::Components::ContactFrom>();
+	msc.registerDeSerializerJson<Message::Components::ContactFrom>();
+	msc.registerSerializerJson<Message::Components::ContactTo>();
+	msc.registerDeSerializerJson<Message::Components::ContactTo>();
+	msc.registerSerializerJson<Message::Components::TagUnread>();
+	msc.registerDeSerializerJson<Message::Components::TagUnread>();
+	msc.registerSerializerJson<Message::Components::Read>();
+	msc.registerDeSerializerJson<Message::Components::Read>();
+	msc.registerSerializerJson<Message::Components::MessageText>();
+	msc.registerDeSerializerJson<Message::Components::MessageText>();
+	msc.registerSerializerJson<Message::Components::TagMessageIsAction>();
+	msc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
+
+	// files
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
+}
+

src/fragment_store/register_mfs_json_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/register_mfs_json_tox_message_components.cpp

@@ -0,0 +1,10 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/tox_message_components.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::ToxGroupMessageID>();
+	msc.registerDeSerializerJson<Message::Components::ToxGroupMessageID>();
+}
+

src/fragment_store/register_mfs_json_tox_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/uuid_generator.cpp

@@ -0,0 +1,68 @@
+#include "./uuid_generator.hpp"
+
+UUIDGenerator_128_128::UUIDGenerator_128_128(void) {
+	{ // random namespace
+		const auto num0 = _rng();
+		const auto num1 = _rng();
+		const auto num2 = _rng();
+		const auto num3 = _rng();
+
+		_uuid_namespace[0+0] = (num0 >> 0) & 0xff;
+		_uuid_namespace[0+1] = (num0 >> 8) & 0xff;
+		_uuid_namespace[0+2] = (num0 >> 16) & 0xff;
+		_uuid_namespace[0+3] = (num0 >> 24) & 0xff;
+
+		_uuid_namespace[4+0] = (num1 >> 0) & 0xff;
+		_uuid_namespace[4+1] = (num1 >> 8) & 0xff;
+		_uuid_namespace[4+2] = (num1 >> 16) & 0xff;
+		_uuid_namespace[4+3] = (num1 >> 24) & 0xff;
+
+		_uuid_namespace[8+0] = (num2 >> 0) & 0xff;
+		_uuid_namespace[8+1] = (num2 >> 8) & 0xff;
+		_uuid_namespace[8+2] = (num2 >> 16) & 0xff;
+		_uuid_namespace[8+3] = (num2 >> 24) & 0xff;
+
+		_uuid_namespace[12+0] = (num3 >> 0) & 0xff;
+		_uuid_namespace[12+1] = (num3 >> 8) & 0xff;
+		_uuid_namespace[12+2] = (num3 >> 16) & 0xff;
+		_uuid_namespace[12+3] = (num3 >> 24) & 0xff;
+	}
+}
+
+UUIDGenerator_128_128::UUIDGenerator_128_128(const std::array<uint8_t, 16>& uuid_namespace) :
+	_uuid_namespace(uuid_namespace)
+{
+}
+
+std::vector<uint8_t> UUIDGenerator_128_128::operator()(void) {
+	std::vector<uint8_t> new_uid(_uuid_namespace.cbegin(), _uuid_namespace.cend());
+	new_uid.resize(new_uid.size() + 16);
+
+	const auto num0 = _rng();
+	const auto num1 = _rng();
+	const auto num2 = _rng();
+	const auto num3 = _rng();
+
+	new_uid[_uuid_namespace.size()+0] = (num0 >> 0) & 0xff;
+	new_uid[_uuid_namespace.size()+1] = (num0 >> 8) & 0xff;
+	new_uid[_uuid_namespace.size()+2] = (num0 >> 16) & 0xff;
+	new_uid[_uuid_namespace.size()+3] = (num0 >> 24) & 0xff;
+
+	new_uid[_uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff;
+	new_uid[_uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff;
+	new_uid[_uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff;
+	new_uid[_uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff;
+
+	new_uid[_uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff;
+	new_uid[_uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff;
+	new_uid[_uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff;
+	new_uid[_uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff;
+
+	new_uid[_uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff;
+	new_uid[_uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff;
+	new_uid[_uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff;
+	new_uid[_uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff;
+
+	return new_uid;
+}
+

src/fragment_store/uuid_generator.hpp

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <vector>
+#include <array>
+#include <random>
+#include <cstdint>
+
+struct UUIDGeneratorI {
+	virtual std::vector<uint8_t> operator()(void) = 0;
+};
+
+// TODO: templates?
+struct UUIDGenerator_128_128 final : public UUIDGeneratorI {
+	private:
+		std::array<uint8_t, 16> _uuid_namespace;
+		std::minstd_rand _rng{std::random_device{}()};
+
+	public:
+		UUIDGenerator_128_128(void); // default randomly initializes namespace
+		UUIDGenerator_128_128(const std::array<uint8_t, 16>& uuid_namespace);
+
+		std::vector<uint8_t> operator()(void) override;
+};
+

src/json/message_components.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/message3/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts)
+	// TODO: SyncedBy
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text)
+
+	namespace Transfer {
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list)
+	} // Transfer
+
+} // Message::Components
+

src/json/tox_message_components.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/tox_messages/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	// TODO: friend msg id, does not have the same qualities
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id)
+	// TODO: transfer stuff, needs content rewrite
+
+} // Message::Components
+

src/main_screen.cpp

@@ -1,5 +1,8 @@
 #include "./main_screen.hpp"
 
+#include "./fragment_store/register_mfs_json_message_components.hpp"
+#include "./fragment_store/register_mfs_json_tox_message_components.hpp"
+
 #include <solanaceae/contact/components.hpp>
 
 #include <imgui/imgui.h>
@@ -13,6 +16,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	renderer(renderer_),
 	rmm(cr),
 	mts(rmm),
+	mfsb(os, "test2_message_store/"),
+	mfs(cr, rmm, os, mfsb),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
@@ -33,6 +38,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	tdch(tpi)
 {
 	tel.subscribeAll(tc);
+	registerMFSJsonMessageComponents(mfs.getMSC());
+	registerMFSJsonToxMessageComponents(mfs.getMSC());
 
 	conf.set("tox", "save_file_path", save_path);
 
@@ -76,6 +83,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	}
 
 	conf.dump();
+
+	mfsb.scanAsync(); // HACK: after plugins and tox contacts got loaded
 }
 
 MainScreen::~MainScreen(void) {
@@ -417,7 +426,8 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 
 	tdch.tick(time_delta); // compute
 
-	mts.iterate(); // compute
+	const float mfs_interval = mfs.tick(time_delta);
+	mts.iterate(); // compute (after mfs)
 
 	_min_tick_interval = std::min<float>(
 		// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
@@ -429,6 +439,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 		_min_tick_interval,
 		fo_interval
 	);
+	_min_tick_interval = std::min<float>(
+		_min_tick_interval,
+		mfs_interval
+	);
 
 	//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";
 

src/main_screen.hpp

@@ -3,10 +3,12 @@
 #include "./screen.hpp"
 
 #include <solanaceae/object_store/object_store.hpp>
+#include <solanaceae/object_store/backends/filesystem_storage.hpp>
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
+#include "./fragment_store/message_fragment_store.hpp"
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@@ -50,6 +52,8 @@ struct MainScreen final : public Screen {
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
 	MessageTimeSort mts;
+	backend::FilesystemStorage mfsb; // message fsb // TODO: make configurable
+	MessageFragmentStore mfs;
 
 	ToxEventLogger tel{std::cout};
 	ToxClient tc;

src/tox_client.hpp

@@ -48,4 +48,3 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase {
 		void saveToxProfile(void);
 };
 
-

src/tox_friend_faux_offline_messaging.cpp

@@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
 		// require
 		if (!mr->all_of<
 				Message::Components::MessageText, // text only for now
-				Message::Components::ContactTo
+				Message::Components::ContactTo,
+				Message::Components::ToxFriendMessageID // yes, needs fake ids
 			>(msg)
 		) {
 			continue; // skip