reduce excessive message frag saving (queue dedup + waiting 10sec)

prepare for frag updates

CMakeLists.txt

@@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			link_libraries(-fsanitize=address)
-			#link_libraries(-fsanitize=address,undefined)
+			#link_libraries(-fsanitize=address)
+			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			message("II enabled ASAN")
 		else()

external/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
 
 add_subdirectory(./entt)
 
@@ -19,3 +19,34 @@ add_subdirectory(./stb)
 add_subdirectory(./libwebp)
 add_subdirectory(./qoi)
 
+if (NOT TARGET nlohmann_json::nlohmann_json)
+	FetchContent_Declare(json
+		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(json)
+endif()
+
+if (NOT TARGET zstd::zstd)
+	# TODO: try find_package() first
+	# TODO: try pkg-config next (will work on most distros)
+
+	set(ZSTD_BUILD_STATIC ON)
+	set(ZSTD_BUILD_SHARED OFF)
+	set(ZSTD_BUILD_PROGRAMS OFF)
+	set(ZSTD_BUILD_CONTRIB OFF)
+	set(ZSTD_BUILD_TESTS OFF)
+	FetchContent_Declare(zstd
+		URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
+		DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+		SOURCE_SUBDIR build/cmake
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(zstd)
+
+	add_library(zstd INTERFACE) # somehow zstd fkd this up
+	target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/)
+	target_link_libraries(zstd INTERFACE libzstd_static)
+	add_library(zstd::zstd ALIAS zstd)
+endif()

flake.nix

@@ -12,13 +12,15 @@
     flake-utils.lib.eachDefaultSystem (system:
     let
       pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
     in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
         pname = "tomato";
         version = "0.0.0";
 
         src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing
 
         nativeBuildInputs = with pkgs; [
           cmake
@@ -58,6 +60,11 @@
         cmakeFlags = [
           "TOMATO_ASAN=1"
           "CMAKE_BUILD_TYPE=RelWithDebInfo"
+
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
+          # do we really care less about the version? do we need a stable abi?
+
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}"
         ];
 
         # TODO: replace with install command
@@ -66,7 +73,7 @@
           mv bin/tomato $out/bin
         '';
 
-        dontStrip = true;
+        dontStrip = true; # does nothing
 
         # copied from nixpkgs's SDL2 default.nix
         # SDL is weird in that instead of just dynamically linking with
@@ -93,6 +100,8 @@
           '';
       };
 
+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
       devShells.${system}.default = pkgs.mkShell {
         #inputsFrom = with pkgs; [ SDL2 ];
         buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?

src/CMakeLists.txt

@@ -1,5 +1,60 @@
 cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
+add_library(fragment_store
+	./fragment_store/fragment_store_i.hpp
+	./fragment_store/fragment_store_i.cpp
+	./fragment_store/types.hpp
+	./fragment_store/meta_components.hpp
+	./fragment_store/meta_components_id.inl
+	./fragment_store/serializer.hpp
+	./fragment_store/fragment_store.hpp
+	./fragment_store/fragment_store.cpp
+
+	./json/message_components.hpp # TODO: move
+	./json/tox_message_components.hpp # TODO: move
+)
+
+target_link_libraries(fragment_store PUBLIC
+	nlohmann_json::nlohmann_json
+	EnTT::EnTT
+	solanaceae_util
+
+	zstd::zstd
+
+	solanaceae_tox_messages # TODO: move
+)
+
+########################################
+
+add_library(message_fragment_store
+	./fragment_store/message_serializer.hpp
+	./fragment_store/message_serializer.cpp
+	./fragment_store/message_fragment_store.hpp
+	./fragment_store/message_fragment_store.cpp
+
+	./fragment_store/register_mfs_json_message_components.hpp
+	./fragment_store/register_mfs_json_message_components.cpp
+	./fragment_store/register_mfs_json_tox_message_components.hpp
+	./fragment_store/register_mfs_json_tox_message_components.cpp
+)
+target_compile_features(message_fragment_store PRIVATE cxx_std_20)
+target_link_libraries(message_fragment_store PUBLIC
+	fragment_store
+	solanaceae_message3
+)
+
+########################################
+
+add_executable(fragment_store_test
+	fragment_store/test_fragstore.cpp
+)
+
+target_link_libraries(fragment_store_test PUBLIC
+	fragment_store
+)
+
+########################################
+
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@@ -82,6 +137,9 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_tox_messages
 
+	fragment_store
+	message_fragment_store
+
 	SDL3::SDL3
 
 	imgui

src/chat_gui4.cpp

@@ -47,6 +47,18 @@ namespace Components {
 
 } // Components
 
+namespace Context {
+
+	// TODO: move back to chat log window and keep per window instead of per contact
+	struct CGView {
+		// set to the ts of the newest rendered msg
+		Message3Handle begin{};
+		// set to the ts of the oldest rendered msg
+		Message3Handle end{};
+	};
+
+} // Context
+
 static constexpr float lerp(float a, float b, float t) {
 	return a + t * (b - a);
 }
@@ -269,28 +281,6 @@ float ChatGui4::render(float time_delta) {
 
 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);
 
-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@@ -303,6 +293,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
 
+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;
 
@@ -389,12 +382,26 @@ float ChatGui4::render(float time_delta) {
 								}
 
 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
-									// get time now
-									const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
-									msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
-									msg_reg.remove<Message::Components::TagUnread>(e);
-									msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+								if (ImGui::IsItemVisible()) {
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
+										// get time now
+										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
+										msg_reg.remove<Message::Components::TagUnread>(e);
+										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+
+										// we remove the unread tag here
+										_rmm.throwEventUpdate(msg_reg, e);
+									}
+
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
 								}
 
 								// highlight self
@@ -559,9 +566,90 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 
+						{ // update view cursers
+							if (!msg_reg.ctx().contains<Context::CGView>()) {
+								msg_reg.ctx().emplace<Context::CGView>();
+							}
+
+							auto& cg_view = msg_reg.ctx().get<Context::CGView>();
+
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view, we setup a view at current time, so the next frags are loaded
+								if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
+									// fix invalid state
+									if (static_cast<bool>(cg_view.begin)) {
+										cg_view.begin.destroy();
+										_rmm.throwEventDestroy(cg_view.begin);
+									}
+									if (static_cast<bool>(cg_view.end)) {
+										cg_view.end.destroy();
+										_rmm.throwEventDestroy(cg_view.end);
+									}
+
+									// create new
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									cg_view.end = {msg_reg, msg_reg.create()};
+
+									cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+									cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+									cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+									cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(cg_view.begin);
+									std::cout << "CG: created view FRONT end ts\n";
+									_rmm.throwEventConstruct(cg_view.end);
+								} // else? we do nothing?
+							} else {
+								bool begin_created {false};
+								if (!static_cast<bool>(cg_view.begin)) {
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								bool end_created {false};
+								if (!static_cast<bool>(cg_view.end)) {
+									cg_view.end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+								cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+								{
+									auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+										old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+										if (begin_created) {
+											std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.begin);
+										} else {
+											//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.begin);
+										}
+									}
+								}
+
+								{
+									auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+										old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+										if (end_created) {
+											std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.end);
+										} else {
+											//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.end);
+										}
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}
 
+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}

src/chat_gui4.hpp

@@ -10,6 +10,9 @@
 #include "./file_selector.hpp"
 #include "./send_image_popup.hpp"
 
+// HACK: move to public msg api?
+#include "./fragment_store/message_fragment_store.hpp"
+
 #include <entt/container/dense_map.hpp>
 
 #include <cstdint>
@@ -32,6 +35,7 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;
 
+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;
 
 	// TODO: per contact

src/fragment_store/README.md

@@ -25,41 +25,41 @@ Just keeps the Fragments in memory.
 
 # File formats
 
-Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
+Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted.
 
 ### Text Json
 
 Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
 Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
 
-A Metadata json object has the following keys:
-- `enc` (uint) Encryption type of the data, if any
-- `comp` (uint) Compression type of the data, if any
-- `metadata` (obj) the 
+A Metadata json object can have arbitrary keys, some are predefined:
+- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any
+- `FragComp::DataCompressionType` (uint) Compression type of the data, if any
 
 ## Binary file headers
 
 ### Split Metadata
 
-file magic bytes `SOLMET` (6 bytes)
+msgpack array:
 
-1 byte encryption type (`0x00` is none)
-
-1 byte compression type (`0x00` is none)
-
-...metadata here...
+- `[0]`: file magic string `SOLMET` (6 bytes)
+- `[1]`: uint8 encryption type (`0x00` is none)
+- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd)
+- `[3]`: binary metadata (optionally compressed and encrypted)
 
 note that the encryption and compression are for the metadata only.
 The metadata itself contains encryption and compression info about the data.
 
 ### Split Data
 
-(none) all the data is in the metadata file.
+All the metadata is in the metadata file. (like encryption and compression)
 This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
 Keep in mind to not use the actual file name as the data/meta file name.
 
 ### Single fragment
 
+Note: this format is unused for now
+
 file magic bytes `SOLFIL` (6 bytes)
 
 1 byte encryption type (`0x00` is none)
@@ -70,3 +70,7 @@ file magic bytes `SOLFIL` (6 bytes)
 
 ...data here...
 
+## Compression types
+
+- `0x00` none
+- `0x01` zstd (without dict)

src/fragment_store/fragment_store.cpp

@@ -0,0 +1,866 @@
+#include "./fragment_store.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <entt/entity/handle.hpp>
+#include <entt/container/dense_set.hpp>
+#include <entt/core/hashed_string.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <zstd.h>
+
+#include <cstdint>
+#include <fstream>
+#include <filesystem>
+#include <memory>
+#include <mutex>
+#include <type_traits>
+#include <utility>
+#include <algorithm>
+
+#include <iostream>
+#include <vector>
+
+static const char* metaFileTypeSuffix(MetaFileType mft) {
+	switch (mft) {
+		case MetaFileType::TEXT_JSON: return ".json";
+		//case MetaFileType::BINARY_ARB: return ".bin";
+		case MetaFileType::BINARY_MSGPACK: return ".msgpack";
+	}
+	return ""; // .unk?
+}
+
+FragmentStore::FragmentStore(void) {
+	{ // random namespace
+		const auto num0 = _rng();
+		const auto num1 = _rng();
+		const auto num2 = _rng();
+		const auto num3 = _rng();
+
+		_session_uuid_namespace[0+0] = (num0 >> 0) & 0xff;
+		_session_uuid_namespace[0+1] = (num0 >> 8) & 0xff;
+		_session_uuid_namespace[0+2] = (num0 >> 16) & 0xff;
+		_session_uuid_namespace[0+3] = (num0 >> 24) & 0xff;
+
+		_session_uuid_namespace[4+0] = (num1 >> 0) & 0xff;
+		_session_uuid_namespace[4+1] = (num1 >> 8) & 0xff;
+		_session_uuid_namespace[4+2] = (num1 >> 16) & 0xff;
+		_session_uuid_namespace[4+3] = (num1 >> 24) & 0xff;
+
+		_session_uuid_namespace[8+0] = (num2 >> 0) & 0xff;
+		_session_uuid_namespace[8+1] = (num2 >> 8) & 0xff;
+		_session_uuid_namespace[8+2] = (num2 >> 16) & 0xff;
+		_session_uuid_namespace[8+3] = (num2 >> 24) & 0xff;
+
+		_session_uuid_namespace[12+0] = (num3 >> 0) & 0xff;
+		_session_uuid_namespace[12+1] = (num3 >> 8) & 0xff;
+		_session_uuid_namespace[12+2] = (num3 >> 16) & 0xff;
+		_session_uuid_namespace[12+3] = (num3 >> 24) & 0xff;
+	}
+	registerSerializers();
+}
+
+FragmentStore::FragmentStore(
+	std::array<uint8_t, 16> session_uuid_namespace
+) : _session_uuid_namespace(std::move(session_uuid_namespace)) {
+	registerSerializers();
+}
+
+FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) {
+	return {_reg, fid};
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(std::array<uint8_t, 16>& uuid_namespace) {
+	std::vector<uint8_t> new_uid(uuid_namespace.cbegin(), uuid_namespace.cend());
+	new_uid.resize(new_uid.size() + 16);
+
+	const auto num0 = _rng();
+	const auto num1 = _rng();
+	const auto num2 = _rng();
+	const auto num3 = _rng();
+
+	new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff;
+
+	return new_uid;
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(void) {
+	return generateNewUID(_session_uuid_namespace);
+}
+
+FragmentID FragmentStore::newFragmentMemoryOwned(
+	const std::vector<uint8_t>& id,
+	size_t initial_size
+) {
+	{ // first check if id is already used
+		auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	{ // next check if space in memory budget
+		const auto free_memory = _memory_budget - _memory_usage;
+		if (initial_size > free_memory) {
+			return entt::null;
+		}
+	}
+
+	// actually allocate and create
+	auto new_data = std::make_unique<std::vector<uint8_t>>(initial_size);
+	if (!static_cast<bool>(new_data)) {
+		// allocation failure
+		return entt::null;
+	}
+	_memory_usage += initial_size;
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+	// TODO: memory comp
+	_reg.emplace<std::unique_ptr<std::vector<uint8_t>>>(new_frag) = std::move(new_data);
+
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft,
+	const std::vector<uint8_t>& id
+) {
+	{ // first check if id is already used
+		const auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	if (store_path.empty()) {
+		store_path = _default_store_path;
+	}
+
+	std::filesystem::create_directories(store_path);
+
+	const auto id_hex = bin2hex(id);
+	std::filesystem::path fragment_file_path;
+
+	if (id_hex.size() < 6) {
+		fragment_file_path = std::filesystem::path{store_path}/id_hex;
+	} else {
+		// use the first 2hex (1byte) as a subfolder
+		std::filesystem::create_directories(std::string{store_path} + id_hex.substr(0, 2));
+		fragment_file_path = std::filesystem::path{std::string{store_path} + id_hex.substr(0, 2)} / id_hex.substr(2);
+	}
+
+	if (std::filesystem::exists(fragment_file_path)) {
+		return entt::null;
+	}
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+
+	// file (info) comp
+	_reg.emplace<FragComp::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
+
+	_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
+
+	// meta needs to be synced to file
+	std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
+	if (!syncToStorage(new_frag, empty_data_cb)) {
+		std::cerr << "FS error: syncToStorage failed while creating new fragment file\n";
+		_reg.destroy(new_frag);
+		return entt::null;
+	}
+
+	// while new metadata might be created here, making sure the file could be created is more important
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft
+) {
+	return newFragmentFile(store_path, mft, generateNewUID());
+}
+
+FragmentID FragmentStore::getFragmentByID(
+	const std::vector<uint8_t>& id
+) {
+	// TODO: accelerate
+	// maybe keep it sorted and binary search? hash table lookup?
+	for (const auto& [frag, id_comp] : _reg.view<FragComp::ID>().each()) {
+		if (id == id_comp.v) {
+			return frag;
+		}
+	}
+
+	return entt::null;
+}
+
+FragmentID FragmentStore::getFragmentCustomMatcher(
+	std::function<bool(FragmentID)>& fn
+) {
+	return entt::null;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		return false;
+	}
+
+	// split object storage
+
+	MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults
+	if (_reg.all_of<FragComp::Ephemeral::MetaFileType>(fid)) {
+		meta_type = _reg.get<FragComp::Ephemeral::MetaFileType>(fid).type;
+	}
+
+	Encryption meta_enc = Encryption::NONE; // TODO: better defaults
+	Compression meta_comp = Compression::NONE; // TODO: better defaults
+
+	if (meta_type != MetaFileType::TEXT_JSON) {
+		if (_reg.all_of<FragComp::Ephemeral::MetaEncryptionType>(fid)) {
+			meta_enc = _reg.get<FragComp::Ephemeral::MetaEncryptionType>(fid).enc;
+		}
+
+		if (_reg.all_of<FragComp::Ephemeral::MetaCompressionType>(fid)) {
+			meta_comp = _reg.get<FragComp::Ephemeral::MetaCompressionType>(fid).comp;
+		}
+	} else {
+		// we cant have encryption or compression
+
+		// TODO: warning/error?
+
+		// TODO: forcing for testing
+		//if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
+		//}
+		//if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
+		//}
+	}
+
+	std::filesystem::path meta_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp";
+	meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string());
+	std::ofstream meta_file{
+		meta_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!meta_file.is_open()) {
+		std::cerr << "FS error: failed to create temporary meta file\n";
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE; // TODO: better defaults
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	std::filesystem::path data_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".tmp";
+	data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string());
+	std::ofstream data_file{
+		data_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		meta_file.close();
+		std::filesystem::remove(meta_tmp_path);
+		std::cerr << "FS error: failed to create temporary data file\n";
+		return false;
+	}
+
+	// sharing code between binary msgpack and text json for now
+	nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed
+	// metadata file
+
+	for (const auto& [type_id, storage] : _reg.storage()) {
+		if (!storage.contains(fid)) {
+			continue;
+		}
+
+		//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
+
+		// use type_id to find serializer
+		auto s_cb_it = _sc._serl_json.find(type_id);
+		if (s_cb_it == _sc._serl_json.end()) {
+			// could not find serializer, not saving
+			continue;
+		}
+
+		// noooo, why cant numbers be keys
+		//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
+			//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
+		//} else if (meta_type == MetaFileType::TEXT_JSON) {
+		s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]);
+		//}
+	}
+
+	if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file
+		const std::vector<uint8_t> meta_data = nlohmann::json::to_msgpack(meta_data_j);
+		std::vector<uint8_t> meta_data_compressed; // empty if none
+		//std::vector<uint8_t> meta_data_encrypted; // empty if none
+
+		if (meta_comp == Compression::ZSTD) {
+			meta_data_compressed.resize(ZSTD_compressBound(meta_data.size()));
+
+			size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3
+			if (ZSTD_isError(cSize)) {
+				std::cerr << "FS error: compressing meta failed\n";
+				meta_data_compressed.clear();
+				meta_comp = Compression::NONE;
+			} else {
+				meta_data_compressed.resize(cSize);
+			}
+		} else if (meta_comp == Compression::NONE) {
+			// do nothing
+		} else {
+			assert(false && "implement me");
+		}
+
+		// TODO: encryption
+
+		// the meta file is itself msgpack data
+		nlohmann::json meta_header_j = nlohmann::json::array();
+		meta_header_j.emplace_back() = "SOLMET";
+		meta_header_j.push_back(meta_enc);
+		meta_header_j.push_back(meta_comp);
+
+		if (false) { // TODO: encryption
+		} else if (!meta_data_compressed.empty()) {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed));
+		} else {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data));
+		}
+
+		const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j);
+		meta_file.write(reinterpret_cast<const char*>(meta_header_data.data()), meta_header_data.size());
+	} else if (meta_type == MetaFileType::TEXT_JSON) {
+		// cant be compressed or encrypted
+		meta_file << meta_data_j.dump(2, ' ', true);
+	}
+
+	// now data
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			if (buffer_actual_size == 0) {
+				break;
+			}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+
+			data_file.write(reinterpret_cast<const char*>(buffer.data()), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> buffer(ZSTD_CStreamInSize());
+		std::vector<uint8_t> compressed_buffer(ZSTD_CStreamOutSize());
+		uint64_t buffer_actual_size {0};
+
+		std::unique_ptr<ZSTD_CCtx, decltype(&ZSTD_freeCCtx)> cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx};
+		ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, 0); // default (3)
+		ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?)
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			//if (buffer_actual_size == 0) {
+				//break;
+			//}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+			bool const lastChunk = (buffer_actual_size < buffer.size());
+
+			ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+			ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 };
+
+			while (input.pos < input.size) {
+				ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 };
+
+				size_t const remaining = ZSTD_compressStream2(cctx.get(), &output , &input, mode);
+				if (ZSTD_isError(remaining)) {
+					std::cerr << "FS error: compressing data failed\n";
+					break;
+				}
+
+				data_file.write(reinterpret_cast<const char*>(compressed_buffer.data()), output.pos);
+
+				if (remaining == 0) {
+					break;
+				}
+			}
+			// same as if lastChunk break;
+		} while (buffer_actual_size == buffer.size());
+	} else {
+		assert(false && "implement me");
+	}
+
+	meta_file.flush();
+	meta_file.close();
+	data_file.flush();
+	data_file.close();
+
+	std::filesystem::rename(
+		meta_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type)
+	);
+
+	std::filesystem::rename(
+		data_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path
+	);
+
+	// TODO: check return value of renames
+
+	if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
+		_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
+	}
+
+	return true;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) {
+	std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+		uint64_t i = 0;
+		for (; i+read < data_size && i < buffer_size; i++) {
+			request_buffer[i] = data[i+read];
+		}
+		read += i;
+
+		return i;
+	};
+	return syncToStorage(fid, fn_cb);
+}
+
+bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		// TODO: memory fragments
+		return false;
+	}
+
+	const auto& frag_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path;
+
+	// TODO: check if metadata dirty?
+	// TODO: what if file changed on disk?
+
+	std::cout << "FS: loading fragment '" << frag_path << "'\n";
+
+	std::ifstream data_file{
+		frag_path,
+		std::ios::in | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n";
+		// error
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE;
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(buffer.data()), buffer.size());
+			buffer_actual_size = data_file.gcount();
+
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			data_cb(buffer.data(), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size() && !data_file.eof());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> in_buffer(ZSTD_DStreamInSize());
+		std::vector<uint8_t> out_buffer(ZSTD_DStreamOutSize());
+		std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx};
+
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(in_buffer.data()), in_buffer.size());
+			buffer_actual_size = data_file.gcount();
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 };
+			do {
+				ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 };
+				size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input);
+				if (ZSTD_isError(ret)) {
+					// error <.<
+					std::cerr << "FS error: decompression error\n";
+					break;
+				}
+
+				data_cb(out_buffer.data(), output.pos);
+			} while (input.pos < input.size);
+		} while (buffer_actual_size == in_buffer.size() && !data_file.eof());
+	} else {
+		assert(false && "implement me");
+	}
+
+	return true;
+}
+
+nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) {
+	std::vector<uint8_t> tmp_buffer;
+	std::function<read_from_storage_put_data_cb> cb = [&tmp_buffer](const uint8_t* buffer, const uint64_t buffer_size) {
+		tmp_buffer.insert(tmp_buffer.end(), buffer, buffer+buffer_size);
+	};
+
+	if (!loadFromStorage(fid, cb)) {
+		return nullptr;
+	}
+
+	return nlohmann::json::parse(tmp_buffer);
+}
+
+size_t FragmentStore::scanStoragePath(std::string_view path) {
+	if (path.empty()) {
+		path = _default_store_path;
+	}
+	// TODO: extract so async can work (or/and make iteratable generator)
+
+	if (!std::filesystem::is_directory(path)) {
+		std::cerr << "FS error: scan path not a directory '" << path << "'\n";
+		return 0;
+	}
+
+	// step 1: make snapshot of files, validate metafiles and save id/path+meta.ext
+	// can be extra thread (if non vfs)
+	struct FragFileEntry {
+		std::string id_str;
+		std::filesystem::path frag_path;
+		std::string meta_ext;
+
+		bool operator==(const FragFileEntry& other) const {
+			// only compare by id
+			return id_str == other.id_str;
+		}
+	};
+	struct FragFileEntryHash {
+		size_t operator()(const FragFileEntry& it) const {
+			return entt::hashed_string(it.id_str.data(), it.id_str.size());
+		}
+	};
+	entt::dense_set<FragFileEntry, FragFileEntryHash> file_frag_list;
+
+	std::filesystem::path storage_path{path};
+
+	auto handle_file = [&](const std::filesystem::path& file_path) {
+		if (!std::filesystem::is_regular_file(file_path)) {
+			return;
+		}
+		// handle file
+
+		if (file_path.has_extension()) {
+			// skip over metadata, assuming only metafiles have extentions (might be wrong?)
+			// also skips temps
+			return;
+		}
+
+		auto relative_path = std::filesystem::proximate(file_path, storage_path);
+		std::string id_str = relative_path.generic_u8string();
+		// delete all '/'
+		id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end());
+		if (id_str.size() % 2 != 0) {
+			std::cerr << "FS error: non hex fragment uid detected: '" << id_str << "'\n";
+		}
+
+		if (file_frag_list.contains(FragFileEntry{id_str, {}, ""})) {
+			std::cerr << "FS error: fragment duplicate detected: '" << id_str << "'\n";
+			return; // skip
+		}
+
+		const char* meta_ext = ".meta.msgpack";
+		{ // find meta
+			// TODO: this as to know all possible extentions
+			bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack");
+			bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json");
+			const size_t meta_sum =
+				(has_meta_msgpack?1:0) +
+				(has_meta_json?1:0)
+			;
+
+			if (meta_sum > 1) { // has multiple
+				std::cerr << "FS error: fragment with multiple meta files detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (meta_sum == 0) {
+				std::cerr << "FS error: fragment missing meta file detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (has_meta_json) {
+				meta_ext = ".meta.json";
+			}
+		}
+
+		file_frag_list.emplace(FragFileEntry{
+			std::move(id_str),
+			file_path,
+			meta_ext
+		});
+	};
+
+	for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) {
+		if (std::filesystem::is_regular_file(outer_path)) {
+			handle_file(outer_path);
+		} else if (std::filesystem::is_directory(outer_path)) {
+			// subdir, part of id
+			for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) {
+				//if (std::filesystem::is_regular_file(inner_path)) {
+
+					//// handle file
+				//} // TODO: support deeper recursion?
+				handle_file(inner_path);
+			}
+		}
+	}
+
+	std::cout << "FS: scan found:\n";
+	for (const auto& it : file_frag_list) {
+		std::cout << "  " << it.id_str << "\n";
+	}
+
+	// step 2: check if files preexist in reg
+	// main thread
+	// (merge into step 3 and do more error checking?)
+	for (auto it = file_frag_list.begin(); it != file_frag_list.end();) {
+		auto id = hex2bin(it->id_str);
+		auto fid = getFragmentByID(id);
+		if (_reg.valid(fid)) {
+			// pre exising (handle differently??)
+			// check if store differs?
+			it = file_frag_list.erase(it);
+		} else {
+			it++;
+		}
+	}
+
+	std::vector<FragmentID> scanned_frags;
+	// step 3: parse meta and insert into reg of non preexising
+	// main thread
+	// TODO: check timestamps of preexisting and reload? mark external/remote dirty?
+	for (const auto& it : file_frag_list) {
+		nlohmann::json j;
+		if (it.meta_ext == ".meta.msgpack") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cout << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			// file is a msgpack within a msgpack
+
+			std::vector<uint8_t> full_meta_data;
+			{ // read meta file
+				// figure out size
+				file.seekg(0, file.end);
+				uint64_t file_size = file.tellg();
+				file.seekg(0, file.beg);
+
+				full_meta_data.resize(file_size);
+
+				file.read(reinterpret_cast<char*>(full_meta_data.data()), full_meta_data.size());
+			}
+
+			const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data);
+
+			if (!meta_header_j.is_array() || meta_header_j.size() < 4) {
+				std::cerr << "FS error: broken binary meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			if (meta_header_j.at(0) != "SOLMET") {
+				std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			Encryption meta_enc = meta_header_j.at(1);
+			if (meta_enc != Encryption::NONE) {
+				std::cerr << "FS error: unknown encryption " << it.frag_path << "\n";
+				continue;
+			}
+
+			Compression meta_comp = meta_header_j.at(2);
+			if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) {
+				std::cerr << "FS error: unknown compression " << it.frag_path << "\n";
+				continue;
+			}
+
+			//const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data");
+			if (!meta_header_j.at(3).is_binary()) {
+				std::cerr << "FS error: meta data not binary " << it.frag_path << "\n";
+				continue;
+			}
+			const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3);
+
+			std::vector<uint8_t> meta_data_decomp;
+			if (meta_comp == Compression::NONE) {
+				// do nothing
+			} else if (meta_comp == Compression::ZSTD) {
+				meta_data_decomp.resize(ZSTD_DStreamOutSize());
+				std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx};
+
+				ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0};
+				ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0};
+				do {
+					size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input);
+					if (ZSTD_isError(ret)) {
+						// error <.<
+						std::cerr << "FS error: decompression error\n";
+						//meta_data_decomp.clear();
+						output.pos = 0; // resize after loop
+						break;
+					}
+				} while (input.pos < input.size);
+				meta_data_decomp.resize(output.pos);
+			} else {
+				assert(false && "implement me");
+			}
+
+			// TODO: enc
+
+			if (!meta_data_decomp.empty()) {
+				j = nlohmann::json::from_msgpack(meta_data_decomp);
+			} else {
+				j = nlohmann::json::from_msgpack(meta_data_ref);
+			}
+		} else if (it.meta_ext == ".meta.json") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cerr << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			file >> j;
+		} else {
+			assert(false);
+		}
+
+		if (!j.is_object()) {
+			std::cerr << "FS error: json in meta is broken " << it.id_str << "\n";
+			continue;
+		}
+
+		// TODO: existing fragment file
+		//newFragmentFile();
+		FragmentHandle fh{_reg, _reg.create()};
+		fh.emplace<FragComp::ID>(hex2bin(it.id_str));
+
+		fh.emplace<FragComp::Ephemeral::FilePath>(it.frag_path.generic_u8string());
+
+		for (const auto& [k, v] : j.items()) {
+			// type id from string hash
+			const auto type_id = entt::hashed_string(k.data(), k.size());
+			const auto deserl_fn_it = _sc._deserl_json.find(type_id);
+			if (deserl_fn_it != _sc._deserl_json.cend()) {
+				// TODO: check return value
+				deserl_fn_it->second(fh, v);
+			} else {
+				std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n";
+			}
+		}
+		scanned_frags.push_back(fh);
+	}
+
+	// TODO: mutex and move code to async and return this list ?
+
+	// throw new frag event here, after loading them all
+	for (const FragmentID fid : scanned_frags) {
+		throwEventConstruct(fid);
+	}
+
+	return scanned_frags.size();
+}
+
+void FragmentStore::scanStoragePathAsync(std::string path) {
+	// add path to queue
+	// HACK: if path is known/any fragment is in the path, this operation blocks (non async)
+	scanStoragePath(path); // TODO: make async and post result
+}
+
+static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Encryption>>(
+		fh.get<FragComp::DataEncryptionType>().enc
+	);
+	return true;
+}
+
+static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataEncryptionType>(
+		static_cast<Encryption>(
+			static_cast<std::underlying_type_t<Encryption>>(in)
+		)
+	);
+	return true;
+}
+
+static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Compression>>(
+		fh.get<FragComp::DataCompressionType>().comp
+	);
+	return true;
+}
+
+static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataCompressionType>(
+		static_cast<Compression>(
+			static_cast<std::underlying_type_t<Compression>>(in)
+		)
+	);
+	return true;
+}
+
+void FragmentStore::registerSerializers(void) {
+	_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
+	_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
+	_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
+	_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
+}
+

src/fragment_store/fragment_store.hpp

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "./fragment_store_i.hpp"
+
+#include "./types.hpp"
+#include "./meta_components.hpp"
+
+#include "./serializer.hpp"
+
+#include <entt/core/type_info.hpp>
+#include <entt/entity/registry.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+#include <vector>
+#include <array>
+#include <cstdint>
+#include <random>
+
+struct FragmentStore : public FragmentStoreI {
+	std::minstd_rand _rng{std::random_device{}()};
+	std::array<uint8_t, 16> _session_uuid_namespace;
+
+	std::string _default_store_path;
+
+	uint64_t _memory_budget {10u*1024u*1024u};
+	uint64_t _memory_usage {0u};
+
+	SerializerCallbacks<FragmentID> _sc;
+
+	FragmentStore(void);
+	FragmentStore(std::array<uint8_t, 16> session_uuid_namespace);
+
+	// HACK: get access to the reg
+	FragmentHandle fragmentHandle(FragmentID fid);
+
+	// TODO: make the frags ref counted
+
+	// TODO: check for exising
+	std::vector<uint8_t> generateNewUID(std::array<uint8_t, 16>& uuid_namespace);
+	std::vector<uint8_t> generateNewUID(void);
+
+	// ========== new fragment ==========
+
+	// memory backed owned
+	FragmentID newFragmentMemoryOwned(
+		const std::vector<uint8_t>& id,
+		size_t initial_size
+	);
+
+	// memory backed view (can only be added? not new?)
+
+	// file backed (rw...)
+	// needs to know which store path to put into
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft,
+		const std::vector<uint8_t>& id
+	);
+	// this variant generate a new, mostly unique, id for us
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft
+	);
+
+	// ========== add fragment ==========
+
+	// ========== get fragment ==========
+	FragmentID getFragmentByID(
+		const std::vector<uint8_t>& id
+	);
+	FragmentID getFragmentCustomMatcher(
+		std::function<bool(FragmentID)>& fn
+	);
+
+	// remove fragment?
+	// unload?
+
+	// ========== sync fragment to storage ==========
+	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
+	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
+	bool syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+	bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size);
+
+	// ========== load fragment data from storage ==========
+	using read_from_storage_put_data_cb = void(const uint8_t* buffer, const uint64_t buffer_size);
+	bool loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb);
+	// convenience function
+	nlohmann::json loadFromStorageNJ(FragmentID fid);
+
+	// fragment discovery?
+	// returns number of new fragments
+	size_t scanStoragePath(std::string_view path);
+	void scanStoragePathAsync(std::string path);
+
+	private:
+		void registerSerializers(void); // internal comps
+		// internal actual backends
+		// TODO: seperate out
+		bool syncToMemory(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+		bool syncToFile(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+};
+

src/fragment_store/fragment_store_i.cpp

@@ -0,0 +1,23 @@
+#include "./fragment_store_i.hpp"
+
+#include <iostream>
+
+void FragmentStoreI::throwEventConstruct(const FragmentID fid) {
+	std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_construct,
+		Fragment::Events::FragmentConstruct{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}
+
+void FragmentStoreI::throwEventUpdate(const FragmentID fid) {
+	std::cout << "FSI debug: event updated " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_updated,
+		Fragment::Events::FragmentUpdated{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}

src/fragment_store/fragment_store_i.hpp

@@ -0,0 +1,59 @@
+#pragma once
+
+#include <solanaceae/util/event_provider.hpp>
+
+#include <entt/entity/registry.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <cstdint>
+
+// internal id
+enum class FragmentID : uint32_t {};
+using FragmentRegistry = entt::basic_registry<FragmentID>;
+using FragmentHandle = entt::basic_handle<FragmentRegistry>;
+
+namespace Fragment::Events {
+
+	struct FragmentConstruct {
+		const FragmentHandle e;
+	};
+	struct FragmentUpdated {
+		const FragmentHandle e;
+	};
+	//struct MessageDestory {
+		//const Message3Handle e;
+	//};
+
+} // Fragment::Events
+
+enum class FragmentStore_Event : uint32_t {
+	fragment_construct,
+	fragment_updated,
+	//message_destroy,
+
+	MAX
+};
+
+struct FragmentStoreEventI {
+	using enumType = FragmentStore_Event;
+
+	virtual ~FragmentStoreEventI(void) {}
+
+	virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; }
+	virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; }
+	//virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; }
+};
+using FragmentStoreEventProviderI = EventProviderI<FragmentStoreEventI>;
+
+struct FragmentStoreI  : public FragmentStoreEventProviderI {
+	static constexpr const char* version {"1"};
+
+	FragmentRegistry _reg;
+
+	virtual ~FragmentStoreI(void) {}
+
+	void throwEventConstruct(const FragmentID fid);
+	void throwEventUpdate(const FragmentID fid);
+	//void throwEventDestroy();
+};
+

src/fragment_store/message_fragment_store.hpp

@@ -0,0 +1,133 @@
+#pragma once
+
+#include "./meta_components.hpp"
+#include "./fragment_store_i.hpp"
+#include "./fragment_store.hpp"
+
+#include "./message_serializer.hpp"
+
+#include <entt/entity/registry.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/container/dense_set.hpp>
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <deque>
+#include <vector>
+#include <cstdint>
+
+namespace Message::Components {
+
+	// unused, consumes too much memory (highly compressable)
+	//using FUID = FragComp::ID;
+
+	struct FID {
+		FragmentID fid {entt::null};
+	};
+
+	// points to the front/newer message
+	// together they define a range that is,
+	// eg the first(end) and last(begin) message being rendered
+	// MFS requires there to be atleast one other fragment after/before,
+	// if not loaded fragment with fitting tsrange(direction) available
+	// uses fragmentAfter/Before()
+	// they can exist standalone
+	// if they are a pair, the inside is filled first
+	// cursers require a timestamp ???
+	struct ViewCurserBegin {
+		Message3 curser_end{entt::null};
+	};
+	struct ViewCurserEnd {
+		Message3 curser_begin{entt::null};
+	};
+
+	// TODO: add adjacency range comp or inside curser
+
+	// TODO: unused
+	// mfs will only load a limited number of fragments per tick (1),
+	// so this tag will be set if we loaded a fragment and
+	// every tick we check all cursers for this tag and continue
+	// and remove once no fragment could be loaded anymore
+	// (internal)
+	struct TagCurserUnsatisfied {};
+
+} // Message::Components
+
+namespace Fragment::Components {
+	struct MessagesTSRange {
+		// timestamp range within the fragment
+		uint64_t begin {0}; // newer msg -> higher number
+		uint64_t end {0};
+	};
+
+	struct MessagesContact {
+		std::vector<uint8_t> id;
+	};
+
+	// TODO: add src contact (self id)
+
+} // Fragment::Components
+
+// handles fragments for messages
+// on new message: assign fuid
+// on new and update: mark as fragment dirty
+// on delete: mark as fragment dirty?
+class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentStoreEventI {
+	protected:
+		Contact3Registry& _cr;
+		RegistryMessageModel& _rmm;
+		FragmentStore& _fs;
+		bool _fs_ignore_event {false};
+
+		// for message components only
+		MessageSerializerCallbacks _sc;
+
+		void handleMessage(const Message3Handle& m);
+
+		void loadFragment(Message3Registry& reg, FragmentHandle fh);
+
+		bool syncFragToStorage(FragmentHandle fh, Message3Registry& reg);
+
+		struct SaveQueueEntry final {
+			uint64_t ts_since_dirty{0};
+			//std::vector<uint8_t> id;
+			FragmentID id;
+			Message3Registry* reg{nullptr};
+		};
+		std::deque<SaveQueueEntry> _fuid_save_queue;
+
+		struct ECQueueEntry final {
+			FragmentID fid;
+			Contact3 c;
+		};
+		std::deque<ECQueueEntry> _event_check_queue;
+
+		// range changed or fragment loaded.
+		// we only load a limited number of fragments at once,
+		// so we need to keep them dirty until nothing was loaded.
+		entt::dense_set<Contact3> _potentially_dirty_contacts;
+
+	public:
+		MessageFragmentStore(
+			Contact3Registry& cr,
+			RegistryMessageModel& rmm,
+			FragmentStore& fs
+		);
+		virtual ~MessageFragmentStore(void);
+
+		MessageSerializerCallbacks& getMSC(void);
+
+		float tick(float time_delta);
+
+		void triggerScan(void);
+
+	protected: // rmm
+		bool onEvent(const Message::Events::MessageConstruct& e) override;
+		bool onEvent(const Message::Events::MessageUpdated& e) override;
+
+	protected: // fs
+		bool onEvent(const Fragment::Events::FragmentConstruct& e) override;
+		bool onEvent(const Fragment::Events::FragmentUpdated& e) override;
+};
+

src/fragment_store/message_serializer.cpp

@@ -0,0 +1,107 @@
+#include "./message_serializer.hpp"
+
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/contact/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <iostream>
+
+static Contact3 findContactByID(Contact3Registry& cr, const std::vector<uint8_t>& id) {
+	// TODO: id lookup table, this is very inefficent
+	for (const auto& [c_it, id_it] : cr.view<Contact::Components::ID>().each()) {
+		if (id == id_it.data) {
+			return c_it;
+		}
+	}
+
+	return entt::null;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactFrom>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactFrom>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactFrom>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactTo>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactTo>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactTo>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}

src/fragment_store/message_serializer.hpp

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+struct MessageSerializerCallbacks {
+	using Registry = Message3Registry;
+	using Handle = Message3Handle;
+
+	Contact3Registry& cr;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+
+// fwd
+namespace Message::Components {
+struct ContactFrom;
+struct ContactTo;
+}
+
+// make specializations known
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);

src/fragment_store/meta_components.hpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "./types.hpp"
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+namespace Fragment::Components {
+
+	// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
+	struct ID {
+		std::vector<uint8_t> v;
+	};
+
+	struct DataEncryptionType {
+		Encryption enc {Encryption::NONE};
+	};
+
+	struct DataCompressionType {
+		Compression comp {Compression::NONE};
+	};
+
+
+	// meta that is not written to (meta-)file
+	namespace Ephemeral {
+
+		// excluded from file meta
+		struct FilePath {
+			// contains store path, if any
+			std::string path;
+		};
+
+		// TODO: seperate into remote and local?
+		// (remote meaning eg. the file on disk was changed by another program)
+		struct DirtyTag {};
+
+
+		// type as comp
+		struct MetaFileType {
+			::MetaFileType type {::MetaFileType::TEXT_JSON};
+		};
+
+		struct MetaEncryptionType {
+			Encryption enc {Encryption::NONE};
+		};
+
+		struct MetaCompressionType {
+			Compression comp {Compression::NONE};
+		};
+
+	} // Ephemeral
+
+} // Components
+
+// shortened to save bytes (until I find a way to save by ID in msgpack)
+namespace FragComp = Fragment::Components;
+
+#include "./meta_components_id.inl"
+

src/fragment_store/meta_components_id.inl

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "./meta_components.hpp"
+
+#include <entt/core/type_info.hpp>
+
+// TODO: move more central
+#define DEFINE_COMP_ID(x) \
+template<> \
+constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
+	using namespace entt::literals; \
+	return #x##_hs; \
+} \
+template<> \
+constexpr std::string_view entt::type_name<x>::value() noexcept { \
+	return #x; \
+}
+
+// cross compiler stable ids
+
+DEFINE_COMP_ID(FragComp::DataEncryptionType)
+DEFINE_COMP_ID(FragComp::DataCompressionType)
+
+#undef DEFINE_COMP_ID
+
+

src/fragment_store/register_mfs_json_message_components.cpp

@@ -0,0 +1,35 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/message_components.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::Timestamp>();
+	msc.registerDeSerializerJson<Message::Components::Timestamp>();
+	msc.registerSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerDeSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerSerializerJson<Message::Components::ContactFrom>();
+	msc.registerDeSerializerJson<Message::Components::ContactFrom>();
+	msc.registerSerializerJson<Message::Components::ContactTo>();
+	msc.registerDeSerializerJson<Message::Components::ContactTo>();
+	msc.registerSerializerJson<Message::Components::TagUnread>();
+	msc.registerDeSerializerJson<Message::Components::TagUnread>();
+	msc.registerSerializerJson<Message::Components::Read>();
+	msc.registerDeSerializerJson<Message::Components::Read>();
+	msc.registerSerializerJson<Message::Components::MessageText>();
+	msc.registerDeSerializerJson<Message::Components::MessageText>();
+	msc.registerSerializerJson<Message::Components::TagMessageIsAction>();
+	msc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
+
+	// files
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
+}
+

src/fragment_store/register_mfs_json_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/register_mfs_json_tox_message_components.cpp

@@ -0,0 +1,10 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/tox_message_components.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::ToxGroupMessageID>();
+	msc.registerDeSerializerJson<Message::Components::ToxGroupMessageID>();
+}
+

src/fragment_store/register_mfs_json_tox_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/serializer.hpp

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+template<typename EntityType = entt::entity>
+struct SerializerCallbacks {
+	using Registry = entt::basic_registry<EntityType>;
+	using Handle = entt::basic_handle<Registry>;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+

src/fragment_store/test_fragstore.cpp

@@ -0,0 +1,80 @@
+#include <cstdint>
+#include <iostream>
+
+#include "./fragment_store.hpp"
+
+#include <nlohmann/json.hpp>
+#include <entt/entity/handle.hpp>
+
+namespace Components {
+	struct MessagesTimestampRange {
+		uint64_t begin {0};
+		uint64_t end {1000};
+	};
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end)
+} // Components
+
+
+int main(void) {
+	FragmentStore fs;
+	fs._default_store_path = "test_store/";
+	fs._sc.registerSerializerJson<Components::MessagesTimestampRange>();
+	fs._sc.registerDeSerializerJson<Components::MessagesTimestampRange>();
+
+	const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9});
+
+	const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	const auto frag2 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	{
+		auto frag0h = fs.fragmentHandle(frag0);
+
+		frag0h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag0h.emplace_or_replace<FragComp::DataEncryptionType>();
+		frag0h.emplace_or_replace<Components::MessagesTimestampRange>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			uint64_t i = 0;
+			for (; i+read < 3000 && i < buffer_size; i++) {
+				request_buffer[i] = uint8_t((i+read) & 0xff);
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag0, fn_cb);
+	}
+
+	{
+		auto frag1h = fs.fragmentHandle(frag1);
+
+		frag1h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag1h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			static constexpr std::string_view text = "This is some random data";
+			uint64_t i = 0;
+			for (; i+read < text.size() && i < buffer_size; i++) {
+				request_buffer[i] = text[i+read];
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag1, fn_cb);
+	}
+
+	{
+		auto frag2h = fs.fragmentHandle(frag2);
+
+		frag2h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag2h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		static constexpr std::string_view text = "This is more random data";
+		fs.syncToStorage(frag2, reinterpret_cast<const uint8_t*>(text.data()), text.size());
+	}
+	return 0;
+}
+

src/fragment_store/types.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+
+enum class Encryption : uint8_t {
+	NONE = 0x00,
+};
+enum class Compression : uint8_t {
+	NONE = 0x00,
+	ZSTD = 0x01,
+	// TODO: zstd without magic
+	// TODO: zstd meta dict
+	// TODO: zstd data(message) dict
+};
+enum class MetaFileType : uint8_t {
+	TEXT_JSON,
+	BINARY_MSGPACK, // msgpacked msgpack
+};
+

src/json/message_components.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/message3/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts)
+	// TODO: SyncedBy
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text)
+
+	namespace Transfer {
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list)
+	} // Transfer
+
+} // Message::Components
+

src/json/tox_message_components.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/tox_messages/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	// TODO: friend msg id, does not have the same qualities
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id)
+	// TODO: transfer stuff, needs content rewrite
+
+} // Message::Components
+

src/main.cpp

@@ -208,10 +208,17 @@ int main(int argc, char** argv) {
 #endif
 	}
 
+	// TODO: use scope for the unique ptrs
+
+	screen.reset();
+
 	ImGui_ImplSDLRenderer3_Shutdown();
 	ImGui_ImplSDL3_Shutdown();
 	ImGui::DestroyContext();
 
+	renderer.reset();
+	window.reset();
+
 	SDL_Quit();
 
 	return 0;

src/main_screen.cpp

@@ -1,5 +1,8 @@
 #include "./main_screen.hpp"
 
+#include "./fragment_store/register_mfs_json_message_components.hpp"
+#include "./fragment_store/register_mfs_json_tox_message_components.hpp"
+
 #include <solanaceae/contact/components.hpp>
 
 #include <imgui/imgui.h>
@@ -13,6 +16,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	renderer(renderer_),
 	rmm(cr),
 	mts(rmm),
+	mfs(cr, rmm, fs),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
@@ -33,6 +37,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	tdch(tpi)
 {
 	tel.subscribeAll(tc);
+	registerMFSJsonMessageComponents(mfs.getMSC());
+	registerMFSJsonToxMessageComponents(mfs.getMSC());
 
 	conf.set("tox", "save_file_path", save_path);
 
@@ -49,6 +55,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";
 
 	{ // setup plugin instances
+		// TODO: make interface useful
+		g_provideInstance<FragmentStoreI>("FragmentStoreI", "host", &fs);
+		g_provideInstance<FragmentStore>("FragmentStore", "host", &fs);
+
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
@@ -74,6 +84,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	}
 
 	conf.dump();
+
+	mfs.triggerScan(); // HACK: after plugins and tox contacts got loaded
 }
 
 MainScreen::~MainScreen(void) {
@@ -215,9 +227,24 @@ Screen* MainScreen::render(float time_delta, bool&) {
 					ImGui::EndMenu();
 				}
 				if (ImGui::BeginMenu("Settings")) {
-					if (ImGui::MenuItem("ImGui Style Editor")) {
+					ImGui::SeparatorText("ImGui");
+
+					if (ImGui::MenuItem("Style Editor")) {
 						_show_tool_style_editor = true;
 					}
+
+					if (ImGui::MenuItem("Metrics")) {
+						_show_tool_metrics = true;
+					}
+
+					if (ImGui::MenuItem("Debug Log")) {
+						_show_tool_debug_log = true;
+					}
+
+					if (ImGui::MenuItem("ID Stack Tool")) {
+						_show_tool_id_stack = true;
+					}
+
 					ImGui::EndMenu();
 				}
 				ImGui::EndMenuBar();
@@ -234,6 +261,18 @@ Screen* MainScreen::render(float time_delta, bool&) {
 		ImGui::End();
 	}
 
+	if (_show_tool_metrics) {
+		ImGui::ShowMetricsWindow(&_show_tool_metrics);
+	}
+
+	if (_show_tool_debug_log) {
+		ImGui::ShowDebugLogWindow(&_show_tool_debug_log);
+	}
+
+	if (_show_tool_id_stack) {
+		ImGui::ShowIDStackToolWindow(&_show_tool_id_stack);
+	}
+
 	if constexpr (false) {
 		ImGui::ShowDemoWindow();
 	}
@@ -388,7 +427,8 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 
 	tdch.tick(time_delta); // compute
 
-	mts.iterate(); // compute
+	const float mfs_interval = mfs.tick(time_delta);
+	mts.iterate(); // compute (after mfs)
 
 	_min_tick_interval = std::min<float>(
 		// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
@@ -400,6 +440,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 		_min_tick_interval,
 		fo_interval
 	);
+	_min_tick_interval = std::min<float>(
+		_min_tick_interval,
+		mfs_interval
+	);
 
 	//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";
 

src/main_screen.hpp

@@ -2,10 +2,12 @@
 
 #include "./screen.hpp"
 
+#include "./fragment_store/fragment_store.hpp"
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
+#include "./fragment_store/message_fragment_store.hpp"
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@@ -43,10 +45,13 @@ extern "C" {
 struct MainScreen final : public Screen {
 	SDL_Renderer* renderer;
 
+	FragmentStore fs;
+
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
 	MessageTimeSort mts;
+	MessageFragmentStore mfs;
 
 	PluginManager pm;
 
@@ -76,6 +81,9 @@ struct MainScreen final : public Screen {
 	ToxDHTCapHisto tdch;
 
 	bool _show_tool_style_editor {false};
+	bool _show_tool_metrics {false};
+	bool _show_tool_debug_log {false};
+	bool _show_tool_id_stack {false};
 
 	bool _window_hidden {false};
 	uint64_t _window_hidden_ts {0};

src/tox_friend_faux_offline_messaging.cpp

@@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
 		// require
 		if (!mr->all_of<
 				Message::Components::MessageText, // text only for now
-				Message::Components::ContactTo
+				Message::Components::ContactTo,
+				Message::Components::ToxFriendMessageID // yes, needs fake ids
 			>(msg)
 		) {
 			continue; // skip