make adjacency loading work, extend range and use loops

disable funky load at first msg

CMakeLists.txt

@@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			link_libraries(-fsanitize=address)
+			#link_libraries(-fsanitize=address)
-			#link_libraries(-fsanitize=address,undefined)
+			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			message("II enabled ASAN")
 		else()

README.md

@@ -2,3 +2,7 @@
 
 ![tomato](res/icon/tomato_v1_256.png)
 
+## Highly experimental solanaceae client with Tox built-in
+
+![group chat](res/tomato_screenshot_group_bot_text_23-02-2024.png)
+

external/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
 
 add_subdirectory(./entt)
 
@@ -18,3 +18,34 @@ add_subdirectory(./imgui)
 add_subdirectory(./stb)
 add_subdirectory(./libwebp)
 
+if (NOT TARGET nlohmann_json::nlohmann_json)
+	FetchContent_Declare(json
+		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(json)
+endif()
+
+if (NOT TARGET zstd::zstd)
+	# TODO: try find_package() first
+	# TODO: try pkg-config next (will work on most distros)
+
+	set(ZSTD_BUILD_STATIC ON)
+	set(ZSTD_BUILD_SHARED OFF)
+	set(ZSTD_BUILD_PROGRAMS OFF)
+	set(ZSTD_BUILD_CONTRIB OFF)
+	set(ZSTD_BUILD_TESTS OFF)
+	FetchContent_Declare(zstd
+		URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
+		DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+		SOURCE_SUBDIR build/cmake
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(zstd)
+
+	add_library(zstd INTERFACE) # somehow zstd fkd this up
+	target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/)
+	target_link_libraries(zstd INTERFACE libzstd_static)
+	add_library(zstd::zstd ALIAS zstd)
+endif()

external/solanaceae_contact

@@ -1 +1 @@
-Subproject commit 6f3f9ef1911ce8189609ba9d6c7a5931ab1efc69
+Subproject commit e40271670b4df96a8d02f32a1ba61a838419db48

external/solanaceae_message3

@@ -1 +1 @@
-Subproject commit c73f727adc99b2b73d09b7ebaa696b882ce535b6
+Subproject commit b8893b1c5ca49178f6ab96db18bf72a50f6b6653

external/solanaceae_tox

@@ -1 +1 @@
-Subproject commit b2a3cb7052041bea2d4793b8446fb2a00411773a
+Subproject commit 32241423ab2783930e1d1e5ba64a67709cf20b9b

flake.nix

@@ -12,13 +12,15 @@
     flake-utils.lib.eachDefaultSystem (system:
     let
       pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
     in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
         pname = "tomato";
         version = "0.0.0";
 
         src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing
 
         nativeBuildInputs = with pkgs; [
           cmake
@@ -58,6 +60,11 @@
         cmakeFlags = [
           "TOMATO_ASAN=1"
           "CMAKE_BUILD_TYPE=RelWithDebInfo"
+
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
+          # do we really care less about the version? do we need a stable abi?
+
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}"
         ];
 
         # TODO: replace with install command
@@ -66,7 +73,7 @@
           mv bin/tomato $out/bin
         '';
 
-        dontStrip = true;
+        dontStrip = true; # does nothing
 
         # copied from nixpkgs's SDL2 default.nix
         # SDL is weird in that instead of just dynamically linking with
@@ -93,6 +100,8 @@
           '';
       };
 
+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
       devShells.${system}.default = pkgs.mkShell {
         #inputsFrom = with pkgs; [ SDL2 ];
         buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?

src/CMakeLists.txt

@@ -1,5 +1,60 @@
 cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
+add_library(fragment_store
+	./fragment_store/fragment_store_i.hpp
+	./fragment_store/fragment_store_i.cpp
+	./fragment_store/types.hpp
+	./fragment_store/meta_components.hpp
+	./fragment_store/meta_components_id.inl
+	./fragment_store/serializer.hpp
+	./fragment_store/fragment_store.hpp
+	./fragment_store/fragment_store.cpp
+
+	./json/message_components.hpp # TODO: move
+	./json/tox_message_components.hpp # TODO: move
+)
+
+target_link_libraries(fragment_store PUBLIC
+	nlohmann_json::nlohmann_json
+	EnTT::EnTT
+	solanaceae_util
+
+	zstd::zstd
+
+	solanaceae_tox_messages # TODO: move
+)
+
+########################################
+
+add_library(message_fragment_store
+	./fragment_store/message_serializer.hpp
+	./fragment_store/message_serializer.cpp
+	./fragment_store/message_fragment_store.hpp
+	./fragment_store/message_fragment_store.cpp
+
+	./fragment_store/register_mfs_json_message_components.hpp
+	./fragment_store/register_mfs_json_message_components.cpp
+	./fragment_store/register_mfs_json_tox_message_components.hpp
+	./fragment_store/register_mfs_json_tox_message_components.cpp
+)
+target_compile_features(message_fragment_store PRIVATE cxx_std_20)
+target_link_libraries(message_fragment_store PUBLIC
+	fragment_store
+	solanaceae_message3
+)
+
+########################################
+
+add_executable(fragment_store_test
+	fragment_store/test_fragstore.cpp
+)
+
+target_link_libraries(fragment_store_test PUBLIC
+	fragment_store
+)
+
+########################################
+
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@@ -80,6 +135,9 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_tox_messages
 
+	fragment_store
+	message_fragment_store
+
 	SDL3::SDL3
 
 	imgui

src/chat_gui4.cpp

@@ -37,6 +37,18 @@ namespace Components {
 
 } // Components
 
+namespace Context {
+
+	// TODO: move back to chat log window and keep per window instead of per contact
+	struct CGView {
+		// set to the ts of the newest rendered msg
+		Message3Handle begin{};
+		// set to the ts of the oldest rendered msg
+		Message3Handle end{};
+	};
+
+} // Context
+
 static constexpr float lerp(float a, float b, float t) {
 	return a + t * (b - a);
 }
@@ -259,28 +271,6 @@ float ChatGui4::render(float time_delta) {
 
 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);
 
-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@@ -293,6 +283,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
 
+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;
 
@@ -381,12 +374,23 @@ float ChatGui4::render(float time_delta) {
 								}
 
 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
+								if (ImGui::IsItemVisible()) {
-									// get time now
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
-									const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+										// get time now
-									msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
+										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
-									msg_reg.remove<Message::Components::TagUnread>(e);
+										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
-									msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+										msg_reg.remove<Message::Components::TagUnread>(e);
+										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+									}
+
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
 								}
 
 								// highlight self
@@ -539,9 +543,90 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 
+						{ // update view cursers
+							if (!msg_reg.ctx().contains<Context::CGView>()) {
+								msg_reg.ctx().emplace<Context::CGView>();
+							}
+
+							auto& cg_view = msg_reg.ctx().get<Context::CGView>();
+
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view, we setup a view at current time, so the next frags are loaded
+								if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
+									// fix invalid state
+									if (static_cast<bool>(cg_view.begin)) {
+										cg_view.begin.destroy();
+										_rmm.throwEventDestroy(cg_view.begin);
+									}
+									if (static_cast<bool>(cg_view.end)) {
+										cg_view.end.destroy();
+										_rmm.throwEventDestroy(cg_view.end);
+									}
+
+									// create new
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									cg_view.end = {msg_reg, msg_reg.create()};
+
+									cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+									cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+									cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+									cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(cg_view.begin);
+									std::cout << "CG: created view FRONT end ts\n";
+									_rmm.throwEventConstruct(cg_view.end);
+								} // else? we do nothing?
+							} else {
+								bool begin_created {false};
+								if (!static_cast<bool>(cg_view.begin)) {
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								bool end_created {false};
+								if (!static_cast<bool>(cg_view.end)) {
+									cg_view.end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+								cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+								{
+									auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+										old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+										if (begin_created) {
+											std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.begin);
+										} else {
+											std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.begin);
+										}
+									}
+								}
+
+								{
+									auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+										old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+										if (end_created) {
+											std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.end);
+										} else {
+											std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.end);
+										}
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}
 
+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}

src/chat_gui4.hpp

@@ -10,6 +10,9 @@
 #include "./file_selector.hpp"
 #include "./send_image_popup.hpp"
 
+// HACK: move to public msg api?
+#include "./fragment_store/message_fragment_store.hpp"
+
 #include <entt/container/dense_map.hpp>
 
 #include <cstdint>
@@ -32,6 +35,7 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;
 
+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;
 
 	// TODO: per contact

src/fragment_store/README.md

@@ -25,41 +25,41 @@ Just keeps the Fragments in memory.
 
 # File formats
 
-Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
+Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted.
 
 ### Text Json
 
 Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
 Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
 
-A Metadata json object has the following keys:
+A Metadata json object can have arbitrary keys, some are predefined:
-- `enc` (uint) Encryption type of the data, if any
+- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any
-- `comp` (uint) Compression type of the data, if any
+- `FragComp::DataCompressionType` (uint) Compression type of the data, if any
-- `metadata` (obj) the 
 
 ## Binary file headers
 
 ### Split Metadata
 
-file magic bytes `SOLMET` (6 bytes)
+msgpack array:
 
-1 byte encryption type (`0x00` is none)
+- `[0]`: file magic string `SOLMET` (6 bytes)
-
+- `[1]`: uint8 encryption type (`0x00` is none)
-1 byte compression type (`0x00` is none)
+- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd)
-
+- `[3]`: binary metadata (optionally compressed and encrypted)
-...metadata here...
 
 note that the encryption and compression are for the metadata only.
 The metadata itself contains encryption and compression info about the data.
 
 ### Split Data
 
-(none) all the data is in the metadata file.
+All the metadata is in the metadata file. (like encryption and compression)
 This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
 Keep in mind to not use the actual file name as the data/meta file name.
 
 ### Single fragment
 
+Note: this format is unused for now
+
 file magic bytes `SOLFIL` (6 bytes)
 
 1 byte encryption type (`0x00` is none)
@@ -70,3 +70,7 @@ file magic bytes `SOLFIL` (6 bytes)
 
 ...data here...
 
+## Compression types
+
+- `0x00` none
+- `0x01` zstd (without dict)

src/fragment_store/fragment_store.cpp

@@ -0,0 +1,869 @@
+#include "./fragment_store.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <entt/entity/handle.hpp>
+#include <entt/container/dense_set.hpp>
+#include <entt/core/hashed_string.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <zstd.h>
+
+#include <cstdint>
+#include <fstream>
+#include <filesystem>
+#include <memory>
+#include <mutex>
+#include <type_traits>
+#include <utility>
+#include <algorithm>
+
+#include <iostream>
+#include <vector>
+
+static const char* metaFileTypeSuffix(MetaFileType mft) {
+	switch (mft) {
+		case MetaFileType::TEXT_JSON: return ".json";
+		//case MetaFileType::BINARY_ARB: return ".bin";
+		case MetaFileType::BINARY_MSGPACK: return ".msgpack";
+	}
+	return ""; // .unk?
+}
+
+FragmentStore::FragmentStore(void) {
+	{ // random namespace
+		const auto num0 = _rng();
+		const auto num1 = _rng();
+		const auto num2 = _rng();
+		const auto num3 = _rng();
+
+		_session_uuid_namespace[0+0] = (num0 >> 0) & 0xff;
+		_session_uuid_namespace[0+1] = (num0 >> 8) & 0xff;
+		_session_uuid_namespace[0+2] = (num0 >> 16) & 0xff;
+		_session_uuid_namespace[0+3] = (num0 >> 24) & 0xff;
+
+		_session_uuid_namespace[4+0] = (num1 >> 0) & 0xff;
+		_session_uuid_namespace[4+1] = (num1 >> 8) & 0xff;
+		_session_uuid_namespace[4+2] = (num1 >> 16) & 0xff;
+		_session_uuid_namespace[4+3] = (num1 >> 24) & 0xff;
+
+		_session_uuid_namespace[8+0] = (num2 >> 0) & 0xff;
+		_session_uuid_namespace[8+1] = (num2 >> 8) & 0xff;
+		_session_uuid_namespace[8+2] = (num2 >> 16) & 0xff;
+		_session_uuid_namespace[8+3] = (num2 >> 24) & 0xff;
+
+		_session_uuid_namespace[12+0] = (num3 >> 0) & 0xff;
+		_session_uuid_namespace[12+1] = (num3 >> 8) & 0xff;
+		_session_uuid_namespace[12+2] = (num3 >> 16) & 0xff;
+		_session_uuid_namespace[12+3] = (num3 >> 24) & 0xff;
+	}
+	registerSerializers();
+}
+
+FragmentStore::FragmentStore(
+	std::array<uint8_t, 16> session_uuid_namespace
+) : _session_uuid_namespace(std::move(session_uuid_namespace)) {
+	registerSerializers();
+}
+
+FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) {
+	return {_reg, fid};
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(std::array<uint8_t, 16>& uuid_namespace) {
+	std::vector<uint8_t> new_uid(uuid_namespace.cbegin(), uuid_namespace.cend());
+	new_uid.resize(new_uid.size() + 16);
+
+	const auto num0 = _rng();
+	const auto num1 = _rng();
+	const auto num2 = _rng();
+	const auto num3 = _rng();
+
+	new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff;
+
+	return new_uid;
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(void) {
+	return generateNewUID(_session_uuid_namespace);
+}
+
+FragmentID FragmentStore::newFragmentMemoryOwned(
+	const std::vector<uint8_t>& id,
+	size_t initial_size
+) {
+	{ // first check if id is already used
+		auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	{ // next check if space in memory budget
+		const auto free_memory = _memory_budget - _memory_usage;
+		if (initial_size > free_memory) {
+			return entt::null;
+		}
+	}
+
+	// actually allocate and create
+	auto new_data = std::make_unique<std::vector<uint8_t>>(initial_size);
+	if (!static_cast<bool>(new_data)) {
+		// allocation failure
+		return entt::null;
+	}
+	_memory_usage += initial_size;
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+	// TODO: memory comp
+	_reg.emplace<std::unique_ptr<std::vector<uint8_t>>>(new_frag) = std::move(new_data);
+
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft,
+	const std::vector<uint8_t>& id
+) {
+	{ // first check if id is already used
+		const auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	if (store_path.empty()) {
+		store_path = _default_store_path;
+	}
+
+	std::filesystem::create_directories(store_path);
+
+	const auto id_hex = bin2hex(id);
+	std::filesystem::path fragment_file_path;
+
+	if (id_hex.size() < 6) {
+		fragment_file_path = std::filesystem::path{store_path}/id_hex;
+	} else {
+		// use the first 2hex (1byte) as a subfolder
+		std::filesystem::create_directories(std::string{store_path} + id_hex.substr(0, 2));
+		fragment_file_path = std::filesystem::path{std::string{store_path} + id_hex.substr(0, 2)} / id_hex.substr(2);
+	}
+
+	if (std::filesystem::exists(fragment_file_path)) {
+		return entt::null;
+	}
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+
+	// file (info) comp
+	_reg.emplace<FragComp::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
+
+	_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
+
+	// meta needs to be synced to file
+	std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
+	if (!syncToStorage(new_frag, empty_data_cb)) {
+		std::cerr << "FS error: syncToStorage failed while creating new fragment file\n";
+		_reg.destroy(new_frag);
+		return entt::null;
+	}
+
+	// while new metadata might be created here, making sure the file could be created is more important
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft
+) {
+	return newFragmentFile(store_path, mft, generateNewUID());
+}
+
+FragmentID FragmentStore::getFragmentByID(
+	const std::vector<uint8_t>& id
+) {
+	// TODO: accelerate
+	// maybe keep it sorted and binary search? hash table lookup?
+	for (const auto& [frag, id_comp] : _reg.view<FragComp::ID>().each()) {
+		if (id == id_comp.v) {
+			return frag;
+		}
+	}
+
+	return entt::null;
+}
+
+FragmentID FragmentStore::getFragmentCustomMatcher(
+	std::function<bool(FragmentID)>& fn
+) {
+	return entt::null;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		return false;
+	}
+
+	// split object storage
+
+	MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults
+	if (_reg.all_of<FragComp::Ephemeral::MetaFileType>(fid)) {
+		meta_type = _reg.get<FragComp::Ephemeral::MetaFileType>(fid).type;
+	}
+
+	Encryption meta_enc = Encryption::NONE; // TODO: better defaults
+	Compression meta_comp = Compression::NONE; // TODO: better defaults
+
+	if (meta_type != MetaFileType::TEXT_JSON) {
+		if (_reg.all_of<FragComp::Ephemeral::MetaEncryptionType>(fid)) {
+			meta_enc = _reg.get<FragComp::Ephemeral::MetaEncryptionType>(fid).enc;
+		}
+
+		if (_reg.all_of<FragComp::Ephemeral::MetaCompressionType>(fid)) {
+			meta_comp = _reg.get<FragComp::Ephemeral::MetaCompressionType>(fid).comp;
+		}
+	} else {
+		// we cant have encryption or compression
+
+		// TODO: warning/error?
+
+		// TODO: forcing for testing
+		//if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
+		//}
+		//if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
+		//}
+	}
+
+	std::filesystem::path meta_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp";
+	meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string());
+	std::ofstream meta_file{
+		meta_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!meta_file.is_open()) {
+		std::cerr << "FS error: failed to create temporary meta file\n";
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE; // TODO: better defaults
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	std::filesystem::path data_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".tmp";
+	data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string());
+	std::ofstream data_file{
+		data_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		meta_file.close();
+		std::filesystem::remove(meta_tmp_path);
+		std::cerr << "FS error: failed to create temporary data file\n";
+		return false;
+	}
+
+	// sharing code between binary msgpack and text json for now
+	nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed
+	// metadata file
+
+	for (const auto& [type_id, storage] : _reg.storage()) {
+		if (!storage.contains(fid)) {
+			continue;
+		}
+
+		//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
+
+		// use type_id to find serializer
+		auto s_cb_it = _sc._serl_json.find(type_id);
+		if (s_cb_it == _sc._serl_json.end()) {
+			// could not find serializer, not saving
+			continue;
+		}
+
+		// noooo, why cant numbers be keys
+		//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
+			//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
+		//} else if (meta_type == MetaFileType::TEXT_JSON) {
+		s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]);
+		//}
+	}
+
+	if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file
+		const std::vector<uint8_t> meta_data = nlohmann::json::to_msgpack(meta_data_j);
+		std::vector<uint8_t> meta_data_compressed; // empty if none
+		//std::vector<uint8_t> meta_data_encrypted; // empty if none
+
+		if (meta_comp == Compression::ZSTD) {
+			meta_data_compressed.resize(ZSTD_compressBound(meta_data.size()));
+
+			size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3
+			if (ZSTD_isError(cSize)) {
+				std::cerr << "FS error: compressing meta failed\n";
+				meta_data_compressed.clear();
+				meta_comp = Compression::NONE;
+			} else {
+				meta_data_compressed.resize(cSize);
+			}
+		} else if (meta_comp == Compression::NONE) {
+			// do nothing
+		} else {
+			assert(false && "implement me");
+		}
+
+		// TODO: encryption
+
+		// the meta file is itself msgpack data
+		nlohmann::json meta_header_j = nlohmann::json::array();
+		meta_header_j.emplace_back() = "SOLMET";
+		meta_header_j.push_back(meta_enc);
+		meta_header_j.push_back(meta_comp);
+
+		if (false) { // TODO: encryption
+		} else if (!meta_data_compressed.empty()) {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed));
+		} else {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data));
+		}
+
+		const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j);
+		meta_file.write(reinterpret_cast<const char*>(meta_header_data.data()), meta_header_data.size());
+	} else if (meta_type == MetaFileType::TEXT_JSON) {
+		// cant be compressed or encrypted
+		meta_file << meta_data_j.dump(2, ' ', true);
+	}
+
+	// now data
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			if (buffer_actual_size == 0) {
+				break;
+			}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+
+			data_file.write(reinterpret_cast<const char*>(buffer.data()), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> buffer(ZSTD_CStreamInSize());
+		std::vector<uint8_t> compressed_buffer(ZSTD_CStreamOutSize());
+		uint64_t buffer_actual_size {0};
+
+		ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+		ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 0); // default (3)
+		ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?)
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			//if (buffer_actual_size == 0) {
+				//break;
+			//}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+			bool const lastChunk = (buffer_actual_size < buffer.size());
+
+			ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+			ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 };
+
+			while (input.pos < input.size) {
+				ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 };
+
+				size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
+				if (ZSTD_isError(remaining)) {
+					std::cerr << "FS error: compressing data failed\n";
+					break;
+				}
+
+				data_file.write(reinterpret_cast<const char*>(compressed_buffer.data()), output.pos);
+
+				if (remaining == 0) {
+					break;
+				}
+			}
+			// same as if lastChunk break;
+		} while (buffer_actual_size == buffer.size());
+	} else {
+		assert(false && "implement me");
+	}
+
+	meta_file.flush();
+	meta_file.close();
+	data_file.flush();
+	data_file.close();
+
+	std::filesystem::rename(
+		meta_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type)
+	);
+
+	std::filesystem::rename(
+		data_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path
+	);
+
+	// TODO: check return value of renames
+
+	if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
+		_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
+	}
+
+	return true;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) {
+	std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+		uint64_t i = 0;
+		for (; i+read < data_size && i < buffer_size; i++) {
+			request_buffer[i] = data[i+read];
+		}
+		read += i;
+
+		return i;
+	};
+	return syncToStorage(fid, fn_cb);
+}
+
+bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		// TODO: memory fragments
+		return false;
+	}
+
+	const auto& frag_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path;
+
+	// TODO: check if metadata dirty?
+	// TODO: what if file changed on disk?
+
+	std::cout << "FS: loading fragment '" << frag_path << "'\n";
+
+	std::ifstream data_file{
+		frag_path,
+		std::ios::in | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n";
+		// error
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE;
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(buffer.data()), buffer.size());
+			buffer_actual_size = data_file.gcount();
+
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			data_cb(buffer.data(), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size() && !data_file.eof());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> in_buffer(ZSTD_DStreamInSize());
+		std::vector<uint8_t> out_buffer(ZSTD_DStreamOutSize());
+		ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(in_buffer.data()), in_buffer.size());
+			buffer_actual_size = data_file.gcount();
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 };
+			do {
+				ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 };
+				size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
+				if (ZSTD_isError(ret)) {
+					// error <.<
+					std::cerr << "FS error: decompression error\n";
+					break;
+				}
+
+				data_cb(out_buffer.data(), output.pos);
+			} while (input.pos < input.size);
+		} while (buffer_actual_size == in_buffer.size() && !data_file.eof());
+
+		ZSTD_freeDCtx(dctx);
+	} else {
+		assert(false && "implement me");
+	}
+
+	return true;
+}
+
+nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) {
+	std::vector<uint8_t> tmp_buffer;
+	std::function<read_from_storage_put_data_cb> cb = [&tmp_buffer](const uint8_t* buffer, const uint64_t buffer_size) {
+		tmp_buffer.insert(tmp_buffer.end(), buffer, buffer+buffer_size);
+	};
+
+	if (!loadFromStorage(fid, cb)) {
+		return nullptr;
+	}
+
+	return nlohmann::json::parse(tmp_buffer);
+}
+
+size_t FragmentStore::scanStoragePath(std::string_view path) {
+	if (path.empty()) {
+		path = _default_store_path;
+	}
+	// TODO: extract so async can work (or/and make iteratable generator)
+
+	if (!std::filesystem::is_directory(path)) {
+		std::cerr << "FS error: scan path not a directory '" << path << "'\n";
+		return 0;
+	}
+
+	// step 1: make snapshot of files, validate metafiles and save id/path+meta.ext
+	// can be extra thread (if non vfs)
+	struct FragFileEntry {
+		std::string id_str;
+		std::filesystem::path frag_path;
+		std::string meta_ext;
+
+		bool operator==(const FragFileEntry& other) const {
+			// only compare by id
+			return id_str == other.id_str;
+		}
+	};
+	struct FragFileEntryHash {
+		size_t operator()(const FragFileEntry& it) const {
+			return entt::hashed_string(it.id_str.data(), it.id_str.size());
+		}
+	};
+	entt::dense_set<FragFileEntry, FragFileEntryHash> file_frag_list;
+
+	std::filesystem::path storage_path{path};
+
+	auto handle_file = [&](const std::filesystem::path& file_path) {
+		if (!std::filesystem::is_regular_file(file_path)) {
+			return;
+		}
+		// handle file
+
+		if (file_path.has_extension()) {
+			// skip over metadata, assuming only metafiles have extentions (might be wrong?)
+			// also skips temps
+			return;
+		}
+
+		auto relative_path = std::filesystem::proximate(file_path, storage_path);
+		std::string id_str = relative_path.generic_u8string();
+		// delete all '/'
+		id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end());
+		if (id_str.size() % 2 != 0) {
+			std::cerr << "FS error: non hex fragment uid detected: '" << id_str << "'\n";
+		}
+
+		if (file_frag_list.contains(FragFileEntry{id_str, {}, ""})) {
+			std::cerr << "FS error: fragment duplicate detected: '" << id_str << "'\n";
+			return; // skip
+		}
+
+		const char* meta_ext = ".meta.msgpack";
+		{ // find meta
+			// TODO: this as to know all possible extentions
+			bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack");
+			bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json");
+			const size_t meta_sum =
+				(has_meta_msgpack?1:0) +
+				(has_meta_json?1:0)
+			;
+
+			if (meta_sum > 1) { // has multiple
+				std::cerr << "FS error: fragment with multiple meta files detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (meta_sum == 0) {
+				std::cerr << "FS error: fragment missing meta file detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (has_meta_json) {
+				meta_ext = ".meta.json";
+			}
+		}
+
+		file_frag_list.emplace(FragFileEntry{
+			std::move(id_str),
+			file_path,
+			meta_ext
+		});
+	};
+
+	for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) {
+		if (std::filesystem::is_regular_file(outer_path)) {
+			handle_file(outer_path);
+		} else if (std::filesystem::is_directory(outer_path)) {
+			// subdir, part of id
+			for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) {
+				//if (std::filesystem::is_regular_file(inner_path)) {
+
+					//// handle file
+				//} // TODO: support deeper recursion?
+				handle_file(inner_path);
+			}
+		}
+	}
+
+	std::cout << "FS: scan found:\n";
+	for (const auto& it : file_frag_list) {
+		std::cout << "  " << it.id_str << "\n";
+	}
+
+	// step 2: check if files preexist in reg
+	// main thread
+	// (merge into step 3 and do more error checking?)
+	for (auto it = file_frag_list.begin(); it != file_frag_list.end();) {
+		auto id = hex2bin(it->id_str);
+		auto fid = getFragmentByID(id);
+		if (_reg.valid(fid)) {
+			// pre exising (handle differently??)
+			// check if store differs?
+			it = file_frag_list.erase(it);
+		} else {
+			it++;
+		}
+	}
+
+	std::vector<FragmentID> scanned_frags;
+	// step 3: parse meta and insert into reg of non preexising
+	// main thread
+	// TODO: check timestamps of preexisting and reload? mark external/remote dirty?
+	for (const auto& it : file_frag_list) {
+		nlohmann::json j;
+		if (it.meta_ext == ".meta.msgpack") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cout << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			// file is a msgpack within a msgpack
+
+			std::vector<uint8_t> full_meta_data;
+			{ // read meta file
+				// figure out size
+				file.seekg(0, file.end);
+				uint64_t file_size = file.tellg();
+				file.seekg(0, file.beg);
+
+				full_meta_data.resize(file_size);
+
+				file.read(reinterpret_cast<char*>(full_meta_data.data()), full_meta_data.size());
+			}
+
+			const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data);
+
+			if (!meta_header_j.is_array() || meta_header_j.size() < 4) {
+				std::cerr << "FS error: broken binary meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			if (meta_header_j.at(0) != "SOLMET") {
+				std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			Encryption meta_enc = meta_header_j.at(1);
+			if (meta_enc != Encryption::NONE) {
+				std::cerr << "FS error: unknown encryption " << it.frag_path << "\n";
+				continue;
+			}
+
+			Compression meta_comp = meta_header_j.at(2);
+			if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) {
+				std::cerr << "FS error: unknown compression " << it.frag_path << "\n";
+				continue;
+			}
+
+			//const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data");
+			if (!meta_header_j.at(3).is_binary()) {
+				std::cerr << "FS error: meta data not binary " << it.frag_path << "\n";
+				continue;
+			}
+			const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3);
+
+			std::vector<uint8_t> meta_data_decomp;
+			if (meta_comp == Compression::NONE) {
+				// do nothing
+			} else if (meta_comp == Compression::ZSTD) {
+				meta_data_decomp.resize(ZSTD_DStreamOutSize());
+				ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+
+				ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0};
+				ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0};
+				do {
+					size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
+					if (ZSTD_isError(ret)) {
+						// error <.<
+						std::cerr << "FS error: decompression error\n";
+						meta_data_decomp.clear();
+						break;
+					}
+				} while (input.pos < input.size);
+				meta_data_decomp.resize(output.pos);
+
+				ZSTD_freeDCtx(dctx);
+			} else {
+				assert(false && "implement me");
+			}
+
+			// TODO: enc
+
+			if (!meta_data_decomp.empty()) {
+				j = nlohmann::json::from_msgpack(meta_data_decomp);
+			} else {
+				j = nlohmann::json::from_msgpack(meta_data_ref);
+			}
+		} else if (it.meta_ext == ".meta.json") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cerr << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			file >> j;
+		} else {
+			assert(false);
+		}
+
+		if (!j.is_object()) {
+			std::cerr << "FS error: json in meta is broken " << it.id_str << "\n";
+			continue;
+		}
+
+		// TODO: existing fragment file
+		//newFragmentFile();
+		FragmentHandle fh{_reg, _reg.create()};
+		fh.emplace<FragComp::ID>(hex2bin(it.id_str));
+
+		fh.emplace<FragComp::Ephemeral::FilePath>(it.frag_path.generic_u8string());
+
+		for (const auto& [k, v] : j.items()) {
+			// type id from string hash
+			const auto type_id = entt::hashed_string(k.data(), k.size());
+			const auto deserl_fn_it = _sc._deserl_json.find(type_id);
+			if (deserl_fn_it != _sc._deserl_json.cend()) {
+				// TODO: check return value
+				deserl_fn_it->second(fh, v);
+			} else {
+				std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n";
+			}
+		}
+		scanned_frags.push_back(fh);
+	}
+
+	// TODO: mutex and move code to async and return this list ?
+
+	// throw new frag event here, after loading them all
+	for (const FragmentID fid : scanned_frags) {
+		throwEventConstruct(fid);
+	}
+
+	return scanned_frags.size();
+}
+
+void FragmentStore::scanStoragePathAsync(std::string path) {
+	// add path to queue
+	// HACK: if path is known/any fragment is in the path, this operation blocks (non async)
+	scanStoragePath(path); // TODO: make async and post result
+}
+
+static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Encryption>>(
+		fh.get<FragComp::DataEncryptionType>().enc
+	);
+	return true;
+}
+
+static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataEncryptionType>(
+		static_cast<Encryption>(
+			static_cast<std::underlying_type_t<Encryption>>(in)
+		)
+	);
+	return true;
+}
+
+static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Compression>>(
+		fh.get<FragComp::DataCompressionType>().comp
+	);
+	return true;
+}
+
+static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataCompressionType>(
+		static_cast<Compression>(
+			static_cast<std::underlying_type_t<Compression>>(in)
+		)
+	);
+	return true;
+}
+
+void FragmentStore::registerSerializers(void) {
+	_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
+	_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
+	_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
+	_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
+}
+

src/fragment_store/fragment_store.hpp

@@ -0,0 +1,103 @@
+#pragma once
+
+#include "./fragment_store_i.hpp"
+
+#include "./types.hpp"
+#include "./meta_components.hpp"
+
+#include "./serializer.hpp"
+
+#include <entt/core/type_info.hpp>
+#include <entt/entity/registry.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+#include <vector>
+#include <array>
+#include <cstdint>
+#include <random>
+
+struct FragmentStore : public FragmentStoreI {
+	std::minstd_rand _rng{std::random_device{}()};
+	std::array<uint8_t, 16> _session_uuid_namespace;
+
+	std::string _default_store_path;
+
+	uint64_t _memory_budget {10u*1024u*1024u};
+	uint64_t _memory_usage {0u};
+
+	SerializerCallbacks<FragmentID> _sc;
+
+	FragmentStore(void);
+	FragmentStore(std::array<uint8_t, 16> session_uuid_namespace);
+
+	// HACK: get access to the reg
+	FragmentHandle fragmentHandle(FragmentID fid);
+
+	// TODO: make the frags ref counted
+
+	// TODO: check for exising
+	std::vector<uint8_t> generateNewUID(std::array<uint8_t, 16>& uuid_namespace);
+	std::vector<uint8_t> generateNewUID(void);
+
+	// ========== new fragment ==========
+
+	// memory backed owned
+	FragmentID newFragmentMemoryOwned(
+		const std::vector<uint8_t>& id,
+		size_t initial_size
+	);
+
+	// memory backed view (can only be added? not new?)
+
+	// file backed (rw...)
+	// needs to know which store path to put into
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft,
+		const std::vector<uint8_t>& id
+	);
+	// this variant generate a new, mostly unique, id for us
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft
+	);
+
+	// ========== add fragment ==========
+
+	// ========== get fragment ==========
+	FragmentID getFragmentByID(
+		const std::vector<uint8_t>& id
+	);
+	FragmentID getFragmentCustomMatcher(
+		std::function<bool(FragmentID)>& fn
+	);
+
+	// remove fragment?
+	// unload?
+
+	// ========== sync fragment to storage ==========
+	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
+	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
+	bool syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+	bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size);
+
+	// ========== load fragment data from storage ==========
+	using read_from_storage_put_data_cb = void(const uint8_t* buffer, const uint64_t buffer_size);
+	bool loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb);
+	// convenience function
+	nlohmann::json loadFromStorageNJ(FragmentID fid);
+
+	// fragment discovery?
+	// returns number of new fragments
+	size_t scanStoragePath(std::string_view path);
+	void scanStoragePathAsync(std::string path);
+
+	private:
+		void registerSerializers(void); // internal comps
+		// internal actual backends
+		// TODO: seperate out
+		bool syncToMemory(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+		bool syncToFile(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+};
+

src/fragment_store/fragment_store_i.cpp

@@ -0,0 +1,23 @@
+#include "./fragment_store_i.hpp"
+
+#include <iostream>
+
+void FragmentStoreI::throwEventConstruct(const FragmentID fid) {
+	std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_construct,
+		Fragment::Events::FragmentConstruct{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}
+
+void FragmentStoreI::throwEventUpdate(const FragmentID fid) {
+	std::cout << "FSI debug: event updated " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_updated,
+		Fragment::Events::FragmentUpdated{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}

src/fragment_store/fragment_store_i.hpp

@@ -0,0 +1,63 @@
+#pragma once
+
+#include <solanaceae/util/event_provider.hpp>
+
+#include <entt/entity/registry.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <cstdint>
+
+// internal id
+enum class FragmentID : uint32_t {};
+using FragmentRegistry = entt::basic_registry<FragmentID>;
+using FragmentHandle = entt::basic_handle<FragmentRegistry>;
+
+namespace Fragment::Events {
+
+	struct FragmentConstruct {
+		const FragmentHandle e;
+	};
+	struct FragmentUpdated {
+		const FragmentHandle e;
+	};
+	//struct MessageDestory {
+		//const Message3Handle e;
+	//};
+
+} // Fragment::Events
+
+enum class FragmentStore_Event : uint32_t {
+	fragment_construct,
+	fragment_updated,
+	//message_destroy,
+
+	MAX
+};
+
+struct FragmentStoreEventI {
+	using enumType = FragmentStore_Event;
+
+	virtual ~FragmentStoreEventI(void) {}
+
+	virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; }
+	virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; }
+	//virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; }
+
+	// mm3
+	// send text
+	// send file path
+};
+using FragmentStoreEventProviderI = EventProviderI<FragmentStoreEventI>;
+
+struct FragmentStoreI  : public FragmentStoreEventProviderI {
+	static constexpr const char* version {"1"};
+
+	FragmentRegistry _reg;
+
+	virtual ~FragmentStoreI(void) {}
+
+	void throwEventConstruct(const FragmentID fid);
+	void throwEventUpdate(const FragmentID fid);
+	//void throwEventDestroy();
+};
+

src/fragment_store/message_fragment_store.cpp

@@ -0,0 +1,894 @@
+#include "./message_fragment_store.hpp"
+
+#include "../json/message_components.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/contact/components.hpp>
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/message3/contact_components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <algorithm>
+#include <string>
+#include <cstdint>
+#include <cassert>
+#include <iostream>
+
+// https://youtu.be/CU2exyhYPfA
+
+// everything assumes a single fragment registry
+
+namespace Message::Components {
+
+	// ctx
+	struct OpenFragments {
+		struct OpenFrag final {
+			std::vector<uint8_t> uid;
+		};
+		// only contains fragments with <1024 messages and <28h tsrage
+		// TODO: this needs to move into the message reg
+		std::vector<OpenFrag> fuid_open;
+	};
+
+	// all message fragments of this contact
+	struct ContactFragments final {
+		// kept up-to-date by events
+		struct InternalEntry {
+			// indecies into the sorted arrays
+			size_t i_b;
+			size_t i_e;
+		};
+		entt::dense_map<FragmentID, InternalEntry> frags;
+
+		// add 2 sorted contact lists for both range begin and end
+		// TODO: adding and removing becomes expensive with enough frags, consider splitting or heap
+		std::vector<FragmentID> sorted_begin;
+		std::vector<FragmentID> sorted_end;
+
+		// api
+		// return true if it was actually inserted
+		bool insert(FragmentHandle frag);
+		bool erase(FragmentID frag);
+		// update? (just erase() + insert())
+
+		// uses range begin to go back in time
+		FragmentID prev(FragmentID frag) const;
+		// uses range end to go forward in time
+		FragmentID next(FragmentID frag) const;
+	};
+
+	// all LOADED message fragments
+	// TODO: merge into ContactFragments (and pull in openfrags)
+	struct LoadedContactFragments final {
+		// kept up-to-date by events
+		entt::dense_set<FragmentID> frags;
+	};
+
+} // Message::Components
+
+namespace Fragment::Components {
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id)
+} // Fragment::Components
+
+void MessageFragmentStore::handleMessage(const Message3Handle& m) {
+	if (_fs_ignore_event) {
+		// message event because of us loading a fragment, ignore
+		// TODO: this barely makes a difference
+		return;
+	}
+
+	if (!static_cast<bool>(m)) {
+		return; // huh?
+	}
+
+	if (!m.all_of<Message::Components::Timestamp>()) {
+		return; // we only handle msg with ts
+	}
+
+	_potentially_dirty_contacts.emplace(m.registry()->ctx().get<Contact3>()); // always mark dirty here
+	if (m.any_of<Message::Components::ViewCurserBegin, Message::Components::ViewCurserEnd>()) {
+		// not an actual message, but we probalby need to check and see if we need to load fragments
+		std::cout << "MFS: new or updated curser\n";
+		return;
+	}
+
+	if (!m.all_of<Message::Components::FUID>()) {
+		std::cout << "MFS: new msg missing FUID\n";
+		if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
+			m.registry()->ctx().emplace<Message::Components::OpenFragments>();
+
+#if 0
+			// TODO: move this to async
+			// TODO: move this to tick and just respect the dirty
+			FragmentHandle most_recent_fag;
+			uint64_t most_recent_ts{0};
+			if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+				for (const auto& [fid, si] : m.registry()->ctx().get<Message::Components::ContactFragments>().frags) {
+					auto fh = _fs.fragmentHandle(fid);
+					if (!static_cast<bool>(fh) || !fh.all_of<FragComp::MessagesTSRange, FragComp::MessagesContact>()) {
+						// TODO: remove at this point?
+						continue;
+					}
+
+					const uint64_t f_ts = fh.get<FragComp::MessagesTSRange>().begin;
+					if (f_ts > most_recent_ts) {
+						// this makes no sense, we retry to load the first fragment on every new message and bail here, bc it was already
+						if (m.registry()->ctx().contains<Message::Components::LoadedContactFragments>()) {
+							if (m.registry()->ctx().get<Message::Components::LoadedContactFragments>().frags.contains(fh)) {
+								continue; // already loaded
+							}
+						}
+						most_recent_ts = f_ts;
+						most_recent_fag = {_fs._reg, fid};
+					}
+				}
+			}
+
+			if (static_cast<bool>(most_recent_fag)) {
+				loadFragment(*m.registry(), most_recent_fag);
+			}
+#endif
+		}
+
+		auto& fuid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fuid_open;
+
+		const auto msg_ts = m.get<Message::Components::Timestamp>().ts;
+		// missing fuid
+		// find closesed non-sealed off fragment
+
+		std::vector<uint8_t> fragment_uid;
+
+		// first search for fragment where the ts falls into the range
+		for (const auto& fuid : fuid_open) {
+			auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fuid.uid));
+			assert(static_cast<bool>(fh));
+
+			// assuming ts range exists
+			auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
+
+			if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) {
+				fragment_uid = fuid.uid;
+				// TODO: check conditions for open here
+				// TODO: mark msg (and frag?) dirty
+			}
+		}
+
+		// if it did not fit into an existing fragment, we next look for fragments that could be extended
+		if (fragment_uid.empty()) {
+			for (const auto& fuid : fuid_open) {
+				auto fh = _fs.fragmentHandle(_fs.getFragmentByID(fuid.uid));
+				assert(static_cast<bool>(fh));
+
+				// assuming ts range exists
+				auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
+
+				const int64_t frag_range = int64_t(fts_comp.end) - int64_t(fts_comp.begin);
+				constexpr static int64_t max_frag_ts_extent {1000*60*60};
+				//constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing
+				const int64_t possible_extention = max_frag_ts_extent - frag_range;
+
+				// which direction
+				if ((fts_comp.begin - possible_extention) <= msg_ts && fts_comp.begin > msg_ts) {
+					fragment_uid = fuid.uid;
+
+					std::cout << "MFS: extended begin from " << fts_comp.begin << " to " << msg_ts << "\n";
+
+					// assuming ts range exists
+					fts_comp.begin = msg_ts; // extend into the past
+
+					if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+						// should be the case
+						m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
+						m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+					}
+
+
+					// TODO: check conditions for open here
+					// TODO: mark msg (and frag?) dirty
+				} else if ((fts_comp.end + possible_extention) >= msg_ts && fts_comp.end < msg_ts) {
+					fragment_uid = fuid.uid;
+
+					std::cout << "MFS: extended end from " << fts_comp.end << " to " << msg_ts << "\n";
+
+					// assuming ts range exists
+					fts_comp.end = msg_ts; // extend into the future
+
+					if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+						// should be the case
+						m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
+						m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+					}
+
+					// TODO: check conditions for open here
+					// TODO: mark msg (and frag?) dirty
+				}
+			}
+		}
+
+		// if its still not found, we need a new fragment
+		if (fragment_uid.empty()) {
+			const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::BINARY_MSGPACK);
+			auto fh = _fs.fragmentHandle(new_fid);
+			if (!static_cast<bool>(fh)) {
+				std::cout << "MFS error: failed to create new fragment for message\n";
+				return;
+			}
+
+			fragment_uid = fh.get<FragComp::ID>().v;
+
+			fh.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>().comp = Compression::ZSTD;
+			fh.emplace_or_replace<FragComp::DataCompressionType>().comp = Compression::ZSTD;
+
+			auto& new_ts_range = fh.emplace_or_replace<FragComp::MessagesTSRange>();
+			new_ts_range.begin = msg_ts;
+			new_ts_range.end = msg_ts;
+
+			{
+				const auto msg_reg_contact = m.registry()->ctx().get<Contact3>();
+				if (_cr.all_of<Contact::Components::ID>(msg_reg_contact)) {
+					fh.emplace<FragComp::MessagesContact>(_cr.get<Contact::Components::ID>(msg_reg_contact).data);
+				} else {
+					// ? rage quit?
+				}
+			}
+
+			// contact frag
+			if (!m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+				m.registry()->ctx().emplace<Message::Components::ContactFragments>();
+			}
+			m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+
+			// loaded contact frag
+			if (!m.registry()->ctx().contains<Message::Components::LoadedContactFragments>()) {
+				m.registry()->ctx().emplace<Message::Components::LoadedContactFragments>();
+			}
+			m.registry()->ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
+
+			fuid_open.emplace_back(Message::Components::OpenFragments::OpenFrag{fragment_uid});
+
+			std::cout << "MFS: created new fragment " << bin2hex(fragment_uid) << "\n";
+
+			_fs_ignore_event = true;
+			_fs.throwEventConstruct(fh);
+			_fs_ignore_event = false;
+		}
+
+		// if this is still empty, something is very wrong and we exit here
+		if (fragment_uid.empty()) {
+			std::cout << "MFS error: failed to find/create fragment for message\n";
+			return;
+		}
+
+		m.emplace<Message::Components::FUID>(fragment_uid);
+
+		// in this case we know the fragment needs an update
+		_fuid_save_queue.push({Message::getTimeMS(), fragment_uid, m.registry()});
+	}
+
+	// TODO: do we use fid?
+
+	// on new message: assign fuid
+	// on new and update: mark as fragment dirty
+}
+
+// assumes not loaded frag
+// need update from frag
+void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) {
+	std::cout << "MFS: loadFragment\n";
+	const auto j = _fs.loadFromStorageNJ(fh);
+
+	if (!j.is_array()) {
+		// wrong data
+		return;
+	}
+
+	// TODO: this should probably never be the case, since we already know here that it is a msg frag
+	if (!reg.ctx().contains<Message::Components::ContactFragments>()) {
+		reg.ctx().emplace<Message::Components::ContactFragments>();
+	}
+	reg.ctx().get<Message::Components::ContactFragments>().insert(fh);
+
+	// mark loaded
+	if (!reg.ctx().contains<Message::Components::LoadedContactFragments>()) {
+		reg.ctx().emplace<Message::Components::LoadedContactFragments>();
+	}
+	reg.ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
+
+	for (const auto& j_entry : j) {
+		auto new_real_msg = Message3Handle{reg, reg.create()};
+		// load into staging reg
+		for (const auto& [k, v] : j_entry.items()) {
+			//std::cout << "K:" << k << " V:" << v.dump() << "\n";
+			const auto type_id = entt::hashed_string(k.data(), k.size());
+			const auto deserl_fn_it = _sc._deserl_json.find(type_id);
+			if (deserl_fn_it != _sc._deserl_json.cend()) {
+				try {
+					if (!deserl_fn_it->second(_sc, new_real_msg, v)) {
+						std::cerr << "MFS error: failed deserializing '" << k << "'\n";
+					}
+				} catch(...) {
+					std::cerr << "MFS error: failed deserializing (threw) '" << k << "'\n";
+				}
+			} else {
+				std::cerr << "MFS warning: missing deserializer for meta key '" << k << "'\n";
+			}
+		}
+
+		new_real_msg.emplace_or_replace<Message::Components::FUID>(fh.get<FragComp::ID>());
+
+		// dup check (hacky, specific to protocols)
+		Message3 dup_msg {entt::null};
+		{
+			// get comparator from contact
+			if (reg.ctx().contains<Contact3>()) {
+				const auto c = reg.ctx().get<Contact3>();
+				if (_cr.all_of<Contact::Components::MessageIsSame>(c)) {
+					auto& comp = _cr.get<Contact::Components::MessageIsSame>(c).comp;
+					// walking EVERY existing message OOF
+					// this needs optimizing
+					for (const Message3 other_msg : reg.view<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
+						if (other_msg == new_real_msg) {
+							continue; // skip self
+						}
+
+						if (comp({reg, other_msg}, new_real_msg)) {
+							// dup
+							dup_msg = other_msg;
+							break;
+						}
+					}
+				}
+			}
+		}
+
+		if (reg.valid(dup_msg)) {
+			//  -> merge with preexisting
+			//  -> throw update
+			reg.destroy(new_real_msg);
+			//_rmm.throwEventUpdate(reg, new_real_msg);
+		} else {
+			if (!new_real_msg.all_of<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
+				// does not have needed components to be stand alone
+				reg.destroy(new_real_msg);
+				std::cerr << "MFS warning: message with missing basic compoments\n";
+				continue;
+			}
+
+			//  -> throw create
+			_rmm.throwEventConstruct(reg, new_real_msg);
+		}
+	}
+}
+
+MessageFragmentStore::MessageFragmentStore(
+	Contact3Registry& cr,
+	RegistryMessageModel& rmm,
+	FragmentStore& fs
+) : _cr(cr), _rmm(rmm), _fs(fs), _sc{_cr, {}, {}} {
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_construct);
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_updated);
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_destroy);
+
+	_fs._sc.registerSerializerJson<FragComp::MessagesTSRange>();
+	_fs._sc.registerDeSerializerJson<FragComp::MessagesTSRange>();
+	_fs._sc.registerSerializerJson<FragComp::MessagesContact>();
+	_fs._sc.registerDeSerializerJson<FragComp::MessagesContact>();
+
+	_fs.subscribe(this, FragmentStore_Event::fragment_construct);
+}
+
+MessageFragmentStore::~MessageFragmentStore(void) {
+	// TODO: sync all dirty fragments
+}
+
+MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) {
+	return _sc;
+}
+
+// checks range against all cursers in msgreg
+static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message3Registry& msg_reg) {
+	// 1D collision checks:
+	//  - for range vs range:
+	//    r1 rhs >= r0 lhs AND r1 lhs <= r0 rhs
+	//  - for range vs point:
+	//    p >= r0 lhs AND p <= r0 rhs
+	// NOTE: directions for us are reversed (begin has larger values as end)
+
+	auto c_b_view = msg_reg.view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
+	c_b_view.use<Message::Components::ViewCurserBegin>();
+	for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) {
+		// p and r1 rhs can be seen as the same
+		// but first we need to know if a curser begin is a point or a range
+
+		// TODO: margin?
+		auto ts_begin = ts_begin_comp.ts;
+		auto ts_end = ts_begin_comp.ts; // simplyfy code by making a single begin curser act as an infinitly small range
+		if (msg_reg.valid(vcb.curser_end) && msg_reg.all_of<Message::Components::ViewCurserEnd>(vcb.curser_end)) {
+			// TODO: respect curser end's begin?
+			// TODO: remember which ends we checked and check remaining
+			ts_end = msg_reg.get<Message::Components::Timestamp>(vcb.curser_end).ts;
+
+			// sanity check curser order
+			if (ts_end > ts_begin) {
+				std::cerr << "MFS warning: begin curser and end curser of view swapped!!\n";
+				std::swap(ts_begin, ts_end);
+			}
+		}
+
+		// perform both checks here
+		if (ts_begin < range_end || ts_end > range_begin) {
+			continue;
+		}
+
+		// range hits a view
+		return true;
+	}
+
+	return false;
+}
+
+static bool isLess(const std::vector<uint8_t>& lhs, const std::vector<uint8_t>& rhs) {
+	size_t i = 0;
+	for (; i < lhs.size() && i < rhs.size(); i++) {
+		if (lhs[i] < rhs[i]) {
+			return true;
+		} else if (lhs[i] > rhs[i]) {
+			return false;
+		}
+		// else continue
+	}
+
+	// here we have equality of common lenths
+
+	// we define smaller arrays to be less
+	return lhs.size() < rhs.size();
+}
+
+float MessageFragmentStore::tick(float time_delta) {
+	// sync dirty fragments here
+	if (!_fuid_save_queue.empty()) {
+		const auto fid = _fs.getFragmentByID(_fuid_save_queue.front().id);
+		auto* reg = _fuid_save_queue.front().reg;
+		assert(reg != nullptr);
+
+		auto fh = _fs.fragmentHandle(fid);
+		auto& ftsrange = fh.get_or_emplace<FragComp::MessagesTSRange>(Message::getTimeMS(), Message::getTimeMS());
+
+		auto j = nlohmann::json::array();
+
+		// TODO: does every message have ts?
+		auto msg_view = reg->view<Message::Components::Timestamp>();
+		// we also assume all messages have fuid (hack: call handle when not?)
+		for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) {
+			const Message3 m = *it;
+
+			if (!reg->all_of<Message::Components::FUID, Message::Components::ContactFrom, Message::Components::ContactTo>(m)) {
+				continue;
+			}
+
+			// require msg for now
+			if (!reg->any_of<Message::Components::MessageText/*, Message::Components::Transfer::FileInfo*/>(m)) {
+				continue;
+			}
+
+			if (_fuid_save_queue.front().id != reg->get<Message::Components::FUID>(m).v) {
+				continue; // not ours
+			}
+
+			{ // potentially adjust tsrange (some external processes can change timestamps)
+				const auto msg_ts = msg_view.get<Message::Components::Timestamp>(m).ts;
+				if (ftsrange.begin > msg_ts) {
+					ftsrange.begin = msg_ts;
+				} else if (ftsrange.end < msg_ts) {
+					ftsrange.end = msg_ts;
+				}
+			}
+
+			auto& j_entry = j.emplace_back(nlohmann::json::object());
+
+			for (const auto& [type_id, storage] : reg->storage()) {
+				if (!storage.contains(m)) {
+					continue;
+				}
+
+				//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
+
+				// use type_id to find serializer
+				auto s_cb_it = _sc._serl_json.find(type_id);
+				if (s_cb_it == _sc._serl_json.end()) {
+					// could not find serializer, not saving
+					//std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n";
+					continue;
+				}
+
+				s_cb_it->second(_sc, {*reg, m}, j_entry[storage.type().name()]);
+			}
+		}
+
+		// if save as binary
+		//nlohmann::json::to_msgpack(j);
+		auto j_dump = j.dump(2, ' ', true);
+		if (_fs.syncToStorage(fid, reinterpret_cast<const uint8_t*>(j_dump.data()), j_dump.size())) {
+			//std::cout << "MFS: dumped " << j_dump << "\n";
+			// succ
+			_fuid_save_queue.pop();
+		}
+	}
+
+	// load needed fragments here
+
+	// last check event frags
+	// only checks if it collides with ranges, not adjacent
+	// bc ~range~ msgreg will be marked dirty and checked next tick
+	const bool had_events = !_event_check_queue.empty();
+	for (size_t i = 0; i < 10 && !_event_check_queue.empty(); i++) {
+		std::cout << "MFS: event check\n";
+		auto fh = _fs.fragmentHandle(_event_check_queue.front().fid);
+		auto c = _event_check_queue.front().c;
+		_event_check_queue.pop();
+
+		if (!static_cast<bool>(fh)) {
+			return 0.05f;
+		}
+
+		if (!fh.all_of<FragComp::MessagesTSRange>()) {
+			return 0.05f;
+		}
+
+		// get ts range of frag and collide with all curser(s/ranges)
+		const auto& frag_range = fh.get<FragComp::MessagesTSRange>();
+
+		auto* msg_reg = _rmm.get(c);
+		if (msg_reg == nullptr) {
+			return 0.05f;
+		}
+
+		if (rangeVisible(frag_range.begin, frag_range.end, !msg_reg)) {
+			loadFragment(*msg_reg, fh);
+			_potentially_dirty_contacts.emplace(c);
+			return 0.05f; // only one but soon again
+		}
+	}
+	if (had_events) {
+		std::cout << "MFS: event check none\n";
+		return 0.05f; // only check events, even if non where hit
+	}
+
+	if (!_potentially_dirty_contacts.empty()) {
+		std::cout << "MFS: pdc\n";
+		// here we check if any view of said contact needs frag loading
+		// only once per tick tho
+
+		// TODO: this makes order depend on internal order and is not fair
+		auto it = _potentially_dirty_contacts.cbegin();
+
+		auto* msg_reg = _rmm.get(*it);
+
+		// first do collision check agains every contact associated fragment
+		// that is not already loaded !!
+		if (msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
+			const auto& cf = msg_reg->ctx().get<Message::Components::ContactFragments>();
+			if (!cf.frags.empty()) {
+				if (!msg_reg->ctx().contains<Message::Components::LoadedContactFragments>()) {
+					msg_reg->ctx().emplace<Message::Components::LoadedContactFragments>();
+				}
+				const auto& loaded_frags = msg_reg->ctx().get<Message::Components::LoadedContactFragments>().frags;
+
+				for (const auto& [fid, si] : msg_reg->ctx().get<Message::Components::ContactFragments>().frags) {
+					if (loaded_frags.contains(fid)) {
+						continue;
+					}
+
+					auto fh = _fs.fragmentHandle(fid);
+
+					if (!static_cast<bool>(fh)) {
+						std::cerr << "MFS error: frag is invalid\n";
+						// WHAT
+						msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
+						return 0.05f;
+					}
+
+					if (!fh.all_of<FragComp::MessagesTSRange>()) {
+						std::cerr << "MFS error: frag has no range\n";
+						// ????
+						msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
+						return 0.05f;
+					}
+
+					// get ts range of frag and collide with all curser(s/ranges)
+					const auto& [range_begin, range_end] = fh.get<FragComp::MessagesTSRange>();
+
+					if (rangeVisible(range_begin, range_end, *msg_reg)) {
+						std::cout << "MFS: frag hit by vis range\n";
+						loadFragment(*msg_reg, fh);
+						return 0.05f;
+					}
+				}
+				// no new visible fragment
+				std::cout << "MFS: no new frag directly visible\n";
+
+				// now, finally, check for adjecent fragments that need to be loaded
+				// we do this by finding the outermost fragment in a rage, and extend it by one
+
+				// TODO: rewrite using some bounding range tree to perform collision checks !!!
+				// (this is now performing better, but still)
+
+
+				// for each view
+				auto c_b_view = msg_reg->view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
+				c_b_view.use<Message::Components::ViewCurserBegin>();
+				for (const auto& [_, ts_begin_comp, vcb] : c_b_view.each()) {
+					// aka "scroll down"
+					{ // find newest(-ish) frag in range
+						// or in reverse frag end <= range begin
+
+
+						// lower bound of frag end and range begin
+						const auto right = std::lower_bound(
+							cf.sorted_end.crbegin(),
+							cf.sorted_end.crend(),
+							ts_begin_comp.ts,
+							[&](const FragmentID element, const auto& value) -> bool {
+								return _fs._reg.get<FragComp::MessagesTSRange>(element).end >= value;
+							}
+						);
+
+						FragmentID next_frag{entt::null};
+						if (right != cf.sorted_end.crend()) {
+							next_frag = cf.next(*right);
+						}
+						// we checked earlier that cf is not empty
+						if (!_fs._reg.valid(next_frag)) {
+							// fall back to closest, cf is not empty
+							next_frag = cf.sorted_end.front();
+						}
+
+						// a single adjacent frag is often not enough
+						// only ok bc next is cheap
+						for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); i++) {
+							if (!loaded_frags.contains(next_frag)) {
+								std::cout << "MFS: next frag of range\n";
+								loadFragment(*msg_reg, {_fs._reg, next_frag});
+								return 0.05f;
+							}
+
+							next_frag = cf.next(next_frag);
+						}
+					}
+
+					// curser end
+					if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of<Message::Components::Timestamp>(vcb.curser_end)) {
+						continue;
+					}
+					const auto ts_end = msg_reg->get<Message::Components::Timestamp>(vcb.curser_end).ts;
+
+					// aka "scroll up"
+					{ // find oldest(-ish) frag in range
+						// frag begin >= range end
+
+						// lower bound of frag begin and range end
+						const auto left = std::lower_bound(
+							cf.sorted_begin.cbegin(),
+							cf.sorted_begin.cend(),
+							ts_end,
+							[&](const FragmentID element, const auto& value) -> bool {
+								return _fs._reg.get<FragComp::MessagesTSRange>(element).begin < value;
+							}
+						);
+
+						FragmentID prev_frag{entt::null};
+						if (left != cf.sorted_begin.cend()) {
+							prev_frag = cf.prev(*left);
+						}
+						// we checked earlier that cf is not empty
+						if (!_fs._reg.valid(prev_frag)) {
+							// fall back to closest, cf is not empty
+							prev_frag = cf.sorted_begin.back();
+						}
+
+						// a single adjacent frag is often not enough
+						// only ok bc next is cheap
+						for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); i++) {
+							if (!loaded_frags.contains(prev_frag)) {
+								std::cout << "MFS: prev frag of range\n";
+								loadFragment(*msg_reg, {_fs._reg, prev_frag});
+								return 0.05f;
+							}
+
+							prev_frag = cf.prev(prev_frag);
+						}
+					}
+				}
+			}
+		} else {
+			// contact has no fragments, skip
+		}
+
+		_potentially_dirty_contacts.erase(it);
+
+		return 0.05f;
+	}
+
+
+	return 1000.f*60.f*60.f;
+}
+
+void MessageFragmentStore::triggerScan(void) {
+	_fs.scanStoragePath("test_message_store/");
+}
+
+bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) {
+	handleMessage(e.e);
+	return false;
+}
+
+bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) {
+	handleMessage(e.e);
+	return false;
+}
+
+// TODO: handle deletes? diff between unload?
+
+bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) {
+	if (_fs_ignore_event) {
+		return false; // skip self
+	}
+
+	if (!e.e.all_of<FragComp::MessagesTSRange, FragComp::MessagesContact>()) {
+		return false; // not for us
+	}
+
+	// TODO: are we sure it is a *new* fragment?
+
+	Contact3 frag_contact = entt::null;
+	{ // get contact
+		const auto& frag_contact_id = e.e.get<FragComp::MessagesContact>().id;
+		// TODO: id lookup table, this is very inefficent
+		for (const auto& [c_it, id_it] : _cr.view<Contact::Components::ID>().each()) {
+			if (frag_contact_id == id_it.data) {
+				frag_contact = c_it;
+				break;
+			}
+		}
+		if (!_cr.valid(frag_contact)) {
+			// unkown contact
+			return false;
+		}
+	}
+
+	// create if not exist
+	auto* msg_reg = _rmm.get(frag_contact);
+	if (msg_reg == nullptr) {
+		// msg reg not created yet
+		// TODO: this is an erroious path
+		return false;
+	}
+
+	if (!msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
+		msg_reg->ctx().emplace<Message::Components::ContactFragments>();
+	}
+	msg_reg->ctx().get<Message::Components::ContactFragments>().erase(e.e); // TODO: check/update/fragment update
+	msg_reg->ctx().get<Message::Components::ContactFragments>().insert(e.e);
+
+	_event_check_queue.push(ECQueueEntry{e.e, frag_contact});
+
+	return false;
+}
+
+bool Message::Components::ContactFragments::insert(FragmentHandle frag) {
+	if (frags.contains(frag)) {
+		return false;
+	}
+
+	// both sorted arrays are sorted ascending
+	// so for insertion we search for the last index that is <= and insert after it
+	// or we search for the first > (or end) and insert before it    <---
+	// since equal fragments are UB, we can assume they are only > or <
+
+	size_t begin_index {0};
+	{ // begin
+		const auto pos = std::find_if(
+			sorted_begin.cbegin(),
+			sorted_begin.cend(),
+			[frag](const FragmentID a) -> bool {
+				const auto begin_a = frag.registry()->get<FragComp::MessagesTSRange>(a).begin;
+				const auto begin_frag = frag.get<FragComp::MessagesTSRange>().begin;
+				if (begin_a > begin_frag) {
+					return true;
+				} else if (begin_a < begin_frag) {
+					return false;
+				} else {
+					// equal ts, we need to fall back to id (id can not be equal)
+					return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
+				}
+			}
+		);
+
+		begin_index = std::distance(sorted_begin.cbegin(), pos);
+
+		// we need to insert before pos (end is valid here)
+		sorted_begin.insert(pos, frag);
+	}
+
+	size_t end_index {0};
+	{ // end
+		const auto pos = std::find_if_not(
+			sorted_end.cbegin(),
+			sorted_end.cend(),
+			[frag](const FragmentID a) -> bool {
+				const auto end_a = frag.registry()->get<FragComp::MessagesTSRange>(a).end;
+				const auto end_frag = frag.get<FragComp::MessagesTSRange>().end;
+				if (end_a > end_frag) {
+					return true;
+				} else if (end_a < end_frag) {
+					return false;
+				} else {
+					// equal ts, we need to fall back to id (id can not be equal)
+					return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
+				}
+			}
+		);
+
+		end_index = std::distance(sorted_end.cbegin(), pos);
+
+		// we need to insert before pos (end is valid here)
+		sorted_end.insert(pos, frag);
+	}
+
+	frags.emplace(frag, InternalEntry{begin_index, end_index});
+
+	return true;
+}
+
+bool Message::Components::ContactFragments::erase(FragmentID frag) {
+	auto frags_it = frags.find(frag);
+	if (frags_it == frags.end()) {
+		return false;
+	}
+
+	assert(sorted_begin.size() == sorted_end.size());
+	assert(sorted_begin.size() > frags_it->second.i_b);
+
+	sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b);
+	sorted_end.erase(sorted_end.begin() + frags_it->second.i_e);
+
+	frags.erase(frags_it);
+
+	return true;
+}
+
+FragmentID Message::Components::ContactFragments::prev(FragmentID frag) const {
+	// uses range begin to go back in time
+
+	auto it = frags.find(frag);
+	if (it == frags.end()) {
+		return entt::null;
+	}
+
+	const auto src_i = it->second.i_b;
+	if (src_i > 0) {
+		return sorted_begin[src_i-1];
+	}
+
+	return entt::null;
+}
+
+FragmentID Message::Components::ContactFragments::next(FragmentID frag) const {
+	// uses range end to go forward in time
+
+	auto it = frags.find(frag);
+	if (it == frags.end()) {
+		return entt::null;
+	}
+
+	const auto src_i = it->second.i_e;
+	if (src_i+1 < sorted_end.size()) {
+		return sorted_end[src_i+1];
+	}
+
+	return entt::null;
+}
+

src/fragment_store/message_fragment_store.hpp

@@ -0,0 +1,126 @@
+#pragma once
+
+#include "./meta_components.hpp"
+#include "./fragment_store_i.hpp"
+#include "./fragment_store.hpp"
+
+#include "./message_serializer.hpp"
+
+#include <entt/entity/registry.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/container/dense_set.hpp>
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <queue>
+#include <vector>
+#include <cstdint>
+
+namespace Message::Components {
+
+	using FUID = FragComp::ID;
+
+	// unused
+	struct FID {
+		FragmentID fid {entt::null};
+	};
+
+	// points to the front/newer message
+	// together they define a range that is,
+	// eg the first(end) and last(begin) message being rendered
+	// MFS requires there to be atleast one other message after/before,
+	// if not loaded fragment with fitting tsrange(direction) available
+	// uses fragmentAfter/Before()
+	// they can exist standalone
+	// if they are a pair, the inside is filled first
+	// cursers require a timestamp ???
+	struct ViewCurserBegin {
+		Message3 curser_end{entt::null};
+	};
+	struct ViewCurserEnd {
+		Message3 curser_begin{entt::null};
+	};
+
+	// mfs will only load a limited number of fragments per tick (1),
+	// so this tag will be set if we loaded a fragment and
+	// every tick we check all cursers for this tag and continue
+	// and remove once no fragment could be loaded anymore
+	// (internal)
+	struct TagCurserUnsatisfied {};
+
+} // Message::Components
+
+namespace Fragment::Components {
+	struct MessagesTSRange {
+		// timestamp range within the fragment
+		uint64_t begin {0}; // newer msg -> higher number
+		uint64_t end {0};
+	};
+
+	struct MessagesContact {
+		std::vector<uint8_t> id;
+	};
+
+	// TODO: add src contact (self id)
+
+} // Fragment::Components
+
+// handles fragments for messages
+// on new message: assign fuid
+// on new and update: mark as fragment dirty
+// on delete: mark as fragment dirty?
+class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentStoreEventI {
+	protected:
+		Contact3Registry& _cr;
+		RegistryMessageModel& _rmm;
+		FragmentStore& _fs;
+		bool _fs_ignore_event {false};
+
+		// for message components only
+		MessageSerializerCallbacks _sc;
+
+		void handleMessage(const Message3Handle& m);
+
+		void loadFragment(Message3Registry& reg, FragmentHandle fh);
+
+		struct SaveQueueEntry final {
+			uint64_t ts_since_dirty{0};
+			std::vector<uint8_t> id;
+			Message3Registry* reg{nullptr};
+		};
+		std::queue<SaveQueueEntry> _fuid_save_queue;
+
+		struct ECQueueEntry final {
+			FragmentID fid;
+			Contact3 c;
+		};
+		std::queue<ECQueueEntry> _event_check_queue;
+
+		// range changed or fragment loaded.
+		// we only load a limited number of fragments at once,
+		// so we need to keep them dirty until nothing was loaded.
+		entt::dense_set<Contact3> _potentially_dirty_contacts;
+
+	public:
+		MessageFragmentStore(
+			Contact3Registry& cr,
+			RegistryMessageModel& rmm,
+			FragmentStore& fs
+		);
+		virtual ~MessageFragmentStore(void);
+
+		MessageSerializerCallbacks& getMSC(void);
+
+		float tick(float time_delta);
+
+		void triggerScan(void);
+
+	protected: // rmm
+		bool onEvent(const Message::Events::MessageConstruct& e) override;
+		bool onEvent(const Message::Events::MessageUpdated& e) override;
+
+	protected: // fs
+		bool onEvent(const Fragment::Events::FragmentConstruct& e) override;
+};
+

src/fragment_store/message_serializer.cpp

@@ -0,0 +1,107 @@
+#include "./message_serializer.hpp"
+
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/contact/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <iostream>
+
+static Contact3 findContactByID(Contact3Registry& cr, const std::vector<uint8_t>& id) {
+	// TODO: id lookup table, this is very inefficent
+	for (const auto& [c_it, id_it] : cr.view<Contact::Components::ID>().each()) {
+		if (id == id_it.data) {
+			return c_it;
+		}
+	}
+
+	return entt::null;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactFrom>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactFrom>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactFrom>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactTo>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactTo>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactTo>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}

src/fragment_store/message_serializer.hpp

@@ -0,0 +1,85 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+struct MessageSerializerCallbacks {
+	using Registry = Message3Registry;
+	using Handle = Message3Handle;
+
+	Contact3Registry& cr;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+
+// fwd
+namespace Message::Components {
+struct ContactFrom;
+struct ContactTo;
+}
+
+// make specializations known
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);

src/fragment_store/meta_components.hpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include "./types.hpp"
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+namespace Fragment::Components {
+
+	// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
+	struct ID {
+		std::vector<uint8_t> v;
+	};
+
+	struct DataEncryptionType {
+		Encryption enc {Encryption::NONE};
+	};
+
+	struct DataCompressionType {
+		Compression comp {Compression::NONE};
+	};
+
+
+	// meta that is not written to (meta-)file
+	namespace Ephemeral {
+
+		// excluded from file meta
+		struct FilePath {
+			// contains store path, if any
+			std::string path;
+		};
+
+		// TODO: seperate into remote and local?
+		// (remote meaning eg. the file on disk was changed by another program)
+		struct DirtyTag {};
+
+
+		// type as comp
+		struct MetaFileType {
+			::MetaFileType type {::MetaFileType::TEXT_JSON};
+		};
+
+		struct MetaEncryptionType {
+			Encryption enc {Encryption::NONE};
+		};
+
+		struct MetaCompressionType {
+			Compression comp {Compression::NONE};
+		};
+
+	} // Ephemeral
+
+} // Components
+
+// shortened to save bytes (until I find a way to save by ID in msgpack)
+namespace FragComp = Fragment::Components;
+
+#include "./meta_components_id.inl"
+

src/fragment_store/meta_components_id.inl

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "./meta_components.hpp"
+
+#include <entt/core/type_info.hpp>
+
+// TODO: move more central
+#define DEFINE_COMP_ID(x) \
+template<> \
+constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
+	using namespace entt::literals; \
+	return #x##_hs; \
+} \
+template<> \
+constexpr std::string_view entt::type_name<x>::value() noexcept { \
+	return #x; \
+}
+
+// cross compiler stable ids
+
+DEFINE_COMP_ID(FragComp::DataEncryptionType)
+DEFINE_COMP_ID(FragComp::DataCompressionType)
+
+#undef DEFINE_COMP_ID
+
+

src/fragment_store/register_mfs_json_message_components.cpp

@@ -0,0 +1,35 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/message_components.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::Timestamp>();
+	msc.registerDeSerializerJson<Message::Components::Timestamp>();
+	msc.registerSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerDeSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerSerializerJson<Message::Components::ContactFrom>();
+	msc.registerDeSerializerJson<Message::Components::ContactFrom>();
+	msc.registerSerializerJson<Message::Components::ContactTo>();
+	msc.registerDeSerializerJson<Message::Components::ContactTo>();
+	msc.registerSerializerJson<Message::Components::TagUnread>();
+	msc.registerDeSerializerJson<Message::Components::TagUnread>();
+	msc.registerSerializerJson<Message::Components::Read>();
+	msc.registerDeSerializerJson<Message::Components::Read>();
+	msc.registerSerializerJson<Message::Components::MessageText>();
+	msc.registerDeSerializerJson<Message::Components::MessageText>();
+	msc.registerSerializerJson<Message::Components::TagMessageIsAction>();
+	msc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
+
+	// files
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
+}
+

src/fragment_store/register_mfs_json_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/register_mfs_json_tox_message_components.cpp

@@ -0,0 +1,10 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/tox_message_components.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::ToxGroupMessageID>();
+	msc.registerDeSerializerJson<Message::Components::ToxGroupMessageID>();
+}
+

src/fragment_store/register_mfs_json_tox_message_components.hpp

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc);
+

src/fragment_store/serializer.hpp

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+template<typename EntityType = entt::entity>
+struct SerializerCallbacks {
+	using Registry = entt::basic_registry<EntityType>;
+	using Handle = entt::basic_handle<Registry>;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+

src/fragment_store/test_fragstore.cpp

@@ -0,0 +1,80 @@
+#include <cstdint>
+#include <iostream>
+
+#include "./fragment_store.hpp"
+
+#include <nlohmann/json.hpp>
+#include <entt/entity/handle.hpp>
+
+namespace Components {
+	struct MessagesTimestampRange {
+		uint64_t begin {0};
+		uint64_t end {1000};
+	};
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end)
+} // Components
+
+
+int main(void) {
+	FragmentStore fs;
+	fs._default_store_path = "test_store/";
+	fs._sc.registerSerializerJson<Components::MessagesTimestampRange>();
+	fs._sc.registerDeSerializerJson<Components::MessagesTimestampRange>();
+
+	const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9});
+
+	const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	const auto frag2 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	{
+		auto frag0h = fs.fragmentHandle(frag0);
+
+		frag0h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag0h.emplace_or_replace<FragComp::DataEncryptionType>();
+		frag0h.emplace_or_replace<Components::MessagesTimestampRange>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			uint64_t i = 0;
+			for (; i+read < 3000 && i < buffer_size; i++) {
+				request_buffer[i] = uint8_t((i+read) & 0xff);
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag0, fn_cb);
+	}
+
+	{
+		auto frag1h = fs.fragmentHandle(frag1);
+
+		frag1h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag1h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			static constexpr std::string_view text = "This is some random data";
+			uint64_t i = 0;
+			for (; i+read < text.size() && i < buffer_size; i++) {
+				request_buffer[i] = text[i+read];
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag1, fn_cb);
+	}
+
+	{
+		auto frag2h = fs.fragmentHandle(frag2);
+
+		frag2h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag2h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		static constexpr std::string_view text = "This is more random data";
+		fs.syncToStorage(frag2, reinterpret_cast<const uint8_t*>(text.data()), text.size());
+	}
+	return 0;
+}
+

src/fragment_store/types.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+
+enum class Encryption : uint8_t {
+	NONE = 0x00,
+};
+enum class Compression : uint8_t {
+	NONE = 0x00,
+	ZSTD = 0x01,
+	// TODO: zstd without magic
+	// TODO: zstd meta dict
+	// TODO: zstd data(message) dict
+};
+enum class MetaFileType : uint8_t {
+	TEXT_JSON,
+	BINARY_MSGPACK, // msgpacked msgpack
+};
+

src/json/message_components.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/message3/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts)
+	// TODO: SyncedBy
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text)
+
+	namespace Transfer {
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list)
+	} // Transfer
+
+} // Message::Components
+

src/json/tox_message_components.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/tox_messages/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	// TODO: friend msg id, does not have the same qualities
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id)
+	// TODO: transfer stuff, needs content rewrite
+
+} // Message::Components
+

src/main_screen.cpp

@@ -1,5 +1,8 @@
 #include "./main_screen.hpp"
 
+#include "./fragment_store/register_mfs_json_message_components.hpp"
+#include "./fragment_store/register_mfs_json_tox_message_components.hpp"
+
 #include <solanaceae/contact/components.hpp>
 
 #include <imgui/imgui.h>
@@ -13,6 +16,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	renderer(renderer_),
 	rmm(cr),
 	mts(rmm),
+	mfs(cr, rmm, fs),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
@@ -33,6 +37,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	tdch(tpi)
 {
 	tel.subscribeAll(tc);
+	registerMFSJsonMessageComponents(mfs.getMSC());
+	registerMFSJsonToxMessageComponents(mfs.getMSC());
 
 	conf.set("tox", "save_file_path", save_path);
 
@@ -49,6 +55,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";
 
 	{ // setup plugin instances
+		// TODO: make interface useful
+		g_provideInstance<FragmentStoreI>("FragmentStoreI", "host", &fs);
+		g_provideInstance<FragmentStore>("FragmentStore", "host", &fs);
+
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
@@ -74,6 +84,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	}
 
 	conf.dump();
+
+	mfs.triggerScan(); // HACK: after plugins and tox contacts got loaded
 }
 
 MainScreen::~MainScreen(void) {
@@ -388,7 +400,8 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 
 	tdch.tick(time_delta); // compute
 
-	mts.iterate(); // compute
+	const float mfs_interval = mfs.tick(time_delta);
+	mts.iterate(); // compute (after mfs)
 
 	_min_tick_interval = std::min<float>(
 		// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
@@ -400,6 +413,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 		_min_tick_interval,
 		fo_interval
 	);
+	_min_tick_interval = std::min<float>(
+		_min_tick_interval,
+		mfs_interval
+	);
 
 	//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";
 

src/main_screen.hpp

@@ -2,10 +2,12 @@
 
 #include "./screen.hpp"
 
+#include "./fragment_store/fragment_store.hpp"
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
+#include "./fragment_store/message_fragment_store.hpp"
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@@ -43,10 +45,13 @@ extern "C" {
 struct MainScreen final : public Screen {
 	SDL_Renderer* renderer;
 
+	FragmentStore fs;
+
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
 	MessageTimeSort mts;
+	MessageFragmentStore mfs;
 
 	PluginManager pm;
 

src/send_image_popup.hpp

@@ -32,7 +32,7 @@ struct SendImagePopup {
 	Rect crop_rect;
 	Rect crop_before_drag;
 
-	bool cropping {true};
+	bool cropping {false};
 	bool dragging_last_frame_ul {false};
 	bool dragging_last_frame_lr {false};