combat memory leaks with smart pointers

refactor saving and save on exit
dirty frag on message updates (if still open)
2024-03-11 12:05:30 +01:00 · 2024-03-11 12:05:30 +01:00 · 2024-03-11 12:05:30 +01:00 · 2024-03-11 12:05:30 +01:00 · 2024-03-11 12:05:30 +01:00 · 2024-03-11 12:05:30 +01:00
58 changed files with 3172 additions and 259 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			link_libraries(-fsanitize=address)
-			#link_libraries(-fsanitize=address,undefined)
+			#link_libraries(-fsanitize=address)
+			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			message("II enabled ASAN")
 		else()
--- a/external/CMakeLists.txt
+++ b/external/CMakeLists.txt
@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)

 add_subdirectory(./entt)

@ -19,3 +19,34 @@ add_subdirectory(./stb)
 add_subdirectory(./libwebp)
 add_subdirectory(./qoi)

+if (NOT TARGET nlohmann_json::nlohmann_json)
+	FetchContent_Declare(json
+		URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+		URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757257b107ae403032760ed6ef121c9d55e81757d
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(json)
+endif()
+
+if (NOT TARGET zstd::zstd)
+	# TODO: try find_package() first
+	# TODO: try pkg-config next (will work on most distros)
+
+	set(ZSTD_BUILD_STATIC ON)
+	set(ZSTD_BUILD_SHARED OFF)
+	set(ZSTD_BUILD_PROGRAMS OFF)
+	set(ZSTD_BUILD_CONTRIB OFF)
+	set(ZSTD_BUILD_TESTS OFF)
+	FetchContent_Declare(zstd
+		URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
+		DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+		SOURCE_SUBDIR build/cmake
+		EXCLUDE_FROM_ALL
+	)
+	FetchContent_MakeAvailable(zstd)
+
+	add_library(zstd INTERFACE) # somehow zstd fkd this up
+	target_include_directories(zstd INTERFACE ${zstd_SOURCE_DIR}/lib/)
+	target_link_libraries(zstd INTERFACE libzstd_static)
+	add_library(zstd::zstd ALIAS zstd)
+endif()
--- a/external/solanaceae_tox
+++ b/external/solanaceae_tox
@ -1 +1 @@
-Subproject commit 4bd7235a739dec020365d216509474f641029113
+Subproject commit ce81ef7cf7cea2fe2091912c9eafe787cbba6100
--- a/external/solanaceae_toxcore
+++ b/external/solanaceae_toxcore
@ -1 +1 @@
-Subproject commit 49ab40a1ba97e884bf43ab5f35cfb48fc51f91de
+Subproject commit cf3679018be3f90db0f2f1e9433a966692976421
--- a/external/toxcore/c-toxcore/auto_tests/group_message_test.c
+++ b/external/toxcore/c-toxcore/auto_tests/group_message_test.c
@ -284,7 +284,7 @@ static void group_private_message_handler(const Tox_Event_Group_Private_Message

    const uint32_t groupnumber = tox_event_group_private_message_get_group_number(event);
    const uint32_t peer_id = tox_event_group_private_message_get_peer_id(event);
-    const Tox_Message_Type type = tox_event_group_private_message_get_type(event);
+    const Tox_Message_Type type = tox_event_group_private_message_get_message_type(event);
    const uint8_t *message = tox_event_group_private_message_get_message(event);
    const size_t length = tox_event_group_private_message_get_message_length(event);
    const Tox_Group_Message_Id pseudo_msg_id = tox_event_group_private_message_get_message_id(event);
--- a/external/toxcore/c-toxcore/other/event_tooling/generate_event_c.cpp
+++ b/external/toxcore/c-toxcore/other/event_tooling/generate_event_c.cpp
@ -686,7 +686,7 @@ int main(int argc, char** argv) {
            {
                EventTypeTrivial{"uint32_t", "group_number"},
                EventTypeTrivial{"uint32_t", "peer_id"},
-                EventTypeByteRange{"name", "name_length", "length"}, // the latter two are ideally the same
+                EventTypeByteRange{"name", "name_length", "name_length"},
            }
        },
        {
@ -702,7 +702,7 @@ int main(int argc, char** argv) {
            {
                EventTypeTrivial{"uint32_t", "group_number"},
                EventTypeTrivial{"uint32_t", "peer_id"},
-                EventTypeByteRange{"topic", "topic_length", "length"}, // the latter two are ideally the same
+                EventTypeByteRange{"topic", "topic_length", "topic_length"},
            }
        },
        {
@ -737,7 +737,7 @@ int main(int argc, char** argv) {
            "Group_Password",
            {
                EventTypeTrivial{"uint32_t", "group_number"},
-                EventTypeByteRange{"password", "password_length", "length"}, // the latter two are ideally the same
+                EventTypeByteRange{"password", "password_length", "password_length"},
            }
        },
        {
@ -745,8 +745,8 @@ int main(int argc, char** argv) {
            {
                EventTypeTrivial{"uint32_t", "group_number"},
                EventTypeTrivial{"uint32_t", "peer_id"},
-                EventTypeTrivial{"Tox_Message_Type", "type"},
-                EventTypeByteRange{"message", "message_length", "length"}, // the latter two are ideally the same
+                EventTypeTrivial{"Tox_Message_Type", "message_type"},
+                EventTypeByteRange{"message", "message_length", "message_length"},
                EventTypeTrivial{"uint32_t", "message_id"},
            }
        },
@ -755,8 +755,8 @@ int main(int argc, char** argv) {
            {
                EventTypeTrivial{"uint32_t", "group_number"},
                EventTypeTrivial{"uint32_t", "peer_id"},
-                EventTypeTrivial{"Tox_Message_Type", "type"},
-                EventTypeByteRange{"message", "message_length", "length"}, // the latter two are ideally the same
+                EventTypeTrivial{"Tox_Message_Type", "message_type"},
+                EventTypeByteRange{"message", "message_length", "message_length"},
                EventTypeTrivial{"uint32_t", "message_id"},
            }
        },
@ -765,7 +765,7 @@ int main(int argc, char** argv) {
            {
                EventTypeTrivial{"uint32_t", "group_number"},
                EventTypeTrivial{"uint32_t", "peer_id"},
-                EventTypeByteRange{"data", "data_length", "length"}, // the latter two are ideally the same
+                EventTypeByteRange{"data", "data_length", "data_length"},
            }
        },
        {
@ -773,15 +773,15 @@ int main(int argc, char** argv) {
            {
                EventTypeTrivial{"uint32_t", "group_number"},
                EventTypeTrivial{"uint32_t", "peer_id"},
-                EventTypeByteRange{"data", "data_length", "length"}, // the latter two are ideally the same
+                EventTypeByteRange{"data", "data_length", "data_length"},
            }
        },
        {
            "Group_Invite",
            {
                EventTypeTrivial{"uint32_t", "friend_number"},
-                EventTypeByteRange{"invite_data", "invite_data_length", "length"}, // the latter two are ideally the same
-                EventTypeByteRange{"group_name", "group_name_length", "group_name_length"}, // they are :)
+                EventTypeByteRange{"invite_data", "invite_data_length", "invite_data_length"},
+                EventTypeByteRange{"group_name", "group_name_length", "group_name_length"},
            }
        },
        {
@ -797,8 +797,8 @@ int main(int argc, char** argv) {
                EventTypeTrivial{"uint32_t", "group_number"},
                EventTypeTrivial{"uint32_t", "peer_id"},
                EventTypeTrivial{"Tox_Group_Exit_Type", "exit_type"},
-                EventTypeByteRange{"name", "name_length", "name_length"}, // they are :)
-                EventTypeByteRange{"part_message", "part_message_length", "part_message_length"}, // they are :)
+                EventTypeByteRange{"name", "name_length", "name_length"},
+                EventTypeByteRange{"part_message", "part_message_length", "part_message_length"},
            }
        },
        {
--- a/external/toxcore/c-toxcore/testing/BUILD.bazel
+++ b/external/toxcore/c-toxcore/testing/BUILD.bazel
@ -14,6 +14,7 @@ sh_test(
    args = ["$(locations %s)" % f for f in CIMPLE_FILES] + [
        "-Wno-boolean-return",
        "-Wno-callback-names",
+        "-Wno-enum-from-int",
        "+RTS",
        "-N4",
        "-RTS",
--- a/external/toxcore/c-toxcore/toxcore/Messenger.c
+++ b/external/toxcore/c-toxcore/toxcore/Messenger.c
@ -775,31 +775,31 @@ int m_set_statusmessage(Messenger *m, const uint8_t *status, uint16_t length)
 }

 non_null()
-static bool userstatus_from_int(uint8_t status, Userstatus *out)
+static bool userstatus_from_int(uint8_t status, Userstatus *out_enum)
 {
    switch (status) {
        case USERSTATUS_NONE: {
-            *out = USERSTATUS_NONE;
+            *out_enum = USERSTATUS_NONE;
            return true;
        }

        case USERSTATUS_AWAY: {
-            *out = USERSTATUS_AWAY;
+            *out_enum = USERSTATUS_AWAY;
            return true;
        }

        case USERSTATUS_BUSY: {
-            *out = USERSTATUS_BUSY;
+            *out_enum = USERSTATUS_BUSY;
            return true;
        }

        case USERSTATUS_INVALID: {
-            *out = USERSTATUS_INVALID;
+            *out_enum = USERSTATUS_INVALID;
            return true;
        }

        default: {
-            *out = USERSTATUS_INVALID;
+            *out_enum = USERSTATUS_INVALID;
            return false;
        }
    }
--- a/external/toxcore/c-toxcore/toxcore/events/group_custom_packet.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_custom_packet.c
@ -220,7 +220,7 @@ static Tox_Event_Group_Custom_Packet *tox_event_group_custom_packet_alloc(void *
 *****************************************************/

 void tox_events_handle_group_custom_packet(
-    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *data, size_t length,
+    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *data, size_t data_length,
    void *user_data)
 {
    Tox_Event_Group_Custom_Packet *group_custom_packet = tox_event_group_custom_packet_alloc(user_data);
@ -231,5 +231,5 @@ void tox_events_handle_group_custom_packet(

    tox_event_group_custom_packet_set_group_number(group_custom_packet, group_number);
    tox_event_group_custom_packet_set_peer_id(group_custom_packet, peer_id);
-    tox_event_group_custom_packet_set_data(group_custom_packet, data, length);
+    tox_event_group_custom_packet_set_data(group_custom_packet, data, data_length);
 }
--- a/external/toxcore/c-toxcore/toxcore/events/group_custom_private_packet.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_custom_private_packet.c
@ -220,7 +220,7 @@ static Tox_Event_Group_Custom_Private_Packet *tox_event_group_custom_private_pac
 *****************************************************/

 void tox_events_handle_group_custom_private_packet(
-    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *data, size_t length,
+    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *data, size_t data_length,
    void *user_data)
 {
    Tox_Event_Group_Custom_Private_Packet *group_custom_private_packet = tox_event_group_custom_private_packet_alloc(user_data);
@ -231,5 +231,5 @@ void tox_events_handle_group_custom_private_packet(

    tox_event_group_custom_private_packet_set_group_number(group_custom_private_packet, group_number);
    tox_event_group_custom_private_packet_set_peer_id(group_custom_private_packet, peer_id);
-    tox_event_group_custom_private_packet_set_data(group_custom_private_packet, data, length);
+    tox_event_group_custom_private_packet_set_data(group_custom_private_packet, data, data_length);
 }
--- a/external/toxcore/c-toxcore/toxcore/events/group_invite.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_invite.c
@ -248,7 +248,7 @@ static Tox_Event_Group_Invite *tox_event_group_invite_alloc(void *user_data)
 *****************************************************/

 void tox_events_handle_group_invite(
-    Tox *tox, uint32_t friend_number, const uint8_t *invite_data, size_t length, const uint8_t *group_name, size_t group_name_length,
+    Tox *tox, uint32_t friend_number, const uint8_t *invite_data, size_t invite_data_length, const uint8_t *group_name, size_t group_name_length,
    void *user_data)
 {
    Tox_Event_Group_Invite *group_invite = tox_event_group_invite_alloc(user_data);
@ -258,6 +258,6 @@ void tox_events_handle_group_invite(
    }

    tox_event_group_invite_set_friend_number(group_invite, friend_number);
-    tox_event_group_invite_set_invite_data(group_invite, invite_data, length);
+    tox_event_group_invite_set_invite_data(group_invite, invite_data, invite_data_length);
    tox_event_group_invite_set_group_name(group_invite, group_name, group_name_length);
 }
--- a/external/toxcore/c-toxcore/toxcore/events/group_message.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_message.c
@ -27,7 +27,7 @@
 struct Tox_Event_Group_Message {
    uint32_t group_number;
    uint32_t peer_id;
-    Tox_Message_Type type;
+    Tox_Message_Type message_type;
    uint8_t *message;
    uint32_t message_length;
    uint32_t message_id;
@ -60,16 +60,16 @@ uint32_t tox_event_group_message_get_peer_id(const Tox_Event_Group_Message *grou
 }

 non_null()
-static void tox_event_group_message_set_type(Tox_Event_Group_Message *group_message,
-        Tox_Message_Type type)
+static void tox_event_group_message_set_message_type(Tox_Event_Group_Message *group_message,
+        Tox_Message_Type message_type)
 {
    assert(group_message != nullptr);
-    group_message->type = type;
+    group_message->message_type = message_type;
 }
-Tox_Message_Type tox_event_group_message_get_type(const Tox_Event_Group_Message *group_message)
+Tox_Message_Type tox_event_group_message_get_message_type(const Tox_Event_Group_Message *group_message)
 {
    assert(group_message != nullptr);
-    return group_message->type;
+    return group_message->message_type;
 }

 non_null(1) nullable(2)
@ -143,7 +143,7 @@ bool tox_event_group_message_pack(
    return bin_pack_array(bp, 5)
           && bin_pack_u32(bp, event->group_number)
           && bin_pack_u32(bp, event->peer_id)
-           && tox_message_type_pack(event->type, bp)
+           && tox_message_type_pack(event->message_type, bp)
           && bin_pack_bin(bp, event->message, event->message_length)
           && bin_pack_u32(bp, event->message_id);
 }
@ -159,7 +159,7 @@ static bool tox_event_group_message_unpack_into(

    return bin_unpack_u32(bu, &event->group_number)
           && bin_unpack_u32(bu, &event->peer_id)
-           && tox_message_type_unpack(&event->type, bu)
+           && tox_message_type_unpack(&event->message_type, bu)
           && bin_unpack_bin(bu, &event->message, &event->message_length)
           && bin_unpack_u32(bu, &event->message_id);
 }
@ -254,7 +254,7 @@ static Tox_Event_Group_Message *tox_event_group_message_alloc(void *user_data)
 *****************************************************/

 void tox_events_handle_group_message(
-    Tox *tox, uint32_t group_number, uint32_t peer_id, Tox_Message_Type type, const uint8_t *message, size_t length, uint32_t message_id,
+    Tox *tox, uint32_t group_number, uint32_t peer_id, Tox_Message_Type message_type, const uint8_t *message, size_t message_length, uint32_t message_id,
    void *user_data)
 {
    Tox_Event_Group_Message *group_message = tox_event_group_message_alloc(user_data);
@ -265,7 +265,7 @@ void tox_events_handle_group_message(

    tox_event_group_message_set_group_number(group_message, group_number);
    tox_event_group_message_set_peer_id(group_message, peer_id);
-    tox_event_group_message_set_type(group_message, type);
-    tox_event_group_message_set_message(group_message, message, length);
+    tox_event_group_message_set_message_type(group_message, message_type);
+    tox_event_group_message_set_message(group_message, message, message_length);
    tox_event_group_message_set_message_id(group_message, message_id);
 }
--- a/external/toxcore/c-toxcore/toxcore/events/group_password.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_password.c
@ -204,7 +204,7 @@ static Tox_Event_Group_Password *tox_event_group_password_alloc(void *user_data)
 *****************************************************/

 void tox_events_handle_group_password(
-    Tox *tox, uint32_t group_number, const uint8_t *password, size_t length,
+    Tox *tox, uint32_t group_number, const uint8_t *password, size_t password_length,
    void *user_data)
 {
    Tox_Event_Group_Password *group_password = tox_event_group_password_alloc(user_data);
@ -214,5 +214,5 @@ void tox_events_handle_group_password(
    }

    tox_event_group_password_set_group_number(group_password, group_number);
-    tox_event_group_password_set_password(group_password, password, length);
+    tox_event_group_password_set_password(group_password, password, password_length);
 }
--- a/external/toxcore/c-toxcore/toxcore/events/group_peer_name.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_peer_name.c
@ -220,7 +220,7 @@ static Tox_Event_Group_Peer_Name *tox_event_group_peer_name_alloc(void *user_dat
 *****************************************************/

 void tox_events_handle_group_peer_name(
-    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *name, size_t length,
+    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *name, size_t name_length,
    void *user_data)
 {
    Tox_Event_Group_Peer_Name *group_peer_name = tox_event_group_peer_name_alloc(user_data);
@ -231,5 +231,5 @@ void tox_events_handle_group_peer_name(

    tox_event_group_peer_name_set_group_number(group_peer_name, group_number);
    tox_event_group_peer_name_set_peer_id(group_peer_name, peer_id);
-    tox_event_group_peer_name_set_name(group_peer_name, name, length);
+    tox_event_group_peer_name_set_name(group_peer_name, name, name_length);
 }
--- a/external/toxcore/c-toxcore/toxcore/events/group_private_message.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_private_message.c
@ -27,7 +27,7 @@
 struct Tox_Event_Group_Private_Message {
    uint32_t group_number;
    uint32_t peer_id;
-    Tox_Message_Type type;
+    Tox_Message_Type message_type;
    uint8_t *message;
    uint32_t message_length;
    uint32_t message_id;
@ -60,16 +60,16 @@ uint32_t tox_event_group_private_message_get_peer_id(const Tox_Event_Group_Priva
 }

 non_null()
-static void tox_event_group_private_message_set_type(Tox_Event_Group_Private_Message *group_private_message,
-        Tox_Message_Type type)
+static void tox_event_group_private_message_set_message_type(Tox_Event_Group_Private_Message *group_private_message,
+        Tox_Message_Type message_type)
 {
    assert(group_private_message != nullptr);
-    group_private_message->type = type;
+    group_private_message->message_type = message_type;
 }
-Tox_Message_Type tox_event_group_private_message_get_type(const Tox_Event_Group_Private_Message *group_private_message)
+Tox_Message_Type tox_event_group_private_message_get_message_type(const Tox_Event_Group_Private_Message *group_private_message)
 {
    assert(group_private_message != nullptr);
-    return group_private_message->type;
+    return group_private_message->message_type;
 }

 non_null(1) nullable(2)
@ -143,7 +143,7 @@ bool tox_event_group_private_message_pack(
    return bin_pack_array(bp, 5)
           && bin_pack_u32(bp, event->group_number)
           && bin_pack_u32(bp, event->peer_id)
-           && tox_message_type_pack(event->type, bp)
+           && tox_message_type_pack(event->message_type, bp)
           && bin_pack_bin(bp, event->message, event->message_length)
           && bin_pack_u32(bp, event->message_id);
 }
@ -159,7 +159,7 @@ static bool tox_event_group_private_message_unpack_into(

    return bin_unpack_u32(bu, &event->group_number)
           && bin_unpack_u32(bu, &event->peer_id)
-           && tox_message_type_unpack(&event->type, bu)
+           && tox_message_type_unpack(&event->message_type, bu)
           && bin_unpack_bin(bu, &event->message, &event->message_length)
           && bin_unpack_u32(bu, &event->message_id);
 }
@ -254,7 +254,7 @@ static Tox_Event_Group_Private_Message *tox_event_group_private_message_alloc(vo
 *****************************************************/

 void tox_events_handle_group_private_message(
-    Tox *tox, uint32_t group_number, uint32_t peer_id, Tox_Message_Type type, const uint8_t *message, size_t length, uint32_t message_id,
+    Tox *tox, uint32_t group_number, uint32_t peer_id, Tox_Message_Type message_type, const uint8_t *message, size_t message_length, uint32_t message_id,
    void *user_data)
 {
    Tox_Event_Group_Private_Message *group_private_message = tox_event_group_private_message_alloc(user_data);
@ -265,7 +265,7 @@ void tox_events_handle_group_private_message(

    tox_event_group_private_message_set_group_number(group_private_message, group_number);
    tox_event_group_private_message_set_peer_id(group_private_message, peer_id);
-    tox_event_group_private_message_set_type(group_private_message, type);
-    tox_event_group_private_message_set_message(group_private_message, message, length);
+    tox_event_group_private_message_set_message_type(group_private_message, message_type);
+    tox_event_group_private_message_set_message(group_private_message, message, message_length);
    tox_event_group_private_message_set_message_id(group_private_message, message_id);
 }
--- a/external/toxcore/c-toxcore/toxcore/events/group_topic.c
+++ b/external/toxcore/c-toxcore/toxcore/events/group_topic.c
@ -220,7 +220,7 @@ static Tox_Event_Group_Topic *tox_event_group_topic_alloc(void *user_data)
 *****************************************************/

 void tox_events_handle_group_topic(
-    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *topic, size_t length,
+    Tox *tox, uint32_t group_number, uint32_t peer_id, const uint8_t *topic, size_t topic_length,
    void *user_data)
 {
    Tox_Event_Group_Topic *group_topic = tox_event_group_topic_alloc(user_data);
@ -231,5 +231,5 @@ void tox_events_handle_group_topic(

    tox_event_group_topic_set_group_number(group_topic, group_number);
    tox_event_group_topic_set_peer_id(group_topic, peer_id);
-    tox_event_group_topic_set_topic(group_topic, topic, length);
+    tox_event_group_topic_set_topic(group_topic, topic, topic_length);
 }
--- a/external/toxcore/c-toxcore/toxcore/group_pack.c
+++ b/external/toxcore/c-toxcore/toxcore/group_pack.c
@ -26,46 +26,46 @@
 #include "network.h"
 #include "util.h"

-bool group_privacy_state_from_int(uint8_t value, Group_Privacy_State *out)
+bool group_privacy_state_from_int(uint8_t value, Group_Privacy_State *out_enum)
 {
    switch (value) {
        case GI_PUBLIC: {
-            *out = GI_PUBLIC;
+            *out_enum = GI_PUBLIC;
            return true;
        }

        case GI_PRIVATE: {
-            *out = GI_PRIVATE;
+            *out_enum = GI_PRIVATE;
            return true;
        }

        default: {
-            *out = GI_PUBLIC;
+            *out_enum = GI_PUBLIC;
            return false;
        }
    }
 }

-bool group_voice_state_from_int(uint8_t value, Group_Voice_State *out)
+bool group_voice_state_from_int(uint8_t value, Group_Voice_State *out_enum)
 {
    switch (value) {
        case GV_ALL: {
-            *out = GV_ALL;
+            *out_enum = GV_ALL;
            return true;
        }

        case GV_MODS: {
-            *out = GV_MODS;
+            *out_enum = GV_MODS;
            return true;
        }

        case GV_FOUNDER: {
-            *out = GV_FOUNDER;
+            *out_enum = GV_FOUNDER;
            return true;
        }

        default: {
-            *out = GV_ALL;
+            *out_enum = GV_ALL;
            return false;
        }
    }
--- a/external/toxcore/c-toxcore/toxcore/group_pack.h
+++ b/external/toxcore/c-toxcore/toxcore/group_pack.h
@ -34,8 +34,8 @@ non_null()
 bool gc_load_unpack_group(GC_Chat *chat, Bin_Unpack *bu);

 non_null()
-bool group_privacy_state_from_int(uint8_t value, Group_Privacy_State *out);
+bool group_privacy_state_from_int(uint8_t value, Group_Privacy_State *out_enum);
 non_null()
-bool group_voice_state_from_int(uint8_t value, Group_Voice_State *out);
+bool group_voice_state_from_int(uint8_t value, Group_Voice_State *out_enum);

 #endif /* C_TOXCORE_TOXCORE_GROUP_PACK_H */
--- a/external/toxcore/c-toxcore/toxcore/tox.c
+++ b/external/toxcore/c-toxcore/toxcore/tox.c
@ -3932,7 +3932,7 @@ bool tox_group_get_password(const Tox *tox, uint32_t group_number, uint8_t *pass
 }

 Tox_Group_Message_Id tox_group_send_message(
-    const Tox *tox, uint32_t group_number, Tox_Message_Type type, const uint8_t *message,
+    const Tox *tox, uint32_t group_number, Tox_Message_Type message_type, const uint8_t *message,
    size_t length, Tox_Err_Group_Send_Message *error)
 {
    assert(tox != nullptr);
@ -3953,7 +3953,7 @@ Tox_Group_Message_Id tox_group_send_message(
    }

    uint32_t message_id = 0;
-    const int ret = gc_send_message(chat, message, length, type, &message_id);
+    const int ret = gc_send_message(chat, message, length, message_type, &message_id);
    tox_unlock(tox);

    switch (ret) {
@ -3995,7 +3995,7 @@ Tox_Group_Message_Id tox_group_send_message(
 }

 Tox_Group_Message_Id tox_group_send_private_message(const Tox *tox, uint32_t group_number, uint32_t peer_id,
-        Tox_Message_Type type, const uint8_t *message, size_t length, Tox_Err_Group_Send_Private_Message *error)
+        Tox_Message_Type message_type, const uint8_t *message, size_t length, Tox_Err_Group_Send_Private_Message *error)
 {
    assert(tox != nullptr);

@ -4015,7 +4015,7 @@ Tox_Group_Message_Id tox_group_send_private_message(const Tox *tox, uint32_t gro
    }

    uint32_t message_id = 0;
-    const int ret = gc_send_private_message(chat, gc_peer_id_from_int(peer_id), type, message, length, &message_id);
+    const int ret = gc_send_private_message(chat, gc_peer_id_from_int(peer_id), message_type, message, length, &message_id);
    tox_unlock(tox);

    switch (ret) {
--- a/external/toxcore/c-toxcore/toxcore/tox.h
+++ b/external/toxcore/c-toxcore/toxcore/tox.h
@ -4094,11 +4094,11 @@ bool tox_group_peer_get_public_key(
 * @param group_number The group number of the group the name change is intended for.
 * @param peer_id The ID of the peer who has changed their name.
 * @param name The name data.
- * @param length The length of the name.
+ * @param name_length The length of the name.
 */
 typedef void tox_group_peer_name_cb(
    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id,
-    const uint8_t name[], size_t length, void *user_data);
+    const uint8_t name[], size_t name_length, void *user_data);

 /**
 * Set the callback for the `group_peer_name` event. Pass NULL to unset.
@ -4235,11 +4235,11 @@ bool tox_group_get_topic(
 * @param peer_id The ID of the peer who changed the topic. If the peer who set the topic
 *   is not present in our peer list this value will be set to 0.
 * @param topic The topic data.
- * @param length The topic length.
+ * @param topic_length The topic length.
 */
 typedef void tox_group_topic_cb(
    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id,
-    const uint8_t topic[], size_t length,
+    const uint8_t topic[], size_t topic_length,
    void *user_data);

 /**
@ -4416,11 +4416,11 @@ bool tox_group_get_password(
 /**
 * @param group_number The group number of the group for which the password has changed.
 * @param password The new group password.
- * @param length The length of the password.
+ * @param password_length The length of the password.
 */
 typedef void tox_group_password_cb(
    Tox *tox, Tox_Group_Number group_number,
-    const uint8_t password[], size_t length,
+    const uint8_t password[], size_t password_length,
    void *user_data);

 /**
@ -4493,7 +4493,7 @@ const char *tox_err_group_send_message_to_string(Tox_Err_Group_Send_Message valu
 * then reassemble the fragments. Messages may not be empty.
 *
 * @param group_number The group number of the group the message is intended for.
- * @param type Message type (normal, action, ...).
+ * @param message_type Message type (normal, action, ...).
 * @param message A non-NULL pointer to the first element of a byte array
 *   containing the message text.
 * @param length Length of the message to be sent.
@ -4502,7 +4502,7 @@ const char *tox_err_group_send_message_to_string(Tox_Err_Group_Send_Message valu
 *   returned message ID value will be undefined.
 */
 Tox_Group_Message_Id tox_group_send_message(
-    const Tox *tox, Tox_Group_Number group_number, Tox_Message_Type type,
+    const Tox *tox, Tox_Group_Number group_number, Tox_Message_Type message_type,
    const uint8_t message[], size_t length,
    Tox_Err_Group_Send_Message *error);

@ -4533,6 +4533,11 @@ typedef enum Tox_Err_Group_Send_Private_Message {
     */
    TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_EMPTY,

+    /**
+     * The message type is invalid.
+     */
+    TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_BAD_TYPE,
+
    /**
     * The caller does not have the required permissions to send group messages.
     */
@ -4548,11 +4553,6 @@ typedef enum Tox_Err_Group_Send_Private_Message {
     */
    TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_DISCONNECTED,

-    /**
-     * The message type is invalid.
-     */
-    TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_BAD_TYPE,
-
 } Tox_Err_Group_Send_Private_Message;

 const char *tox_err_group_send_private_message_to_string(Tox_Err_Group_Send_Private_Message value);
@ -4569,6 +4569,7 @@ const char *tox_err_group_send_private_message_to_string(Tox_Err_Group_Send_Priv
 *
 * @param group_number The group number of the group the message is intended for.
 * @param peer_id The ID of the peer the message is intended for.
+ * @param message_type The type of message (normal, action, ...).
 * @param message A non-NULL pointer to the first element of a byte array
 *   containing the message text.
 * @param length Length of the message to be sent.
@ -4576,7 +4577,7 @@ const char *tox_err_group_send_private_message_to_string(Tox_Err_Group_Send_Priv
 * @return true on success.
 */
 Tox_Group_Message_Id tox_group_send_private_message(
-    const Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id, Tox_Message_Type type,
+    const Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id, Tox_Message_Type message_type,
    const uint8_t message[], size_t length,
    Tox_Err_Group_Send_Private_Message *error);

@ -4729,14 +4730,14 @@ bool tox_group_send_custom_private_packet(const Tox *tox, Tox_Group_Number group
 /**
 * @param group_number The group number of the group the message is intended for.
 * @param peer_id The ID of the peer who sent the message.
- * @param type The type of message (normal, action, ...).
+ * @param message_type The type of message (normal, action, ...).
 * @param message The message data.
+ * @param message_length The length of the message.
 * @param message_id A pseudo message id that clients can use to uniquely identify this group message.
- * @param length The length of the message.
 */
 typedef void tox_group_message_cb(
-    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id, Tox_Message_Type type,
-    const uint8_t message[], size_t length, Tox_Group_Message_Id message_id, void *user_data);
+    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id, Tox_Message_Type message_type,
+    const uint8_t message[], size_t message_length, Tox_Group_Message_Id message_id, void *user_data);

 /**
 * Set the callback for the `group_message` event. Pass NULL to unset.
@ -4748,12 +4749,14 @@ void tox_callback_group_message(Tox *tox, tox_group_message_cb *callback);
 /**
 * @param group_number The group number of the group the private message is intended for.
 * @param peer_id The ID of the peer who sent the private message.
+ * @param message_type The type of message (normal, action, ...).
 * @param message The message data.
- * @param length The length of the message.
+ * @param message_length The length of the message.
+ * @param message_id A pseudo message id that clients can use to uniquely identify this group message.
 */
 typedef void tox_group_private_message_cb(
-    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id, Tox_Message_Type type,
-    const uint8_t message[], size_t length, Tox_Group_Message_Id message_id, void *user_data);
+    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id, Tox_Message_Type message_type,
+    const uint8_t message[], size_t message_length, Tox_Group_Message_Id message_id, void *user_data);

 /**
 * Set the callback for the `group_private_message` event. Pass NULL to unset.
@ -4766,11 +4769,11 @@ void tox_callback_group_private_message(Tox *tox, tox_group_private_message_cb *
 * @param group_number The group number of the group the packet is intended for.
 * @param peer_id The ID of the peer who sent the packet.
 * @param data The packet data.
- * @param length The length of the data.
+ * @param data_length The length of the data.
 */
 typedef void tox_group_custom_packet_cb(
    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id,
-    const uint8_t data[], size_t length, void *user_data);
+    const uint8_t data[], size_t data_length, void *user_data);

 /**
 * Set the callback for the `group_custom_packet` event. Pass NULL to unset.
@ -4783,11 +4786,11 @@ void tox_callback_group_custom_packet(Tox *tox, tox_group_custom_packet_cb *call
 * @param group_number The group number of the group the packet is intended for.
 * @param peer_id The ID of the peer who sent the packet.
 * @param data The packet data.
- * @param length The length of the data.
+ * @param data_length The length of the data.
 */
 typedef void tox_group_custom_private_packet_cb(
    Tox *tox, Tox_Group_Number group_number, Tox_Group_Peer_Number peer_id,
-    const uint8_t data[], size_t length, void *user_data);
+    const uint8_t data[], size_t data_length, void *user_data);

 /**
 * Set the callback for the `group_custom_private_packet` event. Pass NULL to unset.
@ -4923,11 +4926,11 @@ Tox_Group_Number tox_group_invite_accept(
 /**
 * @param friend_number The friend number of the contact who sent the invite.
 * @param invite_data The invite data.
- * @param length The length of invite_data.
+ * @param invite_data_length The length of invite_data.
 */
 typedef void tox_group_invite_cb(
    Tox *tox, Tox_Friend_Number friend_number,
-    const uint8_t invite_data[], size_t length,
+    const uint8_t invite_data[], size_t invite_data_length,
    const uint8_t group_name[], size_t group_name_length,
    void *user_data);

--- a/external/toxcore/c-toxcore/toxcore/tox_api.c
+++ b/external/toxcore/c-toxcore/toxcore/tox_api.c
@ -1340,6 +1340,9 @@ const char *tox_err_group_send_private_message_to_string(Tox_Err_Group_Send_Priv
        case TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_EMPTY:
            return "TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_EMPTY";

+        case TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_BAD_TYPE:
+            return "TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_BAD_TYPE";
+
        case TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_PERMISSIONS:
            return "TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_PERMISSIONS";

@ -1348,9 +1351,6 @@ const char *tox_err_group_send_private_message_to_string(Tox_Err_Group_Send_Priv

        case TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_DISCONNECTED:
            return "TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_DISCONNECTED";
-
-        case TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_BAD_TYPE:
-            return "TOX_ERR_GROUP_SEND_PRIVATE_MESSAGE_BAD_TYPE";
    }

    return "<invalid Tox_Err_Group_Send_Private_Message>";
--- a/external/toxcore/c-toxcore/toxcore/tox_event.c
+++ b/external/toxcore/c-toxcore/toxcore/tox_event.c
@ -725,216 +725,216 @@ bool tox_event_pack(const Tox_Event *event, Bin_Pack *bp)
 }

 non_null()
-static bool tox_event_type_from_int(uint32_t value, Tox_Event_Type *out)
+static bool tox_event_type_from_int(uint32_t value, Tox_Event_Type *out_enum)
 {
    switch (value) {
        case TOX_EVENT_SELF_CONNECTION_STATUS: {
-            *out = TOX_EVENT_SELF_CONNECTION_STATUS;
+            *out_enum = TOX_EVENT_SELF_CONNECTION_STATUS;
            return true;
        }

        case TOX_EVENT_FRIEND_REQUEST: {
-            *out = TOX_EVENT_FRIEND_REQUEST;
+            *out_enum = TOX_EVENT_FRIEND_REQUEST;
            return true;
        }

        case TOX_EVENT_FRIEND_CONNECTION_STATUS: {
-            *out = TOX_EVENT_FRIEND_CONNECTION_STATUS;
+            *out_enum = TOX_EVENT_FRIEND_CONNECTION_STATUS;
            return true;
        }

        case TOX_EVENT_FRIEND_LOSSY_PACKET: {
-            *out = TOX_EVENT_FRIEND_LOSSY_PACKET;
+            *out_enum = TOX_EVENT_FRIEND_LOSSY_PACKET;
            return true;
        }

        case TOX_EVENT_FRIEND_LOSSLESS_PACKET: {
-            *out = TOX_EVENT_FRIEND_LOSSLESS_PACKET;
+            *out_enum = TOX_EVENT_FRIEND_LOSSLESS_PACKET;
            return true;
        }

        case TOX_EVENT_FRIEND_NAME: {
-            *out = TOX_EVENT_FRIEND_NAME;
+            *out_enum = TOX_EVENT_FRIEND_NAME;
            return true;
        }

        case TOX_EVENT_FRIEND_STATUS: {
-            *out = TOX_EVENT_FRIEND_STATUS;
+            *out_enum = TOX_EVENT_FRIEND_STATUS;
            return true;
        }

        case TOX_EVENT_FRIEND_STATUS_MESSAGE: {
-            *out = TOX_EVENT_FRIEND_STATUS_MESSAGE;
+            *out_enum = TOX_EVENT_FRIEND_STATUS_MESSAGE;
            return true;
        }

        case TOX_EVENT_FRIEND_MESSAGE: {
-            *out = TOX_EVENT_FRIEND_MESSAGE;
+            *out_enum = TOX_EVENT_FRIEND_MESSAGE;
            return true;
        }

        case TOX_EVENT_FRIEND_READ_RECEIPT: {
-            *out = TOX_EVENT_FRIEND_READ_RECEIPT;
+            *out_enum = TOX_EVENT_FRIEND_READ_RECEIPT;
            return true;
        }

        case TOX_EVENT_FRIEND_TYPING: {
-            *out = TOX_EVENT_FRIEND_TYPING;
+            *out_enum = TOX_EVENT_FRIEND_TYPING;
            return true;
        }

        case TOX_EVENT_FILE_CHUNK_REQUEST: {
-            *out = TOX_EVENT_FILE_CHUNK_REQUEST;
+            *out_enum = TOX_EVENT_FILE_CHUNK_REQUEST;
            return true;
        }

        case TOX_EVENT_FILE_RECV: {
-            *out = TOX_EVENT_FILE_RECV;
+            *out_enum = TOX_EVENT_FILE_RECV;
            return true;
        }

        case TOX_EVENT_FILE_RECV_CHUNK: {
-            *out = TOX_EVENT_FILE_RECV_CHUNK;
+            *out_enum = TOX_EVENT_FILE_RECV_CHUNK;
            return true;
        }

        case TOX_EVENT_FILE_RECV_CONTROL: {
-            *out = TOX_EVENT_FILE_RECV_CONTROL;
+            *out_enum = TOX_EVENT_FILE_RECV_CONTROL;
            return true;
        }

        case TOX_EVENT_CONFERENCE_INVITE: {
-            *out = TOX_EVENT_CONFERENCE_INVITE;
+            *out_enum = TOX_EVENT_CONFERENCE_INVITE;
            return true;
        }

        case TOX_EVENT_CONFERENCE_CONNECTED: {
-            *out = TOX_EVENT_CONFERENCE_CONNECTED;
+            *out_enum = TOX_EVENT_CONFERENCE_CONNECTED;
            return true;
        }

        case TOX_EVENT_CONFERENCE_PEER_LIST_CHANGED: {
-            *out = TOX_EVENT_CONFERENCE_PEER_LIST_CHANGED;
+            *out_enum = TOX_EVENT_CONFERENCE_PEER_LIST_CHANGED;
            return true;
        }

        case TOX_EVENT_CONFERENCE_PEER_NAME: {
-            *out = TOX_EVENT_CONFERENCE_PEER_NAME;
+            *out_enum = TOX_EVENT_CONFERENCE_PEER_NAME;
            return true;
        }

        case TOX_EVENT_CONFERENCE_TITLE: {
-            *out = TOX_EVENT_CONFERENCE_TITLE;
+            *out_enum = TOX_EVENT_CONFERENCE_TITLE;
            return true;
        }

        case TOX_EVENT_CONFERENCE_MESSAGE: {
-            *out = TOX_EVENT_CONFERENCE_MESSAGE;
+            *out_enum = TOX_EVENT_CONFERENCE_MESSAGE;
            return true;
        }

        case TOX_EVENT_GROUP_PEER_NAME: {
-            *out = TOX_EVENT_GROUP_PEER_NAME;
+            *out_enum = TOX_EVENT_GROUP_PEER_NAME;
            return true;
        }

        case TOX_EVENT_GROUP_PEER_STATUS: {
-            *out = TOX_EVENT_GROUP_PEER_STATUS;
+            *out_enum = TOX_EVENT_GROUP_PEER_STATUS;
            return true;
        }

        case TOX_EVENT_GROUP_TOPIC: {
-            *out = TOX_EVENT_GROUP_TOPIC;
+            *out_enum = TOX_EVENT_GROUP_TOPIC;
            return true;
        }

        case TOX_EVENT_GROUP_PRIVACY_STATE: {
-            *out = TOX_EVENT_GROUP_PRIVACY_STATE;
+            *out_enum = TOX_EVENT_GROUP_PRIVACY_STATE;
            return true;
        }

        case TOX_EVENT_GROUP_VOICE_STATE: {
-            *out = TOX_EVENT_GROUP_VOICE_STATE;
+            *out_enum = TOX_EVENT_GROUP_VOICE_STATE;
            return true;
        }

        case TOX_EVENT_GROUP_TOPIC_LOCK: {
-            *out = TOX_EVENT_GROUP_TOPIC_LOCK;
+            *out_enum = TOX_EVENT_GROUP_TOPIC_LOCK;
            return true;
        }

        case TOX_EVENT_GROUP_PEER_LIMIT: {
-            *out = TOX_EVENT_GROUP_PEER_LIMIT;
+            *out_enum = TOX_EVENT_GROUP_PEER_LIMIT;
            return true;
        }

        case TOX_EVENT_GROUP_PASSWORD: {
-            *out = TOX_EVENT_GROUP_PASSWORD;
+            *out_enum = TOX_EVENT_GROUP_PASSWORD;
            return true;
        }

        case TOX_EVENT_GROUP_MESSAGE: {
-            *out = TOX_EVENT_GROUP_MESSAGE;
+            *out_enum = TOX_EVENT_GROUP_MESSAGE;
            return true;
        }

        case TOX_EVENT_GROUP_PRIVATE_MESSAGE: {
-            *out = TOX_EVENT_GROUP_PRIVATE_MESSAGE;
+            *out_enum = TOX_EVENT_GROUP_PRIVATE_MESSAGE;
            return true;
        }

        case TOX_EVENT_GROUP_CUSTOM_PACKET: {
-            *out = TOX_EVENT_GROUP_CUSTOM_PACKET;
+            *out_enum = TOX_EVENT_GROUP_CUSTOM_PACKET;
            return true;
        }

        case TOX_EVENT_GROUP_CUSTOM_PRIVATE_PACKET: {
-            *out = TOX_EVENT_GROUP_CUSTOM_PRIVATE_PACKET;
+            *out_enum = TOX_EVENT_GROUP_CUSTOM_PRIVATE_PACKET;
            return true;
        }

        case TOX_EVENT_GROUP_INVITE: {
-            *out = TOX_EVENT_GROUP_INVITE;
+            *out_enum = TOX_EVENT_GROUP_INVITE;
            return true;
        }

        case TOX_EVENT_GROUP_PEER_JOIN: {
-            *out = TOX_EVENT_GROUP_PEER_JOIN;
+            *out_enum = TOX_EVENT_GROUP_PEER_JOIN;
            return true;
        }

        case TOX_EVENT_GROUP_PEER_EXIT: {
-            *out = TOX_EVENT_GROUP_PEER_EXIT;
+            *out_enum = TOX_EVENT_GROUP_PEER_EXIT;
            return true;
        }

        case TOX_EVENT_GROUP_SELF_JOIN: {
-            *out = TOX_EVENT_GROUP_SELF_JOIN;
+            *out_enum = TOX_EVENT_GROUP_SELF_JOIN;
            return true;
        }

        case TOX_EVENT_GROUP_JOIN_FAIL: {
-            *out = TOX_EVENT_GROUP_JOIN_FAIL;
+            *out_enum = TOX_EVENT_GROUP_JOIN_FAIL;
            return true;
        }

        case TOX_EVENT_GROUP_MODERATION: {
-            *out = TOX_EVENT_GROUP_MODERATION;
+            *out_enum = TOX_EVENT_GROUP_MODERATION;
            return true;
        }

        case TOX_EVENT_DHT_GET_NODES_RESPONSE: {
-            *out = TOX_EVENT_DHT_GET_NODES_RESPONSE;
+            *out_enum = TOX_EVENT_DHT_GET_NODES_RESPONSE;
            return true;
        }

        case TOX_EVENT_INVALID: {
-            *out = TOX_EVENT_INVALID;
+            *out_enum = TOX_EVENT_INVALID;
            return true;
        }

        default: {
-            *out = TOX_EVENT_INVALID;
+            *out_enum = TOX_EVENT_INVALID;
            return false;
        }
    }
--- a/external/toxcore/c-toxcore/toxcore/tox_events.h
+++ b/external/toxcore/c-toxcore/toxcore/tox_events.h
@ -248,7 +248,7 @@ uint32_t tox_event_group_message_get_group_number(
    const Tox_Event_Group_Message *group_message);
 uint32_t tox_event_group_message_get_peer_id(
    const Tox_Event_Group_Message *group_message);
-Tox_Message_Type tox_event_group_message_get_type(
+Tox_Message_Type tox_event_group_message_get_message_type(
    const Tox_Event_Group_Message *group_message);
 const uint8_t *tox_event_group_message_get_message(
    const Tox_Event_Group_Message *group_message);
@ -262,7 +262,7 @@ uint32_t tox_event_group_private_message_get_group_number(
    const Tox_Event_Group_Private_Message *group_private_message);
 uint32_t tox_event_group_private_message_get_peer_id(
    const Tox_Event_Group_Private_Message *group_private_message);
-Tox_Message_Type tox_event_group_private_message_get_type(
+Tox_Message_Type tox_event_group_private_message_get_message_type(
    const Tox_Event_Group_Private_Message *group_private_message);
 const uint8_t *tox_event_group_private_message_get_message(
    const Tox_Event_Group_Private_Message *group_private_message);
--- a/external/toxcore/c-toxcore/toxcore/tox_unpack.c
+++ b/external/toxcore/c-toxcore/toxcore/tox_unpack.c
@ -11,21 +11,21 @@
 #include "tox.h"

 non_null()
-static bool tox_conference_type_from_int(uint32_t value, Tox_Conference_Type *out)
+static bool tox_conference_type_from_int(uint32_t value, Tox_Conference_Type *out_enum)
 {
    switch (value) {
        case TOX_CONFERENCE_TYPE_TEXT: {
-            *out = TOX_CONFERENCE_TYPE_TEXT;
+            *out_enum = TOX_CONFERENCE_TYPE_TEXT;
            return true;
        }

        case TOX_CONFERENCE_TYPE_AV: {
-            *out = TOX_CONFERENCE_TYPE_AV;
+            *out_enum = TOX_CONFERENCE_TYPE_AV;
            return true;
        }

        default: {
-            *out = TOX_CONFERENCE_TYPE_TEXT;
+            *out_enum = TOX_CONFERENCE_TYPE_TEXT;
            return false;
        }
    }
@ -38,26 +38,26 @@ bool tox_conference_type_unpack(Tox_Conference_Type *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_connection_from_int(uint32_t value, Tox_Connection *out)
+static bool tox_connection_from_int(uint32_t value, Tox_Connection *out_enum)
 {
    switch (value) {
        case TOX_CONNECTION_NONE: {
-            *out = TOX_CONNECTION_NONE;
+            *out_enum = TOX_CONNECTION_NONE;
            return true;
        }

        case TOX_CONNECTION_TCP: {
-            *out = TOX_CONNECTION_TCP;
+            *out_enum = TOX_CONNECTION_TCP;
            return true;
        }

        case TOX_CONNECTION_UDP: {
-            *out = TOX_CONNECTION_UDP;
+            *out_enum = TOX_CONNECTION_UDP;
            return true;
        }

        default: {
-            *out = TOX_CONNECTION_NONE;
+            *out_enum = TOX_CONNECTION_NONE;
            return false;
        }
    }
@ -71,26 +71,26 @@ bool tox_connection_unpack(Tox_Connection *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_file_control_from_int(uint32_t value, Tox_File_Control *out)
+static bool tox_file_control_from_int(uint32_t value, Tox_File_Control *out_enum)
 {
    switch (value) {
        case TOX_FILE_CONTROL_RESUME: {
-            *out = TOX_FILE_CONTROL_RESUME;
+            *out_enum = TOX_FILE_CONTROL_RESUME;
            return true;
        }

        case TOX_FILE_CONTROL_PAUSE: {
-            *out = TOX_FILE_CONTROL_PAUSE;
+            *out_enum = TOX_FILE_CONTROL_PAUSE;
            return true;
        }

        case TOX_FILE_CONTROL_CANCEL: {
-            *out = TOX_FILE_CONTROL_CANCEL;
+            *out_enum = TOX_FILE_CONTROL_CANCEL;
            return true;
        }

        default: {
-            *out = TOX_FILE_CONTROL_RESUME;
+            *out_enum = TOX_FILE_CONTROL_RESUME;
            return false;
        }
    }
@ -104,21 +104,21 @@ bool tox_file_control_unpack(Tox_File_Control *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_message_type_from_int(uint32_t value, Tox_Message_Type *out)
+static bool tox_message_type_from_int(uint32_t value, Tox_Message_Type *out_enum)
 {
    switch (value) {
        case TOX_MESSAGE_TYPE_NORMAL: {
-            *out = TOX_MESSAGE_TYPE_NORMAL;
+            *out_enum = TOX_MESSAGE_TYPE_NORMAL;
            return true;
        }

        case TOX_MESSAGE_TYPE_ACTION: {
-            *out = TOX_MESSAGE_TYPE_ACTION;
+            *out_enum = TOX_MESSAGE_TYPE_ACTION;
            return true;
        }

        default: {
-            *out = TOX_MESSAGE_TYPE_NORMAL;
+            *out_enum = TOX_MESSAGE_TYPE_NORMAL;
            return false;
        }
    }
@ -132,26 +132,26 @@ bool tox_message_type_unpack(Tox_Message_Type *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_user_status_from_int(uint32_t value, Tox_User_Status *out)
+static bool tox_user_status_from_int(uint32_t value, Tox_User_Status *out_enum)
 {
    switch (value) {
        case TOX_USER_STATUS_NONE: {
-            *out = TOX_USER_STATUS_NONE;
+            *out_enum = TOX_USER_STATUS_NONE;
            return true;
        }

        case TOX_USER_STATUS_AWAY: {
-            *out = TOX_USER_STATUS_AWAY;
+            *out_enum = TOX_USER_STATUS_AWAY;
            return true;
        }

        case TOX_USER_STATUS_BUSY: {
-            *out = TOX_USER_STATUS_BUSY;
+            *out_enum = TOX_USER_STATUS_BUSY;
            return true;
        }

        default: {
-            *out = TOX_USER_STATUS_NONE;
+            *out_enum = TOX_USER_STATUS_NONE;
            return false;
        }
    }
@ -165,19 +165,19 @@ bool tox_user_status_unpack(Tox_User_Status *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_group_privacy_state_from_int(uint32_t value, Tox_Group_Privacy_State *out)
+static bool tox_group_privacy_state_from_int(uint32_t value, Tox_Group_Privacy_State *out_enum)
 {
    switch (value) {
        case TOX_GROUP_PRIVACY_STATE_PUBLIC: {
-            *out = TOX_GROUP_PRIVACY_STATE_PUBLIC;
+            *out_enum = TOX_GROUP_PRIVACY_STATE_PUBLIC;
            return true;
        }
        case TOX_GROUP_PRIVACY_STATE_PRIVATE: {
-            *out = TOX_GROUP_PRIVACY_STATE_PRIVATE;
+            *out_enum = TOX_GROUP_PRIVACY_STATE_PRIVATE;
            return true;
        }
        default: {
-            *out = TOX_GROUP_PRIVACY_STATE_PUBLIC;
+            *out_enum = TOX_GROUP_PRIVACY_STATE_PUBLIC;
            return false;
        }
    }
@ -189,23 +189,23 @@ bool tox_group_privacy_state_unpack(Tox_Group_Privacy_State *val, Bin_Unpack *bu
           && tox_group_privacy_state_from_int(u32, val);
 }
 non_null()
-static bool tox_group_voice_state_from_int(uint32_t value, Tox_Group_Voice_State *out)
+static bool tox_group_voice_state_from_int(uint32_t value, Tox_Group_Voice_State *out_enum)
 {
    switch (value) {
        case TOX_GROUP_VOICE_STATE_ALL: {
-            *out = TOX_GROUP_VOICE_STATE_ALL;
+            *out_enum = TOX_GROUP_VOICE_STATE_ALL;
            return true;
        }
        case TOX_GROUP_VOICE_STATE_MODERATOR: {
-            *out = TOX_GROUP_VOICE_STATE_MODERATOR;
+            *out_enum = TOX_GROUP_VOICE_STATE_MODERATOR;
            return true;
        }
        case TOX_GROUP_VOICE_STATE_FOUNDER: {
-            *out = TOX_GROUP_VOICE_STATE_FOUNDER;
+            *out_enum = TOX_GROUP_VOICE_STATE_FOUNDER;
            return true;
        }
        default: {
-            *out = TOX_GROUP_VOICE_STATE_ALL;
+            *out_enum = TOX_GROUP_VOICE_STATE_ALL;
            return false;
        }
    }
@ -218,19 +218,19 @@ bool tox_group_voice_state_unpack(Tox_Group_Voice_State *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_group_topic_lock_from_int(uint32_t value, Tox_Group_Topic_Lock *out)
+static bool tox_group_topic_lock_from_int(uint32_t value, Tox_Group_Topic_Lock *out_enum)
 {
    switch (value) {
        case TOX_GROUP_TOPIC_LOCK_ENABLED: {
-            *out = TOX_GROUP_TOPIC_LOCK_ENABLED;
+            *out_enum = TOX_GROUP_TOPIC_LOCK_ENABLED;
            return true;
        }
        case TOX_GROUP_TOPIC_LOCK_DISABLED: {
-            *out = TOX_GROUP_TOPIC_LOCK_DISABLED;
+            *out_enum = TOX_GROUP_TOPIC_LOCK_DISABLED;
            return true;
        }
        default: {
-            *out = TOX_GROUP_TOPIC_LOCK_ENABLED;
+            *out_enum = TOX_GROUP_TOPIC_LOCK_ENABLED;
            return false;
        }
    }
@ -243,23 +243,23 @@ bool tox_group_topic_lock_unpack(Tox_Group_Topic_Lock *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_group_join_fail_from_int(uint32_t value, Tox_Group_Join_Fail *out)
+static bool tox_group_join_fail_from_int(uint32_t value, Tox_Group_Join_Fail *out_enum)
 {
    switch (value) {
        case TOX_GROUP_JOIN_FAIL_PEER_LIMIT: {
-            *out = TOX_GROUP_JOIN_FAIL_PEER_LIMIT;
+            *out_enum = TOX_GROUP_JOIN_FAIL_PEER_LIMIT;
            return true;
        }
        case TOX_GROUP_JOIN_FAIL_INVALID_PASSWORD: {
-            *out = TOX_GROUP_JOIN_FAIL_INVALID_PASSWORD;
+            *out_enum = TOX_GROUP_JOIN_FAIL_INVALID_PASSWORD;
            return true;
        }
        case TOX_GROUP_JOIN_FAIL_UNKNOWN: {
-            *out = TOX_GROUP_JOIN_FAIL_UNKNOWN;
+            *out_enum = TOX_GROUP_JOIN_FAIL_UNKNOWN;
            return true;
        }
        default: {
-            *out = TOX_GROUP_JOIN_FAIL_PEER_LIMIT;
+            *out_enum = TOX_GROUP_JOIN_FAIL_PEER_LIMIT;
            return false;
        }
    }
@ -272,27 +272,27 @@ bool tox_group_join_fail_unpack(Tox_Group_Join_Fail *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_group_mod_event_from_int(uint32_t value, Tox_Group_Mod_Event *out)
+static bool tox_group_mod_event_from_int(uint32_t value, Tox_Group_Mod_Event *out_enum)
 {
    switch (value) {
        case TOX_GROUP_MOD_EVENT_KICK: {
-            *out = TOX_GROUP_MOD_EVENT_KICK;
+            *out_enum = TOX_GROUP_MOD_EVENT_KICK;
            return true;
        }
        case TOX_GROUP_MOD_EVENT_OBSERVER: {
-            *out = TOX_GROUP_MOD_EVENT_OBSERVER;
+            *out_enum = TOX_GROUP_MOD_EVENT_OBSERVER;
            return true;
        }
        case TOX_GROUP_MOD_EVENT_USER: {
-            *out = TOX_GROUP_MOD_EVENT_USER;
+            *out_enum = TOX_GROUP_MOD_EVENT_USER;
            return true;
        }
        case TOX_GROUP_MOD_EVENT_MODERATOR: {
-            *out = TOX_GROUP_MOD_EVENT_MODERATOR;
+            *out_enum = TOX_GROUP_MOD_EVENT_MODERATOR;
            return true;
        }
        default: {
-            *out = TOX_GROUP_MOD_EVENT_KICK;
+            *out_enum = TOX_GROUP_MOD_EVENT_KICK;
            return false;
        }
    }
@ -305,35 +305,35 @@ bool tox_group_mod_event_unpack(Tox_Group_Mod_Event *val, Bin_Unpack *bu)
 }

 non_null()
-static bool tox_group_exit_type_from_int(uint32_t value, Tox_Group_Exit_Type *out)
+static bool tox_group_exit_type_from_int(uint32_t value, Tox_Group_Exit_Type *out_enum)
 {
    switch (value) {
        case TOX_GROUP_EXIT_TYPE_QUIT: {
-            *out = TOX_GROUP_EXIT_TYPE_QUIT;
+            *out_enum = TOX_GROUP_EXIT_TYPE_QUIT;
            return true;
        }
        case TOX_GROUP_EXIT_TYPE_TIMEOUT: {
-            *out = TOX_GROUP_EXIT_TYPE_TIMEOUT;
+            *out_enum = TOX_GROUP_EXIT_TYPE_TIMEOUT;
            return true;
        }
        case TOX_GROUP_EXIT_TYPE_DISCONNECTED: {
-            *out = TOX_GROUP_EXIT_TYPE_DISCONNECTED;
+            *out_enum = TOX_GROUP_EXIT_TYPE_DISCONNECTED;
            return true;
        }
        case TOX_GROUP_EXIT_TYPE_SELF_DISCONNECTED: {
-            *out = TOX_GROUP_EXIT_TYPE_SELF_DISCONNECTED;
+            *out_enum = TOX_GROUP_EXIT_TYPE_SELF_DISCONNECTED;
            return true;
        }
        case TOX_GROUP_EXIT_TYPE_KICK: {
-            *out = TOX_GROUP_EXIT_TYPE_KICK;
+            *out_enum = TOX_GROUP_EXIT_TYPE_KICK;
            return true;
        }
        case TOX_GROUP_EXIT_TYPE_SYNC_ERROR: {
-            *out = TOX_GROUP_EXIT_TYPE_SYNC_ERROR;
+            *out_enum = TOX_GROUP_EXIT_TYPE_SYNC_ERROR;
            return true;
        }
        default: {
-            *out = TOX_GROUP_EXIT_TYPE_QUIT;
+            *out_enum = TOX_GROUP_EXIT_TYPE_QUIT;
            return false;
        }
    }
--- a/flake.lock
+++ b/flake.lock
@ -20,16 +20,16 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1694553957,
-        "narHash": "sha256-8o15HEax53lBJjjcr5VHMpuuT6vBcrzSNB6y2iGlPaU=",
+        "lastModified": 1709953752,
+        "narHash": "sha256-LW84B4vM1cn7E6cDNQn2LndT9iJXI1dRE5fwbNFbQa8=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "e7fe745d22df5fa282b321e577fe18d4f62e0f0b",
+        "rev": "fcaa81ed3c273237217330cf342ef1873b77c80a",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "release-23.05",
+        "ref": "release-23.11",
        "repo": "nixpkgs",
        "type": "github"
      }
--- a/flake.nix
+++ b/flake.nix
@ -4,7 +4,7 @@
  # append '.?submodules=1' to the nix commands.

  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/release-23.05";
+    nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
    flake-utils.url = "github:numtide/flake-utils";
  };

@ -12,13 +12,15 @@
    flake-utils.lib.eachDefaultSystem (system:
    let
      pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
    in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
        pname = "tomato";
        version = "0.0.0";

        src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing

        nativeBuildInputs = with pkgs; [
          cmake
@ -58,6 +60,11 @@
        cmakeFlags = [
          "TOMATO_ASAN=1"
          "CMAKE_BUILD_TYPE=RelWithDebInfo"
+
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${pkgs.nlohmann_json.src}" # we care less about version here
+          # do we really care less about the version? do we need a stable abi?
+
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}"
        ];

        # TODO: replace with install command
@ -66,7 +73,7 @@
          mv bin/tomato $out/bin
        '';

-        dontStrip = true;
+        dontStrip = true; # does nothing

        # copied from nixpkgs's SDL2 default.nix
        # SDL is weird in that instead of just dynamically linking with
@ -93,6 +100,8 @@
          '';
      };

+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
      devShells.${system}.default = pkgs.mkShell {
        #inputsFrom = with pkgs; [ SDL2 ];
        buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@ -1,5 +1,60 @@
 cmake_minimum_required(VERSION 3.9 FATAL_ERROR)

+add_library(fragment_store
+	./fragment_store/fragment_store_i.hpp
+	./fragment_store/fragment_store_i.cpp
+	./fragment_store/types.hpp
+	./fragment_store/meta_components.hpp
+	./fragment_store/meta_components_id.inl
+	./fragment_store/serializer.hpp
+	./fragment_store/fragment_store.hpp
+	./fragment_store/fragment_store.cpp
+
+	./json/message_components.hpp # TODO: move
+	./json/tox_message_components.hpp # TODO: move
+)
+
+target_link_libraries(fragment_store PUBLIC
+	nlohmann_json::nlohmann_json
+	EnTT::EnTT
+	solanaceae_util
+
+	zstd::zstd
+
+	solanaceae_tox_messages # TODO: move
+)
+
+########################################
+
+add_library(message_fragment_store
+	./fragment_store/message_serializer.hpp
+	./fragment_store/message_serializer.cpp
+	./fragment_store/message_fragment_store.hpp
+	./fragment_store/message_fragment_store.cpp
+
+	./fragment_store/register_mfs_json_message_components.hpp
+	./fragment_store/register_mfs_json_message_components.cpp
+	./fragment_store/register_mfs_json_tox_message_components.hpp
+	./fragment_store/register_mfs_json_tox_message_components.cpp
+)
+target_compile_features(message_fragment_store PRIVATE cxx_std_20)
+target_link_libraries(message_fragment_store PUBLIC
+	fragment_store
+	solanaceae_message3
+)
+
+########################################
+
+add_executable(fragment_store_test
+	fragment_store/test_fragstore.cpp
+)
+
+target_link_libraries(fragment_store_test PUBLIC
+	fragment_store
+)
+
+########################################
+
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@ -82,6 +137,9 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_tox_messages

+	fragment_store
+	message_fragment_store
+
 	SDL3::SDL3

 	imgui
--- a/src/chat_gui4.cpp
+++ b/src/chat_gui4.cpp
@ -37,6 +37,18 @@ namespace Components {

 } // Components

+namespace Context {
+
+	// TODO: move back to chat log window and keep per window instead of per contact
+	struct CGView {
+		// set to the ts of the newest rendered msg
+		Message3Handle begin{};
+		// set to the ts of the oldest rendered msg
+		Message3Handle end{};
+	};
+
+} // Context
+
 static constexpr float lerp(float a, float b, float t) {
 	return a + t * (b - a);
 }
@ -259,28 +271,6 @@ float ChatGui4::render(float time_delta) {

 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);

-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@ -293,6 +283,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);

+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;

@ -381,12 +374,26 @@ float ChatGui4::render(float time_delta) {
 								}

 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
+								if (ImGui::IsItemVisible()) {
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
 										// get time now
 										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
 										msg_reg.remove<Message::Components::TagUnread>(e);
 										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+
+										// we remove the unread tag here
+										_rmm.throwEventUpdate(msg_reg, e);
+									}
+
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
 								}

 								// highlight self
@ -539,9 +546,90 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);

+						{ // update view cursers
+							if (!msg_reg.ctx().contains<Context::CGView>()) {
+								msg_reg.ctx().emplace<Context::CGView>();
+							}
+
+							auto& cg_view = msg_reg.ctx().get<Context::CGView>();
+
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view, we setup a view at current time, so the next frags are loaded
+								if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
+									// fix invalid state
+									if (static_cast<bool>(cg_view.begin)) {
+										cg_view.begin.destroy();
+										_rmm.throwEventDestroy(cg_view.begin);
+									}
+									if (static_cast<bool>(cg_view.end)) {
+										cg_view.end.destroy();
+										_rmm.throwEventDestroy(cg_view.end);
+									}
+
+									// create new
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									cg_view.end = {msg_reg, msg_reg.create()};
+
+									cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+									cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+									cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+									cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(cg_view.begin);
+									std::cout << "CG: created view FRONT end ts\n";
+									_rmm.throwEventConstruct(cg_view.end);
+								} // else? we do nothing?
+							} else {
+								bool begin_created {false};
+								if (!static_cast<bool>(cg_view.begin)) {
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								bool end_created {false};
+								if (!static_cast<bool>(cg_view.end)) {
+									cg_view.end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+								cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+								{
+									auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+										old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+										if (begin_created) {
+											std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.begin);
+										} else {
+											//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.begin);
+										}
+									}
+								}
+
+								{
+									auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+										old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+										if (end_created) {
+											std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.end);
+										} else {
+											//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.end);
+										}
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}

+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}
--- a/src/chat_gui4.hpp
+++ b/src/chat_gui4.hpp
@ -10,6 +10,9 @@
 #include "./file_selector.hpp"
 #include "./send_image_popup.hpp"

+// HACK: move to public msg api?
+#include "./fragment_store/message_fragment_store.hpp"
+
 #include <entt/container/dense_map.hpp>

 #include <cstdint>
@ -32,6 +35,7 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;

+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;

 	// TODO: per contact
--- a/src/fragment_store/README.md
+++ b/src/fragment_store/README.md
@ -25,41 +25,41 @@ Just keeps the Fragments in memory.

 # File formats

-Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
+Files can be compressed and encrypted. Since compression needs the data's structure to work properly, it is applied before it is encrypted.

 ### Text Json

 Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
 Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.

-A Metadata json object has the following keys:
- `enc` (uint) Encryption type of the data, if any
- `comp` (uint) Compression type of the data, if any
- `metadata` (obj) the 
+A Metadata json object can have arbitrary keys, some are predefined:
+- `FragComp::DataEncryptionType` (uint) Encryption type of the data, if any
+- `FragComp::DataCompressionType` (uint) Compression type of the data, if any

 ## Binary file headers

 ### Split Metadata

-file magic bytes `SOLMET` (6 bytes)
+msgpack array:

-1 byte encryption type (`0x00` is none)
-
-1 byte compression type (`0x00` is none)
-
-...metadata here...
+- `[0]`: file magic string `SOLMET` (6 bytes)
+- `[1]`: uint8 encryption type (`0x00` is none)
+- `[2]`: uint8 compression type (`0x00` is none, `0x01` is zstd)
+- `[3]`: binary metadata (optionally compressed and encrypted)

 note that the encryption and compression are for the metadata only.
 The metadata itself contains encryption and compression info about the data.

 ### Split Data

-(none) all the data is in the metadata file.
+All the metadata is in the metadata file. (like encryption and compression)
 This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
 Keep in mind to not use the actual file name as the data/meta file name.

 ### Single fragment

+Note: this format is unused for now
+
 file magic bytes `SOLFIL` (6 bytes)

 1 byte encryption type (`0x00` is none)
@ -70,3 +70,7 @@ file magic bytes `SOLFIL` (6 bytes)

 ...data here...

+## Compression types
+
+- `0x00` none
+- `0x01` zstd (without dict)
--- a/src/fragment_store/fragment_store.cpp
+++ b/src/fragment_store/fragment_store.cpp
@ -0,0 +1,866 @@
+#include "./fragment_store.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <entt/entity/handle.hpp>
+#include <entt/container/dense_set.hpp>
+#include <entt/core/hashed_string.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <zstd.h>
+
+#include <cstdint>
+#include <fstream>
+#include <filesystem>
+#include <memory>
+#include <mutex>
+#include <type_traits>
+#include <utility>
+#include <algorithm>
+
+#include <iostream>
+#include <vector>
+
+static const char* metaFileTypeSuffix(MetaFileType mft) {
+	switch (mft) {
+		case MetaFileType::TEXT_JSON: return ".json";
+		//case MetaFileType::BINARY_ARB: return ".bin";
+		case MetaFileType::BINARY_MSGPACK: return ".msgpack";
+	}
+	return ""; // .unk?
+}
+
+FragmentStore::FragmentStore(void) {
+	{ // random namespace
+		const auto num0 = _rng();
+		const auto num1 = _rng();
+		const auto num2 = _rng();
+		const auto num3 = _rng();
+
+		_session_uuid_namespace[0+0] = (num0 >> 0) & 0xff;
+		_session_uuid_namespace[0+1] = (num0 >> 8) & 0xff;
+		_session_uuid_namespace[0+2] = (num0 >> 16) & 0xff;
+		_session_uuid_namespace[0+3] = (num0 >> 24) & 0xff;
+
+		_session_uuid_namespace[4+0] = (num1 >> 0) & 0xff;
+		_session_uuid_namespace[4+1] = (num1 >> 8) & 0xff;
+		_session_uuid_namespace[4+2] = (num1 >> 16) & 0xff;
+		_session_uuid_namespace[4+3] = (num1 >> 24) & 0xff;
+
+		_session_uuid_namespace[8+0] = (num2 >> 0) & 0xff;
+		_session_uuid_namespace[8+1] = (num2 >> 8) & 0xff;
+		_session_uuid_namespace[8+2] = (num2 >> 16) & 0xff;
+		_session_uuid_namespace[8+3] = (num2 >> 24) & 0xff;
+
+		_session_uuid_namespace[12+0] = (num3 >> 0) & 0xff;
+		_session_uuid_namespace[12+1] = (num3 >> 8) & 0xff;
+		_session_uuid_namespace[12+2] = (num3 >> 16) & 0xff;
+		_session_uuid_namespace[12+3] = (num3 >> 24) & 0xff;
+	}
+	registerSerializers();
+}
+
+FragmentStore::FragmentStore(
+	std::array<uint8_t, 16> session_uuid_namespace
+) : _session_uuid_namespace(std::move(session_uuid_namespace)) {
+	registerSerializers();
+}
+
+FragmentHandle FragmentStore::fragmentHandle(FragmentID fid) {
+	return {_reg, fid};
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(std::array<uint8_t, 16>& uuid_namespace) {
+	std::vector<uint8_t> new_uid(uuid_namespace.cbegin(), uuid_namespace.cend());
+	new_uid.resize(new_uid.size() + 16);
+
+	const auto num0 = _rng();
+	const auto num1 = _rng();
+	const auto num2 = _rng();
+	const auto num3 = _rng();
+
+	new_uid[uuid_namespace.size()+0] = (num0 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+1] = (num0 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+2] = (num0 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+3] = (num0 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+4+0] = (num1 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+4+1] = (num1 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+4+2] = (num1 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+4+3] = (num1 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+8+0] = (num2 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+8+1] = (num2 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+8+2] = (num2 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+8+3] = (num2 >> 24) & 0xff;
+
+	new_uid[uuid_namespace.size()+12+0] = (num3 >> 0) & 0xff;
+	new_uid[uuid_namespace.size()+12+1] = (num3 >> 8) & 0xff;
+	new_uid[uuid_namespace.size()+12+2] = (num3 >> 16) & 0xff;
+	new_uid[uuid_namespace.size()+12+3] = (num3 >> 24) & 0xff;
+
+	return new_uid;
+}
+
+std::vector<uint8_t> FragmentStore::generateNewUID(void) {
+	return generateNewUID(_session_uuid_namespace);
+}
+
+FragmentID FragmentStore::newFragmentMemoryOwned(
+	const std::vector<uint8_t>& id,
+	size_t initial_size
+) {
+	{ // first check if id is already used
+		auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	{ // next check if space in memory budget
+		const auto free_memory = _memory_budget - _memory_usage;
+		if (initial_size > free_memory) {
+			return entt::null;
+		}
+	}
+
+	// actually allocate and create
+	auto new_data = std::make_unique<std::vector<uint8_t>>(initial_size);
+	if (!static_cast<bool>(new_data)) {
+		// allocation failure
+		return entt::null;
+	}
+	_memory_usage += initial_size;
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+	// TODO: memory comp
+	_reg.emplace<std::unique_ptr<std::vector<uint8_t>>>(new_frag) = std::move(new_data);
+
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft,
+	const std::vector<uint8_t>& id
+) {
+	{ // first check if id is already used
+		const auto exising_id = getFragmentByID(id);
+		if (_reg.valid(exising_id)) {
+			return entt::null;
+		}
+	}
+
+	if (store_path.empty()) {
+		store_path = _default_store_path;
+	}
+
+	std::filesystem::create_directories(store_path);
+
+	const auto id_hex = bin2hex(id);
+	std::filesystem::path fragment_file_path;
+
+	if (id_hex.size() < 6) {
+		fragment_file_path = std::filesystem::path{store_path}/id_hex;
+	} else {
+		// use the first 2hex (1byte) as a subfolder
+		std::filesystem::create_directories(std::string{store_path} + id_hex.substr(0, 2));
+		fragment_file_path = std::filesystem::path{std::string{store_path} + id_hex.substr(0, 2)} / id_hex.substr(2);
+	}
+
+	if (std::filesystem::exists(fragment_file_path)) {
+		return entt::null;
+	}
+
+	const auto new_frag = _reg.create();
+
+	_reg.emplace<FragComp::ID>(new_frag, id);
+
+	// file (info) comp
+	_reg.emplace<FragComp::Ephemeral::FilePath>(new_frag, fragment_file_path.generic_u8string());
+
+	_reg.emplace<FragComp::Ephemeral::MetaFileType>(new_frag, mft);
+
+	// meta needs to be synced to file
+	std::function<write_to_storage_fetch_data_cb> empty_data_cb = [](const uint8_t*, uint64_t) -> uint64_t { return 0; };
+	if (!syncToStorage(new_frag, empty_data_cb)) {
+		std::cerr << "FS error: syncToStorage failed while creating new fragment file\n";
+		_reg.destroy(new_frag);
+		return entt::null;
+	}
+
+	// while new metadata might be created here, making sure the file could be created is more important
+	throwEventConstruct(new_frag);
+
+	return new_frag;
+}
+FragmentID FragmentStore::newFragmentFile(
+	std::string_view store_path,
+	MetaFileType mft
+) {
+	return newFragmentFile(store_path, mft, generateNewUID());
+}
+
+FragmentID FragmentStore::getFragmentByID(
+	const std::vector<uint8_t>& id
+) {
+	// TODO: accelerate
+	// maybe keep it sorted and binary search? hash table lookup?
+	for (const auto& [frag, id_comp] : _reg.view<FragComp::ID>().each()) {
+		if (id == id_comp.v) {
+			return frag;
+		}
+	}
+
+	return entt::null;
+}
+
+FragmentID FragmentStore::getFragmentCustomMatcher(
+	std::function<bool(FragmentID)>& fn
+) {
+	return entt::null;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		return false;
+	}
+
+	// split object storage
+
+	MetaFileType meta_type = MetaFileType::TEXT_JSON; // TODO: better defaults
+	if (_reg.all_of<FragComp::Ephemeral::MetaFileType>(fid)) {
+		meta_type = _reg.get<FragComp::Ephemeral::MetaFileType>(fid).type;
+	}
+
+	Encryption meta_enc = Encryption::NONE; // TODO: better defaults
+	Compression meta_comp = Compression::NONE; // TODO: better defaults
+
+	if (meta_type != MetaFileType::TEXT_JSON) {
+		if (_reg.all_of<FragComp::Ephemeral::MetaEncryptionType>(fid)) {
+			meta_enc = _reg.get<FragComp::Ephemeral::MetaEncryptionType>(fid).enc;
+		}
+
+		if (_reg.all_of<FragComp::Ephemeral::MetaCompressionType>(fid)) {
+			meta_comp = _reg.get<FragComp::Ephemeral::MetaCompressionType>(fid).comp;
+		}
+	} else {
+		// we cant have encryption or compression
+
+		// TODO: warning/error?
+
+		// TODO: forcing for testing
+		//if (_reg.all_of<Components::Ephemeral::MetaEncryptionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaEncryptionType>(fid, Encryption::NONE);
+		//}
+		//if (_reg.all_of<Components::Ephemeral::MetaCompressionType>(fid)) {
+			_reg.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>(fid, Compression::NONE);
+		//}
+	}
+
+	std::filesystem::path meta_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type) + ".tmp";
+	meta_tmp_path.replace_filename("." + meta_tmp_path.filename().generic_u8string());
+	std::ofstream meta_file{
+		meta_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!meta_file.is_open()) {
+		std::cerr << "FS error: failed to create temporary meta file\n";
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE; // TODO: better defaults
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	std::filesystem::path data_tmp_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".tmp";
+	data_tmp_path.replace_filename("." + data_tmp_path.filename().generic_u8string());
+	std::ofstream data_file{
+		data_tmp_path,
+		std::ios::out | std::ios::trunc | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		meta_file.close();
+		std::filesystem::remove(meta_tmp_path);
+		std::cerr << "FS error: failed to create temporary data file\n";
+		return false;
+	}
+
+	// sharing code between binary msgpack and text json for now
+	nlohmann::json meta_data_j = nlohmann::json::object(); // metadata needs to be an object, null not allowed
+	// metadata file
+
+	for (const auto& [type_id, storage] : _reg.storage()) {
+		if (!storage.contains(fid)) {
+			continue;
+		}
+
+		//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
+
+		// use type_id to find serializer
+		auto s_cb_it = _sc._serl_json.find(type_id);
+		if (s_cb_it == _sc._serl_json.end()) {
+			// could not find serializer, not saving
+			continue;
+		}
+
+		// noooo, why cant numbers be keys
+		//if (meta_type == MetaFileType::BINARY_MSGPACK) { // msgpack uses the hash id instead
+			//s_cb_it->second(storage.value(fid), meta_data[storage.type().hash()]);
+		//} else if (meta_type == MetaFileType::TEXT_JSON) {
+		s_cb_it->second({_reg, fid}, meta_data_j[storage.type().name()]);
+		//}
+	}
+
+	if (meta_type == MetaFileType::BINARY_MSGPACK) { // binary metadata file
+		const std::vector<uint8_t> meta_data = nlohmann::json::to_msgpack(meta_data_j);
+		std::vector<uint8_t> meta_data_compressed; // empty if none
+		//std::vector<uint8_t> meta_data_encrypted; // empty if none
+
+		if (meta_comp == Compression::ZSTD) {
+			meta_data_compressed.resize(ZSTD_compressBound(meta_data.size()));
+
+			size_t const cSize = ZSTD_compress(meta_data_compressed.data(), meta_data_compressed.size(), meta_data.data(), meta_data.size(), 0); // 0 is default is probably 3
+			if (ZSTD_isError(cSize)) {
+				std::cerr << "FS error: compressing meta failed\n";
+				meta_data_compressed.clear();
+				meta_comp = Compression::NONE;
+			} else {
+				meta_data_compressed.resize(cSize);
+			}
+		} else if (meta_comp == Compression::NONE) {
+			// do nothing
+		} else {
+			assert(false && "implement me");
+		}
+
+		// TODO: encryption
+
+		// the meta file is itself msgpack data
+		nlohmann::json meta_header_j = nlohmann::json::array();
+		meta_header_j.emplace_back() = "SOLMET";
+		meta_header_j.push_back(meta_enc);
+		meta_header_j.push_back(meta_comp);
+
+		if (false) { // TODO: encryption
+		} else if (!meta_data_compressed.empty()) {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data_compressed));
+		} else {
+			meta_header_j.push_back(nlohmann::json::binary(meta_data));
+		}
+
+		const auto meta_header_data = nlohmann::json::to_msgpack(meta_header_j);
+		meta_file.write(reinterpret_cast<const char*>(meta_header_data.data()), meta_header_data.size());
+	} else if (meta_type == MetaFileType::TEXT_JSON) {
+		// cant be compressed or encrypted
+		meta_file << meta_data_j.dump(2, ' ', true);
+	}
+
+	// now data
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			if (buffer_actual_size == 0) {
+				break;
+			}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+
+			data_file.write(reinterpret_cast<const char*>(buffer.data()), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> buffer(ZSTD_CStreamInSize());
+		std::vector<uint8_t> compressed_buffer(ZSTD_CStreamOutSize());
+		uint64_t buffer_actual_size {0};
+
+		std::unique_ptr<ZSTD_CCtx, decltype(&ZSTD_freeCCtx)> cctx{ZSTD_createCCtx(), &ZSTD_freeCCtx};
+		ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_compressionLevel, 0); // default (3)
+		ZSTD_CCtx_setParameter(cctx.get(), ZSTD_c_checksumFlag, 1); // add extra checksums (to frames?)
+		do {
+			buffer_actual_size = data_cb(buffer.data(), buffer.size());
+			//if (buffer_actual_size == 0) {
+				//break;
+			//}
+			if (buffer_actual_size > buffer.size()) {
+				// wtf
+				break;
+			}
+			bool const lastChunk = (buffer_actual_size < buffer.size());
+
+			ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
+			ZSTD_inBuffer input = { buffer.data(), buffer_actual_size, 0 };
+
+			while (input.pos < input.size) {
+				ZSTD_outBuffer output = { compressed_buffer.data(), compressed_buffer.size(), 0 };
+
+				size_t const remaining = ZSTD_compressStream2(cctx.get(), &output , &input, mode);
+				if (ZSTD_isError(remaining)) {
+					std::cerr << "FS error: compressing data failed\n";
+					break;
+				}
+
+				data_file.write(reinterpret_cast<const char*>(compressed_buffer.data()), output.pos);
+
+				if (remaining == 0) {
+					break;
+				}
+			}
+			// same as if lastChunk break;
+		} while (buffer_actual_size == buffer.size());
+	} else {
+		assert(false && "implement me");
+	}
+
+	meta_file.flush();
+	meta_file.close();
+	data_file.flush();
+	data_file.close();
+
+	std::filesystem::rename(
+		meta_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path + ".meta" + metaFileTypeSuffix(meta_type)
+	);
+
+	std::filesystem::rename(
+		data_tmp_path,
+		_reg.get<FragComp::Ephemeral::FilePath>(fid).path
+	);
+
+	// TODO: check return value of renames
+
+	if (_reg.all_of<FragComp::Ephemeral::DirtyTag>(fid)) {
+		_reg.remove<FragComp::Ephemeral::DirtyTag>(fid);
+	}
+
+	return true;
+}
+
+bool FragmentStore::syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size) {
+	std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ull, data, data_size](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+		uint64_t i = 0;
+		for (; i+read < data_size && i < buffer_size; i++) {
+			request_buffer[i] = data[i+read];
+		}
+		read += i;
+
+		return i;
+	};
+	return syncToStorage(fid, fn_cb);
+}
+
+bool FragmentStore::loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb) {
+	if (!_reg.valid(fid)) {
+		return false;
+	}
+
+	if (!_reg.all_of<FragComp::Ephemeral::FilePath>(fid)) {
+		// not a file fragment?
+		// TODO: memory fragments
+		return false;
+	}
+
+	const auto& frag_path = _reg.get<FragComp::Ephemeral::FilePath>(fid).path;
+
+	// TODO: check if metadata dirty?
+	// TODO: what if file changed on disk?
+
+	std::cout << "FS: loading fragment '" << frag_path << "'\n";
+
+	std::ifstream data_file{
+		frag_path,
+		std::ios::in | std::ios::binary // always binary, also for text
+	};
+
+	if (!data_file.is_open()) {
+		std::cerr << "FS error: fragment data file failed to open '" << frag_path << "'\n";
+		// error
+		return false;
+	}
+
+	Compression data_comp = Compression::NONE;
+	if (_reg.all_of<FragComp::DataCompressionType>(fid)) {
+		data_comp = _reg.get<FragComp::DataCompressionType>(fid).comp;
+	}
+
+	if (data_comp == Compression::NONE) {
+		std::array<uint8_t, 1024> buffer;
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(buffer.data()), buffer.size());
+			buffer_actual_size = data_file.gcount();
+
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			data_cb(buffer.data(), buffer_actual_size);
+		} while (buffer_actual_size == buffer.size() && !data_file.eof());
+	} else if (data_comp == Compression::ZSTD) {
+		std::vector<uint8_t> in_buffer(ZSTD_DStreamInSize());
+		std::vector<uint8_t> out_buffer(ZSTD_DStreamOutSize());
+		std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx};
+
+		uint64_t buffer_actual_size {0};
+		do {
+			data_file.read(reinterpret_cast<char*>(in_buffer.data()), in_buffer.size());
+			buffer_actual_size = data_file.gcount();
+			if (buffer_actual_size == 0) {
+				break;
+			}
+
+			ZSTD_inBuffer input {in_buffer.data(), buffer_actual_size, 0 };
+			do {
+				ZSTD_outBuffer output = { out_buffer.data(), out_buffer.size(), 0 };
+				size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input);
+				if (ZSTD_isError(ret)) {
+					// error <.<
+					std::cerr << "FS error: decompression error\n";
+					break;
+				}
+
+				data_cb(out_buffer.data(), output.pos);
+			} while (input.pos < input.size);
+		} while (buffer_actual_size == in_buffer.size() && !data_file.eof());
+	} else {
+		assert(false && "implement me");
+	}
+
+	return true;
+}
+
+nlohmann::json FragmentStore::loadFromStorageNJ(FragmentID fid) {
+	std::vector<uint8_t> tmp_buffer;
+	std::function<read_from_storage_put_data_cb> cb = [&tmp_buffer](const uint8_t* buffer, const uint64_t buffer_size) {
+		tmp_buffer.insert(tmp_buffer.end(), buffer, buffer+buffer_size);
+	};
+
+	if (!loadFromStorage(fid, cb)) {
+		return nullptr;
+	}
+
+	return nlohmann::json::parse(tmp_buffer);
+}
+
+size_t FragmentStore::scanStoragePath(std::string_view path) {
+	if (path.empty()) {
+		path = _default_store_path;
+	}
+	// TODO: extract so async can work (or/and make iteratable generator)
+
+	if (!std::filesystem::is_directory(path)) {
+		std::cerr << "FS error: scan path not a directory '" << path << "'\n";
+		return 0;
+	}
+
+	// step 1: make snapshot of files, validate metafiles and save id/path+meta.ext
+	// can be extra thread (if non vfs)
+	struct FragFileEntry {
+		std::string id_str;
+		std::filesystem::path frag_path;
+		std::string meta_ext;
+
+		bool operator==(const FragFileEntry& other) const {
+			// only compare by id
+			return id_str == other.id_str;
+		}
+	};
+	struct FragFileEntryHash {
+		size_t operator()(const FragFileEntry& it) const {
+			return entt::hashed_string(it.id_str.data(), it.id_str.size());
+		}
+	};
+	entt::dense_set<FragFileEntry, FragFileEntryHash> file_frag_list;
+
+	std::filesystem::path storage_path{path};
+
+	auto handle_file = [&](const std::filesystem::path& file_path) {
+		if (!std::filesystem::is_regular_file(file_path)) {
+			return;
+		}
+		// handle file
+
+		if (file_path.has_extension()) {
+			// skip over metadata, assuming only metafiles have extentions (might be wrong?)
+			// also skips temps
+			return;
+		}
+
+		auto relative_path = std::filesystem::proximate(file_path, storage_path);
+		std::string id_str = relative_path.generic_u8string();
+		// delete all '/'
+		id_str.erase(std::remove(id_str.begin(), id_str.end(), '/'), id_str.end());
+		if (id_str.size() % 2 != 0) {
+			std::cerr << "FS error: non hex fragment uid detected: '" << id_str << "'\n";
+		}
+
+		if (file_frag_list.contains(FragFileEntry{id_str, {}, ""})) {
+			std::cerr << "FS error: fragment duplicate detected: '" << id_str << "'\n";
+			return; // skip
+		}
+
+		const char* meta_ext = ".meta.msgpack";
+		{ // find meta
+			// TODO: this as to know all possible extentions
+			bool has_meta_msgpack = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.msgpack");
+			bool has_meta_json = std::filesystem::is_regular_file(file_path.generic_u8string() + ".meta.json");
+			const size_t meta_sum =
+				(has_meta_msgpack?1:0) +
+				(has_meta_json?1:0)
+			;
+
+			if (meta_sum > 1) { // has multiple
+				std::cerr << "FS error: fragment with multiple meta files detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (meta_sum == 0) {
+				std::cerr << "FS error: fragment missing meta file detected: " << id_str << "\n";
+				return; // skip
+			}
+
+			if (has_meta_json) {
+				meta_ext = ".meta.json";
+			}
+		}
+
+		file_frag_list.emplace(FragFileEntry{
+			std::move(id_str),
+			file_path,
+			meta_ext
+		});
+	};
+
+	for (const auto& outer_path : std::filesystem::directory_iterator(storage_path)) {
+		if (std::filesystem::is_regular_file(outer_path)) {
+			handle_file(outer_path);
+		} else if (std::filesystem::is_directory(outer_path)) {
+			// subdir, part of id
+			for (const auto& inner_path : std::filesystem::directory_iterator(outer_path)) {
+				//if (std::filesystem::is_regular_file(inner_path)) {
+
+					//// handle file
+				//} // TODO: support deeper recursion?
+				handle_file(inner_path);
+			}
+		}
+	}
+
+	std::cout << "FS: scan found:\n";
+	for (const auto& it : file_frag_list) {
+		std::cout << "  " << it.id_str << "\n";
+	}
+
+	// step 2: check if files preexist in reg
+	// main thread
+	// (merge into step 3 and do more error checking?)
+	for (auto it = file_frag_list.begin(); it != file_frag_list.end();) {
+		auto id = hex2bin(it->id_str);
+		auto fid = getFragmentByID(id);
+		if (_reg.valid(fid)) {
+			// pre exising (handle differently??)
+			// check if store differs?
+			it = file_frag_list.erase(it);
+		} else {
+			it++;
+		}
+	}
+
+	std::vector<FragmentID> scanned_frags;
+	// step 3: parse meta and insert into reg of non preexising
+	// main thread
+	// TODO: check timestamps of preexisting and reload? mark external/remote dirty?
+	for (const auto& it : file_frag_list) {
+		nlohmann::json j;
+		if (it.meta_ext == ".meta.msgpack") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cout << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			// file is a msgpack within a msgpack
+
+			std::vector<uint8_t> full_meta_data;
+			{ // read meta file
+				// figure out size
+				file.seekg(0, file.end);
+				uint64_t file_size = file.tellg();
+				file.seekg(0, file.beg);
+
+				full_meta_data.resize(file_size);
+
+				file.read(reinterpret_cast<char*>(full_meta_data.data()), full_meta_data.size());
+			}
+
+			const auto meta_header_j = nlohmann::json::from_msgpack(full_meta_data);
+
+			if (!meta_header_j.is_array() || meta_header_j.size() < 4) {
+				std::cerr << "FS error: broken binary meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			if (meta_header_j.at(0) != "SOLMET") {
+				std::cerr << "FS error: wrong magic '" << meta_header_j.at(0) << "' in meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			Encryption meta_enc = meta_header_j.at(1);
+			if (meta_enc != Encryption::NONE) {
+				std::cerr << "FS error: unknown encryption " << it.frag_path << "\n";
+				continue;
+			}
+
+			Compression meta_comp = meta_header_j.at(2);
+			if (meta_comp != Compression::NONE && meta_comp != Compression::ZSTD) {
+				std::cerr << "FS error: unknown compression " << it.frag_path << "\n";
+				continue;
+			}
+
+			//const auto& meta_data_ref = meta_header_j.at(3).is_binary()?meta_header_j.at(3):meta_header_j.at(3).at("data");
+			if (!meta_header_j.at(3).is_binary()) {
+				std::cerr << "FS error: meta data not binary " << it.frag_path << "\n";
+				continue;
+			}
+			const nlohmann::json::binary_t& meta_data_ref = meta_header_j.at(3);
+
+			std::vector<uint8_t> meta_data_decomp;
+			if (meta_comp == Compression::NONE) {
+				// do nothing
+			} else if (meta_comp == Compression::ZSTD) {
+				meta_data_decomp.resize(ZSTD_DStreamOutSize());
+				std::unique_ptr<ZSTD_DCtx, decltype(&ZSTD_freeDCtx)> dctx{ZSTD_createDCtx(), &ZSTD_freeDCtx};
+
+				ZSTD_inBuffer input {meta_data_ref.data(), meta_data_ref.size(), 0};
+				ZSTD_outBuffer output = {meta_data_decomp.data(), meta_data_decomp.size(), 0};
+				do {
+					size_t const ret = ZSTD_decompressStream(dctx.get(), &output , &input);
+					if (ZSTD_isError(ret)) {
+						// error <.<
+						std::cerr << "FS error: decompression error\n";
+						//meta_data_decomp.clear();
+						output.pos = 0; // resize after loop
+						break;
+					}
+				} while (input.pos < input.size);
+				meta_data_decomp.resize(output.pos);
+			} else {
+				assert(false && "implement me");
+			}
+
+			// TODO: enc
+
+			if (!meta_data_decomp.empty()) {
+				j = nlohmann::json::from_msgpack(meta_data_decomp);
+			} else {
+				j = nlohmann::json::from_msgpack(meta_data_ref);
+			}
+		} else if (it.meta_ext == ".meta.json") {
+			std::ifstream file(it.frag_path.generic_u8string() + it.meta_ext, std::ios::in | std::ios::binary);
+			if (!file.is_open()) {
+				std::cerr << "FS error: failed opening meta " << it.frag_path << "\n";
+				continue;
+			}
+
+			file >> j;
+		} else {
+			assert(false);
+		}
+
+		if (!j.is_object()) {
+			std::cerr << "FS error: json in meta is broken " << it.id_str << "\n";
+			continue;
+		}
+
+		// TODO: existing fragment file
+		//newFragmentFile();
+		FragmentHandle fh{_reg, _reg.create()};
+		fh.emplace<FragComp::ID>(hex2bin(it.id_str));
+
+		fh.emplace<FragComp::Ephemeral::FilePath>(it.frag_path.generic_u8string());
+
+		for (const auto& [k, v] : j.items()) {
+			// type id from string hash
+			const auto type_id = entt::hashed_string(k.data(), k.size());
+			const auto deserl_fn_it = _sc._deserl_json.find(type_id);
+			if (deserl_fn_it != _sc._deserl_json.cend()) {
+				// TODO: check return value
+				deserl_fn_it->second(fh, v);
+			} else {
+				std::cerr << "FS warning: missing deserializer for meta key '" << k << "'\n";
+			}
+		}
+		scanned_frags.push_back(fh);
+	}
+
+	// TODO: mutex and move code to async and return this list ?
+
+	// throw new frag event here, after loading them all
+	for (const FragmentID fid : scanned_frags) {
+		throwEventConstruct(fid);
+	}
+
+	return scanned_frags.size();
+}
+
+void FragmentStore::scanStoragePathAsync(std::string path) {
+	// add path to queue
+	// HACK: if path is known/any fragment is in the path, this operation blocks (non async)
+	scanStoragePath(path); // TODO: make async and post result
+}
+
+static bool serl_json_data_enc_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Encryption>>(
+		fh.get<FragComp::DataEncryptionType>().enc
+	);
+	return true;
+}
+
+static bool deserl_json_data_enc_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataEncryptionType>(
+		static_cast<Encryption>(
+			static_cast<std::underlying_type_t<Encryption>>(in)
+		)
+	);
+	return true;
+}
+
+static bool serl_json_data_comp_type(const FragmentHandle fh, nlohmann::json& out) {
+	out = static_cast<std::underlying_type_t<Compression>>(
+		fh.get<FragComp::DataCompressionType>().comp
+	);
+	return true;
+}
+
+static bool deserl_json_data_comp_type(FragmentHandle fh, const nlohmann::json& in) {
+	fh.emplace_or_replace<FragComp::DataCompressionType>(
+		static_cast<Compression>(
+			static_cast<std::underlying_type_t<Compression>>(in)
+		)
+	);
+	return true;
+}
+
+void FragmentStore::registerSerializers(void) {
+	_sc.registerSerializerJson<FragComp::DataEncryptionType>(serl_json_data_enc_type);
+	_sc.registerDeSerializerJson<FragComp::DataEncryptionType>(deserl_json_data_enc_type);
+	_sc.registerSerializerJson<FragComp::DataCompressionType>(serl_json_data_comp_type);
+	_sc.registerDeSerializerJson<FragComp::DataCompressionType>(deserl_json_data_comp_type);
+}
+
--- a/src/fragment_store/fragment_store.hpp
+++ b/src/fragment_store/fragment_store.hpp
@ -0,0 +1,103 @@
+#pragma once
+
+#include "./fragment_store_i.hpp"
+
+#include "./types.hpp"
+#include "./meta_components.hpp"
+
+#include "./serializer.hpp"
+
+#include <entt/core/type_info.hpp>
+#include <entt/entity/registry.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+#include <vector>
+#include <array>
+#include <cstdint>
+#include <random>
+
+struct FragmentStore : public FragmentStoreI {
+	std::minstd_rand _rng{std::random_device{}()};
+	std::array<uint8_t, 16> _session_uuid_namespace;
+
+	std::string _default_store_path;
+
+	uint64_t _memory_budget {10u*1024u*1024u};
+	uint64_t _memory_usage {0u};
+
+	SerializerCallbacks<FragmentID> _sc;
+
+	FragmentStore(void);
+	FragmentStore(std::array<uint8_t, 16> session_uuid_namespace);
+
+	// HACK: get access to the reg
+	FragmentHandle fragmentHandle(FragmentID fid);
+
+	// TODO: make the frags ref counted
+
+	// TODO: check for exising
+	std::vector<uint8_t> generateNewUID(std::array<uint8_t, 16>& uuid_namespace);
+	std::vector<uint8_t> generateNewUID(void);
+
+	// ========== new fragment ==========
+
+	// memory backed owned
+	FragmentID newFragmentMemoryOwned(
+		const std::vector<uint8_t>& id,
+		size_t initial_size
+	);
+
+	// memory backed view (can only be added? not new?)
+
+	// file backed (rw...)
+	// needs to know which store path to put into
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft,
+		const std::vector<uint8_t>& id
+	);
+	// this variant generate a new, mostly unique, id for us
+	FragmentID newFragmentFile(
+		std::string_view store_path,
+		MetaFileType mft
+	);
+
+	// ========== add fragment ==========
+
+	// ========== get fragment ==========
+	FragmentID getFragmentByID(
+		const std::vector<uint8_t>& id
+	);
+	FragmentID getFragmentCustomMatcher(
+		std::function<bool(FragmentID)>& fn
+	);
+
+	// remove fragment?
+	// unload?
+
+	// ========== sync fragment to storage ==========
+	using write_to_storage_fetch_data_cb = uint64_t(uint8_t* request_buffer, uint64_t buffer_size);
+	// calls data_cb with a buffer to be filled in, cb returns actual count of data. if returned < max, its the last buffer.
+	bool syncToStorage(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+	bool syncToStorage(FragmentID fid, const uint8_t* data, const uint64_t data_size);
+
+	// ========== load fragment data from storage ==========
+	using read_from_storage_put_data_cb = void(const uint8_t* buffer, const uint64_t buffer_size);
+	bool loadFromStorage(FragmentID fid, std::function<read_from_storage_put_data_cb>& data_cb);
+	// convenience function
+	nlohmann::json loadFromStorageNJ(FragmentID fid);
+
+	// fragment discovery?
+	// returns number of new fragments
+	size_t scanStoragePath(std::string_view path);
+	void scanStoragePathAsync(std::string path);
+
+	private:
+		void registerSerializers(void); // internal comps
+		// internal actual backends
+		// TODO: seperate out
+		bool syncToMemory(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+		bool syncToFile(FragmentID fid, std::function<write_to_storage_fetch_data_cb>& data_cb);
+};
+
--- a/src/fragment_store/fragment_store_i.cpp
+++ b/src/fragment_store/fragment_store_i.cpp
@ -0,0 +1,23 @@
+#include "./fragment_store_i.hpp"
+
+#include <iostream>
+
+void FragmentStoreI::throwEventConstruct(const FragmentID fid) {
+	std::cout << "FSI debug: event construct " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_construct,
+		Fragment::Events::FragmentConstruct{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}
+
+void FragmentStoreI::throwEventUpdate(const FragmentID fid) {
+	std::cout << "FSI debug: event updated " << entt::to_integral(fid) << "\n";
+	dispatch(
+		FragmentStore_Event::fragment_updated,
+		Fragment::Events::FragmentUpdated{
+			FragmentHandle{_reg, fid}
+		}
+	);
+}
--- a/src/fragment_store/fragment_store_i.hpp
+++ b/src/fragment_store/fragment_store_i.hpp
@ -0,0 +1,63 @@
+#pragma once
+
+#include <solanaceae/util/event_provider.hpp>
+
+#include <entt/entity/registry.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <cstdint>
+
+// internal id
+enum class FragmentID : uint32_t {};
+using FragmentRegistry = entt::basic_registry<FragmentID>;
+using FragmentHandle = entt::basic_handle<FragmentRegistry>;
+
+namespace Fragment::Events {
+
+	struct FragmentConstruct {
+		const FragmentHandle e;
+	};
+	struct FragmentUpdated {
+		const FragmentHandle e;
+	};
+	//struct MessageDestory {
+		//const Message3Handle e;
+	//};
+
+} // Fragment::Events
+
+enum class FragmentStore_Event : uint32_t {
+	fragment_construct,
+	fragment_updated,
+	//message_destroy,
+
+	MAX
+};
+
+struct FragmentStoreEventI {
+	using enumType = FragmentStore_Event;
+
+	virtual ~FragmentStoreEventI(void) {}
+
+	virtual bool onEvent(const Fragment::Events::FragmentConstruct&) { return false; }
+	virtual bool onEvent(const Fragment::Events::FragmentUpdated&) { return false; }
+	//virtual bool onEvent(const Fragment::Events::MessageDestory&) { return false; }
+
+	// mm3
+	// send text
+	// send file path
+};
+using FragmentStoreEventProviderI = EventProviderI<FragmentStoreEventI>;
+
+struct FragmentStoreI  : public FragmentStoreEventProviderI {
+	static constexpr const char* version {"1"};
+
+	FragmentRegistry _reg;
+
+	virtual ~FragmentStoreI(void) {}
+
+	void throwEventConstruct(const FragmentID fid);
+	void throwEventUpdate(const FragmentID fid);
+	//void throwEventDestroy();
+};
+
--- a/src/fragment_store/fs_binary_msgpack1.png
+++ b/src/fragment_store/fs_binary_msgpack1.png
--- a/src/fragment_store/fs_binary_msgpack2.png
+++ b/src/fragment_store/fs_binary_msgpack2.png
--- a/src/fragment_store/message_fragment_store.cpp
+++ b/src/fragment_store/message_fragment_store.cpp
@ -0,0 +1,900 @@
+#include "./message_fragment_store.hpp"
+
+#include "../json/message_components.hpp"
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/contact/components.hpp>
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/message3/contact_components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <algorithm>
+#include <string>
+#include <cstdint>
+#include <cassert>
+#include <iostream>
+
+// https://youtu.be/CU2exyhYPfA
+
+// everything assumes a single fragment registry
+
+namespace Message::Components {
+
+	// ctx
+	struct OpenFragments {
+		//struct OpenFrag final {
+			////std::vector<uint8_t> uid;
+			//FragmentID id;
+		//};
+		// only contains fragments with <1024 messages and <28h tsrage (or whatever)
+		entt::dense_set<FragmentID> fid_open;
+	};
+
+	// all message fragments of this contact
+	struct ContactFragments final {
+		// kept up-to-date by events
+		struct InternalEntry {
+			// indecies into the sorted arrays
+			size_t i_b;
+			size_t i_e;
+		};
+		entt::dense_map<FragmentID, InternalEntry> frags;
+
+		// add 2 sorted contact lists for both range begin and end
+		// TODO: adding and removing becomes expensive with enough frags, consider splitting or heap
+		std::vector<FragmentID> sorted_begin;
+		std::vector<FragmentID> sorted_end;
+
+		// api
+		// return true if it was actually inserted
+		bool insert(FragmentHandle frag);
+		bool erase(FragmentID frag);
+		// update? (just erase() + insert())
+
+		// uses range begin to go back in time
+		FragmentID prev(FragmentID frag) const;
+		// uses range end to go forward in time
+		FragmentID next(FragmentID frag) const;
+	};
+
+	// all LOADED message fragments
+	// TODO: merge into ContactFragments (and pull in openfrags)
+	struct LoadedContactFragments final {
+		// kept up-to-date by events
+		entt::dense_set<FragmentID> frags;
+	};
+
+} // Message::Components
+
+namespace Fragment::Components {
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTSRange, begin, end)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesContact, id)
+} // Fragment::Components
+
+void MessageFragmentStore::handleMessage(const Message3Handle& m) {
+	if (_fs_ignore_event) {
+		// message event because of us loading a fragment, ignore
+		// TODO: this barely makes a difference
+		return;
+	}
+
+	if (!static_cast<bool>(m)) {
+		return; // huh?
+	}
+
+	if (!m.all_of<Message::Components::Timestamp>()) {
+		return; // we only handle msg with ts
+	}
+
+	_potentially_dirty_contacts.emplace(m.registry()->ctx().get<Contact3>()); // always mark dirty here
+	if (m.any_of<Message::Components::ViewCurserBegin, Message::Components::ViewCurserEnd>()) {
+		// not an actual message, but we probalby need to check and see if we need to load fragments
+		//std::cout << "MFS: new or updated curser\n";
+		return;
+	}
+
+	// TODO: use fid, seving full fuid for every message consumes alot of memory (and heap frag)
+	if (!m.all_of<Message::Components::FID>()) {
+		std::cout << "MFS: new msg missing FID\n";
+		if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
+			m.registry()->ctx().emplace<Message::Components::OpenFragments>();
+		}
+
+		auto& fid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fid_open;
+
+		const auto msg_ts = m.get<Message::Components::Timestamp>().ts;
+		// missing fuid
+		// find closesed non-sealed off fragment
+
+		FragmentID fragment_id{entt::null};
+
+		// first search for fragment where the ts falls into the range
+		for (const auto& fid : fid_open) {
+			auto fh = _fs.fragmentHandle(fid);
+			assert(static_cast<bool>(fh));
+
+			// assuming ts range exists
+			auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
+
+			if (fts_comp.begin <= msg_ts && fts_comp.end >= msg_ts) {
+				fragment_id = fid;
+				// TODO: check conditions for open here
+				// TODO: mark msg (and frag?) dirty
+			}
+		}
+
+		// if it did not fit into an existing fragment, we next look for fragments that could be extended
+		if (!_fs._reg.valid(fragment_id)) {
+			for (const auto& fid : fid_open) {
+				auto fh = _fs.fragmentHandle(fid);
+				assert(static_cast<bool>(fh));
+
+				// assuming ts range exists
+				auto& fts_comp = fh.get<FragComp::MessagesTSRange>();
+
+				const int64_t frag_range = int64_t(fts_comp.end) - int64_t(fts_comp.begin);
+				constexpr static int64_t max_frag_ts_extent {1000*60*60};
+				//constexpr static int64_t max_frag_ts_extent {1000*60*3}; // 3min for testing
+				const int64_t possible_extention = max_frag_ts_extent - frag_range;
+
+				// which direction
+				if ((fts_comp.begin - possible_extention) <= msg_ts && fts_comp.begin > msg_ts) {
+					fragment_id = fid;
+
+					std::cout << "MFS: extended begin from " << fts_comp.begin << " to " << msg_ts << "\n";
+
+					// assuming ts range exists
+					fts_comp.begin = msg_ts; // extend into the past
+
+					if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+						// should be the case
+						m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
+						m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+					}
+
+
+					// TODO: check conditions for open here
+					// TODO: mark msg (and frag?) dirty
+				} else if ((fts_comp.end + possible_extention) >= msg_ts && fts_comp.end < msg_ts) {
+					fragment_id = fid;
+
+					std::cout << "MFS: extended end from " << fts_comp.end << " to " << msg_ts << "\n";
+
+					// assuming ts range exists
+					fts_comp.end = msg_ts; // extend into the future
+
+					if (m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+						// should be the case
+						m.registry()->ctx().get<Message::Components::ContactFragments>().erase(fh);
+						m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+					}
+
+					// TODO: check conditions for open here
+					// TODO: mark msg (and frag?) dirty
+				}
+			}
+		}
+
+		// if its still not found, we need a new fragment
+		if (!_fs._reg.valid(fragment_id)) {
+			const auto new_fid = _fs.newFragmentFile("test_message_store/", MetaFileType::BINARY_MSGPACK);
+			auto fh = _fs.fragmentHandle(new_fid);
+			if (!static_cast<bool>(fh)) {
+				std::cout << "MFS error: failed to create new fragment for message\n";
+				return;
+			}
+
+			fragment_id = fh;
+
+			fh.emplace_or_replace<FragComp::Ephemeral::MetaCompressionType>().comp = Compression::ZSTD;
+			fh.emplace_or_replace<FragComp::DataCompressionType>().comp = Compression::ZSTD;
+
+			auto& new_ts_range = fh.emplace_or_replace<FragComp::MessagesTSRange>();
+			new_ts_range.begin = msg_ts;
+			new_ts_range.end = msg_ts;
+
+			{
+				const auto msg_reg_contact = m.registry()->ctx().get<Contact3>();
+				if (_cr.all_of<Contact::Components::ID>(msg_reg_contact)) {
+					fh.emplace<FragComp::MessagesContact>(_cr.get<Contact::Components::ID>(msg_reg_contact).data);
+				} else {
+					// ? rage quit?
+				}
+			}
+
+			// contact frag
+			if (!m.registry()->ctx().contains<Message::Components::ContactFragments>()) {
+				m.registry()->ctx().emplace<Message::Components::ContactFragments>();
+			}
+			m.registry()->ctx().get<Message::Components::ContactFragments>().insert(fh);
+
+			// loaded contact frag
+			if (!m.registry()->ctx().contains<Message::Components::LoadedContactFragments>()) {
+				m.registry()->ctx().emplace<Message::Components::LoadedContactFragments>();
+			}
+			m.registry()->ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
+
+			fid_open.emplace(fragment_id);
+
+			std::cout << "MFS: created new fragment " << bin2hex(fh.get<FragComp::ID>().v) << "\n";
+
+			_fs_ignore_event = true;
+			_fs.throwEventConstruct(fh);
+			_fs_ignore_event = false;
+		}
+
+		// if this is still empty, something is very wrong and we exit here
+		if (!_fs._reg.valid(fragment_id)) {
+			std::cout << "MFS error: failed to find/create fragment for message\n";
+			return;
+		}
+
+		m.emplace_or_replace<Message::Components::FID>(fragment_id);
+
+		// in this case we know the fragment needs an update
+		_fuid_save_queue.push({Message::getTimeMS(), fragment_id, m.registry()});
+		return; // done
+	}
+
+	const auto msg_fh = _fs.fragmentHandle(m.get<Message::Components::FID>().fid);
+	if (!static_cast<bool>(msg_fh)) {
+		std::cerr << "MFS error: fid in message is invalid\n";
+		return; // TODO: properly handle this case
+	}
+
+	if (!m.registry()->ctx().contains<Message::Components::OpenFragments>()) {
+		m.registry()->ctx().emplace<Message::Components::OpenFragments>();
+	}
+
+	auto& fid_open = m.registry()->ctx().get<Message::Components::OpenFragments>().fid_open;
+
+	if (fid_open.contains(msg_fh)) {
+		// TODO: dedup events
+		// TODO: cooldown per fragsave
+		_fuid_save_queue.push({Message::getTimeMS(), msg_fh, m.registry()});
+		return;
+	}
+
+	// TODO: save updates to old fragments, but writing them to a new fragment that would overwrite on merge
+	// new fragment?, since we dont write to others fragments?
+
+
+	// on new message: assign fuid
+	// on new and update: mark as fragment dirty
+}
+
+// assumes not loaded frag
+// need update from frag
+void MessageFragmentStore::loadFragment(Message3Registry& reg, FragmentHandle fh) {
+	std::cout << "MFS: loadFragment\n";
+	const auto j = _fs.loadFromStorageNJ(fh);
+
+	if (!j.is_array()) {
+		// wrong data
+		return;
+	}
+
+	// TODO: this should probably never be the case, since we already know here that it is a msg frag
+	if (!reg.ctx().contains<Message::Components::ContactFragments>()) {
+		reg.ctx().emplace<Message::Components::ContactFragments>();
+	}
+	reg.ctx().get<Message::Components::ContactFragments>().insert(fh);
+
+	// mark loaded
+	if (!reg.ctx().contains<Message::Components::LoadedContactFragments>()) {
+		reg.ctx().emplace<Message::Components::LoadedContactFragments>();
+	}
+	reg.ctx().get<Message::Components::LoadedContactFragments>().frags.emplace(fh);
+
+	for (const auto& j_entry : j) {
+		auto new_real_msg = Message3Handle{reg, reg.create()};
+		// load into staging reg
+		for (const auto& [k, v] : j_entry.items()) {
+			//std::cout << "K:" << k << " V:" << v.dump() << "\n";
+			const auto type_id = entt::hashed_string(k.data(), k.size());
+			const auto deserl_fn_it = _sc._deserl_json.find(type_id);
+			if (deserl_fn_it != _sc._deserl_json.cend()) {
+				try {
+					if (!deserl_fn_it->second(_sc, new_real_msg, v)) {
+						std::cerr << "MFS error: failed deserializing '" << k << "'\n";
+					}
+				} catch(...) {
+					std::cerr << "MFS error: failed deserializing (threw) '" << k << "'\n";
+				}
+			} else {
+				std::cerr << "MFS warning: missing deserializer for meta key '" << k << "'\n";
+			}
+		}
+
+		new_real_msg.emplace_or_replace<Message::Components::FID>(fh);
+
+		// dup check (hacky, specific to protocols)
+		Message3 dup_msg {entt::null};
+		{
+			// get comparator from contact
+			if (reg.ctx().contains<Contact3>()) {
+				const auto c = reg.ctx().get<Contact3>();
+				if (_cr.all_of<Contact::Components::MessageIsSame>(c)) {
+					auto& comp = _cr.get<Contact::Components::MessageIsSame>(c).comp;
+					// walking EVERY existing message OOF
+					// this needs optimizing
+					for (const Message3 other_msg : reg.view<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
+						if (other_msg == new_real_msg) {
+							continue; // skip self
+						}
+
+						if (comp({reg, other_msg}, new_real_msg)) {
+							// dup
+							dup_msg = other_msg;
+							break;
+						}
+					}
+				}
+			}
+		}
+
+		if (reg.valid(dup_msg)) {
+			//  -> merge with preexisting (needs to be order independent)
+			//  -> throw update
+			reg.destroy(new_real_msg);
+			//_rmm.throwEventUpdate(reg, new_real_msg);
+		} else {
+			if (!new_real_msg.all_of<Message::Components::Timestamp, Message::Components::ContactFrom, Message::Components::ContactTo>()) {
+				// does not have needed components to be stand alone
+				reg.destroy(new_real_msg);
+				std::cerr << "MFS warning: message with missing basic compoments\n";
+				continue;
+			}
+
+			//  -> throw create
+			_rmm.throwEventConstruct(reg, new_real_msg);
+		}
+	}
+}
+
+bool MessageFragmentStore::syncFragToStorage(FragmentHandle fh, Message3Registry& reg) {
+	auto& ftsrange = fh.get_or_emplace<FragComp::MessagesTSRange>(Message::getTimeMS(), Message::getTimeMS());
+
+	auto j = nlohmann::json::array();
+
+	// TODO: does every message have ts?
+	auto msg_view = reg.view<Message::Components::Timestamp>();
+	// we also assume all messages have fid
+	for (auto it = msg_view.rbegin(), it_end = msg_view.rend(); it != it_end; it++) {
+		const Message3 m = *it;
+
+		if (!reg.all_of<Message::Components::FID, Message::Components::ContactFrom, Message::Components::ContactTo>(m)) {
+			continue;
+		}
+
+		// require msg for now
+		if (!reg.any_of<Message::Components::MessageText/*, Message::Components::Transfer::FileInfo*/>(m)) {
+			continue;
+		}
+
+		if (_fuid_save_queue.front().id != reg.get<Message::Components::FID>(m).fid) {
+			continue; // not ours
+		}
+
+		{ // potentially adjust tsrange (some external processes can change timestamps)
+			const auto msg_ts = msg_view.get<Message::Components::Timestamp>(m).ts;
+			if (ftsrange.begin > msg_ts) {
+				ftsrange.begin = msg_ts;
+			} else if (ftsrange.end < msg_ts) {
+				ftsrange.end = msg_ts;
+			}
+		}
+
+		auto& j_entry = j.emplace_back(nlohmann::json::object());
+
+		for (const auto& [type_id, storage] : reg.storage()) {
+			if (!storage.contains(m)) {
+				continue;
+			}
+
+			//std::cout << "storage type: type_id:" << type_id << " name:" << storage.type().name() << "\n";
+
+			// use type_id to find serializer
+			auto s_cb_it = _sc._serl_json.find(type_id);
+			if (s_cb_it == _sc._serl_json.end()) {
+				// could not find serializer, not saving
+				//std::cout << "missing " << storage.type().name() << "(" << type_id << ")\n";
+				continue;
+			}
+
+			s_cb_it->second(_sc, {reg, m}, j_entry[storage.type().name()]);
+		}
+	}
+
+	// we cant skip if array is empty (in theory it will not be empty later on)
+
+	// if save as binary
+	//nlohmann::json::to_msgpack(j);
+	auto j_dump = j.dump(2, ' ', true);
+	if (_fs.syncToStorage(fh, reinterpret_cast<const uint8_t*>(j_dump.data()), j_dump.size())) {
+		//std::cout << "MFS: dumped " << j_dump << "\n";
+		// succ
+		return true;
+	}
+
+	// TODO: error
+	return false;
+}
+
+MessageFragmentStore::MessageFragmentStore(
+	Contact3Registry& cr,
+	RegistryMessageModel& rmm,
+	FragmentStore& fs
+) : _cr(cr), _rmm(rmm), _fs(fs), _sc{_cr, {}, {}} {
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_construct);
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_updated);
+	_rmm.subscribe(this, RegistryMessageModel_Event::message_destroy);
+
+	_fs._sc.registerSerializerJson<FragComp::MessagesTSRange>();
+	_fs._sc.registerDeSerializerJson<FragComp::MessagesTSRange>();
+	_fs._sc.registerSerializerJson<FragComp::MessagesContact>();
+	_fs._sc.registerDeSerializerJson<FragComp::MessagesContact>();
+
+	_fs.subscribe(this, FragmentStore_Event::fragment_construct);
+}
+
+MessageFragmentStore::~MessageFragmentStore(void) {
+	while (!_fuid_save_queue.empty()) {
+		auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id);
+		auto* reg = _fuid_save_queue.front().reg;
+		assert(reg != nullptr);
+		syncFragToStorage(fh, *reg);
+		_fuid_save_queue.pop(); // pop unconditionally
+	}
+}
+
+MessageSerializerCallbacks& MessageFragmentStore::getMSC(void) {
+	return _sc;
+}
+
+// checks range against all cursers in msgreg
+static bool rangeVisible(uint64_t range_begin, uint64_t range_end, const Message3Registry& msg_reg) {
+	// 1D collision checks:
+	//  - for range vs range:
+	//    r1 rhs >= r0 lhs AND r1 lhs <= r0 rhs
+	//  - for range vs point:
+	//    p >= r0 lhs AND p <= r0 rhs
+	// NOTE: directions for us are reversed (begin has larger values as end)
+
+	auto c_b_view = msg_reg.view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
+	c_b_view.use<Message::Components::ViewCurserBegin>();
+	for (const auto& [m, ts_begin_comp, vcb] : c_b_view.each()) {
+		// p and r1 rhs can be seen as the same
+		// but first we need to know if a curser begin is a point or a range
+
+		// TODO: margin?
+		auto ts_begin = ts_begin_comp.ts;
+		auto ts_end = ts_begin_comp.ts; // simplyfy code by making a single begin curser act as an infinitly small range
+		if (msg_reg.valid(vcb.curser_end) && msg_reg.all_of<Message::Components::ViewCurserEnd>(vcb.curser_end)) {
+			// TODO: respect curser end's begin?
+			// TODO: remember which ends we checked and check remaining
+			ts_end = msg_reg.get<Message::Components::Timestamp>(vcb.curser_end).ts;
+
+			// sanity check curser order
+			if (ts_end > ts_begin) {
+				std::cerr << "MFS warning: begin curser and end curser of view swapped!!\n";
+				std::swap(ts_begin, ts_end);
+			}
+		}
+
+		// perform both checks here
+		if (ts_begin < range_end || ts_end > range_begin) {
+			continue;
+		}
+
+		// range hits a view
+		return true;
+	}
+
+	return false;
+}
+
+static bool isLess(const std::vector<uint8_t>& lhs, const std::vector<uint8_t>& rhs) {
+	size_t i = 0;
+	for (; i < lhs.size() && i < rhs.size(); i++) {
+		if (lhs[i] < rhs[i]) {
+			return true;
+		} else if (lhs[i] > rhs[i]) {
+			return false;
+		}
+		// else continue
+	}
+
+	// here we have equality of common lenths
+
+	// we define smaller arrays to be less
+	return lhs.size() < rhs.size();
+}
+
+float MessageFragmentStore::tick(float time_delta) {
+	// sync dirty fragments here
+	if (!_fuid_save_queue.empty()) {
+		auto fh = _fs.fragmentHandle(_fuid_save_queue.front().id);
+		auto* reg = _fuid_save_queue.front().reg;
+		assert(reg != nullptr);
+		if (syncFragToStorage(fh, *reg)) {
+			_fuid_save_queue.pop();
+		}
+	}
+
+	// load needed fragments here
+
+	// last check event frags
+	// only checks if it collides with ranges, not adjacent
+	// bc ~range~ msgreg will be marked dirty and checked next tick
+	const bool had_events = !_event_check_queue.empty();
+	for (size_t i = 0; i < 10 && !_event_check_queue.empty(); i++) {
+		std::cout << "MFS: event check\n";
+		auto fh = _fs.fragmentHandle(_event_check_queue.front().fid);
+		auto c = _event_check_queue.front().c;
+		_event_check_queue.pop();
+
+		if (!static_cast<bool>(fh)) {
+			return 0.05f;
+		}
+
+		if (!fh.all_of<FragComp::MessagesTSRange>()) {
+			return 0.05f;
+		}
+
+		// get ts range of frag and collide with all curser(s/ranges)
+		const auto& frag_range = fh.get<FragComp::MessagesTSRange>();
+
+		auto* msg_reg = _rmm.get(c);
+		if (msg_reg == nullptr) {
+			return 0.05f;
+		}
+
+		if (rangeVisible(frag_range.begin, frag_range.end, !msg_reg)) {
+			loadFragment(*msg_reg, fh);
+			_potentially_dirty_contacts.emplace(c);
+			return 0.05f; // only one but soon again
+		}
+	}
+	if (had_events) {
+		std::cout << "MFS: event check none\n";
+		return 0.05f; // only check events, even if non where hit
+	}
+
+	if (!_potentially_dirty_contacts.empty()) {
+		std::cout << "MFS: pdc\n";
+		// here we check if any view of said contact needs frag loading
+		// only once per tick tho
+
+		// TODO: this makes order depend on internal order and is not fair
+		auto it = _potentially_dirty_contacts.cbegin();
+
+		auto* msg_reg = _rmm.get(*it);
+
+		// first do collision check agains every contact associated fragment
+		// that is not already loaded !!
+		if (msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
+			const auto& cf = msg_reg->ctx().get<Message::Components::ContactFragments>();
+			if (!cf.frags.empty()) {
+				if (!msg_reg->ctx().contains<Message::Components::LoadedContactFragments>()) {
+					msg_reg->ctx().emplace<Message::Components::LoadedContactFragments>();
+				}
+				const auto& loaded_frags = msg_reg->ctx().get<Message::Components::LoadedContactFragments>().frags;
+
+				for (const auto& [fid, si] : msg_reg->ctx().get<Message::Components::ContactFragments>().frags) {
+					if (loaded_frags.contains(fid)) {
+						continue;
+					}
+
+					auto fh = _fs.fragmentHandle(fid);
+
+					if (!static_cast<bool>(fh)) {
+						std::cerr << "MFS error: frag is invalid\n";
+						// WHAT
+						msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
+						return 0.05f;
+					}
+
+					if (!fh.all_of<FragComp::MessagesTSRange>()) {
+						std::cerr << "MFS error: frag has no range\n";
+						// ????
+						msg_reg->ctx().get<Message::Components::ContactFragments>().erase(fid);
+						return 0.05f;
+					}
+
+					// get ts range of frag and collide with all curser(s/ranges)
+					const auto& [range_begin, range_end] = fh.get<FragComp::MessagesTSRange>();
+
+					if (rangeVisible(range_begin, range_end, *msg_reg)) {
+						std::cout << "MFS: frag hit by vis range\n";
+						loadFragment(*msg_reg, fh);
+						return 0.05f;
+					}
+				}
+				// no new visible fragment
+				std::cout << "MFS: no new frag directly visible\n";
+
+				// now, finally, check for adjecent fragments that need to be loaded
+				// we do this by finding the outermost fragment in a rage, and extend it by one
+
+				// TODO: rewrite using some bounding range tree to perform collision checks !!!
+				// (this is now performing better, but still)
+
+
+				// for each view
+				auto c_b_view = msg_reg->view<Message::Components::Timestamp, Message::Components::ViewCurserBegin>();
+				c_b_view.use<Message::Components::ViewCurserBegin>();
+				for (const auto& [_, ts_begin_comp, vcb] : c_b_view.each()) {
+					// aka "scroll down"
+					{ // find newest(-ish) frag in range
+						// or in reverse frag end <= range begin
+
+
+						// lower bound of frag end and range begin
+						const auto right = std::lower_bound(
+							cf.sorted_end.crbegin(),
+							cf.sorted_end.crend(),
+							ts_begin_comp.ts,
+							[&](const FragmentID element, const auto& value) -> bool {
+								return _fs._reg.get<FragComp::MessagesTSRange>(element).end >= value;
+							}
+						);
+
+						FragmentID next_frag{entt::null};
+						if (right != cf.sorted_end.crend()) {
+							next_frag = cf.next(*right);
+						}
+						// we checked earlier that cf is not empty
+						if (!_fs._reg.valid(next_frag)) {
+							// fall back to closest, cf is not empty
+							next_frag = cf.sorted_end.front();
+						}
+
+						// a single adjacent frag is often not enough
+						// only ok bc next is cheap
+						for (size_t i = 0; i < 5 && _fs._reg.valid(next_frag); i++) {
+							if (!loaded_frags.contains(next_frag)) {
+								std::cout << "MFS: next frag of range\n";
+								loadFragment(*msg_reg, {_fs._reg, next_frag});
+								return 0.05f;
+							}
+
+							next_frag = cf.next(next_frag);
+						}
+					}
+
+					// curser end
+					if (!msg_reg->valid(vcb.curser_end) || !msg_reg->all_of<Message::Components::Timestamp>(vcb.curser_end)) {
+						continue;
+					}
+					const auto ts_end = msg_reg->get<Message::Components::Timestamp>(vcb.curser_end).ts;
+
+					// aka "scroll up"
+					{ // find oldest(-ish) frag in range
+						// frag begin >= range end
+
+						// lower bound of frag begin and range end
+						const auto left = std::lower_bound(
+							cf.sorted_begin.cbegin(),
+							cf.sorted_begin.cend(),
+							ts_end,
+							[&](const FragmentID element, const auto& value) -> bool {
+								return _fs._reg.get<FragComp::MessagesTSRange>(element).begin < value;
+							}
+						);
+
+						FragmentID prev_frag{entt::null};
+						if (left != cf.sorted_begin.cend()) {
+							prev_frag = cf.prev(*left);
+						}
+						// we checked earlier that cf is not empty
+						if (!_fs._reg.valid(prev_frag)) {
+							// fall back to closest, cf is not empty
+							prev_frag = cf.sorted_begin.back();
+						}
+
+						// a single adjacent frag is often not enough
+						// only ok bc next is cheap
+						for (size_t i = 0; i < 5 && _fs._reg.valid(prev_frag); i++) {
+							if (!loaded_frags.contains(prev_frag)) {
+								std::cout << "MFS: prev frag of range\n";
+								loadFragment(*msg_reg, {_fs._reg, prev_frag});
+								return 0.05f;
+							}
+
+							prev_frag = cf.prev(prev_frag);
+						}
+					}
+				}
+			}
+		} else {
+			// contact has no fragments, skip
+		}
+
+		_potentially_dirty_contacts.erase(it);
+
+		return 0.05f;
+	}
+
+
+	return 1000.f*60.f*60.f;
+}
+
+void MessageFragmentStore::triggerScan(void) {
+	_fs.scanStoragePath("test_message_store/");
+}
+
+bool MessageFragmentStore::onEvent(const Message::Events::MessageConstruct& e) {
+	handleMessage(e.e);
+	return false;
+}
+
+bool MessageFragmentStore::onEvent(const Message::Events::MessageUpdated& e) {
+	handleMessage(e.e);
+	return false;
+}
+
+// TODO: handle deletes? diff between unload?
+
+bool MessageFragmentStore::onEvent(const Fragment::Events::FragmentConstruct& e) {
+	if (_fs_ignore_event) {
+		return false; // skip self
+	}
+
+	if (!e.e.all_of<FragComp::MessagesTSRange, FragComp::MessagesContact>()) {
+		return false; // not for us
+	}
+
+	// TODO: are we sure it is a *new* fragment?
+
+	Contact3 frag_contact = entt::null;
+	{ // get contact
+		const auto& frag_contact_id = e.e.get<FragComp::MessagesContact>().id;
+		// TODO: id lookup table, this is very inefficent
+		for (const auto& [c_it, id_it] : _cr.view<Contact::Components::ID>().each()) {
+			if (frag_contact_id == id_it.data) {
+				frag_contact = c_it;
+				break;
+			}
+		}
+		if (!_cr.valid(frag_contact)) {
+			// unkown contact
+			return false;
+		}
+	}
+
+	// create if not exist
+	auto* msg_reg = _rmm.get(frag_contact);
+	if (msg_reg == nullptr) {
+		// msg reg not created yet
+		// TODO: this is an erroious path
+		return false;
+	}
+
+	if (!msg_reg->ctx().contains<Message::Components::ContactFragments>()) {
+		msg_reg->ctx().emplace<Message::Components::ContactFragments>();
+	}
+	msg_reg->ctx().get<Message::Components::ContactFragments>().erase(e.e); // TODO: check/update/fragment update
+	msg_reg->ctx().get<Message::Components::ContactFragments>().insert(e.e);
+
+	_event_check_queue.push(ECQueueEntry{e.e, frag_contact});
+
+	return false;
+}
+
+bool Message::Components::ContactFragments::insert(FragmentHandle frag) {
+	if (frags.contains(frag)) {
+		return false;
+	}
+
+	// both sorted arrays are sorted ascending
+	// so for insertion we search for the last index that is <= and insert after it
+	// or we search for the first > (or end) and insert before it    <---
+	// since equal fragments are UB, we can assume they are only > or <
+
+	size_t begin_index {0};
+	{ // begin
+		const auto pos = std::find_if(
+			sorted_begin.cbegin(),
+			sorted_begin.cend(),
+			[frag](const FragmentID a) -> bool {
+				const auto begin_a = frag.registry()->get<FragComp::MessagesTSRange>(a).begin;
+				const auto begin_frag = frag.get<FragComp::MessagesTSRange>().begin;
+				if (begin_a > begin_frag) {
+					return true;
+				} else if (begin_a < begin_frag) {
+					return false;
+				} else {
+					// equal ts, we need to fall back to id (id can not be equal)
+					return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
+				}
+			}
+		);
+
+		begin_index = std::distance(sorted_begin.cbegin(), pos);
+
+		// we need to insert before pos (end is valid here)
+		sorted_begin.insert(pos, frag);
+	}
+
+	size_t end_index {0};
+	{ // end
+		const auto pos = std::find_if_not(
+			sorted_end.cbegin(),
+			sorted_end.cend(),
+			[frag](const FragmentID a) -> bool {
+				const auto end_a = frag.registry()->get<FragComp::MessagesTSRange>(a).end;
+				const auto end_frag = frag.get<FragComp::MessagesTSRange>().end;
+				if (end_a > end_frag) {
+					return true;
+				} else if (end_a < end_frag) {
+					return false;
+				} else {
+					// equal ts, we need to fall back to id (id can not be equal)
+					return isLess(frag.get<FragComp::ID>().v, frag.registry()->get<FragComp::ID>(a).v);
+				}
+			}
+		);
+
+		end_index = std::distance(sorted_end.cbegin(), pos);
+
+		// we need to insert before pos (end is valid here)
+		sorted_end.insert(pos, frag);
+	}
+
+	frags.emplace(frag, InternalEntry{begin_index, end_index});
+
+	return true;
+}
+
+bool Message::Components::ContactFragments::erase(FragmentID frag) {
+	auto frags_it = frags.find(frag);
+	if (frags_it == frags.end()) {
+		return false;
+	}
+
+	assert(sorted_begin.size() == sorted_end.size());
+	assert(sorted_begin.size() > frags_it->second.i_b);
+
+	sorted_begin.erase(sorted_begin.begin() + frags_it->second.i_b);
+	sorted_end.erase(sorted_end.begin() + frags_it->second.i_e);
+
+	frags.erase(frags_it);
+
+	return true;
+}
+
+FragmentID Message::Components::ContactFragments::prev(FragmentID frag) const {
+	// uses range begin to go back in time
+
+	auto it = frags.find(frag);
+	if (it == frags.end()) {
+		return entt::null;
+	}
+
+	const auto src_i = it->second.i_b;
+	if (src_i > 0) {
+		return sorted_begin[src_i-1];
+	}
+
+	return entt::null;
+}
+
+FragmentID Message::Components::ContactFragments::next(FragmentID frag) const {
+	// uses range end to go forward in time
+
+	auto it = frags.find(frag);
+	if (it == frags.end()) {
+		return entt::null;
+	}
+
+	const auto src_i = it->second.i_e;
+	if (src_i+1 < sorted_end.size()) {
+		return sorted_end[src_i+1];
+	}
+
+	return entt::null;
+}
+
--- a/src/fragment_store/message_fragment_store.hpp
+++ b/src/fragment_store/message_fragment_store.hpp
@ -0,0 +1,132 @@
+#pragma once
+
+#include "./meta_components.hpp"
+#include "./fragment_store_i.hpp"
+#include "./fragment_store.hpp"
+
+#include "./message_serializer.hpp"
+
+#include <entt/entity/registry.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/container/dense_set.hpp>
+
+#include <solanaceae/contact/contact_model3.hpp>
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <queue>
+#include <vector>
+#include <cstdint>
+
+namespace Message::Components {
+
+	// unused, consumes too much memory (highly compressable)
+	//using FUID = FragComp::ID;
+
+	struct FID {
+		FragmentID fid {entt::null};
+	};
+
+	// points to the front/newer message
+	// together they define a range that is,
+	// eg the first(end) and last(begin) message being rendered
+	// MFS requires there to be atleast one other fragment after/before,
+	// if not loaded fragment with fitting tsrange(direction) available
+	// uses fragmentAfter/Before()
+	// they can exist standalone
+	// if they are a pair, the inside is filled first
+	// cursers require a timestamp ???
+	struct ViewCurserBegin {
+		Message3 curser_end{entt::null};
+	};
+	struct ViewCurserEnd {
+		Message3 curser_begin{entt::null};
+	};
+
+	// TODO: add adjacency range comp or inside curser
+
+	// TODO: unused
+	// mfs will only load a limited number of fragments per tick (1),
+	// so this tag will be set if we loaded a fragment and
+	// every tick we check all cursers for this tag and continue
+	// and remove once no fragment could be loaded anymore
+	// (internal)
+	struct TagCurserUnsatisfied {};
+
+} // Message::Components
+
+namespace Fragment::Components {
+	struct MessagesTSRange {
+		// timestamp range within the fragment
+		uint64_t begin {0}; // newer msg -> higher number
+		uint64_t end {0};
+	};
+
+	struct MessagesContact {
+		std::vector<uint8_t> id;
+	};
+
+	// TODO: add src contact (self id)
+
+} // Fragment::Components
+
+// handles fragments for messages
+// on new message: assign fuid
+// on new and update: mark as fragment dirty
+// on delete: mark as fragment dirty?
+class MessageFragmentStore : public RegistryMessageModelEventI, public FragmentStoreEventI {
+	protected:
+		Contact3Registry& _cr;
+		RegistryMessageModel& _rmm;
+		FragmentStore& _fs;
+		bool _fs_ignore_event {false};
+
+		// for message components only
+		MessageSerializerCallbacks _sc;
+
+		void handleMessage(const Message3Handle& m);
+
+		void loadFragment(Message3Registry& reg, FragmentHandle fh);
+
+		bool syncFragToStorage(FragmentHandle fh, Message3Registry& reg);
+
+		struct SaveQueueEntry final {
+			uint64_t ts_since_dirty{0};
+			//std::vector<uint8_t> id;
+			FragmentID id;
+			Message3Registry* reg{nullptr};
+		};
+		std::queue<SaveQueueEntry> _fuid_save_queue;
+
+		struct ECQueueEntry final {
+			FragmentID fid;
+			Contact3 c;
+		};
+		std::queue<ECQueueEntry> _event_check_queue;
+
+		// range changed or fragment loaded.
+		// we only load a limited number of fragments at once,
+		// so we need to keep them dirty until nothing was loaded.
+		entt::dense_set<Contact3> _potentially_dirty_contacts;
+
+	public:
+		MessageFragmentStore(
+			Contact3Registry& cr,
+			RegistryMessageModel& rmm,
+			FragmentStore& fs
+		);
+		virtual ~MessageFragmentStore(void);
+
+		MessageSerializerCallbacks& getMSC(void);
+
+		float tick(float time_delta);
+
+		void triggerScan(void);
+
+	protected: // rmm
+		bool onEvent(const Message::Events::MessageConstruct& e) override;
+		bool onEvent(const Message::Events::MessageUpdated& e) override;
+
+	protected: // fs
+		bool onEvent(const Fragment::Events::FragmentConstruct& e) override;
+};
+
--- a/src/fragment_store/message_serializer.cpp
+++ b/src/fragment_store/message_serializer.cpp
@ -0,0 +1,107 @@
+#include "./message_serializer.hpp"
+
+#include <solanaceae/message3/components.hpp>
+#include <solanaceae/contact/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+#include <iostream>
+
+static Contact3 findContactByID(Contact3Registry& cr, const std::vector<uint8_t>& id) {
+	// TODO: id lookup table, this is very inefficent
+	for (const auto& [c_it, id_it] : cr.view<Contact::Components::ID>().each()) {
+		if (id == id_it.data) {
+			return c_it;
+		}
+	}
+
+	return entt::null;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactFrom>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactFrom>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactFrom>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}
+
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j) {
+	const Contact3 c = h.get<Message::Components::ContactTo>().c;
+	if (!msc.cr.valid(c)) {
+		// while this is invalid registry state, it is valid serialization
+		j = nullptr;
+		std::cerr << "MSC warning: encountered invalid contact\n";
+		return true;
+	}
+
+	if (!msc.cr.all_of<Contact::Components::ID>(c)) {
+		// unlucky, this contact is purely ephemeral
+		j = nullptr;
+		std::cerr << "MSC warning: encountered contact without ID\n";
+		return true;
+	}
+
+	j = nlohmann::json::binary(msc.cr.get<Contact::Components::ID>(c).data);
+
+	return true;
+}
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j) {
+	if (j.is_null()) {
+		std::cerr << "MSC warning: encountered null contact\n";
+		h.emplace_or_replace<Message::Components::ContactTo>();
+		return true;
+	}
+
+	const std::vector<uint8_t> id = j.is_binary()?j:j["bytes"];
+
+	Contact3 other_c = findContactByID(msc.cr, id);
+	if (!msc.cr.valid(other_c)) {
+		// create sparse contact with id only
+		other_c = msc.cr.create();
+		msc.cr.emplace_or_replace<Contact::Components::ID>(other_c, id);
+	}
+
+	h.emplace_or_replace<Message::Components::ContactTo>(other_c);
+
+	// TODO: should we return false if the contact is unknown??
+	return true;
+}
--- a/src/fragment_store/message_serializer.hpp
+++ b/src/fragment_store/message_serializer.hpp
@ -0,0 +1,85 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+
+#include <solanaceae/message3/registry_message_model.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+struct MessageSerializerCallbacks {
+	using Registry = Message3Registry;
+	using Handle = Message3Handle;
+
+	Contact3Registry& cr;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(MessageSerializerCallbacks&, const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(MessageSerializerCallbacks&, Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+
+// fwd
+namespace Message::Components {
+struct ContactFrom;
+struct ContactTo;
+}
+
+// make specializations known
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactFrom>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_get_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, const Handle h, nlohmann::json& j);
+template<>
+bool MessageSerializerCallbacks::component_emplace_or_replace_json<Message::Components::ContactTo>(MessageSerializerCallbacks& msc, Handle h, const nlohmann::json& j);
--- a/src/fragment_store/meta_components.hpp
+++ b/src/fragment_store/meta_components.hpp
@ -0,0 +1,60 @@
+#pragma once
+
+#include "./types.hpp"
+
+#include <vector>
+#include <string>
+#include <cstdint>
+
+namespace Fragment::Components {
+
+	// TODO: is this special and should this be saved to meta or not (its already in the file name on disk)
+	struct ID {
+		std::vector<uint8_t> v;
+	};
+
+	struct DataEncryptionType {
+		Encryption enc {Encryption::NONE};
+	};
+
+	struct DataCompressionType {
+		Compression comp {Compression::NONE};
+	};
+
+
+	// meta that is not written to (meta-)file
+	namespace Ephemeral {
+
+		// excluded from file meta
+		struct FilePath {
+			// contains store path, if any
+			std::string path;
+		};
+
+		// TODO: seperate into remote and local?
+		// (remote meaning eg. the file on disk was changed by another program)
+		struct DirtyTag {};
+
+
+		// type as comp
+		struct MetaFileType {
+			::MetaFileType type {::MetaFileType::TEXT_JSON};
+		};
+
+		struct MetaEncryptionType {
+			Encryption enc {Encryption::NONE};
+		};
+
+		struct MetaCompressionType {
+			Compression comp {Compression::NONE};
+		};
+
+	} // Ephemeral
+
+} // Components
+
+// shortened to save bytes (until I find a way to save by ID in msgpack)
+namespace FragComp = Fragment::Components;
+
+#include "./meta_components_id.inl"
+
--- a/src/fragment_store/meta_components_id.inl
+++ b/src/fragment_store/meta_components_id.inl
@ -0,0 +1,26 @@
+#pragma once
+
+#include "./meta_components.hpp"
+
+#include <entt/core/type_info.hpp>
+
+// TODO: move more central
+#define DEFINE_COMP_ID(x) \
+template<> \
+constexpr entt::id_type entt::type_hash<x>::value() noexcept { \
+	using namespace entt::literals; \
+	return #x##_hs; \
+} \
+template<> \
+constexpr std::string_view entt::type_name<x>::value() noexcept { \
+	return #x; \
+}
+
+// cross compiler stable ids
+
+DEFINE_COMP_ID(FragComp::DataEncryptionType)
+DEFINE_COMP_ID(FragComp::DataCompressionType)
+
+#undef DEFINE_COMP_ID
+
+
--- a/src/fragment_store/register_mfs_json_message_components.cpp
+++ b/src/fragment_store/register_mfs_json_message_components.cpp
@ -0,0 +1,35 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/message_components.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::Timestamp>();
+	msc.registerDeSerializerJson<Message::Components::Timestamp>();
+	msc.registerSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerDeSerializerJson<Message::Components::TimestampProcessed>();
+	msc.registerSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerDeSerializerJson<Message::Components::TimestampWritten>();
+	msc.registerSerializerJson<Message::Components::ContactFrom>();
+	msc.registerDeSerializerJson<Message::Components::ContactFrom>();
+	msc.registerSerializerJson<Message::Components::ContactTo>();
+	msc.registerDeSerializerJson<Message::Components::ContactTo>();
+	msc.registerSerializerJson<Message::Components::TagUnread>();
+	msc.registerDeSerializerJson<Message::Components::TagUnread>();
+	msc.registerSerializerJson<Message::Components::Read>();
+	msc.registerDeSerializerJson<Message::Components::Read>();
+	msc.registerSerializerJson<Message::Components::MessageText>();
+	msc.registerDeSerializerJson<Message::Components::MessageText>();
+	msc.registerSerializerJson<Message::Components::TagMessageIsAction>();
+	msc.registerDeSerializerJson<Message::Components::TagMessageIsAction>();
+
+	// files
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileID>()
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfo>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::FileInfoLocal>();
+	//_sc.registerSerializerJson<Message::Components::Transfer::TagHaveAll>();
+	//_sc.registerDeSerializerJson<Message::Components::Transfer::TagHaveAll>();
+}
+
--- a/src/fragment_store/register_mfs_json_message_components.hpp
+++ b/src/fragment_store/register_mfs_json_message_components.hpp
@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonMessageComponents(MessageSerializerCallbacks& msc);
+
--- a/src/fragment_store/register_mfs_json_tox_message_components.cpp
+++ b/src/fragment_store/register_mfs_json_tox_message_components.cpp
@ -0,0 +1,10 @@
+#include "./register_mfs_json_message_components.hpp"
+
+#include "./message_serializer.hpp"
+#include "../json/tox_message_components.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc) {
+	msc.registerSerializerJson<Message::Components::ToxGroupMessageID>();
+	msc.registerDeSerializerJson<Message::Components::ToxGroupMessageID>();
+}
+
--- a/src/fragment_store/register_mfs_json_tox_message_components.hpp
+++ b/src/fragment_store/register_mfs_json_tox_message_components.hpp
@ -0,0 +1,6 @@
+#pragma once
+
+#include "./message_serializer.hpp"
+
+void registerMFSJsonToxMessageComponents(MessageSerializerCallbacks& msc);
+
--- a/src/fragment_store/serializer.hpp
+++ b/src/fragment_store/serializer.hpp
@ -0,0 +1,68 @@
+#pragma once
+
+#include <entt/core/type_info.hpp>
+#include <entt/container/dense_map.hpp>
+#include <entt/entity/handle.hpp>
+
+#include <nlohmann/json_fwd.hpp>
+
+template<typename EntityType = entt::entity>
+struct SerializerCallbacks {
+	using Registry = entt::basic_registry<EntityType>;
+	using Handle = entt::basic_handle<Registry>;
+
+	// nlohmann
+	// json/msgpack
+	using serialize_json_fn = bool(*)(const Handle h, nlohmann::json& out);
+	entt::dense_map<entt::id_type, serialize_json_fn> _serl_json;
+
+	using deserialize_json_fn = bool(*)(Handle h, const nlohmann::json& in);
+	entt::dense_map<entt::id_type, deserialize_json_fn> _deserl_json;
+
+	template<typename T>
+	static bool component_get_json(const Handle h, nlohmann::json& j) {
+		if (h.template all_of<T>()) {
+			if constexpr (!std::is_empty_v<T>) {
+				j = h.template get<T>();
+			}
+			return true;
+		}
+
+		return false;
+	}
+
+	template<typename T>
+	static bool component_emplace_or_replace_json(Handle h, const nlohmann::json& j) {
+		if constexpr (std::is_empty_v<T>) {
+			h.template emplace_or_replace<T>(); // assert empty json?
+		} else {
+			h.template emplace_or_replace<T>(static_cast<T>(j));
+		}
+		return true;
+	}
+
+	void registerSerializerJson(serialize_json_fn fn, const entt::type_info& type_info) {
+		_serl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerSerializerJson(
+		serialize_json_fn fn = component_get_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerSerializerJson(fn, type_info);
+	}
+
+	void registerDeSerializerJson(deserialize_json_fn fn, const entt::type_info& type_info) {
+		_deserl_json[type_info.hash()] = fn;
+	}
+
+	template<typename CompType>
+	void registerDeSerializerJson(
+		deserialize_json_fn fn = component_emplace_or_replace_json<CompType>,
+		const entt::type_info& type_info = entt::type_id<CompType>()
+	) {
+		registerDeSerializerJson(fn, type_info);
+	}
+};
+
--- a/src/fragment_store/test_fragstore.cpp
+++ b/src/fragment_store/test_fragstore.cpp
@ -0,0 +1,80 @@
+#include <cstdint>
+#include <iostream>
+
+#include "./fragment_store.hpp"
+
+#include <nlohmann/json.hpp>
+#include <entt/entity/handle.hpp>
+
+namespace Components {
+	struct MessagesTimestampRange {
+		uint64_t begin {0};
+		uint64_t end {1000};
+	};
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessagesTimestampRange, begin, end)
+} // Components
+
+
+int main(void) {
+	FragmentStore fs;
+	fs._default_store_path = "test_store/";
+	fs._sc.registerSerializerJson<Components::MessagesTimestampRange>();
+	fs._sc.registerDeSerializerJson<Components::MessagesTimestampRange>();
+
+	const auto frag0 = fs.newFragmentFile("", MetaFileType::TEXT_JSON, {0xff, 0xf1, 0xf2, 0xf0, 0xff, 0xff, 0xff, 0xf9});
+
+	const auto frag1 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	const auto frag2 = fs.newFragmentFile("", MetaFileType::BINARY_MSGPACK);
+
+	{
+		auto frag0h = fs.fragmentHandle(frag0);
+
+		frag0h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag0h.emplace_or_replace<FragComp::DataEncryptionType>();
+		frag0h.emplace_or_replace<Components::MessagesTimestampRange>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			uint64_t i = 0;
+			for (; i+read < 3000 && i < buffer_size; i++) {
+				request_buffer[i] = uint8_t((i+read) & 0xff);
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag0, fn_cb);
+	}
+
+	{
+		auto frag1h = fs.fragmentHandle(frag1);
+
+		frag1h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag1h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		std::function<FragmentStore::write_to_storage_fetch_data_cb> fn_cb = [read = 0ul](uint8_t* request_buffer, uint64_t buffer_size) mutable -> uint64_t {
+			static constexpr std::string_view text = "This is some random data";
+			uint64_t i = 0;
+			for (; i+read < text.size() && i < buffer_size; i++) {
+				request_buffer[i] = text[i+read];
+			}
+			read += i;
+
+			return i;
+		};
+		fs.syncToStorage(frag1, fn_cb);
+	}
+
+	{
+		auto frag2h = fs.fragmentHandle(frag2);
+
+		frag2h.emplace_or_replace<FragComp::DataCompressionType>();
+		frag2h.emplace_or_replace<FragComp::DataEncryptionType>();
+
+		static constexpr std::string_view text = "This is more random data";
+		fs.syncToStorage(frag2, reinterpret_cast<const uint8_t*>(text.data()), text.size());
+	}
+	return 0;
+}
+
--- a/src/fragment_store/types.hpp
+++ b/src/fragment_store/types.hpp
@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+
+enum class Encryption : uint8_t {
+	NONE = 0x00,
+};
+enum class Compression : uint8_t {
+	NONE = 0x00,
+	ZSTD = 0x01,
+	// TODO: zstd without magic
+	// TODO: zstd meta dict
+	// TODO: zstd data(message) dict
+};
+enum class MetaFileType : uint8_t {
+	TEXT_JSON,
+	BINARY_MSGPACK, // msgpacked msgpack
+};
+
--- a/src/json/message_components.hpp
+++ b/src/json/message_components.hpp
@ -0,0 +1,27 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/message3/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Timestamp, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampProcessed, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(TimestampWritten, ts)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactFrom, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ContactTo, c)
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Read, ts)
+	// TODO: SyncedBy
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MessageText, text)
+
+	namespace Transfer {
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo::FileDirEntry, file_name, file_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfo, file_list, total_size)
+		NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(FileInfoLocal, file_list)
+	} // Transfer
+
+} // Message::Components
+
--- a/src/json/tox_message_components.hpp
+++ b/src/json/tox_message_components.hpp
@ -0,0 +1,16 @@
+#pragma once
+
+#include <solanaceae/util/utils.hpp>
+
+#include <solanaceae/tox_messages/components.hpp>
+
+#include <nlohmann/json.hpp>
+
+namespace Message::Components {
+
+	// TODO: friend msg id, does not have the same qualities
+	NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(ToxGroupMessageID, id)
+	// TODO: transfer stuff, needs content rewrite
+
+} // Message::Components
+
--- a/src/main_screen.cpp
+++ b/src/main_screen.cpp
@ -1,5 +1,8 @@
 #include "./main_screen.hpp"

+#include "./fragment_store/register_mfs_json_message_components.hpp"
+#include "./fragment_store/register_mfs_json_tox_message_components.hpp"
+
 #include <solanaceae/contact/components.hpp>

 #include <imgui/imgui.h>
@ -13,6 +16,7 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	renderer(renderer_),
 	rmm(cr),
 	mts(rmm),
+	mfs(cr, rmm, fs),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
 	ad(tc),
@ -33,6 +37,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	tdch(tpi)
 {
 	tel.subscribeAll(tc);
+	registerMFSJsonMessageComponents(mfs.getMSC());
+	registerMFSJsonToxMessageComponents(mfs.getMSC());

 	conf.set("tox", "save_file_path", save_path);

@ -49,6 +55,10 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";

 	{ // setup plugin instances
+		// TODO: make interface useful
+		g_provideInstance<FragmentStoreI>("FragmentStoreI", "host", &fs);
+		g_provideInstance<FragmentStore>("FragmentStore", "host", &fs);
+
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
@ -74,6 +84,8 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	}

 	conf.dump();
+
+	mfs.triggerScan(); // HACK: after plugins and tox contacts got loaded
 }

 MainScreen::~MainScreen(void) {
@ -388,7 +400,8 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {

 	tdch.tick(time_delta); // compute

-	mts.iterate(); // compute
+	const float mfs_interval = mfs.tick(time_delta);
+	mts.iterate(); // compute (after mfs)

 	_min_tick_interval = std::min<float>(
 		// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
@ -400,6 +413,10 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 		_min_tick_interval,
 		fo_interval
 	);
+	_min_tick_interval = std::min<float>(
+		_min_tick_interval,
+		mfs_interval
+	);

 	//std::cout << "MS: min tick interval: " << _min_tick_interval << "\n";

--- a/src/main_screen.hpp
+++ b/src/main_screen.hpp
@ -2,10 +2,12 @@

 #include "./screen.hpp"

+#include "./fragment_store/fragment_store.hpp"
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
+#include "./fragment_store/message_fragment_store.hpp"
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@ -43,10 +45,13 @@ extern "C" {
 struct MainScreen final : public Screen {
 	SDL_Renderer* renderer;

+	FragmentStore fs;
+
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
 	MessageTimeSort mts;
+	MessageFragmentStore mfs;

 	PluginManager pm;

--- a/src/sdlrenderer_texture_uploader.cpp
+++ b/src/sdlrenderer_texture_uploader.cpp
@ -1,13 +1,17 @@
 #include "./sdlrenderer_texture_uploader.hpp"

 #include <cassert>
+#include <cstdint>
+#include <cstring>
+
+#include <iostream>

 SDLRendererTextureUploader::SDLRendererTextureUploader(SDL_Renderer* renderer_) :
 	renderer(renderer_)
 {
 }

-uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter) {
+uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) {
 	// TODO: test if pitch is 4 or 4*width
 	SDL_Surface* surf = SDL_CreateSurfaceFrom(
 		(void*)data,
@ -17,8 +21,15 @@ uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t wi
 	);
 	assert(surf); // TODO: add error reporting

-	SDL_Texture* tex = SDL_CreateTextureFromSurface(renderer, surf);
+	SDL_Texture* tex = SDL_CreateTexture(
+		renderer,
+		surf->format->format,
+		access == Access::STREAMING ? SDL_TEXTUREACCESS_STREAMING : SDL_TEXTUREACCESS_STATIC,
+		surf->w, surf->h
+	);
 	assert(tex); // TODO: add error reporting
+	// TODO: error reporting
+	SDL_UpdateTexture(tex, nullptr, surf->pixels, surf->pitch);

 	if (filter == NEAREST) {
 		SDL_SetTextureScaleMode(tex, SDL_SCALEMODE_NEAREST);
@ -31,6 +42,27 @@ uint64_t SDLRendererTextureUploader::uploadRGBA(const uint8_t* data, uint32_t wi
 	return reinterpret_cast<uint64_t>(tex);
 }

+bool SDLRendererTextureUploader::updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) {
+	auto* texture = static_cast<SDL_Texture*>(reinterpret_cast<void*>(tex_id));
+	if (texture == nullptr) {
+		return false;
+	}
+
+	uint8_t* pixels = nullptr;
+	int pitch = 0;
+
+	if (SDL_LockTexture(texture, nullptr, (void**)&pixels, &pitch) != 0) {
+		std::cerr << "SDLRTU error: failed locking texture '" << SDL_GetError() << "'\n";
+		return false;
+	}
+
+	std::memcpy(pixels, data, size);
+
+	SDL_UnlockTexture(texture);
+
+	return true;
+}
+
 void SDLRendererTextureUploader::destroy(uint64_t tex_id) {
 	SDL_DestroyTexture(static_cast<SDL_Texture*>(reinterpret_cast<void*>(tex_id)));
 }
--- a/src/sdlrenderer_texture_uploader.hpp
+++ b/src/sdlrenderer_texture_uploader.hpp
@ -10,7 +10,8 @@ struct SDLRendererTextureUploader : public TextureUploaderI {
 	SDLRendererTextureUploader(SDL_Renderer* renderer_);
 	~SDLRendererTextureUploader(void) = default;

-	uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter) override;
+	uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter, Access access) override;
+	bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) override;
 	void destroy(uint64_t tex_id) override;
 };

--- a/src/texture_uploader.hpp
+++ b/src/texture_uploader.hpp
@ -1,18 +1,29 @@
 #pragma once

 #include <cstdint>
+#include <cstddef>

 struct TextureUploaderI {
-	static constexpr const char* version {"1"};
+	static constexpr const char* version {"2"};

 	enum Filter {
 		NEAREST,
 		LINEAR,
 	};

+	enum Access {
+		STATIC,
+		STREAMING,
+		// target?
+	};
+
 	virtual ~TextureUploaderI(void) {}

-	virtual uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter = LINEAR) = 0;
+	virtual uint64_t uploadRGBA(const uint8_t* data, uint32_t width, uint32_t height, Filter filter = LINEAR, Access access = STATIC) = 0;
+
+	// keeps width height filter
+	// TODO: wh instead of size?
+	virtual bool updateRGBA(uint64_t tex_id, const uint8_t* data, size_t size) = 0;

 	virtual void destroy(uint64_t tex_id) = 0;
 };
--- a/src/tox_friend_faux_offline_messaging.cpp
+++ b/src/tox_friend_faux_offline_messaging.cpp
@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
 		// require
 		if (!mr->all_of<
 				Message::Components::MessageText, // text only for now
-				Message::Components::ContactTo
+				Message::Components::ContactTo,
+				Message::Components::ToxFriendMessageID // yes, needs fake ids
 			>(msg)
 		) {
 			continue; // skip
--- a/src/tox_private_impl.hpp
+++ b/src/tox_private_impl.hpp
@ -16,4 +16,20 @@ struct ToxPrivateImpl : public ToxPrivateI {
 	uint16_t toxDHTGetNumCloselistAnnounceCapable(void) override {
 		return tox_dht_get_num_closelist_announce_capable(_tox);
 	}
+
+	std::tuple<std::optional<std::string>, Tox_Err_Group_Peer_Query> toxGroupPeerGetIPAddress(uint32_t group_number, uint32_t peer_id) override {
+		Tox_Err_Group_Peer_Query err = TOX_ERR_GROUP_PEER_QUERY_OK;
+		size_t str_size = tox_group_peer_get_ip_address_size(_tox, group_number, peer_id, &err);
+		if (err != TOX_ERR_GROUP_PEER_QUERY_OK) {
+			return {std::nullopt, err};
+		}
+		std::string ip_str(str_size, '\0');
+
+		tox_group_peer_get_ip_address(_tox, group_number, peer_id, reinterpret_cast<uint8_t*>(ip_str.data()), &err);
+		if (err == TOX_ERR_GROUP_PEER_QUERY_OK) {
+			return {ip_str, err};
+		} else {
+			return {std::nullopt, err};
+		}
+	}
 };
Author	SHA1	Message	Date
Green Sky	69b3e6d823	combat memory leaks with smart pointers	2024-03-11 12:05:30 +01:00
Green Sky	a211fa27e3	refactor saving and save on exit	2024-03-11 12:05:30 +01:00
Green Sky	98ad43957b	dirty frag on message updates (if still open)	2024-03-11 12:05:30 +01:00
Green Sky	367ee53e07	refactor message fuid -> fid save alot of memory by using fid instead of fuid	2024-03-11 12:05:30 +01:00
Green Sky	f0d320133e	reduce log spam	2024-03-11 12:05:30 +01:00
Green Sky	42a41176e9	remove old code	2024-03-11 12:05:30 +01:00
Green Sky	f9ff02fd3d	forgot to throw update on read	2024-03-11 12:05:29 +01:00
Green Sky	aa793d8e7f	dont sync messages we dont know enough about	2024-03-11 12:05:29 +01:00
Green Sky	ee0cbd7697	make adjacency loading work, extend range and use loops	2024-03-11 12:05:29 +01:00
Green Sky	eb1b4c5d2e	replace old bad prev/next code with way better code	2024-03-11 12:05:29 +01:00
Green Sky	50ec1b5dbf	smaller contact frag fixes	2024-03-11 12:05:29 +01:00
Green Sky	7c0d7b1635	impl new acceleration structure for components, not exploited yet disable funky load at first msg	2024-03-11 12:05:29 +01:00
Green Sky	d98c26882c	forgot to check contact	2024-03-11 12:05:28 +01:00
Green Sky	6dc678a5fa	rework cursers for cg, keep views between switching. will be refactored later	2024-03-11 12:05:28 +01:00
Green Sky	19e45787d0	fix one inverted comparator	2024-03-11 12:05:28 +01:00
Green Sky	acd6ec499a	stop ignoring mfs interval and sort after	2024-03-11 12:05:28 +01:00
Green Sky	363cbe5712	make inital curser a range	2024-03-11 12:05:28 +01:00
Green Sky	974ad0f06e	loading logic implemented but broken (very funky and sometimes even out of contact)	2024-03-11 12:05:28 +01:00
Green Sky	69d0704945	load based on view cursers (untested and not used yet)	2024-03-11 12:05:27 +01:00
Green Sky	480373feee	msg frag before and after helper	2024-03-11 12:05:27 +01:00
Green Sky	d04f4678f8	fix potential tsrange errors and deduplicate state	2024-03-11 12:05:27 +01:00
Green Sky	003070de15	make writing safe (by using a tmp file and moving to actual location)	2024-03-11 12:05:27 +01:00
Green Sky	9d1f8fb9bc	make empty contacts from ids on message load	2024-03-11 12:05:27 +01:00
Green Sky	be79e893e3	change binary meta format and add zstd to metadata	2024-03-11 12:05:27 +01:00
Green Sky	3c6d5b1cf0	switch to streaming compressor for data to drastically improve ratio. would still benefit from a abstract file refactor	2024-03-11 12:05:26 +01:00
Green Sky	b89f3a6779	update fs readme a little	2024-03-11 12:05:26 +01:00
Green Sky	94c1f6ebc2	save msg json zstd compressed (3x compression)	2024-03-11 12:05:26 +01:00
Green Sky	3b034954a7	simplify array cast a little	2024-03-11 12:05:26 +01:00
Green Sky	45e7d42a3f	add zstd dep	2024-03-11 12:05:26 +01:00
Green Sky	52819cab24	comp refactor and make groups work	2024-03-11 12:05:25 +01:00
Green Sky	d821890bde	move json around and disable files for now	2024-03-11 12:05:25 +01:00
Green Sky	d93abaf5b9	reverse message write order	2024-03-11 12:05:25 +01:00
Green Sky	3139b7bbd9	add dup check, would work for ngc if we saved tox group msg id yet	2024-03-11 12:05:25 +01:00
Green Sky	f7114668f4	fix dup on write	2024-03-11 12:05:25 +01:00
Green Sky	791829daf0	basically working, but some dup glitch is still there	2024-03-11 12:05:25 +01:00
Green Sky	7cac05d0ff	scan laters	2024-03-11 12:05:24 +01:00
Green Sky	c715937f40	fragment events + 256bit uuids	2024-03-11 12:05:24 +01:00
Green Sky	e0a98626ea	refactor message serializer to allow access to eg contacts	2024-03-11 12:05:24 +01:00
Green Sky	78ba68c824	further serializer refactoring	2024-03-11 12:05:24 +01:00
Green Sky	172744b665	improve deserialization and provide message comp deserl	2024-03-11 12:05:24 +01:00
Green Sky	de44577d5d	loading fragments mostly working (not notifying anyone yet)	2024-03-11 12:05:24 +01:00
Green Sky	b36b842bdf	add contact id to meta	2024-03-11 12:05:23 +01:00
Green Sky	924a857ed3	more comps	2024-03-11 12:05:23 +01:00
Green Sky	b523696dce	handle empty type	2024-03-11 12:05:23 +01:00
Green Sky	7d6af9d434	dump messages to data (some comps)	2024-03-11 12:05:23 +01:00
Green Sky	0662b2c63e	message fragment meta is saved, but still empty data	2024-03-11 12:05:23 +01:00
Green Sky	81ee11133c	start with messages (no fragments get created yet)	2024-03-11 12:05:23 +01:00
Green Sky	f9077e353d	refactoring, add to mainscreen	2024-03-11 12:05:22 +01:00
Green Sky	30f4053679	random ids	2024-03-11 12:05:22 +01:00
Green Sky	048de9c040	working prototpying code	2024-03-11 12:05:22 +01:00
Green Sky	3cf3097094	toxcore update + private interface update	2024-03-11 11:46:18 +01:00
Green Sky	e801626232	Merge commit 'a5093c4aa36d66049b6b5ca94e3b17412e77391b'	2024-03-11 11:34:13 +01:00
Green Sky	a5093c4aa3	Squashed 'external/toxcore/c-toxcore/' changes from b03b571272..3e05824b80 3e05824b80 refactor: Rename `out` parameters to `out_$something`. 0199c0f17f cleanup: apply the same scheme to types aebbfabe26 cleanup: event length naming inconsistencies 2457125aa8 cleanup: align group send err enum order git-subtree-dir: external/toxcore/c-toxcore git-subtree-split: 3e05824b80eb9bee33e8254cba0780d84c522182	2024-03-11 11:34:13 +01:00
Green Sky	c311bb5c95	update tox interface	2024-03-10 20:50:13 +01:00
Green Sky	887705969f	make texture streaming actually work	2024-03-09 16:26:35 +01:00
Green Sky	316871523d	update nixos target	2024-03-09 11:50:55 +01:00
Green Sky	a3d193516c	add updating to textures	2024-03-08 22:04:58 +01:00