Merge pull request #8 from Green-Sky/fragment_store

ObjectStore backed MessageFragments
Merge branch 'master' into fragment_store
2024-04-14 14:38:49 +02:00 · 2024-04-14 14:32:31 +02:00 · 2024-04-14 14:30:42 +02:00 · 2024-04-14 14:16:53 +02:00 · 2024-04-14 13:58:31 +02:00 · 2024-04-14 12:03:58 +02:00
46 changed files with 1386 additions and 745 deletions
--- a/.github/workflows/cd.yml
+++ b/.github/workflows/cd.yml
@@ -37,10 +37,10 @@ jobs:
      run: |
        SHORT_HASH="$(git rev-parse --short=7 HEAD)"
        if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
        else
          SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
        fi

    - name: Compress artifacts
@@ -88,10 +88,10 @@ jobs:
      run: |
        SHORT_HASH="$(git rev-parse --short=7 HEAD)"
        if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
        else
          SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
        fi

    - name: Compress artifacts
@@ -139,10 +139,10 @@ jobs:
      run: |
        SHORT_HASH="$(git rev-parse --short=7 HEAD)"
        if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
        else
          SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
        fi

    - name: Compress artifacts
@@ -183,10 +183,10 @@ jobs:
      run: |
        SHORT_HASH="$(git rev-parse --short=7 HEAD)"
        if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
        else
          SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
        fi

    - name: Download artifacts
@@ -203,8 +203,8 @@ jobs:
      run: |
        gh release create "$tag" \
            --repo="$GITHUB_REPOSITORY" \
-            --title="nightly ${tag#v}" \
-            --notes="nightly build" \
+            --title="${tag#v}" \
+            --notes="preview build of the latest commit" \
            --prerelease

    - name: Upload artifacts
--- a/.gitmodules
+++ b/.gitmodules
@@ -20,3 +20,9 @@
 [submodule "external/solanaceae_plugin"]
 	path = external/solanaceae_plugin
 	url = https://github.com/Green-Sky/solanaceae_plugin.git
+[submodule "external/solanaceae_object_store"]
+	path = external/solanaceae_object_store
+	url = https://github.com/Green-Sky/solanaceae_object_store.git
+[submodule "external/solanaceae_message_serializer"]
+	path = external/solanaceae_message_serializer
+	url = https://github.com/Green-Sky/solanaceae_message_serializer.git
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			link_libraries(-fsanitize=address)
-			#link_libraries(-fsanitize=address,undefined)
+			#link_libraries(-fsanitize=address)
+			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			message("II enabled ASAN")
 		else()
--- a/external/CMakeLists.txt
+++ b/external/CMakeLists.txt
@@ -1,10 +1,11 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)

 add_subdirectory(./entt)

 add_subdirectory(./solanaceae_util)
 add_subdirectory(./solanaceae_contact)
 add_subdirectory(./solanaceae_message3)
+add_subdirectory(./solanaceae_message_serializer)

 add_subdirectory(./solanaceae_plugin)

@@ -12,6 +13,8 @@ add_subdirectory(./toxcore)
 add_subdirectory(./solanaceae_toxcore)
 add_subdirectory(./solanaceae_tox)

+add_subdirectory(./solanaceae_object_store)
+
 add_subdirectory(./sdl)
 add_subdirectory(./imgui)

--- a/external/solanaceae_message3
+++ b/external/solanaceae_message3
--- a/external/solanaceae_message_serializer
+++ b/external/solanaceae_message_serializer
--- a/external/solanaceae_object_store
+++ b/external/solanaceae_object_store
--- a/external/solanaceae_tox
+++ b/external/solanaceae_tox
--- a/external/solanaceae_util
+++ b/external/solanaceae_util
--- a/external/toxcore/c-toxcore/.cirrus.yml
+++ b/external/toxcore/c-toxcore/.cirrus.yml
@@ -7,13 +7,14 @@ bazel-opt_task:
  configure_script:
    - git submodule update --init --recursive
    - /src/workspace/tools/inject-repo c-toxcore
+    - sed -i -e 's/build --config=remote/#&/' /src/workspace/.bazelrc.local
  test_all_script:
    - cd /src/workspace && bazel test -k
        --build_tag_filters=-haskell
        --test_tag_filters=-haskell
        --
        //c-toxcore/...
-        -//c-toxcore/auto_tests:tcp_relay_test # TODO(robinlinden): Why does this pass locally but not in Cirrus?
+        -//c-toxcore/auto_tests:tcp_relay_test # Cirrus doesn't allow external network connections.

 bazel-dbg_task:
  container:
@@ -30,7 +31,7 @@ bazel-dbg_task:
        --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST
        --
        //c-toxcore/...
-        -//c-toxcore/auto_tests:tcp_relay_test # TODO(robinlinden): Why does this pass locally but not in Cirrus?
+        -//c-toxcore/auto_tests:tcp_relay_test # Cirrus doesn't allow external network connections.

 cimple_task:
  container:
@@ -40,6 +41,7 @@ cimple_task:
  configure_script:
    - git submodule update --init --recursive
    - /src/workspace/tools/inject-repo c-toxcore
+    - sed -i -e 's/build --config=remote/#&/' /src/workspace/.bazelrc.local
  test_all_script:
    - cd /src/workspace && bazel test -k
        --build_tag_filters=haskell
@@ -69,7 +71,7 @@ freebsd_task:
      cmake . \
        -DMIN_LOGGER_LEVEL=TRACE \
        -DMUST_BUILD_TOXAV=ON \
-        -DNON_HERMETIC_TESTS=ON \
+        -DNON_HERMETIC_TESTS=OFF \
        -DTEST_TIMEOUT_SECONDS=50 \
        -DUSE_IPV6=OFF \
        -DAUTOTEST=ON
--- a/external/toxcore/c-toxcore/CHANGELOG.md
+++ b/external/toxcore/c-toxcore/CHANGELOG.md
@@ -1,9 +1,274 @@


+## v0.2.19
+
+### Merged PRs:
+
+- [#2744](https://github.com/TokTok/c-toxcore/pull/2744) docs: Document that group topic lock is default on.
+- [#2743](https://github.com/TokTok/c-toxcore/pull/2743) docs: Add missing param docs for callbacks.
+- [#2742](https://github.com/TokTok/c-toxcore/pull/2742) chore: Add cmake flag to disable unit tests.
+- [#2741](https://github.com/TokTok/c-toxcore/pull/2741) refactor: Don't expose Tox_System in the public API
+- [#2736](https://github.com/TokTok/c-toxcore/pull/2736) cleanup: A more descriptive error for group invite accept function
+- [#2735](https://github.com/TokTok/c-toxcore/pull/2735) chore: Small API doc fixes
+- [#2732](https://github.com/TokTok/c-toxcore/pull/2732) cleanup: event length naming inconsistencies
+- [#2731](https://github.com/TokTok/c-toxcore/pull/2731) cleanup: align group send err enum order
+- [#2729](https://github.com/TokTok/c-toxcore/pull/2729) cleanup: use typedef for private message ID's in callback
+- [#2728](https://github.com/TokTok/c-toxcore/pull/2728) refactor: Observers/ignored peers can now send and receive custom packets
+- [#2727](https://github.com/TokTok/c-toxcore/pull/2727) feat: add message IDs to private group messages
+- [#2726](https://github.com/TokTok/c-toxcore/pull/2726) refactor: Rename `out` parameters to `out_$something`.
+- [#2718](https://github.com/TokTok/c-toxcore/pull/2718) chore: Use a specific non-broken slimcc version.
+- [#2713](https://github.com/TokTok/c-toxcore/pull/2713) feat: Update and improve the Windows cross-compilation
+- [#2712](https://github.com/TokTok/c-toxcore/pull/2712) chore: Update github actions.
+- [#2710](https://github.com/TokTok/c-toxcore/pull/2710) docs: Update the list of CMake options
+- [#2709](https://github.com/TokTok/c-toxcore/pull/2709) refactor: Remove "mod" and "founder" from group API naming scheme
+- [#2708](https://github.com/TokTok/c-toxcore/pull/2708) docs: add the experimental api build option to INSTALL.md
+- [#2705](https://github.com/TokTok/c-toxcore/pull/2705) refactor: Rename Queries to Query to align with other enums.
+- [#2704](https://github.com/TokTok/c-toxcore/pull/2704) fix: Correct type for conference offline peer numbers.
+- [#2700](https://github.com/TokTok/c-toxcore/pull/2700) test: Add FreeBSD VM action on GitHub.
+- [#2699](https://github.com/TokTok/c-toxcore/pull/2699) test: Add pkgsrc build.
+- [#2698](https://github.com/TokTok/c-toxcore/pull/2698) chore: Only install tox_private.h on request.
+- [#2697](https://github.com/TokTok/c-toxcore/pull/2697) test: Build toxcore on NetBSD (VM).
+- [#2696](https://github.com/TokTok/c-toxcore/pull/2696) fix: save_compatibility_test failing on big-endian systems
+- [#2695](https://github.com/TokTok/c-toxcore/pull/2695) fix: Don't serve files from websockify.
+- [#2689](https://github.com/TokTok/c-toxcore/pull/2689) fix: Correctly pass extended public keys to group moderation code.
+- [#2686](https://github.com/TokTok/c-toxcore/pull/2686) chore: Compile libsodium reference implementation with compcert.
+- [#2685](https://github.com/TokTok/c-toxcore/pull/2685) cleanup: correct a few nullable annotations
+- [#2684](https://github.com/TokTok/c-toxcore/pull/2684) cleanup: Don't use `memcpy` to cast arbitrary `struct`s to `uint8_t[]`.
+- [#2683](https://github.com/TokTok/c-toxcore/pull/2683) fix: Pass array, not array pointer, to `memcmp`.
+- [#2682](https://github.com/TokTok/c-toxcore/pull/2682) cleanup: Never pass `void*` directly to `memcpy`.
+- [#2678](https://github.com/TokTok/c-toxcore/pull/2678) chore: Disable NGC saving by default, enable through Tox_Options.
+- [#2675](https://github.com/TokTok/c-toxcore/pull/2675) cleanup: Replace pointer arithmetic with explicit `&arr[i]`.
+- [#2672](https://github.com/TokTok/c-toxcore/pull/2672) refactor: Use `structs` for extended public/secret keys.
+- [#2671](https://github.com/TokTok/c-toxcore/pull/2671) refactor: Use tox rng to seed the keypair generation.
+- [#2666](https://github.com/TokTok/c-toxcore/pull/2666) cleanup: Small improvements found by PVS Studio.
+- [#2664](https://github.com/TokTok/c-toxcore/pull/2664) docs: Run prettier-markdown on markdown files.
+- [#2662](https://github.com/TokTok/c-toxcore/pull/2662) fix: Correct a few potential null derefs in bootstrap daemon.
+- [#2661](https://github.com/TokTok/c-toxcore/pull/2661) fix: Zero out stack-allocated secret key before return.
+- [#2660](https://github.com/TokTok/c-toxcore/pull/2660) fix: Add missing memunlock of local variable when it goes out of scope.
+- [#2659](https://github.com/TokTok/c-toxcore/pull/2659) cleanup: Simplify custom packet length check in NGC.
+- [#2658](https://github.com/TokTok/c-toxcore/pull/2658) refactor: Make prune_gc_sanctions_list more obviously correct.
+- [#2657](https://github.com/TokTok/c-toxcore/pull/2657) fix: Fix some false positive from PVS Studio.
+- [#2656](https://github.com/TokTok/c-toxcore/pull/2656) docs: Add static analysis tool list to README.
+- [#2655](https://github.com/TokTok/c-toxcore/pull/2655) cleanup: Check that WINXP macro exists before comparing it.
+- [#2652](https://github.com/TokTok/c-toxcore/pull/2652) refactor: Make tox mutex non-recursive.
+- [#2648](https://github.com/TokTok/c-toxcore/pull/2648) docs: Add more documentation to crypto_core.
+- [#2647](https://github.com/TokTok/c-toxcore/pull/2647) docs: Fix up doxyfile.
+- [#2645](https://github.com/TokTok/c-toxcore/pull/2645) refactor: Remove `Tox *` from `tox_dispatch`.
+- [#2644](https://github.com/TokTok/c-toxcore/pull/2644) refactor: Don't rely on tox_dispatch passing tox in tests.
+- [#2643](https://github.com/TokTok/c-toxcore/pull/2643) refactor: Use strong typedef for NGC peer id.
+- [#2642](https://github.com/TokTok/c-toxcore/pull/2642) chore: Use C++ mode for clang-tidy.
+- [#2640](https://github.com/TokTok/c-toxcore/pull/2640) refactor: Use strong `typedef` instead of `struct` for `Socket`.
+- [#2637](https://github.com/TokTok/c-toxcore/pull/2637) chore: Check that both gtest and gmock exist for tests.
+- [#2634](https://github.com/TokTok/c-toxcore/pull/2634) chore: Add some comments to the astyle config.
+- [#2629](https://github.com/TokTok/c-toxcore/pull/2629) chore: Reformat sources with astyle.
+- [#2626](https://github.com/TokTok/c-toxcore/pull/2626) chore: Rename C++ headers to .hh suffixes.
+- [#2624](https://github.com/TokTok/c-toxcore/pull/2624) test: Add slimcc compiler compatibility test.
+- [#2622](https://github.com/TokTok/c-toxcore/pull/2622) cleanup: Add more `const` where possible.
+- [#2621](https://github.com/TokTok/c-toxcore/pull/2621) cleanup: Remove implicit bool conversions.
+- [#2620](https://github.com/TokTok/c-toxcore/pull/2620) chore: Only check the bootstrap daemon checksum on release.
+- [#2617](https://github.com/TokTok/c-toxcore/pull/2617) cleanup: Further `#include` cleanups.
+- [#2614](https://github.com/TokTok/c-toxcore/pull/2614) cleanup: Use Bazel modules to enforce proper `#include` hygiene.
+- [#2612](https://github.com/TokTok/c-toxcore/pull/2612) refactor: Move pack/unpack `IP_Port` from DHT into network module.
+- [#2611](https://github.com/TokTok/c-toxcore/pull/2611) chore: Really fix coverage docker image build.
+- [#2610](https://github.com/TokTok/c-toxcore/pull/2610) chore: Fix post-submit coverage image.
+- [#2609](https://github.com/TokTok/c-toxcore/pull/2609) fix: partially fix a bug that prevented group part messages from sending
+- [#2605](https://github.com/TokTok/c-toxcore/pull/2605) fix: Don't use `memcmp` to compare `IP_Port`s.
+- [#2604](https://github.com/TokTok/c-toxcore/pull/2604) chore: Fix rpm build; add a CI check for it.
+- [#2603](https://github.com/TokTok/c-toxcore/pull/2603) chore: Speed up docker builds a bit by reducing layer count.
+- [#2602](https://github.com/TokTok/c-toxcore/pull/2602) cleanup: Add `const` where possible in auto tests.
+- [#2601](https://github.com/TokTok/c-toxcore/pull/2601) fix: a few off by one errors in group autotests
+- [#2598](https://github.com/TokTok/c-toxcore/pull/2598) refactor: Rename `system_{memory,...}` to `os_{memory,...}`.
+- [#2597](https://github.com/TokTok/c-toxcore/pull/2597) test: Add goblint static analyser.
+- [#2594](https://github.com/TokTok/c-toxcore/pull/2594) cleanup: Use `memzero(x, s)` instead of `memset(x, 0, s)`.
+- [#2593](https://github.com/TokTok/c-toxcore/pull/2593) cleanup: Use explicit 0 instead of `PACKET_ID_PADDING`.
+- [#2592](https://github.com/TokTok/c-toxcore/pull/2592) cleanup: Remove all uses of `SIZEOF_VLA`.
+- [#2591](https://github.com/TokTok/c-toxcore/pull/2591) cleanup: Expand the `Tox_Options` accessor macros.
+- [#2590](https://github.com/TokTok/c-toxcore/pull/2590) test: Add a simple new/delete test for Tox.
+- [#2588](https://github.com/TokTok/c-toxcore/pull/2588) cleanup: Remove plan9 support.
+- [#2587](https://github.com/TokTok/c-toxcore/pull/2587) cleanup: Add comment after every `#endif`.
+- [#2583](https://github.com/TokTok/c-toxcore/pull/2583) test: Fix comment I broke in the events test PR.
+- [#2582](https://github.com/TokTok/c-toxcore/pull/2582) cleanup: Rename group to conference in groupav documentation.
+- [#2581](https://github.com/TokTok/c-toxcore/pull/2581) cleanup: Ensure handler params are named after callback params.
+- [#2580](https://github.com/TokTok/c-toxcore/pull/2580) cleanup: Minor cleanup of event unpack code.
+- [#2578](https://github.com/TokTok/c-toxcore/pull/2578) refactor: Allow NULL pointers for byte arrays in events.
+- [#2577](https://github.com/TokTok/c-toxcore/pull/2577) refactor: Add common msgpack array packer with callback.
+- [#2576](https://github.com/TokTok/c-toxcore/pull/2576) cleanup: make some improvements to group moderation test
+- [#2575](https://github.com/TokTok/c-toxcore/pull/2575) refactor: Pass `this` pointer as first param to s11n callbacks.
+- [#2573](https://github.com/TokTok/c-toxcore/pull/2573) cleanup: skip a do_gc iteration before removing peers marked for deletion
+- [#2572](https://github.com/TokTok/c-toxcore/pull/2572) cleanup: Remove `bin_pack_{new,free}`.
+- [#2568](https://github.com/TokTok/c-toxcore/pull/2568) feat: Add dht_get_nodes_response event to the events system.
+- [#2567](https://github.com/TokTok/c-toxcore/pull/2567) refactor: Use enum-specific pack functions for enum values.
+- [#2566](https://github.com/TokTok/c-toxcore/pull/2566) cleanup: Remove empty test doing nothing.
+- [#2565](https://github.com/TokTok/c-toxcore/pull/2565) refactor: Factor out union pack switch from event packer.
+- [#2564](https://github.com/TokTok/c-toxcore/pull/2564) cleanup: Move the 2-element array pack out of individual events.
+- [#2563](https://github.com/TokTok/c-toxcore/pull/2563) test: Add printf log statement to group_moderation_test.
+- [#2562](https://github.com/TokTok/c-toxcore/pull/2562) chore: add clangd files to .gitignore
+- [#2561](https://github.com/TokTok/c-toxcore/pull/2561) refactor: Move file streaming test to its own file.
+- [#2560](https://github.com/TokTok/c-toxcore/pull/2560) fix(ci): window builds now build in parallel
+- [#2559](https://github.com/TokTok/c-toxcore/pull/2559) refactor: Migrate auto_tests to new events API.
+- [#2557](https://github.com/TokTok/c-toxcore/pull/2557) test: Add C++ classes wrapping system interfaces.
+- [#2555](https://github.com/TokTok/c-toxcore/pull/2555) fix: Remove fatal error for non-erroneous case
+- [#2554](https://github.com/TokTok/c-toxcore/pull/2554) fix: Make all the fuzzers work again, and add a test for protodump.
+- [#2552](https://github.com/TokTok/c-toxcore/pull/2552) fix: Make sure there's enough space for CONSUME1 in fuzzers.
+- [#2551](https://github.com/TokTok/c-toxcore/pull/2551) chore: Move from gcov to llvm source-based coverage.
+- [#2549](https://github.com/TokTok/c-toxcore/pull/2549) fix: issues with packet broadcast error reporting
+- [#2547](https://github.com/TokTok/c-toxcore/pull/2547) test: Add fuzz tests to the coverage run.
+- [#2545](https://github.com/TokTok/c-toxcore/pull/2545) refactor: Use `operator==` for equality tests of `Node_format`.
+- [#2543](https://github.com/TokTok/c-toxcore/pull/2543) refactor(test): Slightly nicer C++ interface to tox Random.
+- [#2542](https://github.com/TokTok/c-toxcore/pull/2542) refactor: packet broadcast functions now return errors
+- [#2541](https://github.com/TokTok/c-toxcore/pull/2541) fix: don't pass garbage data buffer to packet send functions
+- [#2540](https://github.com/TokTok/c-toxcore/pull/2540) cleanup: Make group packet entry creation less error-prone
+- [#2539](https://github.com/TokTok/c-toxcore/pull/2539) refactor: Minor refactoring of get_close_nodes functions.
+- [#2538](https://github.com/TokTok/c-toxcore/pull/2538) refactor: Factor out malloc+memcpy into memdup.
+- [#2536](https://github.com/TokTok/c-toxcore/pull/2536) cleanup: Some more test cleanups, removing overly smart code.
+- [#2531](https://github.com/TokTok/c-toxcore/pull/2531) test: Add more unit tests for `add_to_list`.
+- [#2530](https://github.com/TokTok/c-toxcore/pull/2530) refactor: Assign malloc return to a local variable first.
+- [#2529](https://github.com/TokTok/c-toxcore/pull/2529) test: Add "infer" CI check to github, remove from circle.
+- [#2527](https://github.com/TokTok/c-toxcore/pull/2527) cleanup: Add Toxav alias for ToxAV.
+- [#2526](https://github.com/TokTok/c-toxcore/pull/2526) cleanup: Make Tox_Options a typedef.
+- [#2525](https://github.com/TokTok/c-toxcore/pull/2525) cleanup: Add dynamically derived array sizes to the API.
+- [#2524](https://github.com/TokTok/c-toxcore/pull/2524) cleanup: Add explicit array sizes to toxencryptsave.
+- [#2523](https://github.com/TokTok/c-toxcore/pull/2523) cleanup: Move `tox_get_system` out of the public API.
+- [#2522](https://github.com/TokTok/c-toxcore/pull/2522) cleanup: Make setters take non-const `Tox *`.
+- [#2521](https://github.com/TokTok/c-toxcore/pull/2521) cleanup: Make array params in toxav `[]` instead of `*`.
+- [#2520](https://github.com/TokTok/c-toxcore/pull/2520) cleanup: Mark arrays in the tox API as `[]` instead of `*`.
+- [#2519](https://github.com/TokTok/c-toxcore/pull/2519) refactor: Align group message sending with other send functions.
+- [#2518](https://github.com/TokTok/c-toxcore/pull/2518) cleanup: Add typedefs for public API int identifiers.
+- [#2517](https://github.com/TokTok/c-toxcore/pull/2517) chore: Spellcheck tox-bootstrapd
+- [#2516](https://github.com/TokTok/c-toxcore/pull/2516) chore: Remove settings.yml in favour of hs-github-tools.
+- [#2515](https://github.com/TokTok/c-toxcore/pull/2515) chore: Use GPL license with https.
+- [#2513](https://github.com/TokTok/c-toxcore/pull/2513) chore: Add fetch-sha256 script to update bootstrap node hash.
+- [#2512](https://github.com/TokTok/c-toxcore/pull/2512) cleanup: Move all vptr-to-ptr casts to the beginning of a function.
+- [#2510](https://github.com/TokTok/c-toxcore/pull/2510) cleanup: Use github actions matrix to simplify CI.
+- [#2509](https://github.com/TokTok/c-toxcore/pull/2509) fix: Use QueryPerformanceCounter on windows for monotonic time.
+- [#2508](https://github.com/TokTok/c-toxcore/pull/2508) chore: Add `net_(new|kill)_strerror` to cppcheck's allocators.
+- [#2507](https://github.com/TokTok/c-toxcore/pull/2507) cleanup: Run clang-tidy on headers, as well.
+- [#2506](https://github.com/TokTok/c-toxcore/pull/2506) cleanup: Make TCP connection failures a warning instead of error.
+- [#2505](https://github.com/TokTok/c-toxcore/pull/2505) cleanup: Make all .c files include the headers they need.
+- [#2504](https://github.com/TokTok/c-toxcore/pull/2504) cleanup: Upgrade cppcheck, fix some warnings.
+- [#2503](https://github.com/TokTok/c-toxcore/pull/2503) cleanup: Upgrade to clang-tidy-17 and fix some warnings.
+- [#2502](https://github.com/TokTok/c-toxcore/pull/2502) chore: Use `pkg_search_module` directly in cmake.
+- [#2501](https://github.com/TokTok/c-toxcore/pull/2501) chore: Add `IMPORTED_TARGET` to pkg-config packages.
+- [#2499](https://github.com/TokTok/c-toxcore/pull/2499) cleanup: Use target_link_libraries directly in cmake.
+- [#2498](https://github.com/TokTok/c-toxcore/pull/2498) chore: Simplify msvc build using vcpkg.
+- [#2497](https://github.com/TokTok/c-toxcore/pull/2497) cleanup: Remove NaCl support.
+- [#2494](https://github.com/TokTok/c-toxcore/pull/2494) fix: unpack enum function names in event impl generator
+- [#2493](https://github.com/TokTok/c-toxcore/pull/2493) chore: Disable targets for cross-compilation.
+- [#2491](https://github.com/TokTok/c-toxcore/pull/2491) chore: Build a docker image with coverage info in it.
+- [#2490](https://github.com/TokTok/c-toxcore/pull/2490) cleanup: Some portability/warning fixes for Windows builds.
+- [#2488](https://github.com/TokTok/c-toxcore/pull/2488) fix: Correct a use-after-free and fix some memory leaks.
+- [#2487](https://github.com/TokTok/c-toxcore/pull/2487) refactor: Change all enum-like `#define` sequences into enums.
+- [#2486](https://github.com/TokTok/c-toxcore/pull/2486) refactor: Change the `TCP_PACKET_*` defines into an enum.
+- [#2485](https://github.com/TokTok/c-toxcore/pull/2485) refactor: event generation tool for reorder pr
+- [#2484](https://github.com/TokTok/c-toxcore/pull/2484) chore: Fix make_single_file to support core-only.
+- [#2481](https://github.com/TokTok/c-toxcore/pull/2481) chore: Update github actions `uses`.
+- [#2480](https://github.com/TokTok/c-toxcore/pull/2480) feat: add ngc related unpack functions
+- [#2479](https://github.com/TokTok/c-toxcore/pull/2479) feat: Add `to_string` functions for all public enums.
+- [#2477](https://github.com/TokTok/c-toxcore/pull/2477) test: add real timeout test
+- [#2476](https://github.com/TokTok/c-toxcore/pull/2476) chore: Move s390x build to post-merge.
+- [#2475](https://github.com/TokTok/c-toxcore/pull/2475) refactor: Give `enum-from-int` functions the ability to report errors.
+- [#2474](https://github.com/TokTok/c-toxcore/pull/2474) cleanup: Remove test net support.
+- [#2472](https://github.com/TokTok/c-toxcore/pull/2472) feat: Enable ubsan on bootstrap nodes.
+- [#2470](https://github.com/TokTok/c-toxcore/pull/2470) test: Add check-c run to bazel build.
+- [#2468](https://github.com/TokTok/c-toxcore/pull/2468) fix(test): tests use ipv6 by default, even with USE_IPV6 set to 0
+- [#2466](https://github.com/TokTok/c-toxcore/pull/2466) cleanup: Make group saving/loading more forgiving with data errors
+- [#2465](https://github.com/TokTok/c-toxcore/pull/2465) refactor: replace memset with a loop
+- [#2459](https://github.com/TokTok/c-toxcore/pull/2459) fix: Enable debug flag for ubsan.
+- [#2458](https://github.com/TokTok/c-toxcore/pull/2458) fix: also Install header for private/experimental API functions with autotools
+- [#2456](https://github.com/TokTok/c-toxcore/pull/2456) docs: Remove defunct IRC channel from README.md
+- [#2454](https://github.com/TokTok/c-toxcore/pull/2454) fix: memory leaks
+- [#2453](https://github.com/TokTok/c-toxcore/pull/2453) refactor: Install header for private/experimental API functions
+- [#2452](https://github.com/TokTok/c-toxcore/pull/2452) refactor: replace DEFAULT_TCP_RELAY_PORTS_COUNT with a compile-time calculation
+- [#2451](https://github.com/TokTok/c-toxcore/pull/2451) cleanup: clarify disabling of static assert checks
+- [#2449](https://github.com/TokTok/c-toxcore/pull/2449) cleanup: replace tabs with spaces
+- [#2448](https://github.com/TokTok/c-toxcore/pull/2448) feat: group connection queries now return our own connection type
+- [#2447](https://github.com/TokTok/c-toxcore/pull/2447) fix: Docker tox-bootstrapd hash update failing when using BuildKit
+- [#2446](https://github.com/TokTok/c-toxcore/pull/2446) feat: Add groupchat API function that returns an IP address string for a peer
+- [#2442](https://github.com/TokTok/c-toxcore/pull/2442) perf: Slightly reduce bandwidth usage when there are few nodes.
+- [#2439](https://github.com/TokTok/c-toxcore/pull/2439) test: Make esp32 build actually try to instantiate tox.
+- [#2438](https://github.com/TokTok/c-toxcore/pull/2438) cleanup: Remove explicit layering_check feature.
+- [#2437](https://github.com/TokTok/c-toxcore/pull/2437) chore: Upgrade sonar-scan jvm to java 17.
+- [#2436](https://github.com/TokTok/c-toxcore/pull/2436) fix: Add missing `htons` call when adding configured TCP relay.
+- [#2434](https://github.com/TokTok/c-toxcore/pull/2434) chore: Cancel old PR builds on docker and sonar-scan workflows.
+- [#2432](https://github.com/TokTok/c-toxcore/pull/2432) chore: Use C99 on MSVC instead of C11.
+- [#2430](https://github.com/TokTok/c-toxcore/pull/2430) chore: Retry freebsd tests 2 times.
+- [#2429](https://github.com/TokTok/c-toxcore/pull/2429) test: Add an s390x build (on alpine) for CI.
+- [#2428](https://github.com/TokTok/c-toxcore/pull/2428) chore: Add a compcert docker run script.
+- [#2427](https://github.com/TokTok/c-toxcore/pull/2427) cleanup: Use tcc docker image for CI.
+- [#2424](https://github.com/TokTok/c-toxcore/pull/2424) fix: Fix memory leak in the error path of loading savedata.
+- [#2420](https://github.com/TokTok/c-toxcore/pull/2420) refactor: Use Bin_Pack for packing Node_format.
+- [#2416](https://github.com/TokTok/c-toxcore/pull/2416) chore: Add more logging to loading conferences from savedata.
+- [#2415](https://github.com/TokTok/c-toxcore/pull/2415) refactor: Add a `bin_unpack_bin_max` for max-length arrays.
+- [#2414](https://github.com/TokTok/c-toxcore/pull/2414) fix: inversed return values
+- [#2413](https://github.com/TokTok/c-toxcore/pull/2413) cleanup: Fix GCC compatibility.
+- [#2408](https://github.com/TokTok/c-toxcore/pull/2408) fix: Ensure we have allocators available for the error paths.
+- [#2407](https://github.com/TokTok/c-toxcore/pull/2407) cleanup: Remove redundant `-DSODIUM_EXPORT` from definitions.
+- [#2406](https://github.com/TokTok/c-toxcore/pull/2406) cleanup: Fix a few more clang-tidy warnings.
+- [#2405](https://github.com/TokTok/c-toxcore/pull/2405) cleanup: Fix a few more clang-tidy warnings.
+- [#2404](https://github.com/TokTok/c-toxcore/pull/2404) cleanup: Enforce stricter identifier naming using clang-tidy.
+- [#2396](https://github.com/TokTok/c-toxcore/pull/2396) chore: Add devcontainer setup for codespaces.
+- [#2393](https://github.com/TokTok/c-toxcore/pull/2393) refactor: Add `mem` module to allow tests to override allocators.
+- [#2392](https://github.com/TokTok/c-toxcore/pull/2392) refactor: Make event dispatch ordered by receive time.
+- [#2391](https://github.com/TokTok/c-toxcore/pull/2391) docs: Fix doxygen config and remove some redundant comments.
+- [#2390](https://github.com/TokTok/c-toxcore/pull/2390) chore: Fix the Android CI job
+- [#2389](https://github.com/TokTok/c-toxcore/pull/2389) fix: Add missing `#include <array>`.
+- [#2388](https://github.com/TokTok/c-toxcore/pull/2388) chore: Add missing module dependencies.
+- [#2384](https://github.com/TokTok/c-toxcore/pull/2384) feat: increase NGC lossy custom packet size
+- [#2383](https://github.com/TokTok/c-toxcore/pull/2383) fix: add missing ngc constants getter declarations and definitions
+- [#2381](https://github.com/TokTok/c-toxcore/pull/2381) fix: incorrect documentation
+- [#2380](https://github.com/TokTok/c-toxcore/pull/2380) feat: allow for larger incoming NGC packets
+- [#2371](https://github.com/TokTok/c-toxcore/pull/2371) docs: fix group_peer_exit_cb
+- [#2370](https://github.com/TokTok/c-toxcore/pull/2370) fix: behaviour of group api function
+- [#2369](https://github.com/TokTok/c-toxcore/pull/2369) fix: flaky tcp test
+- [#2367](https://github.com/TokTok/c-toxcore/pull/2367) fix: fuzz support for TCP server
+- [#2364](https://github.com/TokTok/c-toxcore/pull/2364) fix: potential endless loop under extremely high load
+- [#2362](https://github.com/TokTok/c-toxcore/pull/2362) test: enable tcp relay for bootstrap fuzzing
+- [#2361](https://github.com/TokTok/c-toxcore/pull/2361) fix: resolve_bootstrap_node() not checking net_getipport() returned count correctly
+- [#2357](https://github.com/TokTok/c-toxcore/pull/2357) feat: get the number of close dht nodes with announce/store support
+- [#2355](https://github.com/TokTok/c-toxcore/pull/2355) fix: group custom packet size limits
+- [#2354](https://github.com/TokTok/c-toxcore/pull/2354) fix: DHTBootstrap should always respond to version packets with toxcore version
+- [#2351](https://github.com/TokTok/c-toxcore/pull/2351) fix: Increase max group message length by four bytes
+- [#2348](https://github.com/TokTok/c-toxcore/pull/2348) refactor: Make some improvements to how often/when we announce a group
+- [#2341](https://github.com/TokTok/c-toxcore/pull/2341) fix: #1144 by forcing misc_tools to be a static lib
+- [#2340](https://github.com/TokTok/c-toxcore/pull/2340) fix: missing net to host conversion of port in logging in group_chat.c
+- [#2339](https://github.com/TokTok/c-toxcore/pull/2339) fix: missing net to host conversion of port in logging
+- [#2338](https://github.com/TokTok/c-toxcore/pull/2338) fix: bug causing friend group invites to sometimes fail & improve logging
+- [#2333](https://github.com/TokTok/c-toxcore/pull/2333) docs: Update README for bootstrap node docker
+- [#2329](https://github.com/TokTok/c-toxcore/pull/2329) refactor: extract each case in handle packet in messenger
+- [#2327](https://github.com/TokTok/c-toxcore/pull/2327) fix: unlock correct dht_friend
+- [#2325](https://github.com/TokTok/c-toxcore/pull/2325) fix: Remove cmake cache files after copying to container build directory
+- [#2323](https://github.com/TokTok/c-toxcore/pull/2323) docs: Update README.md to include cmp submodule info
+- [#2318](https://github.com/TokTok/c-toxcore/pull/2318) chore: disable warning about pre C99 code
+- [#2317](https://github.com/TokTok/c-toxcore/pull/2317) Refactor: Extract shared key cache into separate file
+- [#2311](https://github.com/TokTok/c-toxcore/pull/2311) cleanup: make it more clear that assert and uint32_t increment both only exist if NDEBUG is not defined
+- [#2291](https://github.com/TokTok/c-toxcore/pull/2291) test: Add a protocol dump test to generate initial fuzzer input.
+- [#2271](https://github.com/TokTok/c-toxcore/pull/2271) chore: Migrate from Appveyor to Azure Pipelines
+- [#2269](https://github.com/TokTok/c-toxcore/pull/2269) feat: Merge the remainder of the new groupchats implementation
+- [#2203](https://github.com/TokTok/c-toxcore/pull/2203) refactor: Store time in Mono_Time in milliseconds.
+- [#1944](https://github.com/TokTok/c-toxcore/pull/1944) chore: tox_new() should return null when savedata loading fails
+
+### Closed issues:
+
+- [#2739](https://github.com/TokTok/c-toxcore/issues/2739) Tox_Options.operating_system is not clear about it being an experimental option
+- [#2734](https://github.com/TokTok/c-toxcore/issues/2734) error compiling on fedora
+- [#2649](https://github.com/TokTok/c-toxcore/issues/2649) create_extended_keypair should use Random and be made deterministic for fuzzing
+- [#2358](https://github.com/TokTok/c-toxcore/issues/2358) resolve_bootstrap_node assert hit
+- [#2352](https://github.com/TokTok/c-toxcore/issues/2352) SEGV after infinite loop retrying proxy_socks5_read_connection_response
+- [#2335](https://github.com/TokTok/c-toxcore/issues/2335) ipv6 disabled on kernel cmdline disrupts Tox = most tests fail
+- [#2303](https://github.com/TokTok/c-toxcore/issues/2303) Add new group chats events to tox_events and tox_dispatch modules.
+- [#2302](https://github.com/TokTok/c-toxcore/issues/2302) Run strong fuzz tests against NGC
+- [#2301](https://github.com/TokTok/c-toxcore/issues/2301) Implement NGC save/load in hs-toxcore and run round-trip tests against c-toxcore
+- [#2192](https://github.com/TokTok/c-toxcore/issues/2192) Provide official signed release tarballs
+- [#2118](https://github.com/TokTok/c-toxcore/issues/2118) Add a private "get mono_time" API to `tox_private.h` and use it in auto_tests.
+- [#2029](https://github.com/TokTok/c-toxcore/issues/2029) Migrate all auto tests to the events API.
+- [#2014](https://github.com/TokTok/c-toxcore/issues/2014) Add sodium autotools CI build
+- [#1144](https://github.com/TokTok/c-toxcore/issues/1144) DHT_bootstrap should not link against misc_tools
+
 ## v0.2.18

 ### Merged PRs:

+- [#2300](https://github.com/TokTok/c-toxcore/pull/2300) chore: Release 0.2.18
 - [#2299](https://github.com/TokTok/c-toxcore/pull/2299) fix: remove the assert because buffer can be larger than UINT16_MAX.
 - [#2297](https://github.com/TokTok/c-toxcore/pull/2297) cleanup: remove unused field last_seen from Onion_Friend
 - [#2289](https://github.com/TokTok/c-toxcore/pull/2289) test: Add a Null_System used in toxsave_harness.
@@ -107,6 +372,7 @@

 ### Closed issues:

+- [#2298](https://github.com/TokTok/c-toxcore/issues/2298) assert makes incorrect assumption
 - [#2256](https://github.com/TokTok/c-toxcore/issues/2256) New Defects reported by Coverity Scan for TokTok/c-toxcore
 - [#2109](https://github.com/TokTok/c-toxcore/issues/2109) Assimilate `messenger_test.c`: replace with public API test
 - [#2012](https://github.com/TokTok/c-toxcore/issues/2012) Support building a DLL on Windows
--- a/external/toxcore/c-toxcore/CMakeLists.txt
+++ b/external/toxcore/c-toxcore/CMakeLists.txt
@@ -44,7 +44,7 @@ set_source_files_properties(
 # versions in a synchronised way.
 set(PROJECT_VERSION_MAJOR "0")
 set(PROJECT_VERSION_MINOR "2")
-set(PROJECT_VERSION_PATCH "18")
+set(PROJECT_VERSION_PATCH "19")
 set(PROJECT_VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}")

 # set .so library version / following libtool scheme
@@ -146,6 +146,8 @@ endif()
 option(BUILD_MISC_TESTS "Build additional tests" OFF)
 option(BUILD_FUN_UTILS "Build additional just for fun utilities" OFF)

+option(UNITTEST "Enable unit tests (disable if you don't have a working gmock or gtest)" ON)
+
 option(AUTOTEST "Enable autotests (mainly for CI)" OFF)
 if(AUTOTEST)
  option(NON_HERMETIC_TESTS "Whether to build and run tests that depend on an internet connection" OFF)
@@ -534,7 +536,7 @@ endfunction()

 # The actual unit tests follow.
 #
-if(TARGET GTest::gtest AND TARGET GTest::gmock)
+if(UNITTEST AND TARGET GTest::gtest AND TARGET GTest::gmock)
  unit_test(toxav ring_buffer)
  unit_test(toxav rtp)
  unit_test(toxcore DHT)
--- a/external/toxcore/c-toxcore/INSTALL.md
+++ b/external/toxcore/c-toxcore/INSTALL.md
@@ -151,6 +151,7 @@ There are some options that are available to configure the build.
 | `PROXY_TEST`            | Enable proxy test (requires `other/proxy/proxy_server.go` to be running).                     | ON or OFF                                                                 | OFF                                               |
 | `STRICT_ABI`            | Enforce strict ABI export in dynamic libraries.                                               | ON or OFF                                                                 | OFF                                               |
 | `TEST_TIMEOUT_SECONDS`  | Limit runtime of each test to the number of seconds specified.                                | Positive number or nothing (empty string).                                | Empty string.                                     |
+| `UNITTEST`              | Enable unit tests (disable if you don't have a working gmock or gtest).                       | ON or OFF                                                                 | ON                                                |
 | `USE_IPV6`              | Use IPv6 in tests.                                                                            | ON or OFF                                                                 | ON                                                |

 You can get this list of option using the following commands
--- a/external/toxcore/c-toxcore/configure.ac
+++ b/external/toxcore/c-toxcore/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.

 AC_PREREQ([2.65])
-AC_INIT([tox], [0.2.18])
+AC_INIT([tox], [0.2.19])
 AC_CONFIG_AUX_DIR(configure_aux)
 AC_CONFIG_SRCDIR([toxcore/net_crypto.c])
 AM_INIT_AUTOMAKE([foreign 1.10 -Wall -Werror subdir-objects tar-ustar])
--- a/external/toxcore/c-toxcore/other/bootstrap_daemon/docker/tox-bootstrapd.sha256
+++ b/external/toxcore/c-toxcore/other/bootstrap_daemon/docker/tox-bootstrapd.sha256
@@ -1 +1 @@
-af58a125e5c80d7a19bc7f32868c1edfdf80f366e3bf778728961a50ce63ee26  /usr/local/bin/tox-bootstrapd
+e96f03a89051c5df12c28d0d6941184da2b92742d248bd4c57d31189a0052844  /usr/local/bin/tox-bootstrapd
--- a/external/toxcore/c-toxcore/other/docker/pkgsrc/pkgsrc.Dockerfile
+++ b/external/toxcore/c-toxcore/other/docker/pkgsrc/pkgsrc.Dockerfile
@@ -5,6 +5,7 @@ COPY . /work/c-toxcore-0.2.18
 RUN ["tar", "zcf", "c-toxcore.tar.gz", "c-toxcore-0.2.18"]

 WORKDIR /work/pkgsrc/chat/toxcore
+RUN ["sed", "-i", "-e", "s/libtoxcore.so.2.18.0/libtoxcore.so.2.19.0/g", "PLIST"]
 RUN ["bmake", "clean"]
 RUN ["bmake", "DISTFILES=c-toxcore.tar.gz", "DISTDIR=/work", "NO_CHECKSUM=yes"]
 RUN ["bmake", "install"]
--- a/external/toxcore/c-toxcore/so.version
+++ b/external/toxcore/c-toxcore/so.version
@@ -11,6 +11,6 @@
 # For a full reference see:
 # https://www.gnu.org/software/libtool/manual/libtool.html#Updating-version-info

-CURRENT=20
+CURRENT=21
 REVISION=0
-AGE=18
+AGE=19
--- a/external/toxcore/c-toxcore/testing/fuzzing/bootstrap_fuzz_test.cc
+++ b/external/toxcore/c-toxcore/testing/fuzzing/bootstrap_fuzz_test.cc
@@ -104,7 +104,6 @@ void TestBootstrap(Fuzz_Data &input)

    Ptr<Tox_Options> opts(tox_options_new(nullptr), tox_options_free);
    assert(opts != nullptr);
-    tox_options_set_operating_system(opts.get(), sys.sys.get());

    tox_options_set_log_callback(opts.get(),
        [](Tox *tox, Tox_Log_Level level, const char *file, uint32_t line, const char *func,
@@ -134,8 +133,12 @@ void TestBootstrap(Fuzz_Data &input)
        tox_options_set_tcp_port(opts.get(), 33445);
    }

+    Tox_Options_Testing tox_options_testing;
+    tox_options_testing.operating_system = sys.sys.get();
+
    Tox_Err_New error_new;
-    Tox *tox = tox_new(opts.get(), &error_new);
+    Tox_Err_New_Testing error_new_testing;
+    Tox *tox = tox_new_testing(opts.get(), &error_new, &tox_options_testing, &error_new_testing);

    if (tox == nullptr) {
        // It might fail, because some I/O happens in tox_new, and the fuzzer
@@ -144,6 +147,7 @@ void TestBootstrap(Fuzz_Data &input)
    }

    assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);

    uint8_t pub_key[TOX_PUBLIC_KEY_SIZE] = {0};

--- a/external/toxcore/c-toxcore/testing/fuzzing/e2e_fuzz_test.cc
+++ b/external/toxcore/c-toxcore/testing/fuzzing/e2e_fuzz_test.cc
@@ -138,7 +138,6 @@ void TestEndToEnd(Fuzz_Data &input)

    Ptr<Tox_Options> opts(tox_options_new(nullptr), tox_options_free);
    assert(opts != nullptr);
-    tox_options_set_operating_system(opts.get(), sys.sys.get());
    tox_options_set_local_discovery_enabled(opts.get(), false);

    tox_options_set_log_callback(opts.get(),
@@ -151,8 +150,12 @@ void TestEndToEnd(Fuzz_Data &input)
            }
        });

+    Tox_Options_Testing tox_options_testing;
+    tox_options_testing.operating_system = sys.sys.get();
+
    Tox_Err_New error_new;
-    Tox *tox = tox_new(opts.get(), &error_new);
+    Tox_Err_New_Testing error_new_testing;
+    Tox *tox = tox_new_testing(opts.get(), &error_new, &tox_options_testing, &error_new_testing);

    if (tox == nullptr) {
        // It might fail, because some I/O happens in tox_new, and the fuzzer
@@ -161,6 +164,7 @@ void TestEndToEnd(Fuzz_Data &input)
    }

    assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);

    tox_events_init(tox);

--- a/external/toxcore/c-toxcore/testing/fuzzing/fuzz_support.hh
+++ b/external/toxcore/c-toxcore/testing/fuzzing/fuzz_support.hh
@@ -16,6 +16,7 @@
 #include <vector>

 #include "../../toxcore/tox.h"
+#include "../../toxcore/tox_private.h"

 struct Fuzz_Data {
    static constexpr bool DEBUG = false;
--- a/external/toxcore/c-toxcore/testing/fuzzing/protodump.cc
+++ b/external/toxcore/c-toxcore/testing/fuzzing/protodump.cc
@@ -195,13 +195,16 @@ void RecordBootstrap(const char *init, const char *bootstrap)
        });

    Tox_Err_New error_new;
+    Tox_Err_New_Testing error_new_testing;
+    Tox_Options_Testing tox_options_testing;

    Record_System sys1(global, 4, "tox1");  // fair dice roll
    tox_options_set_log_user_data(opts, &sys1);
-    tox_options_set_operating_system(opts, sys1.sys.get());
-    Tox *tox1 = tox_new(opts, &error_new);
+    tox_options_testing.operating_system = sys1.sys.get();
+    Tox *tox1 = tox_new_testing(opts, &error_new, &tox_options_testing, &error_new_testing);
    assert(tox1 != nullptr);
    assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);
    std::array<uint8_t, TOX_ADDRESS_SIZE> address1;
    tox_self_get_address(tox1, address1.data());
    std::array<uint8_t, TOX_PUBLIC_KEY_SIZE> pk1;
@@ -211,10 +214,11 @@ void RecordBootstrap(const char *init, const char *bootstrap)

    Record_System sys2(global, 5, "tox2");  // unfair dice roll
    tox_options_set_log_user_data(opts, &sys2);
-    tox_options_set_operating_system(opts, sys2.sys.get());
-    Tox *tox2 = tox_new(opts, &error_new);
+    tox_options_testing.operating_system = sys2.sys.get();
+    Tox *tox2 = tox_new_testing(opts, &error_new, &tox_options_testing, &error_new_testing);
    assert(tox2 != nullptr);
    assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);
    std::array<uint8_t, TOX_ADDRESS_SIZE> address2;
    tox_self_get_address(tox2, address2.data());
    std::array<uint8_t, TOX_PUBLIC_KEY_SIZE> pk2;
--- a/external/toxcore/c-toxcore/testing/fuzzing/protodump_reduce.cc
+++ b/external/toxcore/c-toxcore/testing/fuzzing/protodump_reduce.cc
@@ -142,9 +142,11 @@ void TestEndToEnd(Fuzz_Data &input)

    Ptr<Tox_Options> opts(tox_options_new(nullptr), tox_options_free);
    assert(opts != nullptr);
-    tox_options_set_operating_system(opts.get(), sys.sys.get());
    tox_options_set_local_discovery_enabled(opts.get(), false);

+    Tox_Options_Testing tox_options_testing;
+    tox_options_testing.operating_system = sys.sys.get();
+
    tox_options_set_log_callback(opts.get(),
        [](Tox *tox, Tox_Log_Level level, const char *file, uint32_t line, const char *func,
            const char *message, void *user_data) {
@@ -156,7 +158,8 @@ void TestEndToEnd(Fuzz_Data &input)
        });

    Tox_Err_New error_new;
-    Tox *tox = tox_new(opts.get(), &error_new);
+    Tox_Err_New_Testing error_new_testing;
+    Tox *tox = tox_new_testing(opts.get(), &error_new, &tox_options_testing, &error_new_testing);

    if (tox == nullptr) {
        // It might fail, because some I/O happens in tox_new, and the fuzzer
@@ -165,6 +168,7 @@ void TestEndToEnd(Fuzz_Data &input)
    }

    assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);

    tox_events_init(tox);

--- a/external/toxcore/c-toxcore/testing/fuzzing/toxsave_fuzz_test.cc
+++ b/external/toxcore/c-toxcore/testing/fuzzing/toxsave_fuzz_test.cc
@@ -20,14 +20,15 @@ void TestSaveDataLoading(Fuzz_Data &input)
    const size_t savedata_size = input.size();
    CONSUME_OR_RETURN(const uint8_t *savedata, input, savedata_size);

-    Null_System sys;
-    tox_options_set_operating_system(tox_options, sys.sys.get());
-
    // pass test data to Tox
    tox_options_set_savedata_data(tox_options, savedata, savedata_size);
    tox_options_set_savedata_type(tox_options, TOX_SAVEDATA_TYPE_TOX_SAVE);

-    Tox *tox = tox_new(tox_options, nullptr);
+    Tox_Options_Testing tox_options_testing;
+    Null_System sys;
+    tox_options_testing.operating_system = sys.sys.get();
+
+    Tox *tox = tox_new_testing(tox_options, nullptr, &tox_options_testing, nullptr);
    tox_options_free(tox_options);
    if (tox == nullptr) {
        // Tox save was invalid, we're finished here
--- a/external/toxcore/c-toxcore/toxav/toxav.h
+++ b/external/toxcore/c-toxcore/toxav/toxav.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2018 The TokTok team.
+ * Copyright © 2016-2024 The TokTok team.
 * Copyright © 2013-2015 Tox project.
 */

@@ -7,39 +7,40 @@
 * @brief Public audio/video API for Tox clients.
 *
 * This API can handle multiple calls. Each call has its state, in very rare
- * occasions the library can change the state of the call without apps knowledge.
+ * occasions the library can change the state of the call without apps
+ * knowledge.
 *
 * @section av_events Events and callbacks
 *
- * As in Core API, events are handled by callbacks. One callback can be
+ * As in the toxcore API, events are handled by callbacks. One callback can be
 * registered per event. All events have a callback function type named
- * `toxav_{event}_cb` and a function to register it named `toxav_callback_{event}`.
- * Passing a NULL callback will result in no callback being registered for that
- * event. Only one callback per event can be registered, so if a client needs
- * multiple event listeners, it needs to implement the dispatch functionality
- * itself. Unlike Core API, lack of some event handlers will cause the the
- * library to drop calls before they are started. Hanging up call from a
- * callback causes undefined behaviour.
+ * `toxav_{event}_cb` and a function to register it named
+ * `toxav_callback_{event}`. Passing a NULL callback will result in no callback
+ * being registered for that event. Only one callback per event can be
+ * registered, so if a client needs multiple event listeners, it needs to
+ * implement the dispatch functionality itself. Unlike the toxcore API, lack of
+ * some event handlers will cause the the library to drop calls before they are
+ * started. Hanging up call from a callback causes undefined behaviour.
 *
 * @section av_threading Threading implications
 *
 * Only toxav_iterate is thread-safe, all other functions must run from the
- * tox thread.
+ * Tox thread.
 *
 * Important exceptions are the `*_iterate` and `*_iterate_interval`
 * functions. You have to choose either the single thread or the multi thread
 * functions and read their documentation.
 *
 * A common way to run ToxAV (multiple or single instance) is to have a thread,
- * separate from tox instance thread, running a simple toxav_iterate loop,
+ * separate from Tox instance thread, running a simple toxav_iterate loop,
 * sleeping for `toxav_iteration_interval * milliseconds` on each iteration.
 *
- * An important thing to note is that events are triggered from both tox and
- * toxav thread (see above). Audio and video receive frame events are triggered
- * from toxav thread while all the other events are triggered from tox thread.
+ * An important thing to note is that events are triggered from both Tox and
+ * ToxAV thread (see above). Audio and video receive frame events are triggered
+ * from ToxAV thread while all the other events are triggered from Tox thread.
 *
- * Tox thread has priority with mutex mechanisms. Any api function can
- * fail if mutexes are held by tox thread in which case they will set SYNC
+ * Tox thread has priority with mutex mechanisms. Any API function can
+ * fail if mutexes are held by Tox thread in which case they will set SYNC
 * error code.
 *
 * @subsection av_multi_threading Separate audio and video threads
@@ -77,9 +78,9 @@ typedef struct Tox Tox;
 *
 * Each ToxAV instance can be bound to only one Tox instance, and Tox instance
 * can have only one ToxAV instance. One must make sure to close ToxAV instance
- * prior closing Tox instance otherwise undefined behaviour occurs. Upon
- * closing of ToxAV instance, all active calls will be forcibly terminated
- * without notifying peers.
+ * prior closing Tox instance otherwise undefined behaviour occurs. Upon closing
+ * of ToxAV instance, all active calls will be forcibly terminated without
+ * notifying peers.
 */
 typedef struct ToxAV ToxAV;

@@ -100,8 +101,8 @@ typedef enum Toxav_Err_New {
    TOXAV_ERR_NEW_NULL,

    /**
-     * Memory allocation failure while trying to allocate structures required for
-     * the A/V session.
+     * Memory allocation failure while trying to allocate structures required
+     * for the A/V session.
     */
    TOXAV_ERR_NEW_MALLOC,

@@ -203,8 +204,8 @@ typedef enum Toxav_Err_Call {
    TOXAV_ERR_CALL_OK,

    /**
-     * A resource allocation error occurred while trying to create the structures
-     * required for the call.
+     * A resource allocation error occurred while trying to create the
+     * structures required for the call.
     */
    TOXAV_ERR_CALL_MALLOC,

@@ -246,9 +247,9 @@ typedef enum Toxav_Err_Call {
 *
 * @param friend_number The friend number of the friend that should be called.
 * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable
- * audio sending.
+ *   audio sending.
 * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable
- * video sending.
+ *   video sending.
 */
 bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate,
                Toxav_Err_Call *error);
@@ -293,8 +294,9 @@ typedef enum Toxav_Err_Answer {
    TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND,

    /**
-     * The friend was valid, but they are not currently trying to initiate a call.
-     * This is also returned if this client is already in a call with the friend.
+     * The friend was valid, but they are not currently trying to initiate a
+     * call. This is also returned if this client is already in a call with the
+     * friend.
     */
    TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING,

@@ -314,9 +316,9 @@ typedef enum Toxav_Err_Answer {
 *
 * @param friend_number The friend number of the friend that is calling.
 * @param audio_bit_rate Audio bit rate in Kb/sec. Set this to 0 to disable
- * audio sending.
+ *   audio sending.
 * @param video_bit_rate Video bit rate in Kb/sec. Set this to 0 to disable
- * video sending.
+ *   video sending.
 */
 bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate,
                  Toxav_Err_Answer *error);
@@ -337,8 +339,8 @@ enum Toxav_Friend_Call_State {
    /**
     * Set by the AV core if an error occurred on the remote end or if friend
     * timed out. This is the final state after which no more state
-     * transitions can occur for the call. This call state will never be triggered
-     * in combination with other call states.
+     * transitions can occur for the call. This call state will never be
+     * triggered in combination with other call states.
     */
    TOXAV_FRIEND_CALL_STATE_ERROR = 1,

@@ -376,9 +378,9 @@ enum Toxav_Friend_Call_State {
 *
 * @param friend_number The friend number for which the call state changed.
 * @param state The bitmask of the new call state which is guaranteed to be
- * different than the previous state. The state is set to 0 when the call is
- * paused. The bitmask represents all the activities currently performed by the
- * friend.
+ *   different than the previous state. The state is set to 0 when the call is
+ *   paused. The bitmask represents all the activities currently performed by
+ *   the friend.
 */
 typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data);

@@ -397,8 +399,9 @@ void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *callback, void *u
 typedef enum Toxav_Call_Control {

    /**
-     * Resume a previously paused call. Only valid if the pause was caused by this
-     * client, if not, this control is ignored. Not valid before the call is accepted.
+     * Resume a previously paused call. Only valid if the pause was caused by
+     * this client, if not, this control is ignored. Not valid before the call
+     * is accepted.
     */
    TOXAV_CALL_CONTROL_RESUME,

@@ -457,8 +460,8 @@ typedef enum Toxav_Err_Call_Control {
    TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND,

    /**
-     * This client is currently not in a call with the friend. Before the call is
-     * answered, only CANCEL is a valid control.
+     * This client is currently not in a call with the friend. Before the call
+     * is answered, only CANCEL is a valid control.
     */
    TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL,

@@ -474,7 +477,7 @@ typedef enum Toxav_Err_Call_Control {
 * Sends a call control command to a friend.
 *
 * @param friend_number The friend number of the friend this client is in a call
- * with.
+ *   with.
 * @param control The control command to send.
 *
 * @return true on success.
@@ -530,8 +533,8 @@ typedef enum Toxav_Err_Send_Frame {
    TOXAV_ERR_SEND_FRAME_OK,

    /**
-     * In case of video, one of Y, U, or V was NULL. In case of audio, the samples
-     * data pointer was NULL.
+     * In case of video, one of Y, U, or V was NULL. In case of audio, the
+     * samples data pointer was NULL.
     */
    TOXAV_ERR_SEND_FRAME_NULL,

@@ -557,13 +560,13 @@ typedef enum Toxav_Err_Send_Frame {
    TOXAV_ERR_SEND_FRAME_INVALID,

    /**
-     * Either friend turned off audio or video receiving or we turned off sending
-     * for the said payload.
+     * Either friend turned off audio or video receiving or we turned off
+     * sending for the said payload.
     */
    TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED,

    /**
-     * Failed to push frame through rtp interface.
+     * Failed to push frame through RTP interface.
     */
    TOXAV_ERR_SEND_FRAME_RTP_FAILED,

@@ -572,7 +575,8 @@ typedef enum Toxav_Err_Send_Frame {
 /**
 * Send an audio frame to a friend.
 *
- * The expected format of the PCM data is: `[s1c1][s1c2][...][s2c1][s2c2][...]...`
+ * The expected format of the PCM data is:
+ * `[s1c1][s1c2][...][s2c1][s2c2][...]...`
 * Meaning: sample 1 for channel 1, sample 1 for channel 2, ...
 * For mono audio, this has no meaning, every sample is subsequent. For stereo,
 * this means the expected format is LRLRLR... with samples for left and right
@@ -584,10 +588,10 @@ typedef enum Toxav_Err_Send_Frame {
 *   `sample_count * channels`.
 * @param sample_count Number of samples in this frame. Valid numbers here are
 *   `((sample rate) * (audio length) / 1000)`, where audio length can be
- *   2.5, 5, 10, 20, 40 or 60 millseconds.
+ *   2.5, 5, 10, 20, 40 or 60 milliseconds.
 * @param channels Number of audio channels. Supported values are 1 and 2.
 * @param sampling_rate Audio sampling rate used in this frame. Valid sampling
- * rates are 8000, 12000, 16000, 24000, or 48000.
+ *   rates are 8000, 12000, 16000, 24000, or 48000.
 */
 bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t pcm[], size_t sample_count,
                            uint8_t channels, uint32_t sampling_rate, Toxav_Err_Send_Frame *error);
@@ -596,7 +600,7 @@ bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t pcm
 * Set the bit rate to be used in subsequent video frames.
 *
 * @param friend_number The friend number of the friend for which to set the
- * bit rate.
+ *   bit rate.
 * @param bit_rate The new audio bit rate in Kb/sec. Set to 0 to disable.
 *
 * @return true on success.
@@ -605,11 +609,11 @@ bool toxav_audio_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_ra

 /**
 * The function type for the audio_bit_rate callback. The event is triggered
- * when the network becomes too saturated for current bit rates at which
- * point core suggests new bit rates.
+ * when the network becomes too saturated for current bit rates at which point
+ * ToxAV suggests new bit rates.
 *
 * @param friend_number The friend number of the friend for which to set the
- * bit rate.
+ *   bit rate.
 * @param audio_bit_rate Suggested maximum audio bit rate in Kb/sec.
 */
 typedef void toxav_audio_bit_rate_cb(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, void *user_data);
@@ -628,7 +632,7 @@ void toxav_callback_audio_bit_rate(ToxAV *av, toxav_audio_bit_rate_cb *callback,
 * V - plane should be of size: `(height/2) * (width/2)`
 *
 * @param friend_number The friend number of the friend to which to send a video
- * frame.
+ *   frame.
 * @param width Width of the frame in pixels.
 * @param height Height of the frame in pixels.
 * @param y Y (Luminance) plane data.
@@ -646,7 +650,7 @@ bool toxav_video_send_frame(
 * Set the bit rate to be used in subsequent video frames.
 *
 * @param friend_number The friend number of the friend for which to set the
- * bit rate.
+ *   bit rate.
 * @param bit_rate The new video bit rate in Kb/sec. Set to 0 to disable.
 *
 * @return true on success.
@@ -655,11 +659,11 @@ bool toxav_video_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_ra

 /**
 * The function type for the video_bit_rate callback. The event is triggered
- * when the network becomes too saturated for current bit rates at which
- * point core suggests new bit rates.
+ * when the network becomes too saturated for current bit rates at which point
+ * ToxAV suggests new bit rates.
 *
 * @param friend_number The friend number of the friend for which to set the
- * bit rate.
+ *   bit rate.
 * @param video_bit_rate Suggested maximum video bit rate in Kb/sec.
 */
 typedef void toxav_video_bit_rate_cb(ToxAV *av, uint32_t friend_number, uint32_t video_bit_rate, void *user_data);
@@ -714,7 +718,6 @@ void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb
 * @param y Luminosity plane. `Size = MAX(width, abs(ystride)) * height`.
 * @param u U chroma plane. `Size = MAX(width/2, abs(ustride)) * (height/2)`.
 * @param v V chroma plane. `Size = MAX(width/2, abs(vstride)) * (height/2)`.
- *
 * @param ystride Luminosity plane stride.
 * @param ustride U chroma plane stride.
 * @param vstride V chroma plane stride.
@@ -737,26 +740,28 @@ void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb
 #ifndef APIGEN_IGNORE

 /***
- * NOTE Compatibility with old toxav group calls. TODO(iphydf): remove
+ * NOTE Compatibility with old ToxAV group calls. TODO(iphydf): remove
 *
 * TODO(iphydf): Use proper new API guidelines for these. E.g. don't use inline
 * function types, don't have per-callback userdata, especially don't have one
 * userdata per group.
 */

-// TODO(iphydf): Use this better typed one instead of the void-pointer one below.
+// TODO(iphydf): Use this better typed one instead of the void-pointer one
+// below.
 typedef void toxav_group_audio_cb(Tox *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t pcm[],
                                  uint32_t samples, uint8_t channels, uint32_t sample_rate, void *user_data);

 typedef void toxav_audio_data_cb(void *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t pcm[],
                                 uint32_t samples, uint8_t channels, uint32_t sample_rate, void *userdata);

-/** @brief Create a new toxav group.
+/** @brief Create a new ToxAV group.
 *
 * @return group number on success.
 * @retval -1 on failure.
 *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
 */
 int32_t toxav_add_av_groupchat(Tox *tox, toxav_audio_data_cb *audio_callback, void *userdata);

@@ -765,7 +770,8 @@ int32_t toxav_add_av_groupchat(Tox *tox, toxav_audio_data_cb *audio_callback, vo
 * @return group number on success.
 * @retval -1 on failure.
 *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
 */
 int32_t toxav_join_av_groupchat(
    Tox *tox, uint32_t friendnumber, const uint8_t data[], uint16_t length,
@@ -776,7 +782,8 @@ int32_t toxav_join_av_groupchat(
 * @retval 0 on success.
 * @retval -1 on failure.
 *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
 *
 * Valid number of samples are `(sample rate) * (audio length) / 1000`
 *   (Valid values for audio length are: 2.5, 5, 10, 20, 40 or 60 ms)
@@ -794,15 +801,16 @@ int32_t toxav_group_send_audio(
 * A/V must be enabled on a groupchat for audio to be sent to it and for
 * received audio to be handled.
 *
- * An A/V group created with `toxav_add_av_groupchat` or `toxav_join_av_groupchat`
- * will start with A/V enabled.
+ * An A/V group created with `toxav_add_av_groupchat` or
+ * `toxav_join_av_groupchat` will start with A/V enabled.
 *
 * An A/V group loaded from a savefile will start with A/V disabled.
 *
 * @retval 0 on success.
 * @retval -1 on failure.
 *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
 */
 int32_t toxav_groupchat_enable_av(
    Tox *tox, uint32_t groupnumber,
--- a/external/toxcore/c-toxcore/toxcore/tox.c
+++ b/external/toxcore/c-toxcore/toxcore/tox.c
@@ -712,7 +712,8 @@ static int tox_load(Tox *tox, const uint8_t *data, uint32_t length)
                      length - cookie_len, STATE_COOKIE_TYPE);
 }

-Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
+nullable(1, 2, 3)
+static Tox *tox_new_system(const struct Tox_Options *options, Tox_Err_New *error, const Tox_System *sys)
 {
    struct Tox_Options *default_options = nullptr;

@@ -736,7 +737,6 @@ Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
    const struct Tox_Options *const opts = options != nullptr ? options : default_options;
    assert(opts != nullptr);

-    const Tox_System *sys = tox_options_get_operating_system(opts);
    const Tox_System default_system = tox_default_system();

    if (sys == nullptr) {
@@ -1020,6 +1020,37 @@ Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
    return tox;
 }

+Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
+{
+    return tox_new_system(options, error, nullptr);
+}
+
+Tox *tox_new_testing(const Tox_Options *options, Tox_Err_New *error, const Tox_Options_Testing *testing, Tox_Err_New_Testing *testing_error)
+{
+    if (testing == nullptr) {
+        SET_ERROR_PARAMETER(error, TOX_ERR_NEW_NULL);
+        SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_NULL);
+        return nullptr;
+    }
+
+    if (testing->operating_system == nullptr) {
+        SET_ERROR_PARAMETER(error, TOX_ERR_NEW_NULL);
+        SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_NULL);
+        return nullptr;
+    }
+
+    const Tox_System *sys = testing->operating_system;
+
+    if (sys->rng == nullptr || sys->ns == nullptr || sys->mem == nullptr) {
+        SET_ERROR_PARAMETER(error, TOX_ERR_NEW_NULL);
+        SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_NULL);
+        return nullptr;
+    }
+
+    SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_OK);
+    return tox_new_system(options, error, sys);
+}
+
 void tox_kill(Tox *tox)
 {
    if (tox == nullptr) {
@@ -4269,7 +4300,7 @@ uint32_t tox_group_invite_accept(Tox *tox, uint32_t friend_number, const uint8_t
        }

        case -6: {
-            SET_ERROR_PARAMETER(error, TOX_ERR_GROUP_INVITE_ACCEPT_CORE);
+            SET_ERROR_PARAMETER(error, TOX_ERR_GROUP_INVITE_ACCEPT_FRIEND_NOT_FOUND);
            return UINT32_MAX;
        }

--- a/external/toxcore/c-toxcore/toxcore/tox.h
+++ b/external/toxcore/c-toxcore/toxcore/tox.h
--- a/external/toxcore/c-toxcore/toxcore/tox_api.c
+++ b/external/toxcore/c-toxcore/toxcore/tox_api.c
@@ -265,14 +265,6 @@ void tox_options_set_experimental_thread_safety(
 {
    options->experimental_thread_safety = experimental_thread_safety;
 }
-const Tox_System *tox_options_get_operating_system(const Tox_Options *options)
-{
-    return options->operating_system;
-}
-void tox_options_set_operating_system(Tox_Options *options, const Tox_System *operating_system)
-{
-    options->operating_system = operating_system;
-}
 bool tox_options_get_experimental_groups_persistence(const Tox_Options *options)
 {
    return options->experimental_groups_persistence;
@@ -1451,8 +1443,8 @@ const char *tox_err_group_invite_accept_to_string(Tox_Err_Group_Invite_Accept va
        case TOX_ERR_GROUP_INVITE_ACCEPT_PASSWORD:
            return "TOX_ERR_GROUP_INVITE_ACCEPT_PASSWORD";

-        case TOX_ERR_GROUP_INVITE_ACCEPT_CORE:
-            return "TOX_ERR_GROUP_INVITE_ACCEPT_CORE";
+        case TOX_ERR_GROUP_INVITE_ACCEPT_FRIEND_NOT_FOUND:
+            return "TOX_ERR_GROUP_INVITE_ACCEPT_FRIEND_NOT_FOUND";

        case TOX_ERR_GROUP_INVITE_ACCEPT_FAIL_SEND:
            return "TOX_ERR_GROUP_INVITE_ACCEPT_FAIL_SEND";
--- a/external/toxcore/c-toxcore/toxcore/tox_dispatch.h
+++ b/external/toxcore/c-toxcore/toxcore/tox_dispatch.h
@@ -2,6 +2,13 @@
 * Copyright © 2022 The TokTok team.
 */

+/**
+ * WARNING: This is an experimental API and is subject to change.
+ *
+ * At this point, it probably won't change very much anymore, but we may have
+ * small breaking changes before a stable release.
+ */
+
 #ifndef C_TOXCORE_TOXCORE_TOX_DISPATCH_H
 #define C_TOXCORE_TOXCORE_TOX_DISPATCH_H

--- a/external/toxcore/c-toxcore/toxcore/tox_events.h
+++ b/external/toxcore/c-toxcore/toxcore/tox_events.h
@@ -2,6 +2,13 @@
 * Copyright © 2022-2024 The TokTok team.
 */

+/**
+ * WARNING: This is an experimental API and is subject to change.
+ *
+ * At this point, it probably won't change very much anymore, but we may have
+ * small breaking changes before a stable release.
+ */
+
 #ifndef C_TOXCORE_TOXCORE_TOX_EVENTS_H
 #define C_TOXCORE_TOXCORE_TOX_EVENTS_H

@@ -570,6 +577,8 @@ void tox_events_free(Tox_Events *events);
 uint32_t tox_events_bytes_size(const Tox_Events *events);
 bool tox_events_get_bytes(const Tox_Events *events, uint8_t *bytes);

+typedef struct Tox_System Tox_System;
+
 Tox_Events *tox_events_load(const Tox_System *sys, const uint8_t *bytes, uint32_t bytes_size);

 bool tox_events_equal(const Tox_System *sys, const Tox_Events *a, const Tox_Events *b);
--- a/external/toxcore/c-toxcore/toxcore/tox_private.h
+++ b/external/toxcore/c-toxcore/toxcore/tox_private.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2020 The TokTok team.
+ * Copyright © 2016-2024 The TokTok team.
 * Copyright © 2013 Tox project.
 */

@@ -18,38 +18,49 @@ extern "C" {

 typedef uint64_t tox_mono_time_cb(void *user_data);

-struct Tox_System {
+typedef struct Tox_System {
    tox_mono_time_cb *mono_time_callback;
    void *mono_time_user_data;
    const struct Random *rng;
    const struct Network *ns;
    const struct Memory *mem;
-};
+} Tox_System;

 Tox_System tox_default_system(void);

+const Tox_System *tox_get_system(Tox *tox);
+
+typedef struct Tox_Options_Testing {
+    const struct Tox_System *operating_system;
+} Tox_Options_Testing;
+
+typedef enum Tox_Err_New_Testing {
+    TOX_ERR_NEW_TESTING_OK,
+    TOX_ERR_NEW_TESTING_NULL,
+} Tox_Err_New_Testing;
+
+Tox *tox_new_testing(const Tox_Options *options, Tox_Err_New *error, const Tox_Options_Testing *testing, Tox_Err_New_Testing *testing_error);
+
 void tox_lock(const Tox *tox);
 void tox_unlock(const Tox *tox);

-const Tox_System *tox_get_system(Tox *tox);
-
 /**
- * Set the callback for the `friend_lossy_packet` event for a specific packet ID.
- * Pass NULL to unset.
+ * Set the callback for the `friend_lossy_packet` event for a specific packet
+ * ID. Pass NULL to unset.
 *
 * allowed packet ID range:
- * from `PACKET_ID_RANGE_LOSSY_START` to `PACKET_ID_RANGE_LOSSY_END` (both inclusive)
+ * from `PACKET_ID_RANGE_LOSSY_START` to `PACKET_ID_RANGE_LOSSY_END` (both
+ * inclusive)
 */
 void tox_callback_friend_lossy_packet_per_pktid(Tox *tox, tox_friend_lossy_packet_cb *callback, uint8_t pktid);

 /**
- * Set the callback for the `friend_lossless_packet` event for a specific packet ID.
- * Pass NULL to unset.
+ * Set the callback for the `friend_lossless_packet` event for a specific packet
+ * ID. Pass NULL to unset.
 *
 * allowed packet ID range:
- * from `PACKET_ID_RANGE_LOSSLESS_CUSTOM_START` to `PACKET_ID_RANGE_LOSSLESS_CUSTOM_END` (both inclusive)
- * and
- * `PACKET_ID_MSI`
+ * from `PACKET_ID_RANGE_LOSSLESS_CUSTOM_START` to
+ * `PACKET_ID_RANGE_LOSSLESS_CUSTOM_END` (both inclusive) and `PACKET_ID_MSI`
 */
 void tox_callback_friend_lossless_packet_per_pktid(Tox *tox, tox_friend_lossless_packet_cb *callback, uint8_t pktid);

@@ -78,7 +89,7 @@ uint32_t tox_dht_node_public_key_size(void);

 /**
 * @param public_key The node's public key.
- * @param ip The node's IP address, represented as a null terminated string.
+ * @param ip The node's IP address, represented as a NUL-terminated C string.
 * @param port The node's port.
 */
 typedef void tox_dht_get_nodes_response_cb(Tox *tox, const uint8_t *public_key, const char *ip, uint16_t port,
@@ -98,7 +109,8 @@ typedef enum Tox_Err_Dht_Get_Nodes {
    TOX_ERR_DHT_GET_NODES_OK,

    /**
-     * UDP is disabled in tox options; the DHT can only be queried when UDP is enabled.
+     * UDP is disabled in Tox options; the DHT can only be queried when UDP is
+     * enabled.
     */
    TOX_ERR_DHT_GET_NODES_UDP_DISABLED,

@@ -118,21 +130,24 @@ typedef enum Tox_Err_Dht_Get_Nodes {
    TOX_ERR_DHT_GET_NODES_BAD_IP,

    /**
-     * The getnodes request failed. This usually means the packet failed to send.
+     * The getnodes request failed. This usually means the packet failed to
+     * send.
     */
    TOX_ERR_DHT_GET_NODES_FAIL,
 } Tox_Err_Dht_Get_Nodes;

 /**
 * This function sends a getnodes request to a DHT node for its peers that
- * are "close" to the passed target public key according to the distance metric used
- * by the DHT implementation.
+ * are "close" to the passed target public key according to the distance metric
+ * used by the DHT implementation.
 *
- * @param public_key The public key of the node that we wish to query. This key must be
- *   at least `TOX_DHT_NODE_PUBLIC_KEY_SIZE` bytes in length.
- * @param ip A NULL terminated string representing the IP address of the node we wish to query.
+ * @param public_key The public key of the node that we wish to query. This key
+ *   must be at least `TOX_DHT_NODE_PUBLIC_KEY_SIZE` bytes in length.
+ * @param ip A NUL-terminated C string representing the IP address of the node
+ *   we wish to query.
 * @param port The port of the node we wish to query.
- * @param target_public_key The public key for which we want to find close nodes.
+ * @param target_public_key The public key for which we want to find close
+ *   nodes.
 *
 * @return true on success.
 */
@@ -140,8 +155,9 @@ bool tox_dht_get_nodes(const Tox *tox, const uint8_t *public_key, const char *ip
                       const uint8_t *target_public_key, Tox_Err_Dht_Get_Nodes *error);

 /**
- * This function returns the ratio of close dht nodes that are known to support announce/store.
- * This function returns the number of DHT nodes in the closelist.
+ * This function returns the ratio of close dht nodes that are known to support
+ * announce/store. This function returns the number of DHT nodes in the
+ * closelist.
 *
 * @return number
 */
@@ -149,7 +165,7 @@ uint16_t tox_dht_get_num_closelist(const Tox *tox);

 /**
 * This function returns the number of DHT nodes in the closelist,
- * that are capable to store annouce data (introduced in version 0.2.18).
+ * that are capable to store announce data (introduced in version 0.2.18).
 *
 * @return number
 */
@@ -169,30 +185,32 @@ uint16_t tox_dht_get_num_closelist_announce_capable(const Tox *tox);
 uint32_t tox_group_peer_ip_string_max_length(void);

 /**
- * Return the length of the peer's IP address in string form. If the group number or ID
- * is invalid, the return value is unspecified.
+ * Return the length of the peer's IP address in string form. If the group
+ * number or ID is invalid, the return value is unspecified.
 *
 * @param group_number The group number of the group we wish to query.
- * @param peer_id The ID of the peer whose IP address length we want to retrieve.
+ * @param peer_id The ID of the peer whose IP address length we want to
+ *   retrieve.
 */
 size_t tox_group_peer_get_ip_address_size(const Tox *tox, uint32_t group_number, uint32_t peer_id,
        Tox_Err_Group_Peer_Query *error);
 /**
- * Write the IP address associated with the designated peer_id for the designated group number
- * to ip_addr.
+ * Write the IP address associated with the designated peer_id for the
+ * designated group number to ip_addr.
 *
- * If the peer is forcing TCP connections a placeholder value will be written instead,
- * indicating that their real IP address is unknown to us.
+ * If the peer is forcing TCP connections a placeholder value will be written
+ * instead, indicating that their real IP address is unknown to us.
 *
- * If `peer_id` designates ourself, it will write either our own IP address or a placeholder value,
- * depending on whether or not we're forcing TCP connections.
+ * If `peer_id` designates ourself, it will write either our own IP address or a
+ * placeholder value, depending on whether or not we're forcing TCP connections.
 *
- * Call tox_group_peer_get_ip_address_size to determine the allocation size for the `ip_addr` parameter.
+ * Call tox_group_peer_get_ip_address_size to determine the allocation size for
+ * the `ip_addr` parameter.
 *
 * @param group_number The group number of the group we wish to query.
 * @param peer_id The ID of the peer whose public key we wish to retrieve.
- * @param ip_addr A valid memory region large enough to store the IP address string.
- *   If this parameter is NULL, this function call has no effect.
+ * @param ip_addr A valid memory region large enough to store the IP address
+ *   string. If this parameter is NULL, this function call has no effect.
 *
 * @return true on success.
 */
--- a/external/toxcore/c-toxcore/toxencryptsave/toxencryptsave.c
+++ b/external/toxcore/c-toxcore/toxencryptsave/toxencryptsave.c
@@ -67,7 +67,8 @@ void tox_pass_key_free(Tox_Pass_Key *key)
 * produce the same key as was previously used. Any data encrypted with this
 * module can be used as input.
 *
- * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in length.
+ * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in
+ * length.
 * The salt must be TOX_PASS_SALT_LENGTH bytes in length.
 * If the passed byte arrays are smaller than required, the behaviour is
 * undefined.
@@ -182,10 +183,11 @@ Tox_Pass_Key *tox_pass_key_derive_with_salt(
 }

 /**
- * Encrypt a plain text with a key produced by tox_pass_key_derive or tox_pass_key_derive_with_salt.
+ * Encrypt a plain text with a key produced by tox_pass_key_derive or
+ * tox_pass_key_derive_with_salt.
 *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
- * bytes long.
+ * The output array must be at least
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long.
 *
 * @param plaintext A byte array of length `plaintext_len`.
 * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -242,9 +244,9 @@ bool tox_pass_key_encrypt(const Tox_Pass_Key *key, const uint8_t plaintext[], si
 /**
 * Encrypts the given data with the given passphrase.
 *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
- * bytes long. This delegates to tox_pass_key_derive and
- * tox_pass_key_encrypt.
+ * The output array must be at least
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This delegates
+ * to tox_pass_key_derive and tox_pass_key_encrypt.
 *
 * @param plaintext A byte array of length `plaintext_len`.
 * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -280,7 +282,8 @@ bool tox_pass_encrypt(const uint8_t plaintext[], size_t plaintext_len, const uin
 * tox_pass_key_derive or tox_pass_key_derive_with_salt.
 *
 * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
 * @param plaintext The plain text array to write the decrypted data to.
 *
 * @return true on success.
@@ -326,11 +329,13 @@ bool tox_pass_key_decrypt(const Tox_Pass_Key *key, const uint8_t ciphertext[], s
 /**
 * Decrypts the given data with the given passphrase.
 *
- * The output array must be at least `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
- * bytes long. This delegates to tox_pass_key_decrypt.
+ * The output array must be at least
+ * `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This
+ * delegates to tox_pass_key_decrypt.
 *
 * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
 * @param passphrase The user-provided password. Can be empty.
 * @param passphrase_len The length of the password.
 * @param plaintext The plain text array to write the decrypted data to.
--- a/external/toxcore/c-toxcore/toxencryptsave/toxencryptsave.h
+++ b/external/toxcore/c-toxcore/toxencryptsave/toxencryptsave.h
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2018 The TokTok team.
+ * Copyright © 2016-2024 The TokTok team.
 * Copyright © 2013-2016 Tox Developers.
 */

@@ -165,9 +165,9 @@ typedef enum Tox_Err_Decryption {
 /**
 * Encrypts the given data with the given passphrase.
 *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
- * bytes long. This delegates to tox_pass_key_derive and
- * tox_pass_key_encrypt.
+ * The output array must be at least
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This delegates
+ * to tox_pass_key_derive and tox_pass_key_encrypt.
 *
 * @param plaintext A byte array of length `plaintext_len`.
 * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -183,11 +183,13 @@ bool tox_pass_encrypt(const uint8_t plaintext[], size_t plaintext_len, const uin
 /**
 * Decrypts the given data with the given passphrase.
 *
- * The output array must be at least `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
- * bytes long. This delegates to tox_pass_key_decrypt.
+ * The output array must be at least
+ * `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This
+ * delegates to tox_pass_key_decrypt.
 *
 * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
 * @param passphrase The user-provided password. Can be empty.
 * @param passphrase_len The length of the password.
 * @param plaintext The plain text array to write the decrypted data to.
@@ -215,7 +217,8 @@ bool tox_pass_decrypt(const uint8_t ciphertext[], size_t ciphertext_len, const u
 * user-provided password.
 *
 * The Tox_Pass_Key structure is hidden in the implementation. It can be created
- * using tox_pass_key_derive or tox_pass_key_derive_with_salt and must be deallocated using tox_pass_key_free.
+ * using tox_pass_key_derive or tox_pass_key_derive_with_salt and must be
+ * deallocated using tox_pass_key_free.
 */
 #ifndef TOX_PASS_KEY_DEFINED
 #define TOX_PASS_KEY_DEFINED
@@ -261,10 +264,11 @@ Tox_Pass_Key *tox_pass_key_derive_with_salt(
    const uint8_t salt[TOX_PASS_SALT_LENGTH], Tox_Err_Key_Derivation *error);

 /**
- * Encrypt a plain text with a key produced by tox_pass_key_derive or tox_pass_key_derive_with_salt.
+ * Encrypt a plain text with a key produced by tox_pass_key_derive or
+ * tox_pass_key_derive_with_salt.
 *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
- * bytes long.
+ * The output array must be at least
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long.
 *
 * @param plaintext A byte array of length `plaintext_len`.
 * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -280,7 +284,8 @@ bool tox_pass_key_encrypt(const Tox_Pass_Key *key, const uint8_t plaintext[], si
 * tox_pass_key_derive or tox_pass_key_derive_with_salt.
 *
 * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
 * @param plaintext The plain text array to write the decrypted data to.
 *
 * @return true on success.
@@ -315,7 +320,8 @@ typedef enum Tox_Err_Get_Salt {
 * produce the same key as was previously used. Any data encrypted with this
 * module can be used as input.
 *
- * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in length.
+ * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in
+ * length.
 * The salt must be TOX_PASS_SALT_LENGTH bytes in length.
 * If the passed byte arrays are smaller than required, the behaviour is
 * undefined.
--- a/flake.lock
+++ b/flake.lock
@@ -34,10 +34,28 @@
        "type": "github"
      }
    },
+    "nlohmann-json": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1701207391,
+        "narHash": "sha256-7F0Jon+1oWL7uqet5i1IgHX0fUw/+z0QwEcA3zs5xHg=",
+        "owner": "nlohmann",
+        "repo": "json",
+        "rev": "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlohmann",
+        "ref": "v3.11.3",
+        "repo": "json",
+        "type": "github"
+      }
+    },
    "root": {
      "inputs": {
        "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nlohmann-json": "nlohmann-json"
      }
    },
    "systems": {
--- a/flake.nix
+++ b/flake.nix
@@ -6,19 +6,25 @@
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
    flake-utils.url = "github:numtide/flake-utils";
+    nlohmann-json = {
+      url = "github:nlohmann/json/v3.11.3"; # TODO: read version from file
+      flake = false;
+    };
  };

-  outputs = { self, nixpkgs, flake-utils }:
+  outputs = { self, nixpkgs, flake-utils, nlohmann-json }:
    flake-utils.lib.eachDefaultSystem (system:
    let
      pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
    in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
        pname = "tomato";
        version = "0.0.0";

        src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing

        nativeBuildInputs = with pkgs; [
          cmake
@@ -56,8 +62,12 @@
        ] ++ self.packages.${system}.default.dlopenBuildInputs;

        cmakeFlags = [
-          "TOMATO_ASAN=1"
-          "CMAKE_BUILD_TYPE=RelWithDebInfo"
+          "-DTOMATO_ASAN=OFF"
+          "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
+
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${nlohmann-json}" # we care about the version
+          # TODO: use package instead
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # we dont care about the version (we use 1.4.x features)
        ];

        # TODO: replace with install command
@@ -66,7 +76,7 @@
          mv bin/tomato $out/bin
        '';

-        dontStrip = true;
+        dontStrip = true; # does nothing

        # copied from nixpkgs's SDL2 default.nix
        # SDL is weird in that instead of just dynamically linking with
@@ -93,6 +103,8 @@
          '';
      };

+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
      devShells.${system}.default = pkgs.mkShell {
        #inputsFrom = with pkgs; [ SDL2 ];
        buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,5 +1,7 @@
 cmake_minimum_required(VERSION 3.9 FATAL_ERROR)

+########################################
+
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@@ -75,6 +77,7 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_util
 	solanaceae_contact
 	solanaceae_message3
+	solanaceae_message_serializer

 	solanaceae_plugin

@@ -82,6 +85,8 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_tox_messages

+	solanaceae_object_store
+
 	SDL3::SDL3

 	imgui
--- a/src/chat_gui4.cpp
+++ b/src/chat_gui4.cpp
@@ -35,8 +35,30 @@ namespace Components {
 		float fade {1.f};
 	};

+	struct ConvertedTimeCache {
+		// calling localtime is expensive af
+		int tm_year {0};
+		int tm_yday {0};
+		int tm_mon {0};
+		int tm_mday {0};
+		int tm_hour {0};
+		int tm_min {0};
+	};
+
 } // Components

+namespace Context {
+
+	// TODO: move back to chat log window and keep per window instead of per contact
+	struct CGView {
+		// set to the ts of the newest rendered msg
+		Message3Handle begin{};
+		// set to the ts of the oldest rendered msg
+		Message3Handle end{};
+	};
+
+} // Context
+
 static constexpr float lerp(float a, float b, float t) {
 	return a + t * (b - a);
 }
@@ -259,28 +281,6 @@ float ChatGui4::render(float time_delta) {

 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);

-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@@ -293,6 +293,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);

+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;

@@ -324,7 +327,8 @@ float ChatGui4::render(float time_delta) {
 						//tmp_view.use<Message::Components::Timestamp>();
 						//tmp_view.each([&](const Message3 e, Message::Components::ContactFrom& c_from, Message::Components::ContactTo& c_to, Message::Components::Timestamp ts
 						//) {
-						uint64_t prev_ts {0};
+						//uint64_t prev_ts {0};
+						Components::ConvertedTimeCache prev_time {};
 						auto tmp_view = msg_reg.view<Message::Components::Timestamp>();
 						for (auto view_it = tmp_view.rbegin(), view_last = tmp_view.rend(); view_it != view_last; view_it++) {
 							const Message3 e = *view_it;
@@ -342,15 +346,12 @@ float ChatGui4::render(float time_delta) {
 							// TODO: why?
 							ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);

-							{ // check if date changed
-								// TODO: find defined ways of casting to time_t
-								std::time_t prev = prev_ts / 1000;
-								std::time_t next = ts.ts / 1000;
-								std::tm prev_tm = *std::localtime(&prev);
-								std::tm next_tm = *std::localtime(&next);
+							if (msg_reg.all_of<Components::ConvertedTimeCache>(e)) { // check if date changed
+								// TODO: move conversion up?
+								const auto& next_time = msg_reg.get<Components::ConvertedTimeCache>(e);
 								if (
-									prev_tm.tm_yday != next_tm.tm_yday ||
-									prev_tm.tm_year != next_tm.tm_year // making sure
+									prev_time.tm_yday != next_time.tm_yday ||
+									prev_time.tm_year != next_time.tm_year // making sure
 								) {
 									// name
 									if (ImGui::TableNextColumn()) {
@@ -359,14 +360,14 @@ float ChatGui4::render(float time_delta) {
 									// msg
 									if (ImGui::TableNextColumn()) {
 										ImGui::TextDisabled("DATE CHANGED from %d.%d.%d to %d.%d.%d",
-											1900+prev_tm.tm_year, 1+prev_tm.tm_mon, prev_tm.tm_mday,
-											1900+next_tm.tm_year, 1+next_tm.tm_mon, next_tm.tm_mday
+											1900+prev_time.tm_year, 1+prev_time.tm_mon, prev_time.tm_mday,
+											1900+next_time.tm_year, 1+next_time.tm_mon, next_time.tm_mday
 										);
 									}
 									ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 								}

-								prev_ts = ts.ts;
+								prev_time = next_time;
 							}


@@ -381,12 +382,26 @@ float ChatGui4::render(float time_delta) {
 								}

 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
-									// get time now
-									const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
-									msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
-									msg_reg.remove<Message::Components::TagUnread>(e);
-									msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+								if (ImGui::IsItemVisible()) {
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
+										// get time now
+										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
+										msg_reg.remove<Message::Components::TagUnread>(e);
+										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+
+										// we remove the unread tag here
+										_rmm.throwEventUpdate(msg_reg, e);
+									}
+
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
 								}

 								// highlight self
@@ -519,12 +534,24 @@ float ChatGui4::render(float time_delta) {

 							// ts
 							if (ImGui::TableNextColumn()) {
-								auto time = std::chrono::system_clock::to_time_t(
-									std::chrono::time_point<std::chrono::system_clock, std::chrono::milliseconds>{std::chrono::milliseconds{ts.ts}}
-								);
-								auto localtime = std::localtime(&time);
+								if (!msg_reg.all_of<Components::ConvertedTimeCache>(e)) {
+									auto time = std::chrono::system_clock::to_time_t(
+										std::chrono::time_point<std::chrono::system_clock, std::chrono::milliseconds>{std::chrono::milliseconds{ts.ts}}
+									);
+									auto localtime = std::localtime(&time);
+									msg_reg.emplace<Components::ConvertedTimeCache>(
+										e,
+										localtime->tm_year,
+										localtime->tm_yday,
+										localtime->tm_mon,
+										localtime->tm_mday,
+										localtime->tm_hour,
+										localtime->tm_min
+									);
+								}
+								const auto& ctc = msg_reg.get<Components::ConvertedTimeCache>(e);

-								ImGui::Text("%.2d:%.2d", localtime->tm_hour, localtime->tm_min);
+								ImGui::Text("%.2d:%.2d", ctc.tm_hour, ctc.tm_min);
 							}

 							// extra
@@ -539,9 +566,90 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);

+						{ // update view cursers
+							if (!msg_reg.ctx().contains<Context::CGView>()) {
+								msg_reg.ctx().emplace<Context::CGView>();
+							}
+
+							auto& cg_view = msg_reg.ctx().get<Context::CGView>();
+
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view, we setup a view at current time, so the next frags are loaded
+								if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
+									// fix invalid state
+									if (static_cast<bool>(cg_view.begin)) {
+										cg_view.begin.destroy();
+										_rmm.throwEventDestroy(cg_view.begin);
+									}
+									if (static_cast<bool>(cg_view.end)) {
+										cg_view.end.destroy();
+										_rmm.throwEventDestroy(cg_view.end);
+									}
+
+									// create new
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									cg_view.end = {msg_reg, msg_reg.create()};
+
+									cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+									cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+									cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+									cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(cg_view.begin);
+									std::cout << "CG: created view FRONT end ts\n";
+									_rmm.throwEventConstruct(cg_view.end);
+								} // else? we do nothing?
+							} else {
+								bool begin_created {false};
+								if (!static_cast<bool>(cg_view.begin)) {
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								bool end_created {false};
+								if (!static_cast<bool>(cg_view.end)) {
+									cg_view.end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+								cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+								{
+									auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+										old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+										if (begin_created) {
+											std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.begin);
+										} else {
+											//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.begin);
+										}
+									}
+								}
+
+								{
+									auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+										old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+										if (end_created) {
+											std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.end);
+										} else {
+											//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.end);
+										}
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}

+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}
--- a/src/chat_gui4.hpp
+++ b/src/chat_gui4.hpp
@@ -32,6 +32,7 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;

+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;

 	// TODO: per contact
--- a/src/fragment_store/README.md
+++ b/src/fragment_store/README.md
@@ -1,72 +0,0 @@
-# Fragment Store
-
-Fragments are are pieces of information split into Metadata and Data.
-They can be stored seperated or together.
-They can be used as a Transport protocol/logic too.
-
-# Store types
-
-### Object Store
-
-Fragment files are stored with the first 2 hex chars as sub folders:
-eg:
-`objects/` (object store root)
-  - `5f/` (first 2hex subfolder)
-    - `4fffffff` (the fragment file without the first 2 hexchars)
-
-### Split Object Store
-
-Same as Object Store, but medadata and data stored in seperate files.
-Metadata files have the `.meta` suffix. They also have a filetype specific suffix, like `.json`, `.msgpack` etc.
-
-### Memory Store
-
-Just keeps the Fragments in memory.
-
-# File formats
-
-Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
-
-### Text Json
-
-Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
-Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
-
-A Metadata json object has the following keys:
- `enc` (uint) Encryption type of the data, if any
- `comp` (uint) Compression type of the data, if any
- `metadata` (obj) the 
-
-## Binary file headers
-
-### Split Metadata
-
-file magic bytes `SOLMET` (6 bytes)
-
-1 byte encryption type (`0x00` is none)
-
-1 byte compression type (`0x00` is none)
-
-...metadata here...
-
-note that the encryption and compression are for the metadata only.
-The metadata itself contains encryption and compression info about the data.
-
-### Split Data
-
-(none) all the data is in the metadata file.
-This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
-Keep in mind to not use the actual file name as the data/meta file name.
-
-### Single fragment
-
-file magic bytes `SOLFIL` (6 bytes)
-
-1 byte encryption type (`0x00` is none)
-
-1 byte compression type (`0x00` is none)
-
-...metadata here...
-
-...data here...
-
--- a/src/image_loader_webp.cpp
+++ b/src/image_loader_webp.cpp
@@ -1,5 +1,6 @@
 #include "./image_loader_webp.hpp"

+#include <memory>
 #include <webp/demux.h>
 #include <webp/mux.h>
 #include <webp/encode.h>
@@ -21,13 +22,16 @@ ImageLoaderWebP::ImageInfo ImageLoaderWebP::loadInfoFromMemory(const uint8_t* da
 	// Tune 'dec_options' as needed.
 	dec_options.color_mode = MODE_RGBA;

-	WebPAnimDecoder* dec = WebPAnimDecoderNew(&webp_data, &dec_options);
-	if (dec == nullptr) {
+	std::unique_ptr<WebPAnimDecoder, decltype(&WebPAnimDecoderDelete)> dec{
+		WebPAnimDecoderNew(&webp_data, &dec_options),
+		&WebPAnimDecoderDelete
+	};
+	if (!static_cast<bool>(dec)) {
 		return res;
 	}

 	WebPAnimInfo anim_info;
-	WebPAnimDecoderGetInfo(dec, &anim_info);
+	WebPAnimDecoderGetInfo(dec.get(), &anim_info);
 	res.width = anim_info.canvas_width;
 	res.height = anim_info.canvas_height;
 	res.file_ext = "webp";
@@ -48,22 +52,25 @@ ImageLoaderWebP::ImageResult ImageLoaderWebP::loadFromMemoryRGBA(const uint8_t*
 	// Tune 'dec_options' as needed.
 	dec_options.color_mode = MODE_RGBA;

-	WebPAnimDecoder* dec = WebPAnimDecoderNew(&webp_data, &dec_options);
-	if (dec == nullptr) {
+	std::unique_ptr<WebPAnimDecoder, decltype(&WebPAnimDecoderDelete)> dec{
+		WebPAnimDecoderNew(&webp_data, &dec_options),
+		&WebPAnimDecoderDelete
+	};
+	if (!static_cast<bool>(dec)) {
 		return res;
 	}

 	WebPAnimInfo anim_info;
-	WebPAnimDecoderGetInfo(dec, &anim_info);
+	WebPAnimDecoderGetInfo(dec.get(), &anim_info);
 	res.width = anim_info.canvas_width;
 	res.height = anim_info.canvas_height;
 	res.file_ext = "webp";

 	int prev_timestamp = 0;
-	while (WebPAnimDecoderHasMoreFrames(dec)) {
+	while (WebPAnimDecoderHasMoreFrames(dec.get())) {
 		uint8_t* buf;
 		int timestamp;
-		WebPAnimDecoderGetNext(dec, &buf, &timestamp);
+		WebPAnimDecoderGetNext(dec.get(), &buf, &timestamp);
 		// ... (Render 'buf' based on 'timestamp').
 		// ... (Do NOT free 'buf', as it is owned by 'dec').

@@ -74,8 +81,6 @@ ImageLoaderWebP::ImageResult ImageLoaderWebP::loadFromMemoryRGBA(const uint8_t*
 		new_frame.data.insert(new_frame.data.end(), buf, buf+(res.width*res.height*4));
 	}

-	WebPAnimDecoderDelete(dec);
-
 	assert(anim_info.frame_count == res.frames.size());

 	return res;
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -10,7 +10,6 @@
 #include "./start_screen.hpp"

 #include <memory>
-#include <future>
 #include <iostream>
 #include <thread>
 #include <chrono>
@@ -29,8 +28,6 @@ int main(int argc, char** argv) {
 		std::cerr << "SDL_Init failed (" << SDL_GetError() << ")\n";
 		return 1;
 	}
-	// me just messing with RAII cleanup
-	auto sdl_scope = std::async(std::launch::deferred, &SDL_Quit);

 	// more RAII
 	std::unique_ptr<SDL_Window, decltype(&SDL_DestroyWindow)> window {
@@ -86,9 +83,7 @@ int main(int argc, char** argv) {
 	}

 	ImGui_ImplSDL3_InitForSDLRenderer(window.get(), renderer.get());
-	auto imgui_sdl_scope = std::async(std::launch::deferred, &ImGui_ImplSDL3_Shutdown);
 	ImGui_ImplSDLRenderer3_Init(renderer.get());
-	auto imgui_sdlrenderer_scope = std::async(std::launch::deferred, &ImGui_ImplSDLRenderer3_Shutdown);

 	std::unique_ptr<Screen> screen = std::make_unique<StartScreen>(renderer.get());

@@ -213,6 +208,19 @@ int main(int argc, char** argv) {
 #endif
 	}

+	// TODO: use scope for the unique ptrs
+
+	screen.reset();
+
+	ImGui_ImplSDLRenderer3_Shutdown();
+	ImGui_ImplSDL3_Shutdown();
+	ImGui::DestroyContext();
+
+	renderer.reset();
+	window.reset();
+
+	SDL_Quit();
+
 	return 0;
 }

--- a/src/main_screen.cpp
+++ b/src/main_screen.cpp
@@ -1,5 +1,8 @@
 #include "./main_screen.hpp"

+#include <solanaceae/message3/nj/message_components_serializer.hpp>
+#include <solanaceae/tox_messages/nj/tox_message_components_serializer.hpp>
+
 #include <solanaceae/contact/components.hpp>

 #include <imgui/imgui.h>
@@ -12,6 +15,7 @@
 MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::string save_password, std::vector<std::string> plugins) :
 	renderer(renderer_),
 	rmm(cr),
+	msnj{cr, {}, {}},
 	mts(rmm),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
@@ -34,6 +38,9 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 {
 	tel.subscribeAll(tc);

+	registerMessageComponents(msnj);
+	registerToxMessageComponents(msnj);
+
 	conf.set("tox", "save_file_path", save_path);

 	{ // name stuff
@@ -49,9 +56,12 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";

 	{ // setup plugin instances
+		g_provideInstance<ObjectStore2>("ObjectStore2", "host", &os);
+
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
+		g_provideInstance<MessageSerializerNJ>("MessageSerializerNJ", "host", &msnj);

 		g_provideInstance<ToxI>("ToxI", "host", &tc);
 		g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
@@ -215,9 +225,24 @@ Screen* MainScreen::render(float time_delta, bool&) {
 					ImGui::EndMenu();
 				}
 				if (ImGui::BeginMenu("Settings")) {
-					if (ImGui::MenuItem("ImGui Style Editor")) {
+					ImGui::SeparatorText("ImGui");
+
+					if (ImGui::MenuItem("Style Editor")) {
 						_show_tool_style_editor = true;
 					}
+
+					if (ImGui::MenuItem("Metrics")) {
+						_show_tool_metrics = true;
+					}
+
+					if (ImGui::MenuItem("Debug Log")) {
+						_show_tool_debug_log = true;
+					}
+
+					if (ImGui::MenuItem("ID Stack Tool")) {
+						_show_tool_id_stack = true;
+					}
+
 					ImGui::EndMenu();
 				}
 				ImGui::EndMenuBar();
@@ -234,6 +259,18 @@ Screen* MainScreen::render(float time_delta, bool&) {
 		ImGui::End();
 	}

+	if (_show_tool_metrics) {
+		ImGui::ShowMetricsWindow(&_show_tool_metrics);
+	}
+
+	if (_show_tool_debug_log) {
+		ImGui::ShowDebugLogWindow(&_show_tool_debug_log);
+	}
+
+	if (_show_tool_id_stack) {
+		ImGui::ShowIDStackToolWindow(&_show_tool_id_stack);
+	}
+
 	if constexpr (false) {
 		ImGui::ShowDemoWindow();
 	}
@@ -376,7 +413,7 @@ Screen* MainScreen::render(float time_delta, bool&) {
 }

 Screen* MainScreen::tick(float time_delta, bool& quit) {
-	quit = !tc.iterate(); // compute
+	quit = !tc.iterate(time_delta); // compute

 	tcm.iterate(time_delta); // compute

@@ -388,7 +425,7 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {

 	tdch.tick(time_delta); // compute

-	mts.iterate(); // compute
+	mts.iterate(); // compute (after mfs)

 	_min_tick_interval = std::min<float>(
 		// HACK: pow by 1.6 to increase 50 -> ~500 (~522)
--- a/src/main_screen.hpp
+++ b/src/main_screen.hpp
@@ -2,10 +2,12 @@

 #include "./screen.hpp"

+#include <solanaceae/object_store/object_store.hpp>
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
+#include <solanaceae/message3/message_serializer.hpp>
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@@ -43,13 +45,14 @@ extern "C" {
 struct MainScreen final : public Screen {
 	SDL_Renderer* renderer;

+	ObjectStore2 os;
+
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
+	MessageSerializerNJ msnj;
 	MessageTimeSort mts;

-	PluginManager pm;
-
 	ToxEventLogger tel{std::cout};
 	ToxClient tc;
 	ToxPrivateImpl tpi;
@@ -75,7 +78,12 @@ struct MainScreen final : public Screen {
 	ToxUIUtils tuiu;
 	ToxDHTCapHisto tdch;

+	PluginManager pm; // last, so it gets destroyed first
+
 	bool _show_tool_style_editor {false};
+	bool _show_tool_metrics {false};
+	bool _show_tool_debug_log {false};
+	bool _show_tool_id_stack {false};

 	bool _window_hidden {false};
 	uint64_t _window_hidden_ts {0};
--- a/src/send_image_popup.cpp
+++ b/src/send_image_popup.cpp
@@ -83,6 +83,7 @@ bool SendImagePopup::load(void) {
 			}
 		}

+		assert(preview_image.textures.empty());
 		preview_image.timestamp_last_rendered = Message::getTimeMS();
 		preview_image.current_texture = 0;
 		for (const auto& [ms, data] : original_image.frames) {
@@ -173,7 +174,7 @@ void SendImagePopup::render(float time_delta) {

 	// TODO: add cancel shortcut (esc)
 	if (ImGui::BeginPopupModal("send image##SendImagePopup", nullptr/*, ImGuiWindowFlags_NoDecoration*/)) {
-		const auto TEXT_BASE_WIDTH = ImGui::CalcTextSize("A").x;
+		//const auto TEXT_BASE_WIDTH = ImGui::CalcTextSize("A").x;
 		const auto TEXT_BASE_HEIGHT = ImGui::GetTextLineHeightWithSpacing();

 		preview_image.doAnimation(Message::getTimeMS());
--- a/src/tox_client.cpp
+++ b/src/tox_client.cpp
@@ -1,5 +1,4 @@
 #include "./tox_client.hpp"
-#include "toxcore/tox.h"

 // meh, change this
 #include <exception>
@@ -121,10 +120,13 @@ ToxClient::ToxClient(std::string_view save_path, std::string_view save_password)
 }

 ToxClient::~ToxClient(void) {
+	if (_tox_profile_dirty) {
+		saveToxProfile();
+	}
 	tox_kill(_tox);
 }

-bool ToxClient::iterate(void) {
+bool ToxClient::iterate(float time_delta) {
 	Tox_Err_Events_Iterate err_e_it = TOX_ERR_EVENTS_ITERATE_OK;
 	auto* events = tox_events_iterate(_tox, false, &err_e_it);
 	if (err_e_it == TOX_ERR_EVENTS_ITERATE_OK && events != nullptr) {
@@ -136,7 +138,8 @@ bool ToxClient::iterate(void) {

 	tox_events_free(events);

-	if (_tox_profile_dirty) {
+	_save_heat -= time_delta;
+	if (_tox_profile_dirty && _save_heat <= 0.f) {
 		saveToxProfile();
 	}

@@ -180,5 +183,6 @@ void ToxClient::saveToxProfile(void) {
 	}

 	_tox_profile_dirty = false;
+	_save_heat = 10.f;
 }

--- a/src/tox_client.hpp
+++ b/src/tox_client.hpp
@@ -22,6 +22,7 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase {
 		std::string _tox_profile_path;
 		std::string _tox_profile_password;
 		bool _tox_profile_dirty {true}; // set in callbacks
+		float _save_heat {0.f};

 	public:
 		//ToxClient(/*const CommandLine& cl*/);
@@ -34,7 +35,7 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase {
 		void setDirty(void) { _tox_profile_dirty = true; }

 		// returns false when we shoul stop the program
-		bool iterate(void);
+		bool iterate(float time_delta);
 		void stop(void); // let it know it should exit

 		void setToxProfilePath(const std::string& new_path) { _tox_profile_path = new_path; }
@@ -47,4 +48,3 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase {
 		void saveToxProfile(void);
 };

-
--- a/src/tox_friend_faux_offline_messaging.cpp
+++ b/src/tox_friend_faux_offline_messaging.cpp
@@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
 		// require
 		if (!mr->all_of<
 				Message::Components::MessageText, // text only for now
-				Message::Components::ContactTo
+				Message::Components::ContactTo,
+				Message::Components::ToxFriendMessageID // yes, needs fake ids
 			>(msg)
 		) {
 			continue; // skip
Author	SHA1	Message	Date
Erik Scholz	8c24234126	Merge pull request #8 from Green-Sky/fragment_store ObjectStore backed MessageFragments	2024-04-14 14:38:49 +02:00
Green Sky	e0d873d41c	Merge branch 'master' into fragment_store	2024-04-14 14:32:31 +02:00
Green Sky	5728432b76	everything put into their respective repos	2024-04-14 14:30:42 +02:00
Green Sky	0030487613	move converter tool and pics out	2024-04-14 14:16:53 +02:00
Green Sky	f932f5ffb4	mfs and ms moved to their own repo, now only a few files left to clean up	2024-04-14 13:58:31 +02:00
Green Sky	da83065024	make sure we have the right json version in the flake	2024-04-14 12:03:58 +02:00
Green Sky	a6614e76ce	move cursers to public api	2024-04-14 11:13:05 +02:00
Green Sky	cdd67f4779	fix flake and pull in os zstd fixes	2024-04-14 10:57:07 +02:00
Green Sky	a845609660	actually allow loading v2 and enable in converter	2024-04-14 10:23:13 +02:00
Green Sky	de3b8f059e	import os and zstdfile2 bugfixes	2024-04-13 21:06:51 +02:00
Green Sky	7b4af58544	prep convert tool for msgpack transition	2024-04-13 19:47:28 +02:00
Green Sky	f287348550	introduce message fragments version 2 (msgpack) more smaller refactors	2024-04-13 19:13:18 +02:00
Green Sky	498b4435c7	refactor message contexts	2024-04-13 11:38:13 +02:00
Green Sky	a5e67d0ee8	move uuid gen to util	2024-04-12 19:44:24 +02:00
Green Sky	9e30983b22	Merge branch 'master' into fragment_store	2024-04-12 19:24:20 +02:00
Green Sky	195a87b8ab	add object store and expose to plugins	2024-04-12 19:03:30 +02:00
Green Sky	dfbb1dea68	move os and backend to sub	2024-04-12 13:42:08 +02:00
Green Sky	2597edd579	new messages objcomp names	2024-04-12 13:42:08 +02:00
Green Sky	85a29372f4	adding message frag object version and conversion work, but comp name changes incoming	2024-04-12 13:42:07 +02:00
Green Sky	a9f6a5d763	move mfs to os, works, convert tool still incomplete	2024-04-12 13:42:07 +02:00
Green Sky	73180195fe	some more backend interface changes i realized i had to do	2024-04-12 13:42:07 +02:00
Green Sky	10b689ca95	refactor the serializer again	2024-04-12 13:42:07 +02:00
Green Sky	3796841961	conversion improvements	2024-04-12 13:42:07 +02:00
Green Sky	854ed851b4	fixes missing comps and conversion tool is almost working (most of meta is transfered)	2024-04-12 13:42:07 +02:00
Green Sky	5c3b797a99	new object (os + fsb mostly done)	2024-04-12 13:42:06 +02:00
Green Sky	8a580e2fbb	backend fs read	2024-04-12 13:42:06 +02:00
Green Sky	3cede91aa0	more work on backend and moving frags to objs	2024-04-12 13:42:06 +02:00
Green Sky	0610a6a64a	continue os refactor, start with fs backend	2024-04-12 13:42:06 +02:00
Green Sky	26d07b06db	start refactoring in the name of object store	2024-04-12 13:42:06 +02:00
Green Sky	fd0b210bbb	fragment store fully file2 zstd. no more zstd.h in fragment store	2024-04-12 13:42:06 +02:00
Green Sky	268cbe137e	move file stack creation to separate file, removing some scope	2024-04-12 13:42:05 +02:00
Green Sky	31bb0d3e61	refactor meta data decompression using file2 and mem	2024-04-12 13:42:05 +02:00
Green Sky	248f68f6a2	minimal stack creation refactor (wip)	2024-04-12 13:42:05 +02:00
Green Sky	3b010bd16f	catch write errors (eg unsanitzed strings)	2024-04-12 13:42:05 +02:00
Green Sky	7e285290fe	test in memory comp	2024-04-12 13:42:05 +02:00
Green Sky	5767834f71	since its working, remove old commented code	2024-04-12 13:42:05 +02:00
Green Sky	53ce292e82	fix stack overflow (i am stupid) and some file stack error handling	2024-04-12 13:42:04 +02:00
Green Sky	84bd24807d	small change to flush work space. writing new frags is broken rn	2024-04-12 13:42:04 +02:00
Green Sky	8d0518c2e3	harden against some parsing exceptions (by disabling them, since the error case is already handled)	2024-04-12 13:42:04 +02:00
Green Sky	6d150ba441	roll back meta comp (did it wrong) and enable data comp	2024-04-12 13:42:04 +02:00
Green Sky	c737715c66	more refactor and transition meta write	2024-04-12 13:42:04 +02:00
Green Sky	b640b5a06b	variant to span helper	2024-04-12 13:42:04 +02:00
Green Sky	1b9363e7b5	tested and works, cleaning up commented code	2024-04-12 13:42:03 +02:00
Green Sky	16d2238f35	fixes for ci	2024-04-12 13:42:03 +02:00
Green Sky	19844a9423	use file2 zstd wrapper to read frag data (untested)	2024-04-12 13:42:03 +02:00
Green Sky	19fd99f713	refactor out uuidgenerator	2024-04-12 13:42:03 +02:00
Green Sky	f22f523774	minor frag store refactor	2024-04-12 13:42:03 +02:00
Green Sky	8b17ed195f	more testing and file2 zstd now passes tests with varying frame sizes and 1.5gig files	2024-04-12 13:42:03 +02:00
Green Sky	def7fc1959	add file2 impl for zstd (lightly tested and not integrated yet)	2024-04-12 13:42:02 +02:00
Green Sky	318be9cd62	throw update	2024-04-12 13:42:02 +02:00
Green Sky	2772c8ee69	reduce excessive message frag saving (queue dedup + waiting 10sec) prepare for frag updates	2024-04-12 13:42:02 +02:00
Green Sky	eac2927379	try to tame log spam	2024-04-12 13:42:02 +02:00
Green Sky	77a0ae6acd	fix accel structure being wrong and mark empty frags and dont count them	2024-04-12 13:42:02 +02:00
Green Sky	7879a0927b	combat memory leaks with smart pointers	2024-04-12 13:42:02 +02:00
Green Sky	88ea3e177d	refactor saving and save on exit	2024-04-12 13:42:01 +02:00
Green Sky	bc22451524	dirty frag on message updates (if still open)	2024-04-12 13:42:01 +02:00
Green Sky	7b8e93eec3	refactor message fuid -> fid save alot of memory by using fid instead of fuid	2024-04-12 13:42:01 +02:00
Green Sky	71be5c3c6e	reduce log spam	2024-04-12 13:42:01 +02:00
Green Sky	2b8cee6a29	remove old code	2024-04-12 13:42:01 +02:00
Green Sky	5bf4640d61	forgot to throw update on read	2024-04-12 13:42:01 +02:00
Green Sky	0e0e81720b	dont sync messages we dont know enough about	2024-04-12 13:42:00 +02:00
Green Sky	592a4cb9cf	make adjacency loading work, extend range and use loops	2024-04-12 13:42:00 +02:00
Green Sky	93f60bd073	replace old bad prev/next code with way better code	2024-04-12 13:42:00 +02:00
Green Sky	6a6de77ae9	smaller contact frag fixes	2024-04-12 13:42:00 +02:00
Green Sky	89f065a610	impl new acceleration structure for components, not exploited yet disable funky load at first msg	2024-04-12 13:42:00 +02:00
Green Sky	52e95ca654	forgot to check contact	2024-04-12 13:41:59 +02:00
Green Sky	eaa316a2aa	rework cursers for cg, keep views between switching. will be refactored later	2024-04-12 13:41:59 +02:00
Green Sky	bdf4e60f2f	fix one inverted comparator	2024-04-12 13:41:59 +02:00
Green Sky	2e3c779bec	stop ignoring mfs interval and sort after	2024-04-12 13:41:59 +02:00
Green Sky	461a4f1aa7	make inital curser a range	2024-04-12 13:41:59 +02:00
Green Sky	78488daa9b	loading logic implemented but broken (very funky and sometimes even out of contact)	2024-04-12 13:41:59 +02:00
Green Sky	22f2c8f514	load based on view cursers (untested and not used yet)	2024-04-12 13:41:58 +02:00
Green Sky	e442191aad	msg frag before and after helper	2024-04-12 13:41:58 +02:00
Green Sky	795ab2d4e1	fix potential tsrange errors and deduplicate state	2024-04-12 13:41:58 +02:00
Green Sky	0896038dd6	make writing safe (by using a tmp file and moving to actual location)	2024-04-12 13:41:58 +02:00
Green Sky	67c6f9adb0	make empty contacts from ids on message load	2024-04-12 13:41:58 +02:00
Green Sky	6aac44cda9	change binary meta format and add zstd to metadata	2024-04-12 13:41:58 +02:00
Green Sky	4fb2b51b7d	switch to streaming compressor for data to drastically improve ratio. would still benefit from a abstract file refactor	2024-04-12 13:41:57 +02:00
Green Sky	182d844e32	update fs readme a little	2024-04-12 13:41:57 +02:00
Green Sky	6f511016bc	save msg json zstd compressed (3x compression)	2024-04-12 13:41:57 +02:00
Green Sky	fb885b5c21	simplify array cast a little	2024-04-12 13:41:57 +02:00
Green Sky	527a7c63f6	add zstd dep	2024-04-12 13:41:57 +02:00
Green Sky	d21dbb43e2	comp refactor and make groups work	2024-04-12 13:41:57 +02:00
Green Sky	7ac62274f4	move json around and disable files for now	2024-04-12 13:41:56 +02:00
Green Sky	4ec87337c8	reverse message write order	2024-04-12 13:41:56 +02:00
Green Sky	20f7c6d011	add dup check, would work for ngc if we saved tox group msg id yet	2024-04-12 13:41:56 +02:00
Green Sky	24dc5a03f3	fix dup on write	2024-04-12 13:41:56 +02:00
Green Sky	3d0863ff9a	basically working, but some dup glitch is still there	2024-04-12 13:41:56 +02:00
Green Sky	97aedca844	scan laters	2024-04-12 13:41:56 +02:00
Green Sky	2e7d5538d1	fragment events + 256bit uuids	2024-04-12 13:41:55 +02:00
Green Sky	1bfd04680e	refactor message serializer to allow access to eg contacts	2024-04-12 13:41:55 +02:00
Green Sky	73d1d65142	further serializer refactoring	2024-04-12 13:41:55 +02:00
Green Sky	f6e55851cc	improve deserialization and provide message comp deserl	2024-04-12 13:41:55 +02:00
Green Sky	0b0245d844	loading fragments mostly working (not notifying anyone yet)	2024-04-12 13:41:55 +02:00
Green Sky	d278391528	add contact id to meta	2024-04-12 13:41:55 +02:00
Green Sky	aa7a5d6013	more comps	2024-04-12 13:41:54 +02:00
Green Sky	84987216cb	handle empty type	2024-04-12 13:41:54 +02:00
Green Sky	58e9fd5514	dump messages to data (some comps)	2024-04-12 13:41:54 +02:00
Green Sky	3d41eedf48	message fragment meta is saved, but still empty data	2024-04-12 13:41:54 +02:00
Green Sky	2bc30ffcdc	start with messages (no fragments get created yet)	2024-04-12 13:41:54 +02:00
Green Sky	e67d7d37b5	refactoring, add to mainscreen	2024-04-12 13:41:54 +02:00
Green Sky	98ab974515	random ids	2024-04-12 13:41:53 +02:00
Green Sky	267f8dffc1	working prototpying code	2024-04-12 13:41:53 +02:00
Green Sky	b38a1a2507	better github cd	2024-04-12 13:40:58 +02:00
Green Sky	ee87dbc532	update utils	2024-04-12 13:40:35 +02:00
Green Sky	78c7ef5b46	fix main screen destruction order, so plugins can cleanup before their facilities get destroyed	2024-04-06 11:59:05 +02:00
Green Sky	95ec596cfd	Merge commit '47ad96e2b6c5704872b3615a160cd78d29b46b21' toxcore v0.2.19	2024-03-28 16:13:51 +01:00
Green Sky	47ad96e2b6	Squashed 'external/toxcore/c-toxcore/' changes from 3e05824b80..da438763d5 da438763d5 chore: Release 0.2.19 f90417987c chore: Add cmake flag to disable unit tests. 7df3f99417 docs: Document that group topic lock is default on. 9e9ed77390 docs: Add missing param docs for callbacks. 0ec4978de5 refactor: Don't expose Tox_System in the public API a3d1b8595c docs: Public headers, Core/toxcore -> Tox/the Tox library f78d0f3f39 docs: Public headers, events_alloc -> internal 817518949e docs: Public headers, NULL-terminated -> NUL-terminated be085db191 docs: Public headers, spellcheck 4c902955f3 docs: Public headers, 80 column width comments be8a82a818 docs: Public headers, null -> NULL 419d783d95 docs: Public headers, tox -> Tox 5c8aa65e41 docs: Update user data API explanation ad4921dbaa cleanup: A more descriptive error for group invite accept function git-subtree-dir: external/toxcore/c-toxcore git-subtree-split: da438763d5b8e071de6e061a1dcaddd2177dff7d	2024-03-28 16:13:51 +01:00
Green Sky	396477f66b	update utils	2024-03-28 16:11:26 +01:00
Green Sky	01edf9e76e	cleanup	2024-03-16 16:43:45 +01:00
Green Sky	bde0f2c7c3	imgui debug stuff	2024-03-15 13:02:49 +01:00
Green Sky	83bbac2cd1	proper cleanup procedure for main	2024-03-15 13:02:37 +01:00
Green Sky	d5dede5a61	properly cleanup in main	2024-03-12 18:43:36 +01:00
Green Sky	719400068a	cache the localtime result in the message localtime uses global state AND spams allocations like cazy	2024-03-12 18:09:42 +01:00
Green Sky	aaf8c6adc1	save cooldown for tox profile (10sec after last) and save on exit	2024-03-12 18:09:21 +01:00
Green Sky	dc081ae2aa	fix webp mem leak	2024-03-11 20:46:28 +01:00