Merge pull request #8 from Green-Sky/fragment_store

ObjectStore backed MessageFragments

.github/workflows/cd.yml

@@ -37,10 +37,10 @@ jobs:
       run: |
         SHORT_HASH="$(git rev-parse --short=7 HEAD)"
         if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
         else
           SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
         fi
 
     - name: Compress artifacts
@@ -88,10 +88,10 @@ jobs:
       run: |
         SHORT_HASH="$(git rev-parse --short=7 HEAD)"
         if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
         else
           SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
         fi
 
     - name: Compress artifacts
@@ -139,10 +139,10 @@ jobs:
       run: |
         SHORT_HASH="$(git rev-parse --short=7 HEAD)"
         if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
         else
           SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
         fi
 
     - name: Compress artifacts
@@ -183,10 +183,10 @@ jobs:
       run: |
         SHORT_HASH="$(git rev-parse --short=7 HEAD)"
         if [[ "${{ env.BRANCH_NAME }}" == "master" ]]; then
-          echo "name=${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SHORT_HASH}" >> $GITHUB_OUTPUT
         else
           SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
-          echo "name=${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
+          echo "name=dev-${SAFE_NAME}-${SHORT_HASH}" >> $GITHUB_OUTPUT
         fi
 
     - name: Download artifacts
@@ -203,8 +203,8 @@ jobs:
       run: |
         gh release create "$tag" \
             --repo="$GITHUB_REPOSITORY" \
-            --title="nightly ${tag#v}" \
+            --title="${tag#v}" \
-            --notes="nightly build" \
+            --notes="preview build of the latest commit" \
             --prerelease
 
     - name: Upload artifacts

.gitmodules

@@ -20,3 +20,9 @@
 [submodule "external/solanaceae_plugin"]
 	path = external/solanaceae_plugin
 	url = https://github.com/Green-Sky/solanaceae_plugin.git
+[submodule "external/solanaceae_object_store"]
+	path = external/solanaceae_object_store
+	url = https://github.com/Green-Sky/solanaceae_object_store.git
+[submodule "external/solanaceae_message_serializer"]
+	path = external/solanaceae_message_serializer
+	url = https://github.com/Green-Sky/solanaceae_message_serializer.git

CMakeLists.txt

@@ -23,8 +23,8 @@ option(TOMATO_ASAN "Build tomato with asan (gcc/clang/msvc)" OFF)
 if (TOMATO_ASAN)
 	if (${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" OR ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
 		if (NOT WIN32) # exclude mingw
-			link_libraries(-fsanitize=address)
+			#link_libraries(-fsanitize=address)
-			#link_libraries(-fsanitize=address,undefined)
+			link_libraries(-fsanitize=address,undefined)
 			#link_libraries(-fsanitize=undefined)
 			message("II enabled ASAN")
 		else()

external/CMakeLists.txt

@@ -1,10 +1,11 @@
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14...3.24 FATAL_ERROR)
 
 add_subdirectory(./entt)
 
 add_subdirectory(./solanaceae_util)
 add_subdirectory(./solanaceae_contact)
 add_subdirectory(./solanaceae_message3)
+add_subdirectory(./solanaceae_message_serializer)
 
 add_subdirectory(./solanaceae_plugin)
 
@@ -12,6 +13,8 @@ add_subdirectory(./toxcore)
 add_subdirectory(./solanaceae_toxcore)
 add_subdirectory(./solanaceae_tox)
 
+add_subdirectory(./solanaceae_object_store)
+
 add_subdirectory(./sdl)
 add_subdirectory(./imgui)
 

external/toxcore/c-toxcore/.cirrus.yml

@@ -7,13 +7,14 @@ bazel-opt_task:
   configure_script:
     - git submodule update --init --recursive
     - /src/workspace/tools/inject-repo c-toxcore
+    - sed -i -e 's/build --config=remote/#&/' /src/workspace/.bazelrc.local
   test_all_script:
     - cd /src/workspace && bazel test -k
         --build_tag_filters=-haskell
         --test_tag_filters=-haskell
         --
         //c-toxcore/...
-        -//c-toxcore/auto_tests:tcp_relay_test # TODO(robinlinden): Why does this pass locally but not in Cirrus?
+        -//c-toxcore/auto_tests:tcp_relay_test # Cirrus doesn't allow external network connections.
 
 bazel-dbg_task:
   container:
@@ -30,7 +31,7 @@ bazel-dbg_task:
         --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST
         --
         //c-toxcore/...
-        -//c-toxcore/auto_tests:tcp_relay_test # TODO(robinlinden): Why does this pass locally but not in Cirrus?
+        -//c-toxcore/auto_tests:tcp_relay_test # Cirrus doesn't allow external network connections.
 
 cimple_task:
   container:
@@ -40,6 +41,7 @@ cimple_task:
   configure_script:
     - git submodule update --init --recursive
     - /src/workspace/tools/inject-repo c-toxcore
+    - sed -i -e 's/build --config=remote/#&/' /src/workspace/.bazelrc.local
   test_all_script:
     - cd /src/workspace && bazel test -k
         --build_tag_filters=haskell
@@ -69,7 +71,7 @@ freebsd_task:
       cmake . \
         -DMIN_LOGGER_LEVEL=TRACE \
         -DMUST_BUILD_TOXAV=ON \
-        -DNON_HERMETIC_TESTS=ON \
+        -DNON_HERMETIC_TESTS=OFF \
         -DTEST_TIMEOUT_SECONDS=50 \
         -DUSE_IPV6=OFF \
         -DAUTOTEST=ON

external/toxcore/c-toxcore/CHANGELOG.md

@@ -1,9 +1,274 @@
 
 
+## v0.2.19
+
+### Merged PRs:
+
+- [#2744](https://github.com/TokTok/c-toxcore/pull/2744) docs: Document that group topic lock is default on.
+- [#2743](https://github.com/TokTok/c-toxcore/pull/2743) docs: Add missing param docs for callbacks.
+- [#2742](https://github.com/TokTok/c-toxcore/pull/2742) chore: Add cmake flag to disable unit tests.
+- [#2741](https://github.com/TokTok/c-toxcore/pull/2741) refactor: Don't expose Tox_System in the public API
+- [#2736](https://github.com/TokTok/c-toxcore/pull/2736) cleanup: A more descriptive error for group invite accept function
+- [#2735](https://github.com/TokTok/c-toxcore/pull/2735) chore: Small API doc fixes
+- [#2732](https://github.com/TokTok/c-toxcore/pull/2732) cleanup: event length naming inconsistencies
+- [#2731](https://github.com/TokTok/c-toxcore/pull/2731) cleanup: align group send err enum order
+- [#2729](https://github.com/TokTok/c-toxcore/pull/2729) cleanup: use typedef for private message ID's in callback
+- [#2728](https://github.com/TokTok/c-toxcore/pull/2728) refactor: Observers/ignored peers can now send and receive custom packets
+- [#2727](https://github.com/TokTok/c-toxcore/pull/2727) feat: add message IDs to private group messages
+- [#2726](https://github.com/TokTok/c-toxcore/pull/2726) refactor: Rename `out` parameters to `out_$something`.
+- [#2718](https://github.com/TokTok/c-toxcore/pull/2718) chore: Use a specific non-broken slimcc version.
+- [#2713](https://github.com/TokTok/c-toxcore/pull/2713) feat: Update and improve the Windows cross-compilation
+- [#2712](https://github.com/TokTok/c-toxcore/pull/2712) chore: Update github actions.
+- [#2710](https://github.com/TokTok/c-toxcore/pull/2710) docs: Update the list of CMake options
+- [#2709](https://github.com/TokTok/c-toxcore/pull/2709) refactor: Remove "mod" and "founder" from group API naming scheme
+- [#2708](https://github.com/TokTok/c-toxcore/pull/2708) docs: add the experimental api build option to INSTALL.md
+- [#2705](https://github.com/TokTok/c-toxcore/pull/2705) refactor: Rename Queries to Query to align with other enums.
+- [#2704](https://github.com/TokTok/c-toxcore/pull/2704) fix: Correct type for conference offline peer numbers.
+- [#2700](https://github.com/TokTok/c-toxcore/pull/2700) test: Add FreeBSD VM action on GitHub.
+- [#2699](https://github.com/TokTok/c-toxcore/pull/2699) test: Add pkgsrc build.
+- [#2698](https://github.com/TokTok/c-toxcore/pull/2698) chore: Only install tox_private.h on request.
+- [#2697](https://github.com/TokTok/c-toxcore/pull/2697) test: Build toxcore on NetBSD (VM).
+- [#2696](https://github.com/TokTok/c-toxcore/pull/2696) fix: save_compatibility_test failing on big-endian systems
+- [#2695](https://github.com/TokTok/c-toxcore/pull/2695) fix: Don't serve files from websockify.
+- [#2689](https://github.com/TokTok/c-toxcore/pull/2689) fix: Correctly pass extended public keys to group moderation code.
+- [#2686](https://github.com/TokTok/c-toxcore/pull/2686) chore: Compile libsodium reference implementation with compcert.
+- [#2685](https://github.com/TokTok/c-toxcore/pull/2685) cleanup: correct a few nullable annotations
+- [#2684](https://github.com/TokTok/c-toxcore/pull/2684) cleanup: Don't use `memcpy` to cast arbitrary `struct`s to `uint8_t[]`.
+- [#2683](https://github.com/TokTok/c-toxcore/pull/2683) fix: Pass array, not array pointer, to `memcmp`.
+- [#2682](https://github.com/TokTok/c-toxcore/pull/2682) cleanup: Never pass `void*` directly to `memcpy`.
+- [#2678](https://github.com/TokTok/c-toxcore/pull/2678) chore: Disable NGC saving by default, enable through Tox_Options.
+- [#2675](https://github.com/TokTok/c-toxcore/pull/2675) cleanup: Replace pointer arithmetic with explicit `&arr[i]`.
+- [#2672](https://github.com/TokTok/c-toxcore/pull/2672) refactor: Use `structs` for extended public/secret keys.
+- [#2671](https://github.com/TokTok/c-toxcore/pull/2671) refactor: Use tox rng to seed the keypair generation.
+- [#2666](https://github.com/TokTok/c-toxcore/pull/2666) cleanup: Small improvements found by PVS Studio.
+- [#2664](https://github.com/TokTok/c-toxcore/pull/2664) docs: Run prettier-markdown on markdown files.
+- [#2662](https://github.com/TokTok/c-toxcore/pull/2662) fix: Correct a few potential null derefs in bootstrap daemon.
+- [#2661](https://github.com/TokTok/c-toxcore/pull/2661) fix: Zero out stack-allocated secret key before return.
+- [#2660](https://github.com/TokTok/c-toxcore/pull/2660) fix: Add missing memunlock of local variable when it goes out of scope.
+- [#2659](https://github.com/TokTok/c-toxcore/pull/2659) cleanup: Simplify custom packet length check in NGC.
+- [#2658](https://github.com/TokTok/c-toxcore/pull/2658) refactor: Make prune_gc_sanctions_list more obviously correct.
+- [#2657](https://github.com/TokTok/c-toxcore/pull/2657) fix: Fix some false positive from PVS Studio.
+- [#2656](https://github.com/TokTok/c-toxcore/pull/2656) docs: Add static analysis tool list to README.
+- [#2655](https://github.com/TokTok/c-toxcore/pull/2655) cleanup: Check that WINXP macro exists before comparing it.
+- [#2652](https://github.com/TokTok/c-toxcore/pull/2652) refactor: Make tox mutex non-recursive.
+- [#2648](https://github.com/TokTok/c-toxcore/pull/2648) docs: Add more documentation to crypto_core.
+- [#2647](https://github.com/TokTok/c-toxcore/pull/2647) docs: Fix up doxyfile.
+- [#2645](https://github.com/TokTok/c-toxcore/pull/2645) refactor: Remove `Tox *` from `tox_dispatch`.
+- [#2644](https://github.com/TokTok/c-toxcore/pull/2644) refactor: Don't rely on tox_dispatch passing tox in tests.
+- [#2643](https://github.com/TokTok/c-toxcore/pull/2643) refactor: Use strong typedef for NGC peer id.
+- [#2642](https://github.com/TokTok/c-toxcore/pull/2642) chore: Use C++ mode for clang-tidy.
+- [#2640](https://github.com/TokTok/c-toxcore/pull/2640) refactor: Use strong `typedef` instead of `struct` for `Socket`.
+- [#2637](https://github.com/TokTok/c-toxcore/pull/2637) chore: Check that both gtest and gmock exist for tests.
+- [#2634](https://github.com/TokTok/c-toxcore/pull/2634) chore: Add some comments to the astyle config.
+- [#2629](https://github.com/TokTok/c-toxcore/pull/2629) chore: Reformat sources with astyle.
+- [#2626](https://github.com/TokTok/c-toxcore/pull/2626) chore: Rename C++ headers to .hh suffixes.
+- [#2624](https://github.com/TokTok/c-toxcore/pull/2624) test: Add slimcc compiler compatibility test.
+- [#2622](https://github.com/TokTok/c-toxcore/pull/2622) cleanup: Add more `const` where possible.
+- [#2621](https://github.com/TokTok/c-toxcore/pull/2621) cleanup: Remove implicit bool conversions.
+- [#2620](https://github.com/TokTok/c-toxcore/pull/2620) chore: Only check the bootstrap daemon checksum on release.
+- [#2617](https://github.com/TokTok/c-toxcore/pull/2617) cleanup: Further `#include` cleanups.
+- [#2614](https://github.com/TokTok/c-toxcore/pull/2614) cleanup: Use Bazel modules to enforce proper `#include` hygiene.
+- [#2612](https://github.com/TokTok/c-toxcore/pull/2612) refactor: Move pack/unpack `IP_Port` from DHT into network module.
+- [#2611](https://github.com/TokTok/c-toxcore/pull/2611) chore: Really fix coverage docker image build.
+- [#2610](https://github.com/TokTok/c-toxcore/pull/2610) chore: Fix post-submit coverage image.
+- [#2609](https://github.com/TokTok/c-toxcore/pull/2609) fix: partially fix a bug that prevented group part messages from sending
+- [#2605](https://github.com/TokTok/c-toxcore/pull/2605) fix: Don't use `memcmp` to compare `IP_Port`s.
+- [#2604](https://github.com/TokTok/c-toxcore/pull/2604) chore: Fix rpm build; add a CI check for it.
+- [#2603](https://github.com/TokTok/c-toxcore/pull/2603) chore: Speed up docker builds a bit by reducing layer count.
+- [#2602](https://github.com/TokTok/c-toxcore/pull/2602) cleanup: Add `const` where possible in auto tests.
+- [#2601](https://github.com/TokTok/c-toxcore/pull/2601) fix: a few off by one errors in group autotests
+- [#2598](https://github.com/TokTok/c-toxcore/pull/2598) refactor: Rename `system_{memory,...}` to `os_{memory,...}`.
+- [#2597](https://github.com/TokTok/c-toxcore/pull/2597) test: Add goblint static analyser.
+- [#2594](https://github.com/TokTok/c-toxcore/pull/2594) cleanup: Use `memzero(x, s)` instead of `memset(x, 0, s)`.
+- [#2593](https://github.com/TokTok/c-toxcore/pull/2593) cleanup: Use explicit 0 instead of `PACKET_ID_PADDING`.
+- [#2592](https://github.com/TokTok/c-toxcore/pull/2592) cleanup: Remove all uses of `SIZEOF_VLA`.
+- [#2591](https://github.com/TokTok/c-toxcore/pull/2591) cleanup: Expand the `Tox_Options` accessor macros.
+- [#2590](https://github.com/TokTok/c-toxcore/pull/2590) test: Add a simple new/delete test for Tox.
+- [#2588](https://github.com/TokTok/c-toxcore/pull/2588) cleanup: Remove plan9 support.
+- [#2587](https://github.com/TokTok/c-toxcore/pull/2587) cleanup: Add comment after every `#endif`.
+- [#2583](https://github.com/TokTok/c-toxcore/pull/2583) test: Fix comment I broke in the events test PR.
+- [#2582](https://github.com/TokTok/c-toxcore/pull/2582) cleanup: Rename group to conference in groupav documentation.
+- [#2581](https://github.com/TokTok/c-toxcore/pull/2581) cleanup: Ensure handler params are named after callback params.
+- [#2580](https://github.com/TokTok/c-toxcore/pull/2580) cleanup: Minor cleanup of event unpack code.
+- [#2578](https://github.com/TokTok/c-toxcore/pull/2578) refactor: Allow NULL pointers for byte arrays in events.
+- [#2577](https://github.com/TokTok/c-toxcore/pull/2577) refactor: Add common msgpack array packer with callback.
+- [#2576](https://github.com/TokTok/c-toxcore/pull/2576) cleanup: make some improvements to group moderation test
+- [#2575](https://github.com/TokTok/c-toxcore/pull/2575) refactor: Pass `this` pointer as first param to s11n callbacks.
+- [#2573](https://github.com/TokTok/c-toxcore/pull/2573) cleanup: skip a do_gc iteration before removing peers marked for deletion
+- [#2572](https://github.com/TokTok/c-toxcore/pull/2572) cleanup: Remove `bin_pack_{new,free}`.
+- [#2568](https://github.com/TokTok/c-toxcore/pull/2568) feat: Add dht_get_nodes_response event to the events system.
+- [#2567](https://github.com/TokTok/c-toxcore/pull/2567) refactor: Use enum-specific pack functions for enum values.
+- [#2566](https://github.com/TokTok/c-toxcore/pull/2566) cleanup: Remove empty test doing nothing.
+- [#2565](https://github.com/TokTok/c-toxcore/pull/2565) refactor: Factor out union pack switch from event packer.
+- [#2564](https://github.com/TokTok/c-toxcore/pull/2564) cleanup: Move the 2-element array pack out of individual events.
+- [#2563](https://github.com/TokTok/c-toxcore/pull/2563) test: Add printf log statement to group_moderation_test.
+- [#2562](https://github.com/TokTok/c-toxcore/pull/2562) chore: add clangd files to .gitignore
+- [#2561](https://github.com/TokTok/c-toxcore/pull/2561) refactor: Move file streaming test to its own file.
+- [#2560](https://github.com/TokTok/c-toxcore/pull/2560) fix(ci): window builds now build in parallel
+- [#2559](https://github.com/TokTok/c-toxcore/pull/2559) refactor: Migrate auto_tests to new events API.
+- [#2557](https://github.com/TokTok/c-toxcore/pull/2557) test: Add C++ classes wrapping system interfaces.
+- [#2555](https://github.com/TokTok/c-toxcore/pull/2555) fix: Remove fatal error for non-erroneous case
+- [#2554](https://github.com/TokTok/c-toxcore/pull/2554) fix: Make all the fuzzers work again, and add a test for protodump.
+- [#2552](https://github.com/TokTok/c-toxcore/pull/2552) fix: Make sure there's enough space for CONSUME1 in fuzzers.
+- [#2551](https://github.com/TokTok/c-toxcore/pull/2551) chore: Move from gcov to llvm source-based coverage.
+- [#2549](https://github.com/TokTok/c-toxcore/pull/2549) fix: issues with packet broadcast error reporting
+- [#2547](https://github.com/TokTok/c-toxcore/pull/2547) test: Add fuzz tests to the coverage run.
+- [#2545](https://github.com/TokTok/c-toxcore/pull/2545) refactor: Use `operator==` for equality tests of `Node_format`.
+- [#2543](https://github.com/TokTok/c-toxcore/pull/2543) refactor(test): Slightly nicer C++ interface to tox Random.
+- [#2542](https://github.com/TokTok/c-toxcore/pull/2542) refactor: packet broadcast functions now return errors
+- [#2541](https://github.com/TokTok/c-toxcore/pull/2541) fix: don't pass garbage data buffer to packet send functions
+- [#2540](https://github.com/TokTok/c-toxcore/pull/2540) cleanup: Make group packet entry creation less error-prone
+- [#2539](https://github.com/TokTok/c-toxcore/pull/2539) refactor: Minor refactoring of get_close_nodes functions.
+- [#2538](https://github.com/TokTok/c-toxcore/pull/2538) refactor: Factor out malloc+memcpy into memdup.
+- [#2536](https://github.com/TokTok/c-toxcore/pull/2536) cleanup: Some more test cleanups, removing overly smart code.
+- [#2531](https://github.com/TokTok/c-toxcore/pull/2531) test: Add more unit tests for `add_to_list`.
+- [#2530](https://github.com/TokTok/c-toxcore/pull/2530) refactor: Assign malloc return to a local variable first.
+- [#2529](https://github.com/TokTok/c-toxcore/pull/2529) test: Add "infer" CI check to github, remove from circle.
+- [#2527](https://github.com/TokTok/c-toxcore/pull/2527) cleanup: Add Toxav alias for ToxAV.
+- [#2526](https://github.com/TokTok/c-toxcore/pull/2526) cleanup: Make Tox_Options a typedef.
+- [#2525](https://github.com/TokTok/c-toxcore/pull/2525) cleanup: Add dynamically derived array sizes to the API.
+- [#2524](https://github.com/TokTok/c-toxcore/pull/2524) cleanup: Add explicit array sizes to toxencryptsave.
+- [#2523](https://github.com/TokTok/c-toxcore/pull/2523) cleanup: Move `tox_get_system` out of the public API.
+- [#2522](https://github.com/TokTok/c-toxcore/pull/2522) cleanup: Make setters take non-const `Tox *`.
+- [#2521](https://github.com/TokTok/c-toxcore/pull/2521) cleanup: Make array params in toxav `[]` instead of `*`.
+- [#2520](https://github.com/TokTok/c-toxcore/pull/2520) cleanup: Mark arrays in the tox API as `[]` instead of `*`.
+- [#2519](https://github.com/TokTok/c-toxcore/pull/2519) refactor: Align group message sending with other send functions.
+- [#2518](https://github.com/TokTok/c-toxcore/pull/2518) cleanup: Add typedefs for public API int identifiers.
+- [#2517](https://github.com/TokTok/c-toxcore/pull/2517) chore: Spellcheck tox-bootstrapd
+- [#2516](https://github.com/TokTok/c-toxcore/pull/2516) chore: Remove settings.yml in favour of hs-github-tools.
+- [#2515](https://github.com/TokTok/c-toxcore/pull/2515) chore: Use GPL license with https.
+- [#2513](https://github.com/TokTok/c-toxcore/pull/2513) chore: Add fetch-sha256 script to update bootstrap node hash.
+- [#2512](https://github.com/TokTok/c-toxcore/pull/2512) cleanup: Move all vptr-to-ptr casts to the beginning of a function.
+- [#2510](https://github.com/TokTok/c-toxcore/pull/2510) cleanup: Use github actions matrix to simplify CI.
+- [#2509](https://github.com/TokTok/c-toxcore/pull/2509) fix: Use QueryPerformanceCounter on windows for monotonic time.
+- [#2508](https://github.com/TokTok/c-toxcore/pull/2508) chore: Add `net_(new|kill)_strerror` to cppcheck's allocators.
+- [#2507](https://github.com/TokTok/c-toxcore/pull/2507) cleanup: Run clang-tidy on headers, as well.
+- [#2506](https://github.com/TokTok/c-toxcore/pull/2506) cleanup: Make TCP connection failures a warning instead of error.
+- [#2505](https://github.com/TokTok/c-toxcore/pull/2505) cleanup: Make all .c files include the headers they need.
+- [#2504](https://github.com/TokTok/c-toxcore/pull/2504) cleanup: Upgrade cppcheck, fix some warnings.
+- [#2503](https://github.com/TokTok/c-toxcore/pull/2503) cleanup: Upgrade to clang-tidy-17 and fix some warnings.
+- [#2502](https://github.com/TokTok/c-toxcore/pull/2502) chore: Use `pkg_search_module` directly in cmake.
+- [#2501](https://github.com/TokTok/c-toxcore/pull/2501) chore: Add `IMPORTED_TARGET` to pkg-config packages.
+- [#2499](https://github.com/TokTok/c-toxcore/pull/2499) cleanup: Use target_link_libraries directly in cmake.
+- [#2498](https://github.com/TokTok/c-toxcore/pull/2498) chore: Simplify msvc build using vcpkg.
+- [#2497](https://github.com/TokTok/c-toxcore/pull/2497) cleanup: Remove NaCl support.
+- [#2494](https://github.com/TokTok/c-toxcore/pull/2494) fix: unpack enum function names in event impl generator
+- [#2493](https://github.com/TokTok/c-toxcore/pull/2493) chore: Disable targets for cross-compilation.
+- [#2491](https://github.com/TokTok/c-toxcore/pull/2491) chore: Build a docker image with coverage info in it.
+- [#2490](https://github.com/TokTok/c-toxcore/pull/2490) cleanup: Some portability/warning fixes for Windows builds.
+- [#2488](https://github.com/TokTok/c-toxcore/pull/2488) fix: Correct a use-after-free and fix some memory leaks.
+- [#2487](https://github.com/TokTok/c-toxcore/pull/2487) refactor: Change all enum-like `#define` sequences into enums.
+- [#2486](https://github.com/TokTok/c-toxcore/pull/2486) refactor: Change the `TCP_PACKET_*` defines into an enum.
+- [#2485](https://github.com/TokTok/c-toxcore/pull/2485) refactor: event generation tool for reorder pr
+- [#2484](https://github.com/TokTok/c-toxcore/pull/2484) chore: Fix make_single_file to support core-only.
+- [#2481](https://github.com/TokTok/c-toxcore/pull/2481) chore: Update github actions `uses`.
+- [#2480](https://github.com/TokTok/c-toxcore/pull/2480) feat: add ngc related unpack functions
+- [#2479](https://github.com/TokTok/c-toxcore/pull/2479) feat: Add `to_string` functions for all public enums.
+- [#2477](https://github.com/TokTok/c-toxcore/pull/2477) test: add real timeout test
+- [#2476](https://github.com/TokTok/c-toxcore/pull/2476) chore: Move s390x build to post-merge.
+- [#2475](https://github.com/TokTok/c-toxcore/pull/2475) refactor: Give `enum-from-int` functions the ability to report errors.
+- [#2474](https://github.com/TokTok/c-toxcore/pull/2474) cleanup: Remove test net support.
+- [#2472](https://github.com/TokTok/c-toxcore/pull/2472) feat: Enable ubsan on bootstrap nodes.
+- [#2470](https://github.com/TokTok/c-toxcore/pull/2470) test: Add check-c run to bazel build.
+- [#2468](https://github.com/TokTok/c-toxcore/pull/2468) fix(test): tests use ipv6 by default, even with USE_IPV6 set to 0
+- [#2466](https://github.com/TokTok/c-toxcore/pull/2466) cleanup: Make group saving/loading more forgiving with data errors
+- [#2465](https://github.com/TokTok/c-toxcore/pull/2465) refactor: replace memset with a loop
+- [#2459](https://github.com/TokTok/c-toxcore/pull/2459) fix: Enable debug flag for ubsan.
+- [#2458](https://github.com/TokTok/c-toxcore/pull/2458) fix: also Install header for private/experimental API functions with autotools
+- [#2456](https://github.com/TokTok/c-toxcore/pull/2456) docs: Remove defunct IRC channel from README.md
+- [#2454](https://github.com/TokTok/c-toxcore/pull/2454) fix: memory leaks
+- [#2453](https://github.com/TokTok/c-toxcore/pull/2453) refactor: Install header for private/experimental API functions
+- [#2452](https://github.com/TokTok/c-toxcore/pull/2452) refactor: replace DEFAULT_TCP_RELAY_PORTS_COUNT with a compile-time calculation
+- [#2451](https://github.com/TokTok/c-toxcore/pull/2451) cleanup: clarify disabling of static assert checks
+- [#2449](https://github.com/TokTok/c-toxcore/pull/2449) cleanup: replace tabs with spaces
+- [#2448](https://github.com/TokTok/c-toxcore/pull/2448) feat: group connection queries now return our own connection type
+- [#2447](https://github.com/TokTok/c-toxcore/pull/2447) fix: Docker tox-bootstrapd hash update failing when using BuildKit
+- [#2446](https://github.com/TokTok/c-toxcore/pull/2446) feat: Add groupchat API function that returns an IP address string for a peer
+- [#2442](https://github.com/TokTok/c-toxcore/pull/2442) perf: Slightly reduce bandwidth usage when there are few nodes.
+- [#2439](https://github.com/TokTok/c-toxcore/pull/2439) test: Make esp32 build actually try to instantiate tox.
+- [#2438](https://github.com/TokTok/c-toxcore/pull/2438) cleanup: Remove explicit layering_check feature.
+- [#2437](https://github.com/TokTok/c-toxcore/pull/2437) chore: Upgrade sonar-scan jvm to java 17.
+- [#2436](https://github.com/TokTok/c-toxcore/pull/2436) fix: Add missing `htons` call when adding configured TCP relay.
+- [#2434](https://github.com/TokTok/c-toxcore/pull/2434) chore: Cancel old PR builds on docker and sonar-scan workflows.
+- [#2432](https://github.com/TokTok/c-toxcore/pull/2432) chore: Use C99 on MSVC instead of C11.
+- [#2430](https://github.com/TokTok/c-toxcore/pull/2430) chore: Retry freebsd tests 2 times.
+- [#2429](https://github.com/TokTok/c-toxcore/pull/2429) test: Add an s390x build (on alpine) for CI.
+- [#2428](https://github.com/TokTok/c-toxcore/pull/2428) chore: Add a compcert docker run script.
+- [#2427](https://github.com/TokTok/c-toxcore/pull/2427) cleanup: Use tcc docker image for CI.
+- [#2424](https://github.com/TokTok/c-toxcore/pull/2424) fix: Fix memory leak in the error path of loading savedata.
+- [#2420](https://github.com/TokTok/c-toxcore/pull/2420) refactor: Use Bin_Pack for packing Node_format.
+- [#2416](https://github.com/TokTok/c-toxcore/pull/2416) chore: Add more logging to loading conferences from savedata.
+- [#2415](https://github.com/TokTok/c-toxcore/pull/2415) refactor: Add a `bin_unpack_bin_max` for max-length arrays.
+- [#2414](https://github.com/TokTok/c-toxcore/pull/2414) fix: inversed return values
+- [#2413](https://github.com/TokTok/c-toxcore/pull/2413) cleanup: Fix GCC compatibility.
+- [#2408](https://github.com/TokTok/c-toxcore/pull/2408) fix: Ensure we have allocators available for the error paths.
+- [#2407](https://github.com/TokTok/c-toxcore/pull/2407) cleanup: Remove redundant `-DSODIUM_EXPORT` from definitions.
+- [#2406](https://github.com/TokTok/c-toxcore/pull/2406) cleanup: Fix a few more clang-tidy warnings.
+- [#2405](https://github.com/TokTok/c-toxcore/pull/2405) cleanup: Fix a few more clang-tidy warnings.
+- [#2404](https://github.com/TokTok/c-toxcore/pull/2404) cleanup: Enforce stricter identifier naming using clang-tidy.
+- [#2396](https://github.com/TokTok/c-toxcore/pull/2396) chore: Add devcontainer setup for codespaces.
+- [#2393](https://github.com/TokTok/c-toxcore/pull/2393) refactor: Add `mem` module to allow tests to override allocators.
+- [#2392](https://github.com/TokTok/c-toxcore/pull/2392) refactor: Make event dispatch ordered by receive time.
+- [#2391](https://github.com/TokTok/c-toxcore/pull/2391) docs: Fix doxygen config and remove some redundant comments.
+- [#2390](https://github.com/TokTok/c-toxcore/pull/2390) chore: Fix the Android CI job
+- [#2389](https://github.com/TokTok/c-toxcore/pull/2389) fix: Add missing `#include <array>`.
+- [#2388](https://github.com/TokTok/c-toxcore/pull/2388) chore: Add missing module dependencies.
+- [#2384](https://github.com/TokTok/c-toxcore/pull/2384) feat: increase NGC lossy custom packet size
+- [#2383](https://github.com/TokTok/c-toxcore/pull/2383) fix: add missing ngc constants getter declarations and definitions
+- [#2381](https://github.com/TokTok/c-toxcore/pull/2381) fix: incorrect documentation
+- [#2380](https://github.com/TokTok/c-toxcore/pull/2380) feat: allow for larger incoming NGC packets
+- [#2371](https://github.com/TokTok/c-toxcore/pull/2371) docs: fix group_peer_exit_cb
+- [#2370](https://github.com/TokTok/c-toxcore/pull/2370) fix: behaviour of group api function
+- [#2369](https://github.com/TokTok/c-toxcore/pull/2369) fix: flaky tcp test
+- [#2367](https://github.com/TokTok/c-toxcore/pull/2367) fix: fuzz support for TCP server
+- [#2364](https://github.com/TokTok/c-toxcore/pull/2364) fix: potential endless loop under extremely high load
+- [#2362](https://github.com/TokTok/c-toxcore/pull/2362) test: enable tcp relay for bootstrap fuzzing
+- [#2361](https://github.com/TokTok/c-toxcore/pull/2361) fix: resolve_bootstrap_node() not checking net_getipport() returned count correctly
+- [#2357](https://github.com/TokTok/c-toxcore/pull/2357) feat: get the number of close dht nodes with announce/store support
+- [#2355](https://github.com/TokTok/c-toxcore/pull/2355) fix: group custom packet size limits
+- [#2354](https://github.com/TokTok/c-toxcore/pull/2354) fix: DHTBootstrap should always respond to version packets with toxcore version
+- [#2351](https://github.com/TokTok/c-toxcore/pull/2351) fix: Increase max group message length by four bytes
+- [#2348](https://github.com/TokTok/c-toxcore/pull/2348) refactor: Make some improvements to how often/when we announce a group
+- [#2341](https://github.com/TokTok/c-toxcore/pull/2341) fix: #1144 by forcing misc_tools to be a static lib
+- [#2340](https://github.com/TokTok/c-toxcore/pull/2340) fix: missing net to host conversion of port in logging in group_chat.c
+- [#2339](https://github.com/TokTok/c-toxcore/pull/2339) fix: missing net to host conversion of port in logging
+- [#2338](https://github.com/TokTok/c-toxcore/pull/2338) fix: bug causing friend group invites to sometimes fail & improve logging
+- [#2333](https://github.com/TokTok/c-toxcore/pull/2333) docs: Update README for bootstrap node docker
+- [#2329](https://github.com/TokTok/c-toxcore/pull/2329) refactor: extract each case in handle packet in messenger
+- [#2327](https://github.com/TokTok/c-toxcore/pull/2327) fix: unlock correct dht_friend
+- [#2325](https://github.com/TokTok/c-toxcore/pull/2325) fix: Remove cmake cache files after copying to container build directory
+- [#2323](https://github.com/TokTok/c-toxcore/pull/2323) docs: Update README.md to include cmp submodule info
+- [#2318](https://github.com/TokTok/c-toxcore/pull/2318) chore: disable warning about pre C99 code
+- [#2317](https://github.com/TokTok/c-toxcore/pull/2317) Refactor: Extract shared key cache into separate file
+- [#2311](https://github.com/TokTok/c-toxcore/pull/2311) cleanup: make it more clear that assert and uint32_t increment both only exist if NDEBUG is not defined
+- [#2291](https://github.com/TokTok/c-toxcore/pull/2291) test: Add a protocol dump test to generate initial fuzzer input.
+- [#2271](https://github.com/TokTok/c-toxcore/pull/2271) chore: Migrate from Appveyor to Azure Pipelines
+- [#2269](https://github.com/TokTok/c-toxcore/pull/2269) feat: Merge the remainder of the new groupchats implementation
+- [#2203](https://github.com/TokTok/c-toxcore/pull/2203) refactor: Store time in Mono_Time in milliseconds.
+- [#1944](https://github.com/TokTok/c-toxcore/pull/1944) chore: tox_new() should return null when savedata loading fails
+
+### Closed issues:
+
+- [#2739](https://github.com/TokTok/c-toxcore/issues/2739) Tox_Options.operating_system is not clear about it being an experimental option
+- [#2734](https://github.com/TokTok/c-toxcore/issues/2734) error compiling on fedora
+- [#2649](https://github.com/TokTok/c-toxcore/issues/2649) create_extended_keypair should use Random and be made deterministic for fuzzing
+- [#2358](https://github.com/TokTok/c-toxcore/issues/2358) resolve_bootstrap_node assert hit
+- [#2352](https://github.com/TokTok/c-toxcore/issues/2352) SEGV after infinite loop retrying proxy_socks5_read_connection_response
+- [#2335](https://github.com/TokTok/c-toxcore/issues/2335) ipv6 disabled on kernel cmdline disrupts Tox = most tests fail
+- [#2303](https://github.com/TokTok/c-toxcore/issues/2303) Add new group chats events to tox_events and tox_dispatch modules.
+- [#2302](https://github.com/TokTok/c-toxcore/issues/2302) Run strong fuzz tests against NGC
+- [#2301](https://github.com/TokTok/c-toxcore/issues/2301) Implement NGC save/load in hs-toxcore and run round-trip tests against c-toxcore
+- [#2192](https://github.com/TokTok/c-toxcore/issues/2192) Provide official signed release tarballs
+- [#2118](https://github.com/TokTok/c-toxcore/issues/2118) Add a private "get mono_time" API to `tox_private.h` and use it in auto_tests.
+- [#2029](https://github.com/TokTok/c-toxcore/issues/2029) Migrate all auto tests to the events API.
+- [#2014](https://github.com/TokTok/c-toxcore/issues/2014) Add sodium autotools CI build
+- [#1144](https://github.com/TokTok/c-toxcore/issues/1144) DHT_bootstrap should not link against misc_tools
+
 ## v0.2.18
 
 ### Merged PRs:
 
+- [#2300](https://github.com/TokTok/c-toxcore/pull/2300) chore: Release 0.2.18
 - [#2299](https://github.com/TokTok/c-toxcore/pull/2299) fix: remove the assert because buffer can be larger than UINT16_MAX.
 - [#2297](https://github.com/TokTok/c-toxcore/pull/2297) cleanup: remove unused field last_seen from Onion_Friend
 - [#2289](https://github.com/TokTok/c-toxcore/pull/2289) test: Add a Null_System used in toxsave_harness.
@@ -107,6 +372,7 @@
 
 ### Closed issues:
 
+- [#2298](https://github.com/TokTok/c-toxcore/issues/2298) assert makes incorrect assumption
 - [#2256](https://github.com/TokTok/c-toxcore/issues/2256) New Defects reported by Coverity Scan for TokTok/c-toxcore
 - [#2109](https://github.com/TokTok/c-toxcore/issues/2109) Assimilate `messenger_test.c`: replace with public API test
 - [#2012](https://github.com/TokTok/c-toxcore/issues/2012) Support building a DLL on Windows

external/toxcore/c-toxcore/CMakeLists.txt

@@ -44,7 +44,7 @@ set_source_files_properties(
 # versions in a synchronised way.
 set(PROJECT_VERSION_MAJOR "0")
 set(PROJECT_VERSION_MINOR "2")
-set(PROJECT_VERSION_PATCH "18")
+set(PROJECT_VERSION_PATCH "19")
 set(PROJECT_VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}")
 
 # set .so library version / following libtool scheme
@@ -146,6 +146,8 @@ endif()
 option(BUILD_MISC_TESTS "Build additional tests" OFF)
 option(BUILD_FUN_UTILS "Build additional just for fun utilities" OFF)
 
+option(UNITTEST "Enable unit tests (disable if you don't have a working gmock or gtest)" ON)
+
 option(AUTOTEST "Enable autotests (mainly for CI)" OFF)
 if(AUTOTEST)
   option(NON_HERMETIC_TESTS "Whether to build and run tests that depend on an internet connection" OFF)
@@ -534,7 +536,7 @@ endfunction()
 
 # The actual unit tests follow.
 #
-if(TARGET GTest::gtest AND TARGET GTest::gmock)
+if(UNITTEST AND TARGET GTest::gtest AND TARGET GTest::gmock)
   unit_test(toxav ring_buffer)
   unit_test(toxav rtp)
   unit_test(toxcore DHT)

external/toxcore/c-toxcore/INSTALL.md

@@ -151,6 +151,7 @@ There are some options that are available to configure the build.
 | `PROXY_TEST`            | Enable proxy test (requires `other/proxy/proxy_server.go` to be running).                     | ON or OFF                                                                 | OFF                                               |
 | `STRICT_ABI`            | Enforce strict ABI export in dynamic libraries.                                               | ON or OFF                                                                 | OFF                                               |
 | `TEST_TIMEOUT_SECONDS`  | Limit runtime of each test to the number of seconds specified.                                | Positive number or nothing (empty string).                                | Empty string.                                     |
+| `UNITTEST`              | Enable unit tests (disable if you don't have a working gmock or gtest).                       | ON or OFF                                                                 | ON                                                |
 | `USE_IPV6`              | Use IPv6 in tests.                                                                            | ON or OFF                                                                 | ON                                                |
 
 You can get this list of option using the following commands

external/toxcore/c-toxcore/configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.65])
-AC_INIT([tox], [0.2.18])
+AC_INIT([tox], [0.2.19])
 AC_CONFIG_AUX_DIR(configure_aux)
 AC_CONFIG_SRCDIR([toxcore/net_crypto.c])
 AM_INIT_AUTOMAKE([foreign 1.10 -Wall -Werror subdir-objects tar-ustar])

external/toxcore/c-toxcore/other/bootstrap_daemon/docker/tox-bootstrapd.sha256

@@ -1 +1 @@
-af58a125e5c80d7a19bc7f32868c1edfdf80f366e3bf778728961a50ce63ee26  /usr/local/bin/tox-bootstrapd
+e96f03a89051c5df12c28d0d6941184da2b92742d248bd4c57d31189a0052844  /usr/local/bin/tox-bootstrapd

external/toxcore/c-toxcore/other/docker/pkgsrc/pkgsrc.Dockerfile

@@ -5,6 +5,7 @@ COPY . /work/c-toxcore-0.2.18
 RUN ["tar", "zcf", "c-toxcore.tar.gz", "c-toxcore-0.2.18"]
 
 WORKDIR /work/pkgsrc/chat/toxcore
+RUN ["sed", "-i", "-e", "s/libtoxcore.so.2.18.0/libtoxcore.so.2.19.0/g", "PLIST"]
 RUN ["bmake", "clean"]
 RUN ["bmake", "DISTFILES=c-toxcore.tar.gz", "DISTDIR=/work", "NO_CHECKSUM=yes"]
 RUN ["bmake", "install"]

external/toxcore/c-toxcore/so.version

@@ -11,6 +11,6 @@
 # For a full reference see:
 # https://www.gnu.org/software/libtool/manual/libtool.html#Updating-version-info
 
-CURRENT=20
+CURRENT=21
 REVISION=0
-AGE=18
+AGE=19

external/toxcore/c-toxcore/testing/fuzzing/bootstrap_fuzz_test.cc

@@ -104,7 +104,6 @@ void TestBootstrap(Fuzz_Data &input)
 
     Ptr<Tox_Options> opts(tox_options_new(nullptr), tox_options_free);
     assert(opts != nullptr);
-    tox_options_set_operating_system(opts.get(), sys.sys.get());
 
     tox_options_set_log_callback(opts.get(),
         [](Tox *tox, Tox_Log_Level level, const char *file, uint32_t line, const char *func,
@@ -134,8 +133,12 @@ void TestBootstrap(Fuzz_Data &input)
         tox_options_set_tcp_port(opts.get(), 33445);
     }
 
+    Tox_Options_Testing tox_options_testing;
+    tox_options_testing.operating_system = sys.sys.get();
+
     Tox_Err_New error_new;
-    Tox *tox = tox_new(opts.get(), &error_new);
+    Tox_Err_New_Testing error_new_testing;
+    Tox *tox = tox_new_testing(opts.get(), &error_new, &tox_options_testing, &error_new_testing);
 
     if (tox == nullptr) {
         // It might fail, because some I/O happens in tox_new, and the fuzzer
@@ -144,6 +147,7 @@ void TestBootstrap(Fuzz_Data &input)
     }
 
     assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);
 
     uint8_t pub_key[TOX_PUBLIC_KEY_SIZE] = {0};
 

external/toxcore/c-toxcore/testing/fuzzing/e2e_fuzz_test.cc

@@ -138,7 +138,6 @@ void TestEndToEnd(Fuzz_Data &input)
 
     Ptr<Tox_Options> opts(tox_options_new(nullptr), tox_options_free);
     assert(opts != nullptr);
-    tox_options_set_operating_system(opts.get(), sys.sys.get());
     tox_options_set_local_discovery_enabled(opts.get(), false);
 
     tox_options_set_log_callback(opts.get(),
@@ -151,8 +150,12 @@ void TestEndToEnd(Fuzz_Data &input)
             }
         });
 
+    Tox_Options_Testing tox_options_testing;
+    tox_options_testing.operating_system = sys.sys.get();
+
     Tox_Err_New error_new;
-    Tox *tox = tox_new(opts.get(), &error_new);
+    Tox_Err_New_Testing error_new_testing;
+    Tox *tox = tox_new_testing(opts.get(), &error_new, &tox_options_testing, &error_new_testing);
 
     if (tox == nullptr) {
         // It might fail, because some I/O happens in tox_new, and the fuzzer
@@ -161,6 +164,7 @@ void TestEndToEnd(Fuzz_Data &input)
     }
 
     assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);
 
     tox_events_init(tox);
 

external/toxcore/c-toxcore/testing/fuzzing/fuzz_support.hh

@@ -16,6 +16,7 @@
 #include <vector>
 
 #include "../../toxcore/tox.h"
+#include "../../toxcore/tox_private.h"
 
 struct Fuzz_Data {
     static constexpr bool DEBUG = false;

external/toxcore/c-toxcore/testing/fuzzing/protodump.cc

@@ -195,13 +195,16 @@ void RecordBootstrap(const char *init, const char *bootstrap)
         });
 
     Tox_Err_New error_new;
+    Tox_Err_New_Testing error_new_testing;
+    Tox_Options_Testing tox_options_testing;
 
     Record_System sys1(global, 4, "tox1");  // fair dice roll
     tox_options_set_log_user_data(opts, &sys1);
-    tox_options_set_operating_system(opts, sys1.sys.get());
+    tox_options_testing.operating_system = sys1.sys.get();
-    Tox *tox1 = tox_new(opts, &error_new);
+    Tox *tox1 = tox_new_testing(opts, &error_new, &tox_options_testing, &error_new_testing);
     assert(tox1 != nullptr);
     assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);
     std::array<uint8_t, TOX_ADDRESS_SIZE> address1;
     tox_self_get_address(tox1, address1.data());
     std::array<uint8_t, TOX_PUBLIC_KEY_SIZE> pk1;
@@ -211,10 +214,11 @@ void RecordBootstrap(const char *init, const char *bootstrap)
 
     Record_System sys2(global, 5, "tox2");  // unfair dice roll
     tox_options_set_log_user_data(opts, &sys2);
-    tox_options_set_operating_system(opts, sys2.sys.get());
+    tox_options_testing.operating_system = sys2.sys.get();
-    Tox *tox2 = tox_new(opts, &error_new);
+    Tox *tox2 = tox_new_testing(opts, &error_new, &tox_options_testing, &error_new_testing);
     assert(tox2 != nullptr);
     assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);
     std::array<uint8_t, TOX_ADDRESS_SIZE> address2;
     tox_self_get_address(tox2, address2.data());
     std::array<uint8_t, TOX_PUBLIC_KEY_SIZE> pk2;

external/toxcore/c-toxcore/testing/fuzzing/protodump_reduce.cc

@@ -142,9 +142,11 @@ void TestEndToEnd(Fuzz_Data &input)
 
     Ptr<Tox_Options> opts(tox_options_new(nullptr), tox_options_free);
     assert(opts != nullptr);
-    tox_options_set_operating_system(opts.get(), sys.sys.get());
     tox_options_set_local_discovery_enabled(opts.get(), false);
 
+    Tox_Options_Testing tox_options_testing;
+    tox_options_testing.operating_system = sys.sys.get();
+
     tox_options_set_log_callback(opts.get(),
         [](Tox *tox, Tox_Log_Level level, const char *file, uint32_t line, const char *func,
             const char *message, void *user_data) {
@@ -156,7 +158,8 @@ void TestEndToEnd(Fuzz_Data &input)
         });
 
     Tox_Err_New error_new;
-    Tox *tox = tox_new(opts.get(), &error_new);
+    Tox_Err_New_Testing error_new_testing;
+    Tox *tox = tox_new_testing(opts.get(), &error_new, &tox_options_testing, &error_new_testing);
 
     if (tox == nullptr) {
         // It might fail, because some I/O happens in tox_new, and the fuzzer
@@ -165,6 +168,7 @@ void TestEndToEnd(Fuzz_Data &input)
     }
 
     assert(error_new == TOX_ERR_NEW_OK);
+    assert(error_new_testing == TOX_ERR_NEW_TESTING_OK);
 
     tox_events_init(tox);
 

external/toxcore/c-toxcore/testing/fuzzing/toxsave_fuzz_test.cc

@@ -20,14 +20,15 @@ void TestSaveDataLoading(Fuzz_Data &input)
     const size_t savedata_size = input.size();
     CONSUME_OR_RETURN(const uint8_t *savedata, input, savedata_size);
 
-    Null_System sys;
-    tox_options_set_operating_system(tox_options, sys.sys.get());
-
     // pass test data to Tox
     tox_options_set_savedata_data(tox_options, savedata, savedata_size);
     tox_options_set_savedata_type(tox_options, TOX_SAVEDATA_TYPE_TOX_SAVE);
 
-    Tox *tox = tox_new(tox_options, nullptr);
+    Tox_Options_Testing tox_options_testing;
+    Null_System sys;
+    tox_options_testing.operating_system = sys.sys.get();
+
+    Tox *tox = tox_new_testing(tox_options, nullptr, &tox_options_testing, nullptr);
     tox_options_free(tox_options);
     if (tox == nullptr) {
         // Tox save was invalid, we're finished here

external/toxcore/c-toxcore/toxav/toxav.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2018 The TokTok team.
+ * Copyright © 2016-2024 The TokTok team.
  * Copyright © 2013-2015 Tox project.
  */
 
@@ -7,39 +7,40 @@
  * @brief Public audio/video API for Tox clients.
  *
  * This API can handle multiple calls. Each call has its state, in very rare
- * occasions the library can change the state of the call without apps knowledge.
+ * occasions the library can change the state of the call without apps
+ * knowledge.
  *
  * @section av_events Events and callbacks
  *
- * As in Core API, events are handled by callbacks. One callback can be
+ * As in the toxcore API, events are handled by callbacks. One callback can be
  * registered per event. All events have a callback function type named
- * `toxav_{event}_cb` and a function to register it named `toxav_callback_{event}`.
+ * `toxav_{event}_cb` and a function to register it named
- * Passing a NULL callback will result in no callback being registered for that
+ * `toxav_callback_{event}`. Passing a NULL callback will result in no callback
- * event. Only one callback per event can be registered, so if a client needs
+ * being registered for that event. Only one callback per event can be
- * multiple event listeners, it needs to implement the dispatch functionality
+ * registered, so if a client needs multiple event listeners, it needs to
- * itself. Unlike Core API, lack of some event handlers will cause the the
+ * implement the dispatch functionality itself. Unlike the toxcore API, lack of
- * library to drop calls before they are started. Hanging up call from a
+ * some event handlers will cause the the library to drop calls before they are
- * callback causes undefined behaviour.
+ * started. Hanging up call from a callback causes undefined behaviour.
  *
  * @section av_threading Threading implications
  *
  * Only toxav_iterate is thread-safe, all other functions must run from the
- * tox thread.
+ * Tox thread.
  *
  * Important exceptions are the `*_iterate` and `*_iterate_interval`
  * functions. You have to choose either the single thread or the multi thread
  * functions and read their documentation.
  *
  * A common way to run ToxAV (multiple or single instance) is to have a thread,
- * separate from tox instance thread, running a simple toxav_iterate loop,
+ * separate from Tox instance thread, running a simple toxav_iterate loop,
  * sleeping for `toxav_iteration_interval * milliseconds` on each iteration.
  *
- * An important thing to note is that events are triggered from both tox and
+ * An important thing to note is that events are triggered from both Tox and
- * toxav thread (see above). Audio and video receive frame events are triggered
+ * ToxAV thread (see above). Audio and video receive frame events are triggered
- * from toxav thread while all the other events are triggered from tox thread.
+ * from ToxAV thread while all the other events are triggered from Tox thread.
  *
- * Tox thread has priority with mutex mechanisms. Any api function can
+ * Tox thread has priority with mutex mechanisms. Any API function can
- * fail if mutexes are held by tox thread in which case they will set SYNC
+ * fail if mutexes are held by Tox thread in which case they will set SYNC
  * error code.
  *
  * @subsection av_multi_threading Separate audio and video threads
@@ -77,9 +78,9 @@ typedef struct Tox Tox;
  *
  * Each ToxAV instance can be bound to only one Tox instance, and Tox instance
  * can have only one ToxAV instance. One must make sure to close ToxAV instance
- * prior closing Tox instance otherwise undefined behaviour occurs. Upon
+ * prior closing Tox instance otherwise undefined behaviour occurs. Upon closing
- * closing of ToxAV instance, all active calls will be forcibly terminated
+ * of ToxAV instance, all active calls will be forcibly terminated without
- * without notifying peers.
+ * notifying peers.
  */
 typedef struct ToxAV ToxAV;
 
@@ -100,8 +101,8 @@ typedef enum Toxav_Err_New {
     TOXAV_ERR_NEW_NULL,
 
     /**
-     * Memory allocation failure while trying to allocate structures required for
+     * Memory allocation failure while trying to allocate structures required
-     * the A/V session.
+     * for the A/V session.
      */
     TOXAV_ERR_NEW_MALLOC,
 
@@ -203,8 +204,8 @@ typedef enum Toxav_Err_Call {
     TOXAV_ERR_CALL_OK,
 
     /**
-     * A resource allocation error occurred while trying to create the structures
+     * A resource allocation error occurred while trying to create the
-     * required for the call.
+     * structures required for the call.
      */
     TOXAV_ERR_CALL_MALLOC,
 
@@ -293,8 +294,9 @@ typedef enum Toxav_Err_Answer {
     TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND,
 
     /**
-     * The friend was valid, but they are not currently trying to initiate a call.
+     * The friend was valid, but they are not currently trying to initiate a
-     * This is also returned if this client is already in a call with the friend.
+     * call. This is also returned if this client is already in a call with the
+     * friend.
      */
     TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING,
 
@@ -337,8 +339,8 @@ enum Toxav_Friend_Call_State {
     /**
      * Set by the AV core if an error occurred on the remote end or if friend
      * timed out. This is the final state after which no more state
-     * transitions can occur for the call. This call state will never be triggered
+     * transitions can occur for the call. This call state will never be
-     * in combination with other call states.
+     * triggered in combination with other call states.
      */
     TOXAV_FRIEND_CALL_STATE_ERROR = 1,
 
@@ -377,8 +379,8 @@ enum Toxav_Friend_Call_State {
  * @param friend_number The friend number for which the call state changed.
  * @param state The bitmask of the new call state which is guaranteed to be
  *   different than the previous state. The state is set to 0 when the call is
- * paused. The bitmask represents all the activities currently performed by the
+ *   paused. The bitmask represents all the activities currently performed by
- * friend.
+ *   the friend.
  */
 typedef void toxav_call_state_cb(ToxAV *av, uint32_t friend_number, uint32_t state, void *user_data);
 
@@ -397,8 +399,9 @@ void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *callback, void *u
 typedef enum Toxav_Call_Control {
 
     /**
-     * Resume a previously paused call. Only valid if the pause was caused by this
+     * Resume a previously paused call. Only valid if the pause was caused by
-     * client, if not, this control is ignored. Not valid before the call is accepted.
+     * this client, if not, this control is ignored. Not valid before the call
+     * is accepted.
      */
     TOXAV_CALL_CONTROL_RESUME,
 
@@ -457,8 +460,8 @@ typedef enum Toxav_Err_Call_Control {
     TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND,
 
     /**
-     * This client is currently not in a call with the friend. Before the call is
+     * This client is currently not in a call with the friend. Before the call
-     * answered, only CANCEL is a valid control.
+     * is answered, only CANCEL is a valid control.
      */
     TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL,
 
@@ -530,8 +533,8 @@ typedef enum Toxav_Err_Send_Frame {
     TOXAV_ERR_SEND_FRAME_OK,
 
     /**
-     * In case of video, one of Y, U, or V was NULL. In case of audio, the samples
+     * In case of video, one of Y, U, or V was NULL. In case of audio, the
-     * data pointer was NULL.
+     * samples data pointer was NULL.
      */
     TOXAV_ERR_SEND_FRAME_NULL,
 
@@ -557,13 +560,13 @@ typedef enum Toxav_Err_Send_Frame {
     TOXAV_ERR_SEND_FRAME_INVALID,
 
     /**
-     * Either friend turned off audio or video receiving or we turned off sending
+     * Either friend turned off audio or video receiving or we turned off
-     * for the said payload.
+     * sending for the said payload.
      */
     TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED,
 
     /**
-     * Failed to push frame through rtp interface.
+     * Failed to push frame through RTP interface.
      */
     TOXAV_ERR_SEND_FRAME_RTP_FAILED,
 
@@ -572,7 +575,8 @@ typedef enum Toxav_Err_Send_Frame {
 /**
  * Send an audio frame to a friend.
  *
- * The expected format of the PCM data is: `[s1c1][s1c2][...][s2c1][s2c2][...]...`
+ * The expected format of the PCM data is:
+ * `[s1c1][s1c2][...][s2c1][s2c2][...]...`
  * Meaning: sample 1 for channel 1, sample 1 for channel 2, ...
  * For mono audio, this has no meaning, every sample is subsequent. For stereo,
  * this means the expected format is LRLRLR... with samples for left and right
@@ -584,7 +588,7 @@ typedef enum Toxav_Err_Send_Frame {
  *   `sample_count * channels`.
  * @param sample_count Number of samples in this frame. Valid numbers here are
  *   `((sample rate) * (audio length) / 1000)`, where audio length can be
- *   2.5, 5, 10, 20, 40 or 60 millseconds.
+ *   2.5, 5, 10, 20, 40 or 60 milliseconds.
  * @param channels Number of audio channels. Supported values are 1 and 2.
  * @param sampling_rate Audio sampling rate used in this frame. Valid sampling
  *   rates are 8000, 12000, 16000, 24000, or 48000.
@@ -605,8 +609,8 @@ bool toxav_audio_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_ra
 
 /**
  * The function type for the audio_bit_rate callback. The event is triggered
- * when the network becomes too saturated for current bit rates at which
+ * when the network becomes too saturated for current bit rates at which point
- * point core suggests new bit rates.
+ * ToxAV suggests new bit rates.
  *
  * @param friend_number The friend number of the friend for which to set the
  *   bit rate.
@@ -655,8 +659,8 @@ bool toxav_video_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_ra
 
 /**
  * The function type for the video_bit_rate callback. The event is triggered
- * when the network becomes too saturated for current bit rates at which
+ * when the network becomes too saturated for current bit rates at which point
- * point core suggests new bit rates.
+ * ToxAV suggests new bit rates.
  *
  * @param friend_number The friend number of the friend for which to set the
  *   bit rate.
@@ -714,7 +718,6 @@ void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb
  * @param y Luminosity plane. `Size = MAX(width, abs(ystride)) * height`.
  * @param u U chroma plane. `Size = MAX(width/2, abs(ustride)) * (height/2)`.
  * @param v V chroma plane. `Size = MAX(width/2, abs(vstride)) * (height/2)`.
- *
  * @param ystride Luminosity plane stride.
  * @param ustride U chroma plane stride.
  * @param vstride V chroma plane stride.
@@ -737,26 +740,28 @@ void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb
 #ifndef APIGEN_IGNORE
 
 /***
- * NOTE Compatibility with old toxav group calls. TODO(iphydf): remove
+ * NOTE Compatibility with old ToxAV group calls. TODO(iphydf): remove
  *
  * TODO(iphydf): Use proper new API guidelines for these. E.g. don't use inline
  * function types, don't have per-callback userdata, especially don't have one
  * userdata per group.
  */
 
-// TODO(iphydf): Use this better typed one instead of the void-pointer one below.
+// TODO(iphydf): Use this better typed one instead of the void-pointer one
+// below.
 typedef void toxav_group_audio_cb(Tox *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t pcm[],
                                   uint32_t samples, uint8_t channels, uint32_t sample_rate, void *user_data);
 
 typedef void toxav_audio_data_cb(void *tox, uint32_t groupnumber, uint32_t peernumber, const int16_t pcm[],
                                  uint32_t samples, uint8_t channels, uint32_t sample_rate, void *userdata);
 
-/** @brief Create a new toxav group.
+/** @brief Create a new ToxAV group.
  *
  * @return group number on success.
  * @retval -1 on failure.
  *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
  */
 int32_t toxav_add_av_groupchat(Tox *tox, toxav_audio_data_cb *audio_callback, void *userdata);
 
@@ -765,7 +770,8 @@ int32_t toxav_add_av_groupchat(Tox *tox, toxav_audio_data_cb *audio_callback, vo
  * @return group number on success.
  * @retval -1 on failure.
  *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
  */
 int32_t toxav_join_av_groupchat(
     Tox *tox, uint32_t friendnumber, const uint8_t data[], uint16_t length,
@@ -776,7 +782,8 @@ int32_t toxav_join_av_groupchat(
  * @retval 0 on success.
  * @retval -1 on failure.
  *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
  *
  * Valid number of samples are `(sample rate) * (audio length) / 1000`
  *   (Valid values for audio length are: 2.5, 5, 10, 20, 40 or 60 ms)
@@ -794,15 +801,16 @@ int32_t toxav_group_send_audio(
  * A/V must be enabled on a groupchat for audio to be sent to it and for
  * received audio to be handled.
  *
- * An A/V group created with `toxav_add_av_groupchat` or `toxav_join_av_groupchat`
+ * An A/V group created with `toxav_add_av_groupchat` or
- * will start with A/V enabled.
+ * `toxav_join_av_groupchat` will start with A/V enabled.
  *
  * An A/V group loaded from a savefile will start with A/V disabled.
  *
  * @retval 0 on success.
  * @retval -1 on failure.
  *
- * Note that total size of pcm in bytes is equal to `samples * channels * sizeof(int16_t)`.
+ * Note that total size of pcm in bytes is equal to
+ * `samples * channels * sizeof(int16_t)`.
  */
 int32_t toxav_groupchat_enable_av(
     Tox *tox, uint32_t groupnumber,

external/toxcore/c-toxcore/toxcore/tox.c

@@ -712,7 +712,8 @@ static int tox_load(Tox *tox, const uint8_t *data, uint32_t length)
                       length - cookie_len, STATE_COOKIE_TYPE);
 }
 
-Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
+nullable(1, 2, 3)
+static Tox *tox_new_system(const struct Tox_Options *options, Tox_Err_New *error, const Tox_System *sys)
 {
     struct Tox_Options *default_options = nullptr;
 
@@ -736,7 +737,6 @@ Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
     const struct Tox_Options *const opts = options != nullptr ? options : default_options;
     assert(opts != nullptr);
 
-    const Tox_System *sys = tox_options_get_operating_system(opts);
     const Tox_System default_system = tox_default_system();
 
     if (sys == nullptr) {
@@ -1020,6 +1020,37 @@ Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
     return tox;
 }
 
+Tox *tox_new(const struct Tox_Options *options, Tox_Err_New *error)
+{
+    return tox_new_system(options, error, nullptr);
+}
+
+Tox *tox_new_testing(const Tox_Options *options, Tox_Err_New *error, const Tox_Options_Testing *testing, Tox_Err_New_Testing *testing_error)
+{
+    if (testing == nullptr) {
+        SET_ERROR_PARAMETER(error, TOX_ERR_NEW_NULL);
+        SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_NULL);
+        return nullptr;
+    }
+
+    if (testing->operating_system == nullptr) {
+        SET_ERROR_PARAMETER(error, TOX_ERR_NEW_NULL);
+        SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_NULL);
+        return nullptr;
+    }
+
+    const Tox_System *sys = testing->operating_system;
+
+    if (sys->rng == nullptr || sys->ns == nullptr || sys->mem == nullptr) {
+        SET_ERROR_PARAMETER(error, TOX_ERR_NEW_NULL);
+        SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_NULL);
+        return nullptr;
+    }
+
+    SET_ERROR_PARAMETER(testing_error, TOX_ERR_NEW_TESTING_OK);
+    return tox_new_system(options, error, sys);
+}
+
 void tox_kill(Tox *tox)
 {
     if (tox == nullptr) {
@@ -4269,7 +4300,7 @@ uint32_t tox_group_invite_accept(Tox *tox, uint32_t friend_number, const uint8_t
         }
 
         case -6: {
-            SET_ERROR_PARAMETER(error, TOX_ERR_GROUP_INVITE_ACCEPT_CORE);
+            SET_ERROR_PARAMETER(error, TOX_ERR_GROUP_INVITE_ACCEPT_FRIEND_NOT_FOUND);
             return UINT32_MAX;
         }
 

external/toxcore/c-toxcore/toxcore/tox_api.c

@@ -265,14 +265,6 @@ void tox_options_set_experimental_thread_safety(
 {
     options->experimental_thread_safety = experimental_thread_safety;
 }
-const Tox_System *tox_options_get_operating_system(const Tox_Options *options)
-{
-    return options->operating_system;
-}
-void tox_options_set_operating_system(Tox_Options *options, const Tox_System *operating_system)
-{
-    options->operating_system = operating_system;
-}
 bool tox_options_get_experimental_groups_persistence(const Tox_Options *options)
 {
     return options->experimental_groups_persistence;
@@ -1451,8 +1443,8 @@ const char *tox_err_group_invite_accept_to_string(Tox_Err_Group_Invite_Accept va
         case TOX_ERR_GROUP_INVITE_ACCEPT_PASSWORD:
             return "TOX_ERR_GROUP_INVITE_ACCEPT_PASSWORD";
 
-        case TOX_ERR_GROUP_INVITE_ACCEPT_CORE:
+        case TOX_ERR_GROUP_INVITE_ACCEPT_FRIEND_NOT_FOUND:
-            return "TOX_ERR_GROUP_INVITE_ACCEPT_CORE";
+            return "TOX_ERR_GROUP_INVITE_ACCEPT_FRIEND_NOT_FOUND";
 
         case TOX_ERR_GROUP_INVITE_ACCEPT_FAIL_SEND:
             return "TOX_ERR_GROUP_INVITE_ACCEPT_FAIL_SEND";

external/toxcore/c-toxcore/toxcore/tox_dispatch.h

@@ -2,6 +2,13 @@
  * Copyright © 2022 The TokTok team.
  */
 
+/**
+ * WARNING: This is an experimental API and is subject to change.
+ *
+ * At this point, it probably won't change very much anymore, but we may have
+ * small breaking changes before a stable release.
+ */
+
 #ifndef C_TOXCORE_TOXCORE_TOX_DISPATCH_H
 #define C_TOXCORE_TOXCORE_TOX_DISPATCH_H
 

external/toxcore/c-toxcore/toxcore/tox_events.h

@@ -2,6 +2,13 @@
  * Copyright © 2022-2024 The TokTok team.
  */
 
+/**
+ * WARNING: This is an experimental API and is subject to change.
+ *
+ * At this point, it probably won't change very much anymore, but we may have
+ * small breaking changes before a stable release.
+ */
+
 #ifndef C_TOXCORE_TOXCORE_TOX_EVENTS_H
 #define C_TOXCORE_TOXCORE_TOX_EVENTS_H
 
@@ -570,6 +577,8 @@ void tox_events_free(Tox_Events *events);
 uint32_t tox_events_bytes_size(const Tox_Events *events);
 bool tox_events_get_bytes(const Tox_Events *events, uint8_t *bytes);
 
+typedef struct Tox_System Tox_System;
+
 Tox_Events *tox_events_load(const Tox_System *sys, const uint8_t *bytes, uint32_t bytes_size);
 
 bool tox_events_equal(const Tox_System *sys, const Tox_Events *a, const Tox_Events *b);

external/toxcore/c-toxcore/toxcore/tox_private.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2020 The TokTok team.
+ * Copyright © 2016-2024 The TokTok team.
  * Copyright © 2013 Tox project.
  */
 
@@ -18,38 +18,49 @@ extern "C" {
 
 typedef uint64_t tox_mono_time_cb(void *user_data);
 
-struct Tox_System {
+typedef struct Tox_System {
     tox_mono_time_cb *mono_time_callback;
     void *mono_time_user_data;
     const struct Random *rng;
     const struct Network *ns;
     const struct Memory *mem;
-};
+} Tox_System;
 
 Tox_System tox_default_system(void);
 
+const Tox_System *tox_get_system(Tox *tox);
+
+typedef struct Tox_Options_Testing {
+    const struct Tox_System *operating_system;
+} Tox_Options_Testing;
+
+typedef enum Tox_Err_New_Testing {
+    TOX_ERR_NEW_TESTING_OK,
+    TOX_ERR_NEW_TESTING_NULL,
+} Tox_Err_New_Testing;
+
+Tox *tox_new_testing(const Tox_Options *options, Tox_Err_New *error, const Tox_Options_Testing *testing, Tox_Err_New_Testing *testing_error);
+
 void tox_lock(const Tox *tox);
 void tox_unlock(const Tox *tox);
 
-const Tox_System *tox_get_system(Tox *tox);
-
 /**
- * Set the callback for the `friend_lossy_packet` event for a specific packet ID.
+ * Set the callback for the `friend_lossy_packet` event for a specific packet
- * Pass NULL to unset.
+ * ID. Pass NULL to unset.
  *
  * allowed packet ID range:
- * from `PACKET_ID_RANGE_LOSSY_START` to `PACKET_ID_RANGE_LOSSY_END` (both inclusive)
+ * from `PACKET_ID_RANGE_LOSSY_START` to `PACKET_ID_RANGE_LOSSY_END` (both
+ * inclusive)
  */
 void tox_callback_friend_lossy_packet_per_pktid(Tox *tox, tox_friend_lossy_packet_cb *callback, uint8_t pktid);
 
 /**
- * Set the callback for the `friend_lossless_packet` event for a specific packet ID.
+ * Set the callback for the `friend_lossless_packet` event for a specific packet
- * Pass NULL to unset.
+ * ID. Pass NULL to unset.
  *
  * allowed packet ID range:
- * from `PACKET_ID_RANGE_LOSSLESS_CUSTOM_START` to `PACKET_ID_RANGE_LOSSLESS_CUSTOM_END` (both inclusive)
+ * from `PACKET_ID_RANGE_LOSSLESS_CUSTOM_START` to
- * and
+ * `PACKET_ID_RANGE_LOSSLESS_CUSTOM_END` (both inclusive) and `PACKET_ID_MSI`
- * `PACKET_ID_MSI`
  */
 void tox_callback_friend_lossless_packet_per_pktid(Tox *tox, tox_friend_lossless_packet_cb *callback, uint8_t pktid);
 
@@ -78,7 +89,7 @@ uint32_t tox_dht_node_public_key_size(void);
 
 /**
  * @param public_key The node's public key.
- * @param ip The node's IP address, represented as a null terminated string.
+ * @param ip The node's IP address, represented as a NUL-terminated C string.
  * @param port The node's port.
  */
 typedef void tox_dht_get_nodes_response_cb(Tox *tox, const uint8_t *public_key, const char *ip, uint16_t port,
@@ -98,7 +109,8 @@ typedef enum Tox_Err_Dht_Get_Nodes {
     TOX_ERR_DHT_GET_NODES_OK,
 
     /**
-     * UDP is disabled in tox options; the DHT can only be queried when UDP is enabled.
+     * UDP is disabled in Tox options; the DHT can only be queried when UDP is
+     * enabled.
      */
     TOX_ERR_DHT_GET_NODES_UDP_DISABLED,
 
@@ -118,21 +130,24 @@ typedef enum Tox_Err_Dht_Get_Nodes {
     TOX_ERR_DHT_GET_NODES_BAD_IP,
 
     /**
-     * The getnodes request failed. This usually means the packet failed to send.
+     * The getnodes request failed. This usually means the packet failed to
+     * send.
      */
     TOX_ERR_DHT_GET_NODES_FAIL,
 } Tox_Err_Dht_Get_Nodes;
 
 /**
  * This function sends a getnodes request to a DHT node for its peers that
- * are "close" to the passed target public key according to the distance metric used
+ * are "close" to the passed target public key according to the distance metric
- * by the DHT implementation.
+ * used by the DHT implementation.
  *
- * @param public_key The public key of the node that we wish to query. This key must be
+ * @param public_key The public key of the node that we wish to query. This key
- *   at least `TOX_DHT_NODE_PUBLIC_KEY_SIZE` bytes in length.
+ *   must be at least `TOX_DHT_NODE_PUBLIC_KEY_SIZE` bytes in length.
- * @param ip A NULL terminated string representing the IP address of the node we wish to query.
+ * @param ip A NUL-terminated C string representing the IP address of the node
+ *   we wish to query.
  * @param port The port of the node we wish to query.
- * @param target_public_key The public key for which we want to find close nodes.
+ * @param target_public_key The public key for which we want to find close
+ *   nodes.
  *
  * @return true on success.
  */
@@ -140,8 +155,9 @@ bool tox_dht_get_nodes(const Tox *tox, const uint8_t *public_key, const char *ip
                        const uint8_t *target_public_key, Tox_Err_Dht_Get_Nodes *error);
 
 /**
- * This function returns the ratio of close dht nodes that are known to support announce/store.
+ * This function returns the ratio of close dht nodes that are known to support
- * This function returns the number of DHT nodes in the closelist.
+ * announce/store. This function returns the number of DHT nodes in the
+ * closelist.
  *
  * @return number
  */
@@ -149,7 +165,7 @@ uint16_t tox_dht_get_num_closelist(const Tox *tox);
 
 /**
  * This function returns the number of DHT nodes in the closelist,
- * that are capable to store annouce data (introduced in version 0.2.18).
+ * that are capable to store announce data (introduced in version 0.2.18).
  *
  * @return number
  */
@@ -169,30 +185,32 @@ uint16_t tox_dht_get_num_closelist_announce_capable(const Tox *tox);
 uint32_t tox_group_peer_ip_string_max_length(void);
 
 /**
- * Return the length of the peer's IP address in string form. If the group number or ID
+ * Return the length of the peer's IP address in string form. If the group
- * is invalid, the return value is unspecified.
+ * number or ID is invalid, the return value is unspecified.
  *
  * @param group_number The group number of the group we wish to query.
- * @param peer_id The ID of the peer whose IP address length we want to retrieve.
+ * @param peer_id The ID of the peer whose IP address length we want to
+ *   retrieve.
  */
 size_t tox_group_peer_get_ip_address_size(const Tox *tox, uint32_t group_number, uint32_t peer_id,
         Tox_Err_Group_Peer_Query *error);
 /**
- * Write the IP address associated with the designated peer_id for the designated group number
+ * Write the IP address associated with the designated peer_id for the
- * to ip_addr.
+ * designated group number to ip_addr.
  *
- * If the peer is forcing TCP connections a placeholder value will be written instead,
+ * If the peer is forcing TCP connections a placeholder value will be written
- * indicating that their real IP address is unknown to us.
+ * instead, indicating that their real IP address is unknown to us.
  *
- * If `peer_id` designates ourself, it will write either our own IP address or a placeholder value,
+ * If `peer_id` designates ourself, it will write either our own IP address or a
- * depending on whether or not we're forcing TCP connections.
+ * placeholder value, depending on whether or not we're forcing TCP connections.
  *
- * Call tox_group_peer_get_ip_address_size to determine the allocation size for the `ip_addr` parameter.
+ * Call tox_group_peer_get_ip_address_size to determine the allocation size for
+ * the `ip_addr` parameter.
  *
  * @param group_number The group number of the group we wish to query.
  * @param peer_id The ID of the peer whose public key we wish to retrieve.
- * @param ip_addr A valid memory region large enough to store the IP address string.
+ * @param ip_addr A valid memory region large enough to store the IP address
- *   If this parameter is NULL, this function call has no effect.
+ *   string. If this parameter is NULL, this function call has no effect.
  *
  * @return true on success.
  */

external/toxcore/c-toxcore/toxencryptsave/toxencryptsave.c

@@ -67,7 +67,8 @@ void tox_pass_key_free(Tox_Pass_Key *key)
  * produce the same key as was previously used. Any data encrypted with this
  * module can be used as input.
  *
- * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in length.
+ * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in
+ * length.
  * The salt must be TOX_PASS_SALT_LENGTH bytes in length.
  * If the passed byte arrays are smaller than required, the behaviour is
  * undefined.
@@ -182,10 +183,11 @@ Tox_Pass_Key *tox_pass_key_derive_with_salt(
 }
 
 /**
- * Encrypt a plain text with a key produced by tox_pass_key_derive or tox_pass_key_derive_with_salt.
+ * Encrypt a plain text with a key produced by tox_pass_key_derive or
+ * tox_pass_key_derive_with_salt.
  *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
+ * The output array must be at least
- * bytes long.
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long.
  *
  * @param plaintext A byte array of length `plaintext_len`.
  * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -242,9 +244,9 @@ bool tox_pass_key_encrypt(const Tox_Pass_Key *key, const uint8_t plaintext[], si
 /**
  * Encrypts the given data with the given passphrase.
  *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
+ * The output array must be at least
- * bytes long. This delegates to tox_pass_key_derive and
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This delegates
- * tox_pass_key_encrypt.
+ * to tox_pass_key_derive and tox_pass_key_encrypt.
  *
  * @param plaintext A byte array of length `plaintext_len`.
  * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -280,7 +282,8 @@ bool tox_pass_encrypt(const uint8_t plaintext[], size_t plaintext_len, const uin
  * tox_pass_key_derive or tox_pass_key_derive_with_salt.
  *
  * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
  * @param plaintext The plain text array to write the decrypted data to.
  *
  * @return true on success.
@@ -326,11 +329,13 @@ bool tox_pass_key_decrypt(const Tox_Pass_Key *key, const uint8_t ciphertext[], s
 /**
  * Decrypts the given data with the given passphrase.
  *
- * The output array must be at least `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
+ * The output array must be at least
- * bytes long. This delegates to tox_pass_key_decrypt.
+ * `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This
+ * delegates to tox_pass_key_decrypt.
  *
  * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
  * @param passphrase The user-provided password. Can be empty.
  * @param passphrase_len The length of the password.
  * @param plaintext The plain text array to write the decrypted data to.

external/toxcore/c-toxcore/toxencryptsave/toxencryptsave.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2018 The TokTok team.
+ * Copyright © 2016-2024 The TokTok team.
  * Copyright © 2013-2016 Tox Developers.
  */
 
@@ -165,9 +165,9 @@ typedef enum Tox_Err_Decryption {
 /**
  * Encrypts the given data with the given passphrase.
  *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
+ * The output array must be at least
- * bytes long. This delegates to tox_pass_key_derive and
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This delegates
- * tox_pass_key_encrypt.
+ * to tox_pass_key_derive and tox_pass_key_encrypt.
  *
  * @param plaintext A byte array of length `plaintext_len`.
  * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -183,11 +183,13 @@ bool tox_pass_encrypt(const uint8_t plaintext[], size_t plaintext_len, const uin
 /**
  * Decrypts the given data with the given passphrase.
  *
- * The output array must be at least `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
+ * The output array must be at least
- * bytes long. This delegates to tox_pass_key_decrypt.
+ * `ciphertext_len - TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long. This
+ * delegates to tox_pass_key_decrypt.
  *
  * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
  * @param passphrase The user-provided password. Can be empty.
  * @param passphrase_len The length of the password.
  * @param plaintext The plain text array to write the decrypted data to.
@@ -215,7 +217,8 @@ bool tox_pass_decrypt(const uint8_t ciphertext[], size_t ciphertext_len, const u
  * user-provided password.
  *
  * The Tox_Pass_Key structure is hidden in the implementation. It can be created
- * using tox_pass_key_derive or tox_pass_key_derive_with_salt and must be deallocated using tox_pass_key_free.
+ * using tox_pass_key_derive or tox_pass_key_derive_with_salt and must be
+ * deallocated using tox_pass_key_free.
  */
 #ifndef TOX_PASS_KEY_DEFINED
 #define TOX_PASS_KEY_DEFINED
@@ -261,10 +264,11 @@ Tox_Pass_Key *tox_pass_key_derive_with_salt(
     const uint8_t salt[TOX_PASS_SALT_LENGTH], Tox_Err_Key_Derivation *error);
 
 /**
- * Encrypt a plain text with a key produced by tox_pass_key_derive or tox_pass_key_derive_with_salt.
+ * Encrypt a plain text with a key produced by tox_pass_key_derive or
+ * tox_pass_key_derive_with_salt.
  *
- * The output array must be at least `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH`
+ * The output array must be at least
- * bytes long.
+ * `plaintext_len + TOX_PASS_ENCRYPTION_EXTRA_LENGTH` bytes long.
  *
  * @param plaintext A byte array of length `plaintext_len`.
  * @param plaintext_len The length of the plain text array. Bigger than 0.
@@ -280,7 +284,8 @@ bool tox_pass_key_encrypt(const Tox_Pass_Key *key, const uint8_t plaintext[], si
  * tox_pass_key_derive or tox_pass_key_derive_with_salt.
  *
  * @param ciphertext A byte array of length `ciphertext_len`.
- * @param ciphertext_len The length of the cipher text array. At least TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
+ * @param ciphertext_len The length of the cipher text array. At least
+ *   TOX_PASS_ENCRYPTION_EXTRA_LENGTH.
  * @param plaintext The plain text array to write the decrypted data to.
  *
  * @return true on success.
@@ -315,7 +320,8 @@ typedef enum Tox_Err_Get_Salt {
  * produce the same key as was previously used. Any data encrypted with this
  * module can be used as input.
  *
- * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in length.
+ * The cipher text must be at least TOX_PASS_ENCRYPTION_EXTRA_LENGTH bytes in
+ * length.
  * The salt must be TOX_PASS_SALT_LENGTH bytes in length.
  * If the passed byte arrays are smaller than required, the behaviour is
  * undefined.

flake.lock

@@ -34,10 +34,28 @@
         "type": "github"
       }
     },
+    "nlohmann-json": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1701207391,
+        "narHash": "sha256-7F0Jon+1oWL7uqet5i1IgHX0fUw/+z0QwEcA3zs5xHg=",
+        "owner": "nlohmann",
+        "repo": "json",
+        "rev": "9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlohmann",
+        "ref": "v3.11.3",
+        "repo": "json",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nlohmann-json": "nlohmann-json"
       }
     },
     "systems": {

flake.nix

@@ -6,19 +6,25 @@
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/release-23.11";
     flake-utils.url = "github:numtide/flake-utils";
+    nlohmann-json = {
+      url = "github:nlohmann/json/v3.11.3"; # TODO: read version from file
+      flake = false;
+    };
   };
 
-  outputs = { self, nixpkgs, flake-utils }:
+  outputs = { self, nixpkgs, flake-utils, nlohmann-json }:
     flake-utils.lib.eachDefaultSystem (system:
     let
       pkgs = import nixpkgs { inherit system; };
+      stdenv = (pkgs.stdenvAdapters.keepDebugInfo pkgs.stdenv);
     in {
-      packages.default = pkgs.stdenv.mkDerivation {
+      #packages.default = pkgs.stdenv.mkDerivation {
+      packages.default = stdenv.mkDerivation {
         pname = "tomato";
         version = "0.0.0";
 
         src = ./.;
-        submodules = 1;
+        submodules = 1; # does nothing
 
         nativeBuildInputs = with pkgs; [
           cmake
@@ -56,8 +62,12 @@
         ] ++ self.packages.${system}.default.dlopenBuildInputs;
 
         cmakeFlags = [
-          "TOMATO_ASAN=1"
+          "-DTOMATO_ASAN=OFF"
-          "CMAKE_BUILD_TYPE=RelWithDebInfo"
+          "-DCMAKE_BUILD_TYPE=RelWithDebInfo"
+
+          "-DFETCHCONTENT_SOURCE_DIR_JSON=${nlohmann-json}" # we care about the version
+          # TODO: use package instead
+          "-DFETCHCONTENT_SOURCE_DIR_ZSTD=${pkgs.zstd.src}" # we dont care about the version (we use 1.4.x features)
         ];
 
         # TODO: replace with install command
@@ -66,7 +76,7 @@
           mv bin/tomato $out/bin
         '';
 
-        dontStrip = true;
+        dontStrip = true; # does nothing
 
         # copied from nixpkgs's SDL2 default.nix
         # SDL is weird in that instead of just dynamically linking with
@@ -93,6 +103,8 @@
           '';
       };
 
+      #packages.debug = pkgs.enableDebugging self.packages.${system}.default;
+
       devShells.${system}.default = pkgs.mkShell {
         #inputsFrom = with pkgs; [ SDL2 ];
         buildInputs = [ self.packages.${system}.default ]; # this makes a prebuild tomato available in the shell, do we want this?

src/CMakeLists.txt

@@ -1,5 +1,7 @@
 cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
 
+########################################
+
 add_executable(tomato
 	./main.cpp
 	./icon.rc
@@ -75,6 +77,7 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_util
 	solanaceae_contact
 	solanaceae_message3
+	solanaceae_message_serializer
 
 	solanaceae_plugin
 
@@ -82,6 +85,8 @@ target_link_libraries(tomato PUBLIC
 	solanaceae_tox_contacts
 	solanaceae_tox_messages
 
+	solanaceae_object_store
+
 	SDL3::SDL3
 
 	imgui

src/chat_gui4.cpp

@@ -47,6 +47,18 @@ namespace Components {
 
 } // Components
 
+namespace Context {
+
+	// TODO: move back to chat log window and keep per window instead of per contact
+	struct CGView {
+		// set to the ts of the newest rendered msg
+		Message3Handle begin{};
+		// set to the ts of the oldest rendered msg
+		Message3Handle end{};
+	};
+
+} // Context
+
 static constexpr float lerp(float a, float b, float t) {
 	return a + t * (b - a);
 }
@@ -269,28 +281,6 @@ float ChatGui4::render(float time_delta) {
 
 					auto* msg_reg_ptr = _rmm.get(*_selected_contact);
 
-					if (msg_reg_ptr != nullptr) {
-						const auto& mm = *msg_reg_ptr;
-						//const auto& unread_storage = mm.storage<Message::Components::TagUnread>();
-						if (const auto* unread_storage = mm.storage<Message::Components::TagUnread>(); unread_storage != nullptr && !unread_storage->empty()) {
-							//assert(unread_storage->size() == 0);
-							//assert(unread_storage.cbegin() == unread_storage.cend());
-
-#if 0
-							std::cout << "UNREAD ";
-							Message3 prev_ent = entt::null;
-							for (const Message3 e : mm.view<Message::Components::TagUnread>()) {
-								std::cout << entt::to_integral(e) << " ";
-								if (prev_ent == e) {
-									assert(false && "dup");
-								}
-								prev_ent = e;
-							}
-							std::cout << "\n";
-#endif
-						}
-					}
-
 					constexpr ImGuiTableFlags table_flags =
 						ImGuiTableFlags_BordersInnerV |
 						ImGuiTableFlags_RowBg |
@@ -303,6 +293,9 @@ float ChatGui4::render(float time_delta) {
 						ImGui::TableSetupColumn("timestamp");
 						ImGui::TableSetupColumn("extra_info", _show_chat_extra_info ? ImGuiTableColumnFlags_None : ImGuiTableColumnFlags_Disabled);
 
+						Message3Handle message_view_oldest; // oldest visible message
+						Message3Handle message_view_newest; // last visible message
+
 						// very hacky, and we have variable hight entries
 						//ImGuiListClipper clipper;
 
@@ -389,12 +382,26 @@ float ChatGui4::render(float time_delta) {
 								}
 
 								// use username as visibility test
-								if (ImGui::IsItemVisible() && msg_reg.all_of<Message::Components::TagUnread>(e)) {
+								if (ImGui::IsItemVisible()) {
+									if (msg_reg.all_of<Message::Components::TagUnread>(e)) {
 										// get time now
 										const uint64_t ts_now = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
 										msg_reg.emplace_or_replace<Message::Components::Read>(e, ts_now);
 										msg_reg.remove<Message::Components::TagUnread>(e);
 										msg_reg.emplace_or_replace<Components::UnreadFade>(e, 1.f);
+
+										// we remove the unread tag here
+										_rmm.throwEventUpdate(msg_reg, e);
+									}
+
+									// track view
+									if (!static_cast<bool>(message_view_oldest)) {
+										message_view_oldest = {msg_reg, e};
+										message_view_newest = {msg_reg, e};
+									} else if (static_cast<bool>(message_view_newest)) {
+										// update to latest
+										message_view_newest = {msg_reg, e};
+									}
 								}
 
 								// highlight self
@@ -559,9 +566,90 @@ float ChatGui4::render(float time_delta) {
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 						//ImGui::TableNextRow(0, TEXT_BASE_HEIGHT);
 
+						{ // update view cursers
+							if (!msg_reg.ctx().contains<Context::CGView>()) {
+								msg_reg.ctx().emplace<Context::CGView>();
+							}
+
+							auto& cg_view = msg_reg.ctx().get<Context::CGView>();
+
+							// any message in view
+							if (!static_cast<bool>(message_view_oldest)) {
+								// no message in view, we setup a view at current time, so the next frags are loaded
+								if (!static_cast<bool>(cg_view.begin) || !static_cast<bool>(cg_view.end)) {
+									// fix invalid state
+									if (static_cast<bool>(cg_view.begin)) {
+										cg_view.begin.destroy();
+										_rmm.throwEventDestroy(cg_view.begin);
+									}
+									if (static_cast<bool>(cg_view.end)) {
+										cg_view.end.destroy();
+										_rmm.throwEventDestroy(cg_view.end);
+									}
+
+									// create new
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									cg_view.end = {msg_reg, msg_reg.create()};
+
+									cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+									cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+									cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+									cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts = Message::getTimeMS();
+
+									std::cout << "CG: created view FRONT begin ts\n";
+									_rmm.throwEventConstruct(cg_view.begin);
+									std::cout << "CG: created view FRONT end ts\n";
+									_rmm.throwEventConstruct(cg_view.end);
+								} // else? we do nothing?
+							} else {
+								bool begin_created {false};
+								if (!static_cast<bool>(cg_view.begin)) {
+									cg_view.begin = {msg_reg, msg_reg.create()};
+									begin_created = true;
+								}
+								bool end_created {false};
+								if (!static_cast<bool>(cg_view.end)) {
+									cg_view.end = {msg_reg, msg_reg.create()};
+									end_created = true;
+								}
+								cg_view.begin.emplace_or_replace<Message::Components::ViewCurserBegin>(cg_view.end);
+								cg_view.end.emplace_or_replace<Message::Components::ViewCurserEnd>(cg_view.begin);
+
+								{
+									auto& old_begin_ts = cg_view.begin.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_begin_ts != message_view_newest.get<Message::Components::Timestamp>().ts) {
+										old_begin_ts = message_view_newest.get<Message::Components::Timestamp>().ts;
+										if (begin_created) {
+											std::cout << "CG: created view begin ts with " << old_begin_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.begin);
+										} else {
+											//std::cout << "CG: updated view begin ts to " << old_begin_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.begin);
+										}
+									}
+								}
+
+								{
+									auto& old_end_ts = cg_view.end.get_or_emplace<Message::Components::Timestamp>().ts;
+									if (old_end_ts != message_view_oldest.get<Message::Components::Timestamp>().ts) {
+										old_end_ts = message_view_oldest.get<Message::Components::Timestamp>().ts;
+										if (end_created) {
+											std::cout << "CG: created view end ts with " << old_end_ts << "\n";
+											_rmm.throwEventConstruct(cg_view.end);
+										} else {
+											//std::cout << "CG: updated view end ts to " << old_end_ts << "\n";
+											_rmm.throwEventUpdate(cg_view.end);
+										}
+									}
+								}
+							}
+						}
+
 						ImGui::EndTable();
 					}
 
+
 					if (ImGui::GetScrollY() >= ImGui::GetScrollMaxY()) {
 						ImGui::SetScrollHereY(1.f);
 					}

src/chat_gui4.hpp

@@ -32,6 +32,7 @@ class ChatGui4 {
 	FileSelector _fss;
 	SendImagePopup _sip;
 
+	// TODO: refactor this to allow multiple open contacts
 	std::optional<Contact3> _selected_contact;
 
 	// TODO: per contact

src/fragment_store/README.md

@@ -1,72 +0,0 @@
-# Fragment Store
-
-Fragments are are pieces of information split into Metadata and Data.
-They can be stored seperated or together.
-They can be used as a Transport protocol/logic too.
-
-# Store types
-
-### Object Store
-
-Fragment files are stored with the first 2 hex chars as sub folders:
-eg:
-`objects/` (object store root)
-  - `5f/` (first 2hex subfolder)
-    - `4fffffff` (the fragment file without the first 2 hexchars)
-
-### Split Object Store
-
-Same as Object Store, but medadata and data stored in seperate files.
-Metadata files have the `.meta` suffix. They also have a filetype specific suffix, like `.json`, `.msgpack` etc.
-
-### Memory Store
-
-Just keeps the Fragments in memory.
-
-# File formats
-
-Files can be compressed and encrypted. Since compression needs the data structure to funcion, it is applied before it is encrypted.
-
-### Text Json
-
-Text json only makes sense for metadata if it's neither compressed nor encrypted. (otherwise its binary on disk anyway, so why waste bytes).
-Since the content of data is not looked at, nothing stops you from using text json and ecrypt it, but atleast basic compression is advised.
-
-A Metadata json object has the following keys:
-- `enc` (uint) Encryption type of the data, if any
-- `comp` (uint) Compression type of the data, if any
-- `metadata` (obj) the 
-
-## Binary file headers
-
-### Split Metadata
-
-file magic bytes `SOLMET` (6 bytes)
-
-1 byte encryption type (`0x00` is none)
-
-1 byte compression type (`0x00` is none)
-
-...metadata here...
-
-note that the encryption and compression are for the metadata only.
-The metadata itself contains encryption and compression info about the data.
-
-### Split Data
-
-(none) all the data is in the metadata file.
-This is mostly to allow direct storage for files in the Fragment store without excessive duplication.
-Keep in mind to not use the actual file name as the data/meta file name.
-
-### Single fragment
-
-file magic bytes `SOLFIL` (6 bytes)
-
-1 byte encryption type (`0x00` is none)
-
-1 byte compression type (`0x00` is none)
-
-...metadata here...
-
-...data here...
-

src/main.cpp

@@ -208,10 +208,17 @@ int main(int argc, char** argv) {
 #endif
 	}
 
+	// TODO: use scope for the unique ptrs
+
+	screen.reset();
+
 	ImGui_ImplSDLRenderer3_Shutdown();
 	ImGui_ImplSDL3_Shutdown();
 	ImGui::DestroyContext();
 
+	renderer.reset();
+	window.reset();
+
 	SDL_Quit();
 
 	return 0;

src/main_screen.cpp

@@ -1,5 +1,8 @@
 #include "./main_screen.hpp"
 
+#include <solanaceae/message3/nj/message_components_serializer.hpp>
+#include <solanaceae/tox_messages/nj/tox_message_components_serializer.hpp>
+
 #include <solanaceae/contact/components.hpp>
 
 #include <imgui/imgui.h>
@@ -12,6 +15,7 @@
 MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::string save_password, std::vector<std::string> plugins) :
 	renderer(renderer_),
 	rmm(cr),
+	msnj{cr, {}, {}},
 	mts(rmm),
 	tc(save_path, save_password),
 	tpi(tc.getTox()),
@@ -34,6 +38,9 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 {
 	tel.subscribeAll(tc);
 
+	registerMessageComponents(msnj);
+	registerToxMessageComponents(msnj);
+
 	conf.set("tox", "save_file_path", save_path);
 
 	{ // name stuff
@@ -49,9 +56,12 @@ MainScreen::MainScreen(SDL_Renderer* renderer_, std::string save_path, std::stri
 	std::cout << "own address: " << tc.toxSelfGetAddressStr() << "\n";
 
 	{ // setup plugin instances
+		g_provideInstance<ObjectStore2>("ObjectStore2", "host", &os);
+
 		g_provideInstance<ConfigModelI>("ConfigModelI", "host", &conf);
 		g_provideInstance<Contact3Registry>("Contact3Registry", "1", "host", &cr);
 		g_provideInstance<RegistryMessageModel>("RegistryMessageModel", "host", &rmm);
+		g_provideInstance<MessageSerializerNJ>("MessageSerializerNJ", "host", &msnj);
 
 		g_provideInstance<ToxI>("ToxI", "host", &tc);
 		g_provideInstance<ToxPrivateI>("ToxPrivateI", "host", &tpi);
@@ -215,9 +225,24 @@ Screen* MainScreen::render(float time_delta, bool&) {
 					ImGui::EndMenu();
 				}
 				if (ImGui::BeginMenu("Settings")) {
-					if (ImGui::MenuItem("ImGui Style Editor")) {
+					ImGui::SeparatorText("ImGui");
+
+					if (ImGui::MenuItem("Style Editor")) {
 						_show_tool_style_editor = true;
 					}
+
+					if (ImGui::MenuItem("Metrics")) {
+						_show_tool_metrics = true;
+					}
+
+					if (ImGui::MenuItem("Debug Log")) {
+						_show_tool_debug_log = true;
+					}
+
+					if (ImGui::MenuItem("ID Stack Tool")) {
+						_show_tool_id_stack = true;
+					}
+
 					ImGui::EndMenu();
 				}
 				ImGui::EndMenuBar();
@@ -234,6 +259,18 @@ Screen* MainScreen::render(float time_delta, bool&) {
 		ImGui::End();
 	}
 
+	if (_show_tool_metrics) {
+		ImGui::ShowMetricsWindow(&_show_tool_metrics);
+	}
+
+	if (_show_tool_debug_log) {
+		ImGui::ShowDebugLogWindow(&_show_tool_debug_log);
+	}
+
+	if (_show_tool_id_stack) {
+		ImGui::ShowIDStackToolWindow(&_show_tool_id_stack);
+	}
+
 	if constexpr (false) {
 		ImGui::ShowDemoWindow();
 	}
@@ -388,7 +425,7 @@ Screen* MainScreen::tick(float time_delta, bool& quit) {
 
 	tdch.tick(time_delta); // compute
 
-	mts.iterate(); // compute
+	mts.iterate(); // compute (after mfs)
 
 	_min_tick_interval = std::min<float>(
 		// HACK: pow by 1.6 to increase 50 -> ~500 (~522)

src/main_screen.hpp

@@ -2,10 +2,12 @@
 
 #include "./screen.hpp"
 
+#include <solanaceae/object_store/object_store.hpp>
 #include <solanaceae/util/simple_config_model.hpp>
 #include <solanaceae/contact/contact_model3.hpp>
 #include <solanaceae/message3/registry_message_model.hpp>
 #include <solanaceae/message3/message_time_sort.hpp>
+#include <solanaceae/message3/message_serializer.hpp>
 #include <solanaceae/plugin/plugin_manager.hpp>
 #include <solanaceae/toxcore/tox_event_logger.hpp>
 #include "./tox_private_impl.hpp"
@@ -43,13 +45,14 @@ extern "C" {
 struct MainScreen final : public Screen {
 	SDL_Renderer* renderer;
 
+	ObjectStore2 os;
+
 	SimpleConfigModel conf;
 	Contact3Registry cr;
 	RegistryMessageModel rmm;
+	MessageSerializerNJ msnj;
 	MessageTimeSort mts;
 
-	PluginManager pm;
-
 	ToxEventLogger tel{std::cout};
 	ToxClient tc;
 	ToxPrivateImpl tpi;
@@ -75,7 +78,12 @@ struct MainScreen final : public Screen {
 	ToxUIUtils tuiu;
 	ToxDHTCapHisto tdch;
 
+	PluginManager pm; // last, so it gets destroyed first
+
 	bool _show_tool_style_editor {false};
+	bool _show_tool_metrics {false};
+	bool _show_tool_debug_log {false};
+	bool _show_tool_id_stack {false};
 
 	bool _window_hidden {false};
 	uint64_t _window_hidden_ts {0};

src/tox_client.hpp

@@ -48,4 +48,3 @@ class ToxClient : public ToxDefaultImpl, public ToxEventProviderBase {
 		void saveToxProfile(void);
 };
 
-

src/tox_friend_faux_offline_messaging.cpp

@@ -120,7 +120,8 @@ ToxFriendFauxOfflineMessaging::dfmc_Ret ToxFriendFauxOfflineMessaging::doFriendM
 		// require
 		if (!mr->all_of<
 				Message::Components::MessageText, // text only for now
-				Message::Components::ContactTo
+				Message::Components::ContactTo,
+				Message::Components::ToxFriendMessageID // yes, needs fake ids
 			>(msg)
 		) {
 			continue; // skip