Merge commit '9ddeea3d06045c8ae38cd2d6eed0fc2891c6e146'

adbd5b32d8 feat: add ngc events
15ee46d431 add simple test for max sized lossy custom group packet
01e7950c67 increase lossy custom packet size in ngc to the toxcore common max of 1373
9b3c1089f1 Make group saving/loading more forgiving with data errors
55a76003b0 Replace memset(int32_t*, -1, _) with a for-loop
66453439ac fix: also Install header for private/experimental API functions with autotools
3983369103 fix: Enable debug flag for ubsan.
4d1db21102 Update tox-boostrapd hash
e700c31b70 Fix memory leak in group connection
2994441d9c Fix memory leak in save-generator
d0400df13d Fix memory leak in tox-bootstrapd
7a6d50ebe3 Install header for private/experimental API functions
d89677fb5f Remove defunct IRC channel from README.md
26d41fc604 Replace DEFAULT_TCP_RELAY_PORTS_COUNT with a compile-time calculation
63fb2941ca Clarify disabling of static assert checks
65b3375b98 refactor: Use Bin_Pack for packing Node_format.
84ba154f6a group connection queries now return our own connection type
a4df2862ed Replace tabs with spaces
1b6dee7594 Update tox-bootstrapd's base Docker images
a030cdee5c Fix Docker tox-bootstrapd hash update failing when using BuildKit
7cfe35dff2 cleanup: Remove explicit layering_check feature.
d390947245 chore: Upgrade sonar-scan jvm to java 17.
d1e850c56c fix: Add missing `htons` call when adding configured TCP relay.
814090f2b8 chore: Cancel old PR builds on docker and sonar-scan workflows.
83efb17367 perf: Add a KVM FreeBSD build on cirrus ci.
a927183233 test: Add a test for encrypting 100MB of data.
28f39049f6 chore: Retry freebsd tests 2 times.
47e77d1bb0 chore: Use C99 on MSVC instead of C11.
7155f7f60e test: Add an s390x build (on alpine) for CI.
6c35cef63f chore: Add a compcert docker run script.
41e6ea865e cleanup: Use tcc docker image for CI.
e726b197b0 refactor: Store time in Mono_Time in milliseconds.
REVERT: d4b06edc2a feat: add ngc events

git-subtree-dir: external/toxcore/c-toxcore
git-subtree-split: adbd5b32d85d9c13800f5ece17c0a9dce99faacd

external/toxcore/c-toxcore/.circleci/bazel-test

@@ -4,11 +4,9 @@ set -eux
 
 git submodule update --init --recursive
 /src/workspace/tools/inject-repo c-toxcore
+# TODO(iphydf): Re-enable fuzz-test when https://github.com/tweag/rules_nixpkgs/issues/442 is fixed.
 cd /src/workspace && bazel test -k \
-  --config=remote \
+  --build_tag_filters=-haskell,-fuzz-test \
-  --build_tag_filters=-haskell \
+  --test_tag_filters=-haskell,-fuzz-test \
-  --test_tag_filters=-haskell \
-  --remote_download_minimal \
   -- \
-  //c-toxcore/... \
   "$@"

external/toxcore/c-toxcore/.circleci/cmake-ubsan

@@ -6,6 +6,7 @@ CACHEDIR="$HOME/cache"
 
 . ".github/scripts/flags-$CC.sh"
 add_flag -Werror
+add_flag -D_DEBUG
 add_flag -fdiagnostics-color=always
 add_flag -fno-omit-frame-pointer
 add_flag -fno-sanitize-recover=all

external/toxcore/c-toxcore/.circleci/config.yml

@@ -7,11 +7,11 @@ workflows:
     jobs:
       # Dynamic analysis in the Bazel build
       - bazel-asan
+      - bazel-msan
       - bazel-tsan
       # Dynamic analysis with CMake
       - asan
       - tsan
-      - msan
       - ubsan
       # Static analysis
       - clang-analyze
@@ -29,6 +29,7 @@ jobs:
     steps:
       - checkout
       - run: .circleci/bazel-test
+          //c-toxcore/...
 
   bazel-tsan:
     working_directory: /tmp/cirrus-ci-build
@@ -38,11 +39,22 @@ jobs:
     steps:
       - checkout
       - run: .circleci/bazel-test
+          //c-toxcore/...
           -//c-toxcore/auto_tests:conference_av_test
           -//c-toxcore/auto_tests:conference_test
           -//c-toxcore/auto_tests:onion_test
           -//c-toxcore/auto_tests:tox_many_test
 
+  bazel-msan:
+    working_directory: /tmp/cirrus-ci-build
+    docker:
+      - image: toxchat/toktok-stack:latest-msan
+
+    steps:
+      - checkout
+      - run: .circleci/bazel-test
+          //c-toxcore/auto_tests:lossless_packet_test
+
   asan:
     working_directory: ~/work
     docker:
@@ -91,21 +103,6 @@ jobs:
       - run: git submodule update --init --recursive
       - run: CC=clang .circleci/cmake-ubsan
 
-  msan:
-    working_directory: ~/work
-    docker:
-      - image: toxchat/toktok-stack:latest-msan
-
-    steps:
-      - checkout
-      - run: git submodule update --init --recursive
-      - run: rm -rf /src/workspace/c-toxcore/* && mv * /src/workspace/c-toxcore/
-      - run:
-          cd /src/workspace && bazel test
-            //c-toxcore/auto_tests:lossless_packet_test
-            //c-toxcore/toxav/...
-            //c-toxcore/toxcore/...
-
   infer:
     working_directory: ~/work
     docker:

external/toxcore/c-toxcore/.cirrus.yml

@@ -9,10 +9,8 @@ bazel-opt_task:
     - /src/workspace/tools/inject-repo c-toxcore
   test_all_script:
     - cd /src/workspace && bazel test -k
-        --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST
         --build_tag_filters=-haskell
         --test_tag_filters=-haskell
-        --remote_download_minimal
         --
         //c-toxcore/...
         -//c-toxcore/auto_tests:tcp_relay_test # TODO(robinlinden): Why does this pass locally but not in Cirrus?
@@ -27,10 +25,8 @@ bazel-dbg_task:
     - /src/workspace/tools/inject-repo c-toxcore
   test_all_script:
     - cd /src/workspace && bazel test -k
-        --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST
         --build_tag_filters=-haskell
         --test_tag_filters=-haskell
-        --remote_download_minimal
         --
         //c-toxcore/...
         -//c-toxcore/auto_tests:tcp_relay_test # TODO(robinlinden): Why does this pass locally but not in Cirrus?
@@ -45,8 +41,19 @@ cimple_task:
     - /src/workspace/tools/inject-repo c-toxcore
   test_all_script:
     - cd /src/workspace && bazel test -k
-        --remote_http_cache=http://$CIRRUS_HTTP_CACHE_HOST
         --build_tag_filters=haskell
         --test_tag_filters=haskell
         --
         //c-toxcore/...
+
+freebsd_task:
+  container:
+    image: toxchat/freebsd:latest
+    cpu: 2
+    memory: 4G
+    kvm: true
+  configure_script:
+    - git submodule update --init --recursive
+    - cd .. && mv cirrus-ci-build /work/c-toxcore && mkdir cirrus-ci-build
+  test_all_script:
+    - cd /work/c-toxcore && .github/scripts/cmake-freebsd

external/toxcore/c-toxcore/.dockerignore

@@ -0,0 +1,2 @@
+/_build
+/_install

external/toxcore/c-toxcore/.github/scripts/cmake-alpine-s390x

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright (C) 2018-2023 nurupo
+
+# Toxcore building
+
+set -eux
+
+cd .. # /work
+. cmake-alpine-run.sh
+
+# === Get VM ready to build the code ===
+
+start_vm
+
+RUN apk add cmake g++ ninja
+
+mv c-toxcore /
+
+# Copy over toxcore code from host to qemu
+scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P "$SSH_PORT" -r /c-toxcore root@localhost:~
+
+cd /c-toxcore
+. ".github/scripts/flags-gcc.sh"
+
+# Make compilation error on a warning
+add_flag -Werror
+
+# - disabling toxav because vpx doesn't work on s390x.
+# - disabling bootstrap daemons because we don't need them for testing (saving time).
+# - disabling shared libraries because it saves a lot of time on building PIC objects.
+# - enable unity build because it saves a lot of time as well (fewer objects to build).
+RUN "cmake -B_build -Hc-toxcore -GNinja \
+  -DCMAKE_C_FLAGS='$C_FLAGS' \
+  -DCMAKE_CXX_FLAGS='$CXX_FLAGS' \
+  -DCMAKE_EXE_LINKER_FLAGS='$LD_FLAGS' \
+  -DCMAKE_SHARED_LINKER_FLAGS='$LD_FLAGS' \
+  -DCMAKE_INSTALL_PREFIX:PATH='_install' \
+  -DCMAKE_UNITY_BUILD=ON \
+  -DMIN_LOGGER_LEVEL=TRACE \
+  -DNON_HERMETIC_TESTS=ON \
+  -DENABLE_SHARED=OFF \
+  -DBUILD_TOXAV=OFF \
+  -DDHT_BOOTSTRAP=OFF \
+  -DBOOTSTRAP_DAEMON=OFF \
+  -DSTRICT_ABI=ON \
+  -DTEST_TIMEOUT_SECONDS=90 \
+  -DUSE_IPV6=OFF \
+  -DAUTOTEST=ON"
+
+RUN 'cmake --build _build --parallel 2 --target install -- -k 0'
+RUN 'cd _build && ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:1 --timeout 90 || true' &
+
+# Give the tests 5 minutes to run. Sometimes, the per-test timeout doesn't
+# work, so we put a global timeout here for everything.
+sleep 300

external/toxcore/c-toxcore/.github/scripts/cmake-freebsd

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Copyright (C) 2018-2023 nurupo
+
+# Toxcore building
+
+set -eux
+
+cd .. # /work
+. cmake-freebsd-run.sh
+
+# === Get VM ready to build the code ===
+
+# Unpack image only if it's compressed.
+if [ -f "$IMAGE_NAME.gz" ]; then
+  gunzip "$IMAGE_NAME.gz"
+fi
+
+mv c-toxcore /
+
+start_vm
+
+# Copy over toxcore code from host to qemu
+scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P "$SSH_PORT" -r /c-toxcore root@localhost:~
+
+RUN ls -lh
+
+cd /c-toxcore
+. '.github/scripts/flags-clang.sh'
+
+add_ld_flag -Wl,-z,defs
+
+# Make compilation error on a warning.
+add_flag -Werror
+
+# This triggers on FreeBSD's clang.
+add_flag -Wno-format
+add_flag -Wno-unsafe-buffer-usage
+
+RUN "cmake -B_build -Hc-toxcore \
+  -DCMAKE_C_FLAGS='$C_FLAGS' \
+  -DCMAKE_CXX_FLAGS='$CXX_FLAGS' \
+  -DCMAKE_EXE_LINKER_FLAGS='$LD_FLAGS' \
+  -DCMAKE_SHARED_LINKER_FLAGS='$LD_FLAGS' \
+  -DCMAKE_INSTALL_PREFIX:PATH='_install' \
+  -DMIN_LOGGER_LEVEL=TRACE \
+  -DMUST_BUILD_TOXAV=ON \
+  -DNON_HERMETIC_TESTS=ON \
+  -DSTRICT_ABI=ON \
+  -DTEST_TIMEOUT_SECONDS=120 \
+  -DUSE_IPV6=OFF \
+  -DAUTOTEST=ON"
+
+# We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine.
+RUN 'cmake --build _build --parallel "$NPROC" --target install -- -k'
+RUN 'cd _build && ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:2 --timeout 120 || true'
+
+# Gracefully shut down the VM.
+stop_vm

external/toxcore/c-toxcore/.github/scripts/cmake-freebsd-stage2

@@ -1,52 +0,0 @@
-#!/bin/bash
-# Copyright (C) 2018-2021 nurupo
-
-# Toxcore building
-
-set -eux
-
-if [ "$PWD" != "/work" ]; then
-  cd ..
-  mv c-toxcore /
-  mkdir c-toxcore
-  cd /work
-fi
-
-. cmake-freebsd-run.sh
-
-# === Get VM ready to build the code ===
-
-gunzip "$IMAGE_NAME.gz"
-
-start_vm
-
-# Copy over toxcore code from host to qemu
-scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P "$SSH_PORT" -r /c-toxcore root@localhost:~
-
-RUN ls -lh
-
-cd /c-toxcore
-. ".github/scripts/flags-clang.sh"
-
-add_ld_flag -Wl,-z,defs
-
-# Make compilation error on a warning
-add_flag -Werror
-
-RUN 'cmake -B_build -Hc-toxcore \
-  -DCMAKE_C_FLAGS="$C_FLAGS" \
-  -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
-  -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
-  -DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \
-  -DCMAKE_INSTALL_PREFIX:PATH="_install" \
-  -DMIN_LOGGER_LEVEL=TRACE \
-  -DMUST_BUILD_TOXAV=ON \
-  -DNON_HERMETIC_TESTS=ON \
-  -DSTRICT_ABI=ON \
-  -DTEST_TIMEOUT_SECONDS=90 \
-  -DUSE_IPV6=OFF \
-  -DAUTOTEST=ON'
-
-# We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine
-RUN 'gmake "-j$NPROC" -k install -C_build'
-RUN 'gmake "-j$NPROC" test ARGS="-j50" -C_build || true'

external/toxcore/c-toxcore/.github/scripts/flags.sh

@@ -28,8 +28,6 @@ add_flag -O3 -march=native
 
 # Warn on non-ISO C.
 add_c_flag -pedantic
-add_c_flag -std=c99
-add_cxx_flag -std=c++11
 
 add_flag -g3
 add_flag -ftrapv

external/toxcore/c-toxcore/.github/settings.yml

@@ -12,36 +12,52 @@ branches:
     protection:
       required_status_checks:
         contexts:
-          - "bazel-asan"
           - "bazel-dbg"
           - "bazel-opt"
-          - "bazel-tsan"
+          - "build-alpine-s390x"
+          - "build-android"
           - "build-compcert"
           - "build-macos"
           - "build-nacl"
           - "build-tcc"
           - "build-win32"
           - "build-win64"
-          - "CodeFactor"
-          - "common / buildifier"
-          - "coverage-linux"
           - "ci/circleci: asan"
+          - "ci/circleci: bazel-asan"
+          - "ci/circleci: bazel-msan"
+          - "ci/circleci: bazel-tsan"
           - "ci/circleci: clang-analyze"
           - "ci/circleci: clang-tidy"
           - "ci/circleci: cpplint"
           - "ci/circleci: infer"
-          - "ci/circleci: msan"
           - "ci/circleci: static-analysis"
           - "ci/circleci: tsan"
           - "ci/circleci: ubsan"
           - "cimple"
+          - "cimplefmt"
+          - "CodeFactor"
           - "code-review/reviewable"
-          - "continuous-integration/appveyor/pr"
+          - "common / buildifier"
+          - "coverage-linux"
           - "docker-bootstrap-node"
           - "docker-bootstrap-node-websocket"
+          - "docker-clusterfuzz"
+          - "docker-esp32"
+          - "docker-fuzzer"
           - "docker-toxcore-js"
+          - "docker-win32"
+          - "docker-win64"
+          - "doxygen"
+          - "freebsd"
+          - "Hound"
+          - "misra"
           - "mypy"
+          - "program-analysis"
           - "sonar-scan"
+          - "tokstyle"
+          - "TokTok.c-toxcore"
+          - "TokTok.c-toxcore (windows_msvc_conan shared)"
+          - "TokTok.c-toxcore (windows_msvc_conan static)"
 
 # Labels specific to c-toxcore.
 labels:

external/toxcore/c-toxcore/.github/workflows/ci.yml

@@ -60,15 +60,6 @@ jobs:
         with:
           file: other/docker/misra/Dockerfile
 
-  cimplefmt:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Run cimplefmt
-        run: other/docker/cimplefmt/run -u $(find tox* -name "*.[ch]")
-
   build-nacl:
     runs-on: ubuntu-latest
     steps:
@@ -79,6 +70,45 @@ jobs:
         with:
           file: other/docker/autotools/Dockerfile
 
+  build-tcc:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Docker Build
+        uses: docker/build-push-action@v2
+        with:
+          file: other/docker/tcc/Dockerfile
+
+  build-compcert:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Docker Build
+        uses: docker/build-push-action@v2
+        with:
+          file: other/docker/compcert/Dockerfile
+
+  build-alpine-s390x:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Docker Build
+        uses: docker/build-push-action@v2
+        with:
+          file: other/docker/alpine-s390x/Dockerfile
+
+  cimplefmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Run cimplefmt
+        run: other/docker/cimplefmt/run -u $(find tox* -name "*.[ch]")
+
   build-win32:
     runs-on: ubuntu-latest
     steps:
@@ -97,16 +127,6 @@ jobs:
       - name: Cross compilation
         run: .github/scripts/cmake-win64 script
 
-  build-freebsd:
-    runs-on: ubuntu-latest
-    container: toxchat/freebsd
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Build on FreeBSD
-        run: .github/scripts/cmake-freebsd-stage2
-
   build-macos:
     runs-on: macos-latest
     steps:
@@ -125,81 +145,6 @@ jobs:
       - name: Build, test, and upload coverage
         run: .github/scripts/coverage-linux
 
-  build-tcc:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Install dependencies
-        run:
-          sudo apt-get install -y --no-install-recommends
-            tcc
-            libconfig-dev
-            libopus-dev
-            libsodium-dev
-            libvpx-dev
-      - name: Build with TCC
-        run:
-          tcc
-            -Dinline=static
-            -o send_message_test
-            -Wall -Werror
-            -bench -g
-            auto_tests/auto_test_support.c
-            auto_tests/send_message_test.c
-            testing/misc_tools.c
-            toxav/*.c
-            toxcore/*.c
-            toxcore/*/*.c
-            toxencryptsave/*.c
-            third_party/cmp/*.c
-            $(pkg-config --cflags --libs libsodium opus vpx)
-      - name: Run the test
-        run: "./send_message_test | grep 'tox clients connected'"
-      - name: Build amalgamation file with TCC
-        run:
-          other/make_single_file
-            auto_tests/auto_test_support.c
-            auto_tests/send_message_test.c
-            testing/misc_tools.c |
-          tcc -
-            -o send_message_test
-            -Wall -Werror
-            -bench -g
-            $(pkg-config --cflags --libs libsodium opus vpx)
-      - name: Run the test again
-        run: "./send_message_test | grep 'tox clients connected'"
-
-  build-compcert:
-    runs-on: ubuntu-latest
-    container: toxchat/compcert
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Build with CompCert
-        run:
-          ccomp
-            -o send_message_test
-            -Wall -Werror
-            -Wno-c11-extensions
-            -Wno-unknown-pragmas
-            -Wno-unused-variable
-            -fstruct-passing -fno-unprototyped -g
-            auto_tests/auto_test_support.c
-            auto_tests/send_message_test.c
-            testing/misc_tools.c
-            toxav/*.c
-            toxcore/*.c
-            toxcore/*/*.c
-            toxencryptsave/*.c
-            third_party/cmp/*.c
-            -D__COMPCERT__ -DDISABLE_VLA -Dinline=
-            -lpthread $(pkg-config --cflags --libs libsodium opus vpx)
-      - name: Run the test
-        run: "./send_message_test | grep 'tox clients connected'"
-
   build-android:
     runs-on: ubuntu-latest
     steps:

external/toxcore/c-toxcore/.github/workflows/docker.yml

@@ -6,6 +6,11 @@ on:
   pull_request:
     branches: [master]
 
+# Cancel old PR builds when pushing new commits.
+concurrency:
+  group: docker-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   docker-bootstrap-node:
     runs-on: ubuntu-latest

external/toxcore/c-toxcore/.github/workflows/sonar-scan.yml

@@ -6,11 +6,16 @@ on:
   pull_request:
     branches: [master]
 
+# Cancel old PR builds when pushing new commits.
+concurrency:
+  group: scan-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   sonar-scan:
     runs-on: ubuntu-latest
     env:
-      SONAR_SCANNER_VERSION: 4.4.0.2170
+      SONAR_SCANNER_VERSION: 5.0.1.3006
       SONAR_SERVER_URL: "https://sonarcloud.io"
       BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
     steps:
@@ -18,10 +23,10 @@ jobs:
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
           submodules: recursive
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v1
         with:
-          java-version: 11
+          java-version: 17
       - name: Download and set up sonar-scanner
         env:
           SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip

external/toxcore/c-toxcore/BUILD.bazel

@@ -1,8 +1,6 @@
 load("@rules_cc//cc:defs.bzl", "cc_library")
 load("//tools/project:build_defs.bzl", "project")
 
-package(features = ["layering_check"])
-
 project()
 
 genrule(
@@ -10,16 +8,19 @@ genrule(
     srcs = [
         "//c-toxcore/toxav:toxav.h",
         "//c-toxcore/toxcore:tox.h",
+        "//c-toxcore/toxcore:tox_private.h",
         "//c-toxcore/toxencryptsave:toxencryptsave.h",
     ],
     outs = [
         "tox/toxav.h",
         "tox/tox.h",
+        "tox/tox_private.h",
         "tox/toxencryptsave.h",
     ],
     cmd = """
         cp $(location //c-toxcore/toxav:toxav.h) $(GENDIR)/c-toxcore/tox/toxav.h
         cp $(location //c-toxcore/toxcore:tox.h) $(GENDIR)/c-toxcore/tox/tox.h
+        cp $(location //c-toxcore/toxcore:tox_private.h) $(GENDIR)/c-toxcore/tox/tox_private.h
         cp $(location //c-toxcore/toxencryptsave:toxencryptsave.h) $(GENDIR)/c-toxcore/tox/toxencryptsave.h
     """,
     visibility = ["//visibility:public"],

external/toxcore/c-toxcore/CMakeLists.txt

@@ -14,8 +14,8 @@
 #
 ################################################################################
 
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 3.5)
-cmake_policy(VERSION 2.8.12)
+cmake_policy(VERSION 3.5)
 project(toxcore)
 
 list(APPEND CMAKE_MODULE_PATH ${toxcore_SOURCE_DIR}/cmake)
@@ -79,21 +79,19 @@ enable_testing()
 
 set(CMAKE_MACOSX_RPATH ON)
 
-if(${CMAKE_VERSION} VERSION_LESS "3.1.0")
+# Set standard version for compiler.
-  if(NOT MSVC)
+if(MSVC)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  # https://developercommunity.visualstudio.com/t/older-winsdk-headers-are-incompatible-with-zcprepr/1593479
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
-  endif()
-else()
-  # Set standard version for compiler.
   set(CMAKE_C_STANDARD 99)
-  set(CMAKE_CXX_STANDARD 17)
+else()
-  set(CMAKE_C_EXTENSIONS OFF)
+  set(CMAKE_C_STANDARD 11)
-  set(CMAKE_CXX_EXTENSIONS OFF)
-
-  message(STATUS "Supported C compiler features = ${CMAKE_C_COMPILE_FEATURES}")
-  message(STATUS "Supported C++ compiler features = ${CMAKE_CXX_COMPILE_FEATURES}")
 endif()
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_C_EXTENSIONS OFF)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+message(STATUS "Supported C compiler features = ${CMAKE_C_COMPILE_FEATURES}")
+message(STATUS "Supported C++ compiler features = ${CMAKE_CXX_COMPILE_FEATURES}")
 
 set(MIN_LOGGER_LEVEL "" CACHE STRING "Logging level to use (TRACE, DEBUG, INFO, WARNING, ERROR)")
 if(MIN_LOGGER_LEVEL)
@@ -336,7 +334,8 @@ set(toxcore_PKGCONFIG_REQUIRES ${toxcore_PKGCONFIG_REQUIRES} libsodium)
 set(toxcore_API_HEADERS
   ${toxcore_SOURCE_DIR}/toxcore/tox.h^tox
   ${toxcore_SOURCE_DIR}/toxcore/tox_events.h^tox
-  ${toxcore_SOURCE_DIR}/toxcore/tox_dispatch.h^tox)
+  ${toxcore_SOURCE_DIR}/toxcore/tox_dispatch.h^tox
+  ${toxcore_SOURCE_DIR}/toxcore/tox_private.h^tox)
 
 ################################################################################
 #

external/toxcore/c-toxcore/README.md

@@ -4,8 +4,6 @@
 
 [**Website**](https://tox.chat) **|** [**Wiki**](https://wiki.tox.chat/) **|** [**Blog**](https://blog.tox.chat/) **|** [**FAQ**](https://wiki.tox.chat/doku.php?id=users:faq) **|** [**Binaries/Downloads**](https://tox.chat/download.html) **|** [**Clients**](https://wiki.tox.chat/doku.php?id=clients) **|** [**Compiling**](/INSTALL.md)
 
-**IRC Channels:** Users: [#tox@libera.chat](https://web.libera.chat/#tox), Developers: [#toktok@libera.chat](https://web.libera.chat/#toktok)
-
 ## What is Tox
 
 Tox is a peer to peer (serverless) instant messenger aimed at making security
@@ -39,7 +37,7 @@ on the website, where they are updated at least once every 24 hours:
 
 Detailed installation instructions can be found in [INSTALL.md](INSTALL.md).
 
-Be advised that due to the addition of `cmp` as a submodule, you now also need to initialize the git submodules required by toxcore. This can be done by cloning the repo with the following command: `git clone --recurse-submodules https://github.com/Toktok/c-toxcore` or by running `git submodule update --init` in the root directory of the repo. 
+Be advised that due to the addition of `cmp` as a submodule, you now also need to initialize the git submodules required by toxcore. This can be done by cloning the repo with the following command: `git clone --recurse-submodules https://github.com/Toktok/c-toxcore` or by running `git submodule update --init` in the root directory of the repo.
 
 In a nutshell, if you have [libsodium](https://github.com/jedisct1/libsodium)
 installed, run:

external/toxcore/c-toxcore/auto_tests/BUILD.bazel

@@ -1,7 +1,5 @@
 load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
 
-package(features = ["layering_check"])
-
 cc_library(
     name = "check_compat",
     testonly = True,
@@ -29,12 +27,26 @@ flaky_tests = {
     "tox_many_tcp_test": True,
 }
 
+extra_args = {
+    "proxy_test": ["$(location //c-toxcore/other/proxy)"],
+}
+
+extra_data = {
+    "proxy_test": ["//c-toxcore/other/proxy"],
+}
+
 [cc_test(
     name = src[:-2],
     size = "small",
     srcs = [src],
-    args = ["$(location %s)" % src] + ["$(location //c-toxcore/other/proxy)"],
+    args = ["$(location %s)" % src] + extra_args.get(
-    data = glob(["data/*"]) + ["//c-toxcore/other/proxy"],
+        src[:-2],
+        [],
+    ),
+    data = glob(["data/*"]) + extra_data.get(
+        src[:-2],
+        [],
+    ),
     flaky = flaky_tests.get(
         src[:-2],
         False,

external/toxcore/c-toxcore/auto_tests/crypto_test.c

@@ -269,6 +269,30 @@ static void test_large_data_symmetric(void)
     free(m1);
 }
 
+static void test_very_large_data(void)
+{
+    const Random *rng = system_random();
+    ck_assert(rng != nullptr);
+
+    uint8_t nonce[CRYPTO_NONCE_SIZE] = {0};
+    uint8_t pk[CRYPTO_PUBLIC_KEY_SIZE];
+    uint8_t sk[CRYPTO_SECRET_KEY_SIZE];
+    crypto_new_keypair(rng, pk, sk);
+
+    // 100 MiB of data (all zeroes, doesn't matter what's inside).
+    const uint32_t plain_size = 100 * 1024 * 1024;
+    uint8_t *plain = (uint8_t *)malloc(plain_size);
+    uint8_t *encrypted = (uint8_t *)malloc(plain_size + CRYPTO_MAC_SIZE);
+
+    ck_assert(plain != nullptr);
+    ck_assert(encrypted != nullptr);
+
+    encrypt_data(pk, sk, nonce, plain, plain_size, encrypted);
+
+    free(encrypted);
+    free(plain);
+}
+
 static void increment_nonce_number_cmp(uint8_t *nonce, uint32_t num)
 {
     uint32_t num1 = 0;
@@ -340,6 +364,7 @@ int main(void)
     test_endtoend(); /* waiting up to 15 seconds */
     test_large_data();
     test_large_data_symmetric();
+    test_very_large_data();
     test_increment_nonce();
     test_memzero();
 

external/toxcore/c-toxcore/auto_tests/dht_getnodes_api_test.c

@@ -122,6 +122,12 @@ static void test_dht_getnodes(AutoTox *autotoxes)
 
         tox_self_get_dht_id(autotoxes[i].tox, public_key_list[i]);
         tox_callback_dht_get_nodes_response(autotoxes[i].tox, getnodes_response_cb);
+
+        printf("Peer %zu dht closenode count total/announce-capable: %d/%d\n",
+            i,
+            tox_dht_get_num_closelist(autotoxes[i].tox),
+            tox_dht_get_num_closelist_announce_capable(autotoxes[i].tox)
+        );
     }
 
     while (!all_nodes_crawled(autotoxes, NUM_TOXES, public_key_list)) {

external/toxcore/c-toxcore/auto_tests/group_message_test.c

@@ -44,6 +44,10 @@ typedef struct State {
 #define TEST_CUSTOM_PACKET "Why'd ya spill yer beans?"
 #define TEST_CUSTOM_PACKET_LEN (sizeof(TEST_CUSTOM_PACKET) - 1)
 
+#define TEST_CUSTOM_PACKET_LARGE "Where is it I've read that someone condemned to death says or thinks, an hour before his death, that if he had to live on some high rock, on such a narrow ledge that he'd only room to stand, and the ocean, everlasting darkness, everlasting solitude, everlasting tempest around him, if he had to remain standing on a square yard of space all his life, a thousand years, eternity, it were better to live so than to die at once. Only to live, to live and live! Life, whatever it may be! ...............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................0123456789"
+#define TEST_CUSTOM_PACKET_LARGE_LEN (sizeof(TEST_CUSTOM_PACKET_LARGE) - 1)
+static_assert(TEST_CUSTOM_PACKET_LARGE_LEN == TOX_GROUP_MAX_CUSTOM_LOSSY_PACKET_LENGTH, "Should be max");
+
 #define TEST_CUSTOM_PRIVATE_PACKET "This is a custom private packet. Enjoy."
 #define TEST_CUSTOM_PRIVATE_PACKET_LEN (sizeof(TEST_CUSTOM_PRIVATE_PACKET) - 1)
 
@@ -187,6 +191,21 @@ static void group_custom_packet_handler(Tox *tox, uint32_t groupnumber, uint32_t
     ++state->custom_packets_received;
 }
 
+static void group_custom_packet_large_handler(Tox *tox, uint32_t groupnumber, uint32_t peer_id, const uint8_t *data,
+                                        size_t length, void *user_data)
+{
+    ck_assert_msg(length == TEST_CUSTOM_PACKET_LARGE_LEN, "Failed to receive large custom packet. Invalid length: %zu\n", length);
+
+    ck_assert(memcmp(data, TEST_CUSTOM_PACKET_LARGE, length) == 0);
+
+    AutoTox *autotox = (AutoTox *)user_data;
+    ck_assert(autotox != nullptr);
+
+    State *state = (State *)autotox->state;
+
+    ++state->custom_packets_received;
+}
+
 static void group_message_handler(Tox *tox, uint32_t groupnumber, uint32_t peer_id, TOX_MESSAGE_TYPE type,
                                   const uint8_t *message, size_t length, uint32_t pseudo_msg_id, void *user_data)
 {
@@ -450,6 +469,19 @@ static void group_message_test(AutoTox *autotoxes)
         iterate_all_wait(autotoxes, NUM_GROUP_TOXES, ITERATION_INTERVAL);
     }
 
+    // tox0 sends a large max sized lossy custom packet
+
+    // overwrite callback for larger packet
+    tox_callback_group_custom_packet(tox0, group_custom_packet_large_handler);
+
+    tox_group_send_custom_packet(tox1, group_number, false, (const uint8_t *)TEST_CUSTOM_PACKET_LARGE, TEST_CUSTOM_PACKET_LARGE_LEN,
+                                 &c_err);
+    ck_assert_msg(c_err == TOX_ERR_GROUP_SEND_CUSTOM_PACKET_OK, "%d", c_err);
+
+    while (state0->custom_packets_received < 3) {
+        iterate_all_wait(autotoxes, NUM_GROUP_TOXES, ITERATION_INTERVAL);
+    }
+
     uint8_t m[TOX_GROUP_MAX_MESSAGE_LENGTH] = {0};
 
     fprintf(stderr, "Doing lossless packet test...\n");
@@ -538,6 +570,8 @@ int main(void)
 #undef TEST_PRIVATE_MESSAGE_LEN
 #undef TEST_CUSTOM_PACKET
 #undef TEST_CUSTOM_PACKET_LEN
+#undef TEST_CUSTOM_PACKET_LARGE
+#undef TEST_CUSTOM_PACKET_LARGE_LEN
 #undef TEST_CUSTOM_PRIVATE_PACKET
 #undef TEST_CUSTOM_PRIVATE_PACKET_LEN
 #undef IGNORE_MESSAGE

external/toxcore/c-toxcore/azure-pipelines.yml

@@ -12,4 +12,4 @@ jobs:
       - bash: python -m pip install conan==1.59.0
       - bash: git submodule update --init --recursive
       - bash: conan install -if _build -o with_tests=True -o shared=$(conan.shared) .
-      - bash: CONAN_CPU_COUNT=50 CTEST_OUTPUT_ON_FAILURE=1 conan build -bf _build -if _build .
+      - bash: CONAN_CPU_COUNT=50 CTEST_OUTPUT_ON_FAILURE=1 conan build -bf _build -if _build . || true

external/toxcore/c-toxcore/cmake/StrictAbi.cmake

@@ -29,7 +29,7 @@ function(_make_version_script target)
     list(GET sublib 1 ns)
 
     execute_process(
-      COMMAND ${SHELL} -c "egrep '^\\w' ${header} | grep '${ns}_[a-z0-9_]*(' | grep -v '^typedef' | grep -o '${ns}_[a-z0-9_]*(' | egrep -o '\\w+' | sort -u"
+      COMMAND ${SHELL} -c "egrep '^\\w' ${header} | grep '${ns}_[a-z0-9_]*(' | grep -v '^typedef' | grep -o '${ns}_[a-z0-9_]*(' | egrep -o '[a-z0-9_]+' | sort -u"
       OUTPUT_VARIABLE sublib_SYMS
       OUTPUT_STRIP_TRAILING_WHITESPACE)
     string(REPLACE "\n" ";" sublib_SYMS ${sublib_SYMS})

external/toxcore/c-toxcore/other/BUILD.bazel

@@ -1,8 +1,6 @@
 load("@rules_cc//cc:defs.bzl", "cc_binary")
 load("//tools:no_undefined.bzl", "cc_library")
 
-package(features = ["layering_check"])
-
 cc_library(
     name = "bootstrap_node_packets",
     srcs = ["bootstrap_node_packets.c"],
@@ -16,6 +14,7 @@ cc_binary(
     testonly = 1,
     srcs = ["DHT_bootstrap.c"],
     deps = [
+        ":bootstrap_node_packets",
         "//c-toxcore/testing:misc_tools",
         "//c-toxcore/toxcore:DHT",
         "//c-toxcore/toxcore:LAN_discovery",

external/toxcore/c-toxcore/other/DHT_bootstrap.c

@@ -31,11 +31,16 @@
 
 #include "../testing/misc_tools.h"
 
+#define DHT_NODE_EXTRA_PACKETS
+
 #ifdef DHT_NODE_EXTRA_PACKETS
 #include "./bootstrap_node_packets.h"
 
-#define DHT_VERSION_NUMBER 1
+#ifndef DAEMON_VERSION_NUMBER
-#define DHT_MOTD "This is a test motd"
+#define DAEMON_VERSION_NUMBER (1000000000UL + TOX_VERSION_MAJOR*1000000UL + TOX_VERSION_MINOR*1000UL + TOX_VERSION_PATCH*1UL)
+#endif
+
+static const char *motd_str = ""; //Change this to anything within 256 bytes(but 96 bytes maximum prefered)
 #endif
 
 #define PORT 33445
@@ -152,7 +157,7 @@ int main(int argc, char *argv[])
     Onion_Announce *onion_a = new_onion_announce(logger, mem, rng, mono_time, dht);
 
 #ifdef DHT_NODE_EXTRA_PACKETS
-    bootstrap_set_callbacks(dht_get_net(dht), DHT_VERSION_NUMBER, DHT_MOTD, sizeof(DHT_MOTD));
+    bootstrap_set_callbacks(dht_get_net(dht), (uint32_t)DAEMON_VERSION_NUMBER, (const uint8_t *) motd_str, strlen(motd_str)+1);
 #endif
 
     if (!(onion && forwarding && onion_a)) {

external/toxcore/c-toxcore/other/analysis/run-clang-tidy

@@ -2,6 +2,9 @@
 
 CHECKS="*"
 
+# We don't use memcpy_s.
+CHECKS="$CHECKS,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling"
+
 # __attribute__((nonnull)) causes this warning on defensive null checks.
 CHECKS="$CHECKS,-clang-diagnostic-pointer-bool-conversion"
 CHECKS="$CHECKS,-clang-diagnostic-tautological-pointer-compare"
@@ -57,6 +60,27 @@ CHECKS="$CHECKS,-readability-redundant-control-flow"
 CHECKS="$CHECKS,-bugprone-narrowing-conversions"
 CHECKS="$CHECKS,-cppcoreguidelines-narrowing-conversions"
 
+# Mistakenly thinks that
+#   const int a = 0, b = 1;
+#   assert(a < b);
+# is a constant expression in C (it is in C++ though, which is probably why it's
+# mistaken), suggesting to replace 'assert()' with 'static_assert()' in cases
+# where that won't work.
+#
+# There are ways to make 'static_assert()' work, but they are rather annoying --
+# they are somewhat ugly, hurting the readability, and some are error-prone:
+#
+# - Turning 'a' and 'b' into enum constants would make it work, but this falls
+#   apart if the enum types are compared against non-enums down the line
+#     error: enumerated and non-enumerated type in conditional expression [-Werror=extra]
+#
+# - Turning 'a' and 'b' into pre-processor macros is the only option left, but
+#   #defines and #undefs in the middle of a function hurt the readability and
+#   are less idiomatic than simply using 'const int'.
+CHECKS="$CHECKS,-cert-dcl03-c"
+CHECKS="$CHECKS,-hicpp-static-assert"
+CHECKS="$CHECKS,-misc-static-assert"
+
 # TODO(iphydf): We might want some of these. For the ones we don't want, add a
 # comment explaining why not.
 CHECKS="$CHECKS,-clang-analyzer-optin.performance.Padding"

external/toxcore/c-toxcore/other/bootstrap_daemon/BUILD.bazel

@@ -1,7 +1,5 @@
 load("@rules_cc//cc:defs.bzl", "cc_binary")
 
-package(features = ["layering_check"])
-
 cc_binary(
     name = "bootstrap_daemon",
     srcs = glob([

external/toxcore/c-toxcore/other/bootstrap_daemon/docker/Dockerfile

@@ -1,6 +1,6 @@
 ###########################################################
 # Builder image: we compile the code here (static build)
-FROM alpine:3.15.0 AS build
+FROM alpine:3.18.5 AS build
 
 RUN ["apk", "--no-cache", "add",\
  "build-base",\
@@ -62,7 +62,7 @@ RUN ["other/bootstrap_daemon/docker/get-nodes.py", "other/bootstrap_daemon/tox-b
 
 ###########################################################
 # Final image build: this is what runs the bootstrap node
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 COPY --from=build /usr/local/bin/tox-bootstrapd /usr/local/bin/
 COPY --from=build /src/c-toxcore/other/bootstrap_daemon/tox-bootstrapd.conf /etc/tox-bootstrapd.conf

external/toxcore/c-toxcore/other/bootstrap_daemon/docker/tox-bootstrapd.sha256

@@ -1 +1 @@
-4f5b47978dc26aed78719526f862a44693f821db12f5ff6d70b338d67fb6f784  /usr/local/bin/tox-bootstrapd
+5aac1df4d6c1de289e8e9f646d06099c84fd4d9b80d19f45e3254eec3ece2bff  /usr/local/bin/tox-bootstrapd

external/toxcore/c-toxcore/other/bootstrap_daemon/docker/update-sha256

@@ -3,7 +3,7 @@
 set -eux
 
 docker_build() {
-  docker build -f other/bootstrap_daemon/docker/Dockerfile -t toxchat/bootstrap-node .
+  DOCKER_BUILDKIT=1 docker build --progress=plain -f other/bootstrap_daemon/docker/Dockerfile -t toxchat/bootstrap-node .
 }
 
 # Run Docker build once. If it succeeds, we're good.
@@ -12,12 +12,11 @@ if docker_build; then
 fi
 
 # We're not good. Run it again, but now capture the output.
-OUTPUT=$(docker_build || true 2>&1)
+OUTPUT=$(docker_build 2>&1 || true)
 
 if echo "$OUTPUT" | grep '/usr/local/bin/tox-bootstrapd: FAILED'; then
   # This is a checksum warning, so we need to update it.
-  IMAGE=$(echo "$OUTPUT" | grep '^ ---> [0-9a-f]*$' | grep -o '[0-9a-f]*$' | tail -n1)
+  echo "$OUTPUT" | grep -Eo '[0-9a-f]{64}  /usr/local/bin/tox-bootstrapd' | tail -n1 >other/bootstrap_daemon/docker/tox-bootstrapd.sha256
-  docker run --rm "$IMAGE" sha256sum /usr/local/bin/tox-bootstrapd >other/bootstrap_daemon/docker/tox-bootstrapd.sha256
 fi
 
 # Run once last time to complete the build.

external/toxcore/c-toxcore/other/bootstrap_daemon/src/config.c

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2018 The TokTok team.
+ * Copyright © 2016-2023 The TokTok team.
  * Copyright © 2014-2016 Tox project.
  */
 
@@ -39,22 +39,28 @@ static void parse_tcp_relay_ports_config(config_t *cfg, uint16_t **tcp_relay_por
         log_write(LOG_LEVEL_WARNING, "No '%s' setting in the configuration file.\n", NAME_TCP_RELAY_PORTS);
         log_write(LOG_LEVEL_WARNING, "Using default '%s':\n", NAME_TCP_RELAY_PORTS);
 
-        uint16_t default_ports[DEFAULT_TCP_RELAY_PORTS_COUNT] = {DEFAULT_TCP_RELAY_PORTS};
+        uint16_t default_ports[] = {DEFAULT_TCP_RELAY_PORTS};
 
-        for (int i = 0; i < DEFAULT_TCP_RELAY_PORTS_COUNT; ++i) {
+        // Check to avoid calling malloc(0) later on
-            log_write(LOG_LEVEL_INFO, "Port #%d: %u\n", i, default_ports[i]);
+        // NOLINTNEXTLINE, clang-tidy: error: suspicious comparison of 'sizeof(expr)' to a constant [bugprone-sizeof-expression,-warnings-as-errors]
+        static_assert(sizeof(default_ports) > 0, "At least one default TCP relay port should be provided");
+
+        const size_t default_ports_count = sizeof(default_ports)/sizeof(*default_ports);
+
+        for (size_t i = 0; i < default_ports_count; ++i) {
+            log_write(LOG_LEVEL_INFO, "Port #%zu: %u\n", i, default_ports[i]);
         }
 
         // similar procedure to the one of reading config file below
-        *tcp_relay_ports = (uint16_t *)malloc(DEFAULT_TCP_RELAY_PORTS_COUNT * sizeof(uint16_t));
+        *tcp_relay_ports = (uint16_t *)malloc(default_ports_count * sizeof(uint16_t));
 
-        for (int i = 0; i < DEFAULT_TCP_RELAY_PORTS_COUNT; ++i) {
+        for (size_t i = 0; i < default_ports_count; ++i) {
 
             (*tcp_relay_ports)[*tcp_relay_port_count] = default_ports[i];
 
             if ((*tcp_relay_ports)[*tcp_relay_port_count] < MIN_ALLOWED_PORT
                     || (*tcp_relay_ports)[*tcp_relay_port_count] > MAX_ALLOWED_PORT) {
-                log_write(LOG_LEVEL_WARNING, "Port #%d: Invalid port: %u, should be in [%d, %d]. Skipping.\n", i,
+                log_write(LOG_LEVEL_WARNING, "Port #%zu: Invalid port: %u, should be in [%d, %d]. Skipping.\n", i,
                           (*tcp_relay_ports)[*tcp_relay_port_count], MIN_ALLOWED_PORT, MAX_ALLOWED_PORT);
                 continue;
             }

external/toxcore/c-toxcore/other/bootstrap_daemon/src/config_defaults.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-3.0-or-later
- * Copyright © 2016-2018 The TokTok team.
+ * Copyright © 2016-2023 The TokTok team.
  * Copyright © 2014-2016 Tox project.
  */
 
@@ -19,8 +19,7 @@
 #define DEFAULT_ENABLE_IPV4_FALLBACK  1 // 1 - true, 0 - false
 #define DEFAULT_ENABLE_LAN_DISCOVERY  1 // 1 - true, 0 - false
 #define DEFAULT_ENABLE_TCP_RELAY      1 // 1 - true, 0 - false
-#define DEFAULT_TCP_RELAY_PORTS       443, 3389, 33445 // comma-separated list of ports. make sure to adjust DEFAULT_TCP_RELAY_PORTS_COUNT accordingly
+#define DEFAULT_TCP_RELAY_PORTS       443, 3389, 33445 // comma-separated list of ports
-#define DEFAULT_TCP_RELAY_PORTS_COUNT 3
 #define DEFAULT_ENABLE_MOTD           1 // 1 - true, 0 - false
 #define DEFAULT_MOTD                  DAEMON_NAME
 

external/toxcore/c-toxcore/other/bootstrap_daemon/src/tox-bootstrapd.c

@@ -375,6 +375,7 @@ int main(int argc, char *argv[])
 
     if (!onion) {
         log_write(LOG_LEVEL_ERROR, "Couldn't initialize Tox Onion. Exiting.\n");
+        kill_gca(group_announce);
         kill_announcements(announce);
         kill_forwarding(forwarding);
         kill_dht(dht);
@@ -454,7 +455,7 @@ int main(int argc, char *argv[])
         if (tcp_relay_port_count == 0) {
             log_write(LOG_LEVEL_ERROR, "No TCP relay ports read. Exiting.\n");
             kill_onion_announce(onion_a);
-	        kill_gca(group_announce);
+            kill_gca(group_announce);
             kill_announcements(announce);
             kill_forwarding(forwarding);
             kill_onion(onion);

external/toxcore/c-toxcore/other/bootstrap_daemon/websocket/websockify/BUILD.bazel

@@ -1,7 +1,5 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 
-package(features = ["-layering_check"])
-
 go_library(
     name = "go_default_library",
     srcs = ["websockify.go"],

external/toxcore/c-toxcore/other/docker/alpine-s390x/Dockerfile

@@ -0,0 +1,6 @@
+FROM toxchat/alpine-s390x:latest
+
+WORKDIR /work/c-toxcore
+COPY . /work/c-toxcore/
+
+RUN [".github/scripts/cmake-alpine-s390x"]

external/toxcore/c-toxcore/other/docker/alpine-s390x/run

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -eux
+BUILD=alpine-s390x
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .

external/toxcore/c-toxcore/other/docker/autotools/run

@@ -1,3 +1,5 @@
 #!/bin/sh
 
-docker build -t toxchat/c-toxcore:autotools -f other/docker/autotools/Dockerfile .
+set -eux
+BUILD=autotools
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .

external/toxcore/c-toxcore/other/docker/cimplefmt/run

@@ -5,4 +5,4 @@ if [ "$1" = "-u" ]; then
   docker pull toxchat/haskell:hs-cimple
   docker build -t toxchat/cimplefmt -f other/docker/cimplefmt/Dockerfile .
 fi
-docker run --rm -v "$PWD:/work" toxchat/cimplefmt "$@"
+docker run --name toxcore-cimplefmt --rm -v "$PWD:/work" toxchat/cimplefmt "$@"

external/toxcore/c-toxcore/other/docker/circleci/run

@@ -3,4 +3,4 @@
 SANITIZER="${1:-asan}"
 
 docker build -t toxchat/c-toxcore:circleci other/docker/circleci
-docker run --rm -it -v "$PWD:/c-toxcore" toxchat/c-toxcore:circleci "$SANITIZER"
+docker run --name toxcore-circleci --rm -it -v "$PWD:/c-toxcore" toxchat/c-toxcore:circleci "$SANITIZER"

external/toxcore/c-toxcore/other/docker/compcert/Dockerfile

@@ -0,0 +1,30 @@
+FROM toxchat/compcert:latest
+
+WORKDIR /work
+COPY auto_tests/ /work/auto_tests/
+COPY testing/ /work/testing/
+COPY toxav/ /work/toxav/
+COPY toxcore/ /work/toxcore/
+COPY toxencryptsave/ /work/toxencryptsave/
+COPY third_party/ /work/third_party/
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN ccomp \
+ -o send_message_test \
+ -Wall -Werror \
+ -Wno-c11-extensions \
+ -Wno-unknown-pragmas \
+ -Wno-unused-variable \
+ -fstruct-passing -fno-unprototyped -g \
+ auto_tests/auto_test_support.c \
+ auto_tests/send_message_test.c \
+ testing/misc_tools.c \
+ toxav/*.c \
+ toxcore/*.c \
+ toxcore/*/*.c \
+ toxencryptsave/*.c \
+ third_party/cmp/*.c \
+ -D__COMPCERT__ -DDISABLE_VLA -Dinline= \
+ -lpthread $(pkg-config --cflags --libs libsodium opus vpx) \
+ && ./send_message_test | grep 'tox clients connected'

external/toxcore/c-toxcore/other/docker/compcert/run

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -eux
+BUILD=compcert
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .

external/toxcore/c-toxcore/other/docker/coverage/run

@@ -4,4 +4,4 @@ set -eux
 
 docker build -t toxchat/c-toxcore:sources -f other/docker/sources/Dockerfile .
 docker build -t toxchat/c-toxcore:coverage other/docker/coverage
-docker run --rm -it -p "28192:80" toxchat/c-toxcore:coverage
+docker run --name toxcore-coverage --rm -it -p "28192:80" toxchat/c-toxcore:coverage

external/toxcore/c-toxcore/other/docker/doxygen/run

@@ -3,4 +3,4 @@
 set -eux
 
 docker build -t toxchat/c-toxcore:docs -f other/docker/doxygen/Dockerfile .
-docker run --rm -it -p "28192:80" toxchat/c-toxcore:docs
+docker run --name toxcore-docs --rm -it -p "28192:80" toxchat/c-toxcore:docs

external/toxcore/c-toxcore/other/docker/freebsd/Dockerfile

@@ -0,0 +1,6 @@
+FROM toxchat/freebsd:latest
+
+WORKDIR /work/c-toxcore
+COPY . /work/c-toxcore/
+
+RUN [".github/scripts/cmake-freebsd"]

external/toxcore/c-toxcore/other/docker/freebsd/run

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -eux
+BUILD=freebsd
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .

external/toxcore/c-toxcore/other/docker/misra/run

@@ -1,3 +1,5 @@
 #!/bin/sh
 
-docker build -f other/docker/misra/Dockerfile .
+set -eux
+BUILD=misra
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .

external/toxcore/c-toxcore/other/docker/perf/run

@@ -3,7 +3,7 @@
 set -eux
 
 docker build -t toxchat/c-toxcore:perf -f other/docker/perf/Dockerfile .
-docker run --privileged --rm -it \
+docker run --name toxcore-perf --privileged --rm -it \
   -v "$PWD:/work/c-toxcore" \
   toxchat/c-toxcore:perf \
   "$@"

external/toxcore/c-toxcore/other/docker/tcc/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 RUN apt-get update && \
  DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
@@ -17,6 +17,9 @@ COPY testing/ /work/testing/
 COPY toxav/ /work/toxav/
 COPY toxcore/ /work/toxcore/
 COPY toxencryptsave/ /work/toxencryptsave/
+COPY third_party/ /work/third_party/
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN tcc \
  -Dinline=static \
@@ -31,9 +34,9 @@ RUN tcc \
  toxcore/*/*.c \
  toxencryptsave/*.c \
  third_party/cmp/*.c \
- $(pkg-config --cflags --libs libsodium opus vpx)
+ $(pkg-config --cflags --libs libsodium opus vpx) \
+ && ./send_message_test | grep 'tox clients connected'
 
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 COPY other/make_single_file /work/other/
 RUN \
  other/make_single_file \
@@ -44,4 +47,5 @@ RUN \
    -o send_message_test \
    -Wall -Werror \
    -bench -g \
-   $(pkg-config --cflags --libs libsodium opus vpx)
+   $(pkg-config --cflags --libs libsodium opus vpx) \
+ && ./send_message_test | grep 'tox clients connected'

external/toxcore/c-toxcore/other/docker/tcc/run

@@ -1,5 +1,5 @@
 #!/bin/sh
 
 set -eux
-
+BUILD=tcc
-docker build -t toxchat/c-toxcore:tcc -f other/docker/tcc/Dockerfile .
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .

external/toxcore/c-toxcore/other/docker/tokstyle/run

@@ -1,3 +1,5 @@
 #!/bin/sh
 
-docker build -f other/docker/tokstyle/Dockerfile .
+set -eux
+BUILD=tokstyle
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .

external/toxcore/c-toxcore/other/event_tooling/generate_event_c.cpp

@@ -306,7 +306,7 @@ void generate_event_impl(const std::string& event_name, std::vector<EventType> e
     f << "    Tox_Event_" << event_name << " *event, Bin_Unpack *bu)\n{\n";
     f << "    assert(event != nullptr);\n";
     if (event_types.size() > 1) {
-        f << "    if (!bin_unpack_array_fixed(bu, " << event_types.size() << ")) {\n        return false;\n    }\n\n";
+        f << "    if (!bin_unpack_array_fixed(bu, " << event_types.size() << ", nullptr)) {\n        return false;\n    }\n\n";
     }
 
     bool first = true;

external/toxcore/c-toxcore/other/fun/BUILD.bazel

@@ -1,7 +1,5 @@
 load("@rules_cc//cc:defs.bzl", "cc_binary")
 
-package(features = ["layering_check"])
-
 #cc_binary(
 #    name = "cracker",
 #    testonly = 1,

external/toxcore/c-toxcore/other/fun/save-generator.c

@@ -73,10 +73,15 @@ static void print_information(Tox *tox)
     int length = snprintf(nospam_str, sizeof(nospam_str), "%08X", nospam);
     nospam_str[length] = '\0';
 
-    uint8_t *name = (uint8_t *)malloc(tox_self_get_name_size(tox) + 1);
+    size_t name_size = tox_self_get_name_size(tox);
-    assert(name != nullptr);
+    uint8_t *name = (uint8_t *)malloc(name_size + 1);
+
+    if (!name) {
+        return;
+    }
+
     tox_self_get_name(tox, name);
-    name[tox_self_get_name_size(tox)] = '\0';
+    name[name_size] = '\0';
 
     printf("INFORMATION\n");
     printf("----------------------------------\n");
@@ -86,6 +91,8 @@ static void print_information(Tox *tox)
     printf("Status message: %s.\n", GENERATED_STATUS_MESSAGE);
     printf("Number of friends: %zu.\n", tox_self_get_friend_list_size(tox));
     printf("----------------------------------\n");
+
+    free(name);
 }
 
 int main(int argc, char *argv[])

external/toxcore/c-toxcore/other/proxy/BUILD.bazel

@@ -1,7 +1,5 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
 
-package(features = ["layering_check"])
-
 go_library(
     name = "go_default_library",
     srcs = ["proxy_server.go"],

external/toxcore/c-toxcore/super_donators/BUILD.bazel

@@ -1,7 +1,5 @@
 load("@rules_cc//cc:defs.bzl", "cc_binary")
 
-package(features = ["layering_check"])
-
 cc_binary(
     name = "grencez_tok5",
     srcs = ["grencez_tok5.c"],

external/toxcore/c-toxcore/testing/BUILD.bazel

@@ -1,7 +1,5 @@
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
 
-package(features = ["layering_check"])
-
 CIMPLE_FILES = [
     "//c-toxcore/toxav:cimple_files",
     "//c-toxcore/toxcore:cimple_files",

external/toxcore/c-toxcore/testing/fuzzing/BUILD.bazel

@@ -2,8 +2,6 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
 load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test")
 load("@rules_fuzzing//fuzzing/private:binary.bzl", "fuzzing_binary")  # buildifier: disable=bzl-visibility
 
-package(features = ["layering_check"])
-
 cc_library(
     name = "fuzz_support",
     srcs = [
@@ -28,7 +26,7 @@ cc_library(
 
 cc_fuzz_test(
     name = "bootstrap_fuzz_test",
-    #size = "small",
+    size = "small",
     srcs = ["bootstrap_harness.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:bootstrap_fuzzer"],
@@ -43,7 +41,7 @@ cc_fuzz_test(
 
 cc_fuzz_test(
     name = "e2e_fuzz_test",
-    #size = "small",
+    size = "small",
     srcs = ["e2e_fuzz_test.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:e2e_fuzz_test"],
@@ -59,7 +57,7 @@ cc_fuzz_test(
 
 cc_fuzz_test(
     name = "toxsave_fuzz_test",
-    #size = "small",
+    size = "small",
     srcs = ["toxsave_harness.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:toxsave_fuzzer"],
@@ -92,7 +90,7 @@ fuzzing_binary(
 
 cc_fuzz_test(
     name = "protodump_reduce",
-    #size = "small",
+    size = "small",
     srcs = ["protodump_reduce.cc"],
     copts = ["-UNDEBUG"],
     deps = [

external/toxcore/c-toxcore/testing/fuzzing/fuzz_support.cc

@@ -22,6 +22,8 @@
 
 const bool DEBUG = false;
 
+static constexpr tox_mono_time_cb *get_self_clock = ![](Fuzz_System *self) { return self->clock; };
+
 // TODO(iphydf): Put this somewhere shared.
 struct Network_Addr {
     struct sockaddr_storage addr;
@@ -182,7 +184,7 @@ Fuzz_System::Fuzz_System(Fuzz_Data &input)
     }
     , data(input)
 {
-    sys->mono_time_callback = ![](Fuzz_System *self) { return self->clock; };
+    sys->mono_time_callback = get_self_clock;
     sys->mono_time_user_data = this;
     sys->mem = mem.get();
     sys->ns = ns.get();
@@ -269,7 +271,7 @@ Null_System::Null_System()
         std::make_unique<Random>(Random{&null_random_funcs, this}),
     }
 {
-    sys->mono_time_callback = ![](Fuzz_System *self) { return self->clock; };
+    sys->mono_time_callback = get_self_clock;
     sys->mono_time_user_data = this;
     sys->mem = mem.get();
     sys->ns = ns.get();
@@ -400,7 +402,7 @@ Record_System::Record_System(Global &global, uint64_t seed, const char *name)
     , seed_(seed)
     , name_(name)
 {
-    sys->mono_time_callback = ![](Fuzz_System *self) { return self->clock; };
+    sys->mono_time_callback = get_self_clock;
     sys->mono_time_user_data = this;
     sys->mem = mem.get();
     sys->ns = ns.get();

external/toxcore/c-toxcore/third_party/BUILD.bazel

@@ -1,7 +1,5 @@
 load("//tools:no_undefined.bzl", "cc_library")
 
-package(features = ["layering_check"])
-
 cc_library(
     name = "cmp",
     srcs = ["cmp/cmp.c"],

external/toxcore/c-toxcore/toxav/BUILD.bazel

@@ -1,8 +1,6 @@
 load("@rules_cc//cc:defs.bzl", "cc_test")
 load("//tools:no_undefined.bzl", "cc_library")
 
-package(features = ["layering_check"])
-
 exports_files(
     srcs = ["toxav.h"],
     visibility = ["//c-toxcore:__pkg__"],

external/toxcore/c-toxcore/toxcore/BUILD.bazel

@@ -2,10 +2,11 @@ load("@rules_cc//cc:defs.bzl", "cc_test")
 load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test")
 load("//tools:no_undefined.bzl", "cc_library")
 
-package(features = ["layering_check"])
-
 exports_files(
-    srcs = ["tox.h"],
+    srcs = [
+        "tox.h",
+        "tox_private.h",
+    ],
     visibility = ["//c-toxcore:__pkg__"],
 )
 
@@ -76,6 +77,22 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "logger",
+    srcs = ["logger.c"],
+    hdrs = ["logger.h"],
+    visibility = [
+        "//c-toxcore/auto_tests:__pkg__",
+        "//c-toxcore/other:__pkg__",
+        "//c-toxcore/other/bootstrap_daemon:__pkg__",
+        "//c-toxcore/toxav:__pkg__",
+    ],
+    deps = [
+        ":attributes",
+        ":ccompat",
+    ],
+)
+
 cc_library(
     name = "bin_pack",
     srcs = ["bin_pack.c"],
@@ -84,6 +101,7 @@ cc_library(
     deps = [
         ":attributes",
         ":ccompat",
+        ":logger",
         "//c-toxcore/third_party:cmp",
     ],
 )
@@ -158,22 +176,6 @@ cc_test(
     ],
 )
 
-cc_library(
-    name = "logger",
-    srcs = ["logger.c"],
-    hdrs = ["logger.h"],
-    visibility = [
-        "//c-toxcore/auto_tests:__pkg__",
-        "//c-toxcore/other:__pkg__",
-        "//c-toxcore/other/bootstrap_daemon:__pkg__",
-        "//c-toxcore/toxav:__pkg__",
-    ],
-    deps = [
-        ":attributes",
-        ":ccompat",
-    ],
-)
-
 cc_library(
     name = "state",
     srcs = ["state.c"],
@@ -339,6 +341,7 @@ cc_library(
     deps = [
         ":LAN_discovery",
         ":attributes",
+        ":bin_pack",
         ":ccompat",
         ":crypto_core",
         ":logger",
@@ -366,6 +369,7 @@ cc_test(
 
 cc_fuzz_test(
     name = "DHT_fuzz_test",
+    size = "small",
     srcs = ["DHT_fuzz_test.cc"],
     corpus = ["//tools/toktok-fuzzer/corpus:DHT_fuzz_test"],
     deps = [
@@ -405,6 +409,7 @@ cc_library(
 
 cc_fuzz_test(
     name = "forwarding_fuzz_test",
+    size = "small",
     srcs = ["forwarding_fuzz_test.cc"],
     #corpus = ["//tools/toktok-fuzzer/corpus:forwarding_fuzz_test"],
     deps = [
@@ -603,6 +608,7 @@ cc_library(
 
 cc_fuzz_test(
     name = "group_announce_fuzz_test",
+    size = "small",
     srcs = ["group_announce_fuzz_test.cc"],
     #corpus = ["//tools/toktok-fuzzer/corpus:group_announce_fuzz_test"],
     deps = [
@@ -706,6 +712,7 @@ cc_test(
 
 cc_fuzz_test(
     name = "group_moderation_fuzz_test",
+    size = "small",
     srcs = ["group_moderation_fuzz_test.cc"],
     corpus = ["//tools/toktok-fuzzer/corpus:group_moderation_fuzz_test"],
     deps = [
@@ -862,6 +869,7 @@ cc_test(
 
 cc_fuzz_test(
     name = "tox_events_fuzz_test",
+    size = "small",
     srcs = ["tox_events_fuzz_test.cc"],
     corpus = ["//tools/toktok-fuzzer/corpus:tox_events_fuzz_test"],
     deps = [

external/toxcore/c-toxcore/toxcore/DHT.c

@@ -9,10 +9,12 @@
 #include "DHT.h"
 
 #include <assert.h>
+#include <limits.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "LAN_discovery.h"
+#include "bin_pack.h"
 #include "ccompat.h"
 #include "logger.h"
 #include "mono_time.h"
@@ -360,12 +362,32 @@ int packed_node_size(Family ip_family)
 }
 
 
-int pack_ip_port(const Logger *logger, uint8_t *data, uint16_t length, const IP_Port *ip_port)
+/** @brief Packs an IP structure.
+ *
+ * It's the caller's responsibility to make sure `is_ipv4` tells the truth. This
+ * function is an implementation detail of @ref bin_pack_ip_port.
+ *
+ * @param is_ipv4 whether this IP is an IP4 or IP6.
+ *
+ * @retval true on success.
+ */
+non_null()
+static bool bin_pack_ip(Bin_Pack *bp, const IP *ip, bool is_ipv4)
 {
-    if (data == nullptr) {
+    if (is_ipv4) {
-        return -1;
+        return bin_pack_bin_b(bp, ip->ip.v4.uint8, SIZE_IP4);
+    } else {
+        return bin_pack_bin_b(bp, ip->ip.v6.uint8, SIZE_IP6);
     }
+}
 
+/** @brief Packs an IP_Port structure.
+ *
+ * @retval true on success.
+ */
+non_null()
+static bool bin_pack_ip_port(Bin_Pack *bp, const Logger *logger, const IP_Port *ip_port)
+{
     bool is_ipv4;
     uint8_t family;
 
@@ -387,32 +409,34 @@ int pack_ip_port(const Logger *logger, uint8_t *data, uint16_t length, const IP_
         // TODO(iphydf): Find out why we're trying to pack invalid IPs, stop
         // doing that, and turn this into an error.
         LOGGER_TRACE(logger, "cannot pack invalid IP: %s", net_ip_ntoa(&ip_port->ip, &ip_str));
+        return false;
+    }
+
+    return bin_pack_u08_b(bp, family)
+           && bin_pack_ip(bp, &ip_port->ip, is_ipv4)
+           && bin_pack_u16_b(bp, net_ntohs(ip_port->port));
+}
+
+non_null()
+static bool bin_pack_ip_port_handler(Bin_Pack *bp, const Logger *logger, const void *obj)
+{
+    return bin_pack_ip_port(bp, logger, (const IP_Port *)obj);
+}
+
+int pack_ip_port(const Logger *logger, uint8_t *data, uint16_t length, const IP_Port *ip_port)
+{
+    const uint32_t size = bin_pack_obj_size(bin_pack_ip_port_handler, logger, ip_port);
+
+    if (size > length) {
         return -1;
     }
 
-    if (is_ipv4) {
+    if (!bin_pack_obj(bin_pack_ip_port_handler, logger, ip_port, data, length)) {
-        const uint32_t size = 1 + SIZE_IP4 + sizeof(uint16_t);
+        return -1;
-
-        if (size > length) {
-            return -1;
-        }
-
-        data[0] = family;
-        memcpy(data + 1, &ip_port->ip.ip.v4, SIZE_IP4);
-        memcpy(data + 1 + SIZE_IP4, &ip_port->port, sizeof(uint16_t));
-        return size;
-    } else {
-        const uint32_t size = 1 + SIZE_IP6 + sizeof(uint16_t);
-
-        if (size > length) {
-            return -1;
-        }
-
-        data[0] = family;
-        memcpy(data + 1, &ip_port->ip.ip.v6, SIZE_IP6);
-        memcpy(data + 1 + SIZE_IP6, &ip_port->port, sizeof(uint16_t));
-        return size;
     }
+
+    assert(size < INT_MAX);
+    return (int)size;
 }
 
 int dht_create_packet(const Memory *mem, const Random *rng,
@@ -511,33 +535,25 @@ int unpack_ip_port(IP_Port *ip_port, const uint8_t *data, uint16_t length, bool
     }
 }
 
+/** @brief Pack a single node from a node array.
+ *
+ * @retval true on success.
+ */
+non_null()
+static bool bin_pack_node_handler(Bin_Pack *bp, const Logger *logger, const void *arr, uint32_t index)
+{
+    const Node_format *nodes = (const Node_format *)arr;
+    return bin_pack_ip_port(bp, logger, &nodes[index].ip_port)
+           && bin_pack_bin_b(bp, nodes[index].public_key, CRYPTO_PUBLIC_KEY_SIZE);
+}
+
 int pack_nodes(const Logger *logger, uint8_t *data, uint16_t length, const Node_format *nodes, uint16_t number)
 {
-    uint32_t packed_length = 0;
+    const uint32_t size = bin_pack_obj_array_size(bin_pack_node_handler, logger, nodes, number);
-
+    if (!bin_pack_obj_array(bin_pack_node_handler, logger, nodes, number, data, length)) {
-    for (uint32_t i = 0; i < number && packed_length < length; ++i) {
+        return -1;
-        const int ipp_size = pack_ip_port(logger, data + packed_length, length - packed_length, &nodes[i].ip_port);
-
-        if (ipp_size == -1) {
-            return -1;
-        }
-
-        packed_length += ipp_size;
-
-        if (packed_length + CRYPTO_PUBLIC_KEY_SIZE > length) {
-            return -1;
-        }
-
-        memcpy(data + packed_length, nodes[i].public_key, CRYPTO_PUBLIC_KEY_SIZE);
-        packed_length += CRYPTO_PUBLIC_KEY_SIZE;
-
-#ifndef NDEBUG
-        const uint32_t increment = ipp_size + CRYPTO_PUBLIC_KEY_SIZE;
-#endif
-        assert(increment == PACKED_NODE_SIZE_IP4 || increment == PACKED_NODE_SIZE_IP6);
     }
-
+    return size;
-    return packed_length;
 }
 
 int unpack_nodes(Node_format *nodes, uint16_t max_num_nodes, uint16_t *processed_data_len, const uint8_t *data,
@@ -565,8 +581,8 @@ int unpack_nodes(Node_format *nodes, uint16_t max_num_nodes, uint16_t *processed
 
 #ifndef NDEBUG
         const uint32_t increment = ipp_size + CRYPTO_PUBLIC_KEY_SIZE;
-#endif
         assert(increment == PACKED_NODE_SIZE_IP4 || increment == PACKED_NODE_SIZE_IP6);
+#endif
     }
 
     if (processed_data_len != nullptr) {
@@ -2829,8 +2845,9 @@ void dht_save(const DHT *dht, uint8_t *data)
         }
     }
 
-    state_write_section_header(old_data, DHT_STATE_COOKIE_TYPE, pack_nodes(dht->log, data, sizeof(Node_format) * num,
+    state_write_section_header(
-                               clients, num), DHT_STATE_TYPE_NODES);
+            old_data, DHT_STATE_COOKIE_TYPE, pack_nodes(dht->log, data, sizeof(Node_format) * num, clients, num),
+            DHT_STATE_TYPE_NODES);
 
     mem_delete(dht->mem, clients);
 }
@@ -2965,6 +2982,34 @@ bool dht_non_lan_connected(const DHT *dht)
     return false;
 }
 
+uint16_t dht_get_num_closelist(const DHT *dht) {
+    uint16_t num_valid_close_clients = 0;
+    for (uint32_t i = 0; i < LCLIENT_LIST; ++i) {
+        const Client_data *const client = dht_get_close_client(dht, i);
+
+        // check if client is valid
+        if (!(assoc_timeout(dht->cur_time, &client->assoc4) && assoc_timeout(dht->cur_time, &client->assoc6))) {
+            ++num_valid_close_clients;
+        }
+    }
+
+    return num_valid_close_clients;
+}
+
+uint16_t dht_get_num_closelist_announce_capable(const DHT *dht) {
+    uint16_t num_valid_close_clients_with_cap = 0;
+    for (uint32_t i = 0; i < LCLIENT_LIST; ++i) {
+        const Client_data *const client = dht_get_close_client(dht, i);
+
+        // check if client is valid
+        if (!(assoc_timeout(dht->cur_time, &client->assoc4) && assoc_timeout(dht->cur_time, &client->assoc6)) && client->announce_node) {
+            ++num_valid_close_clients_with_cap;
+        }
+    }
+
+    return num_valid_close_clients_with_cap;
+}
+
 unsigned int ipport_self_copy(const DHT *dht, IP_Port *dest)
 {
     ipport_reset(dest);

external/toxcore/c-toxcore/toxcore/DHT.h

@@ -214,6 +214,16 @@ int packed_node_size(Family ip_family);
 non_null()
 int pack_ip_port(const Logger *logger, uint8_t *data, uint16_t length, const IP_Port *ip_port);
 
+/** @brief Unpack IP_Port structure from data of max size length into ip_port.
+ *
+ * len_processed is the offset of data currently unpacked.
+ *
+ * @return size of unpacked ip_port on success.
+ * @retval -1 on failure.
+ */
+non_null()
+int unpack_ip_port(IP_Port *ip_port, const uint8_t *data, uint16_t length, bool tcp_enabled);
+
 /** @brief Encrypt plain and write resulting DHT packet into packet with max size length.
  *
  * @return size of packet on success.
@@ -226,16 +236,6 @@ int dht_create_packet(const Memory *mem, const Random *rng,
                       const uint8_t *plain, size_t plain_length,
                       uint8_t *packet, size_t length);
 
-/** @brief Unpack IP_Port structure from data of max size length into ip_port.
- *
- * len_processed is the offset of data currently unpacked.
- *
- * @return size of unpacked ip_port on success.
- * @retval -1 on failure.
- */
-non_null()
-int unpack_ip_port(IP_Port *ip_port, const uint8_t *data, uint16_t length, bool tcp_enabled);
-
 /** @brief Pack number of nodes into data of maxlength length.
  *
  * @return length of packed nodes on success.
@@ -515,6 +515,24 @@ bool dht_isconnected(const DHT *dht);
 non_null()
 bool dht_non_lan_connected(const DHT *dht);
 
+/**
+ * This function returns the ratio of close dht nodes that are known to support announce/store.
+ * This function returns the number of DHT nodes in the closelist.
+ *
+ * @return number
+ */
+non_null()
+uint16_t dht_get_num_closelist(const DHT *dht);
+
+/**
+ * This function returns the number of DHT nodes in the closelist,
+ * that are capable to store annouce data (introduced in version 0.2.18).
+ *
+ * @return number
+ */
+non_null()
+uint16_t dht_get_num_closelist_announce_capable(const DHT *dht);
+
 /** @brief Attempt to add client with ip_port and public_key to the friends client list
  * and close_clientlist.
  *

external/toxcore/c-toxcore/toxcore/DHT_fuzz_test.cc

@@ -1,6 +1,8 @@
 #include "DHT.h"
 
+#include <cassert>
 #include <cstdlib>
+#include <cstring>
 #include <vector>
 
 #include "../testing/fuzzing/fuzz_support.h"
@@ -36,6 +38,16 @@ void TestUnpackNodes(Fuzz_Data &input)
         LOGGER_ASSERT(logger, packed_size == processed_data_len,
             "packed size (%d) != unpacked size (%d)", packed_size, processed_data_len);
         logger_kill(logger);
+
+        // Check that packed nodes can be unpacked again and result in the
+        // original unpacked nodes.
+        Node_format nodes2[node_count];
+        uint16_t processed_data_len2;
+        const int packed_count2 = unpack_nodes(
+            nodes2, node_count, &processed_data_len2, packed.data(), packed.size(), tcp_enabled);
+        assert(processed_data_len2 == processed_data_len);
+        assert(packed_count2 == packed_count);
+        assert(memcmp(nodes, nodes2, sizeof(Node_format) * packed_count) == 0);
     }
 }
 

external/toxcore/c-toxcore/toxcore/Makefile.inc

@@ -1,7 +1,8 @@
 lib_LTLIBRARIES += libtoxcore.la
 
 libtoxcore_la_include_HEADERS = \
-                        ../toxcore/tox.h
+                        ../toxcore/tox.h \
+                        ../toxcore/tox_private.h
 
 libtoxcore_la_includedir = $(includedir)/tox
 

external/toxcore/c-toxcore/toxcore/Messenger.c

@@ -757,6 +757,20 @@ int m_set_statusmessage(Messenger *m, const uint8_t *status, uint16_t length)
     return 0;
 }
 
+static Userstatus userstatus_from_int(uint8_t status)
+{
+    switch (status) {
+        case 0:
+            return USERSTATUS_NONE;
+        case 1:
+            return USERSTATUS_AWAY;
+        case 2:
+            return USERSTATUS_BUSY;
+        default:
+            return USERSTATUS_INVALID;
+    }
+}
+
 int m_set_userstatus(Messenger *m, uint8_t status)
 {
     if (status >= USERSTATUS_INVALID) {
@@ -767,7 +781,7 @@ int m_set_userstatus(Messenger *m, uint8_t status)
         return 0;
     }
 
-    m->userstatus = (Userstatus)status;
+    m->userstatus = userstatus_from_int(status);
 
     for (uint32_t i = 0; i < m->numfriends; ++i) {
         m->friendlist[i].userstatus_sent = false;
@@ -923,7 +937,7 @@ static int set_friend_statusmessage(const Messenger *m, int32_t friendnumber, co
 non_null()
 static void set_friend_userstatus(const Messenger *m, int32_t friendnumber, uint8_t status)
 {
-    m->friendlist[friendnumber].userstatus = (Userstatus)status;
+    m->friendlist[friendnumber].userstatus = userstatus_from_int(status);
 }
 
 non_null()
@@ -2024,7 +2038,7 @@ non_null(1, 3) nullable(5)
 static int m_handle_packet_offline(Messenger *m, const int i, const uint8_t *data, const uint16_t data_length, void *userdata)
 {
     if (data_length == 0) {
-    	set_friend_status(m, i, FRIEND_CONFIRMED, userdata);
+        set_friend_status(m, i, FRIEND_CONFIRMED, userdata);
     }
 
     return 0;
@@ -2081,9 +2095,9 @@ static int m_handle_packet_userstatus(Messenger *m, const int i, const uint8_t *
         return 0;
     }
 
-    const Userstatus status = (Userstatus)data[0];
+    const Userstatus status = userstatus_from_int(data[0]);
 
-    if (status >= USERSTATUS_INVALID) {
+    if (status == USERSTATUS_INVALID) {
         return 0;
     }
 
@@ -2403,8 +2417,8 @@ static int m_handle_packet(void *object, int i, const uint8_t *temp, uint16_t le
             return m_handle_packet_file_data(m, i, data, data_length, userdata);
         case PACKET_ID_MSI:
             return m_handle_packet_msi(m, i, data, data_length, userdata);
-	    case PACKET_ID_INVITE_GROUPCHAT:
+        case PACKET_ID_INVITE_GROUPCHAT:
-	        return m_handle_packet_invite_groupchat(m, i, data, data_length, userdata);
+            return m_handle_packet_invite_groupchat(m, i, data, data_length, userdata);
     }
 
     return handle_custom_lossless_packet(object, i, temp, len, userdata);
@@ -2627,7 +2641,7 @@ void do_messenger(Messenger *m, void *userdata)
         if (m->tcp_server != nullptr) {
             /* Add self tcp server. */
             IP_Port local_ip_port;
-            local_ip_port.port = m->options.tcp_server_port;
+            local_ip_port.port = net_htons(m->options.tcp_server_port);
             local_ip_port.ip.family = net_family_ipv4();
             local_ip_port.ip.ip.v4 = get_ip4_loopback();
             add_tcp_relay(m->net_crypto, &local_ip_port, tcp_server_public_key(m->tcp_server));
@@ -2988,6 +3002,7 @@ static State_Load_Status load_nospam_keys(Messenger *m, const uint8_t *data, uin
     load_secret_key(m->net_crypto, data + sizeof(uint32_t) + CRYPTO_PUBLIC_KEY_SIZE);
 
     if (!pk_equal(data + sizeof(uint32_t), nc_get_self_public_key(m->net_crypto))) {
+        LOGGER_ERROR(m->log, "public key stored in savedata does not match its secret key");
         return STATE_LOAD_STATUS_ERROR;
     }
 
@@ -3152,7 +3167,7 @@ static void pack_groupchats(const GC_Session *c, Bin_Pack *bp)
 }
 
 non_null()
-static bool pack_groupchats_handler(Bin_Pack *bp, const void *obj)
+static bool pack_groupchats_handler(Bin_Pack *bp, const Logger *log, const void *obj)
 {
     pack_groupchats((const GC_Session *)obj, bp);
     return true;  // TODO(iphydf): Return bool from pack functions.
@@ -3162,7 +3177,7 @@ non_null()
 static uint32_t saved_groups_size(const Messenger *m)
 {
     GC_Session *c = m->group_handler;
-    return bin_pack_obj_size(pack_groupchats_handler, c);
+    return bin_pack_obj_size(pack_groupchats_handler, m->log, c);
 }
 
 non_null()
@@ -3184,7 +3199,7 @@ static uint8_t *groups_save(const Messenger *m, uint8_t *data)
 
     data = state_write_section_header(data, STATE_COOKIE_TYPE, len, STATE_TYPE_GROUPS);
 
-    if (!bin_pack_obj(pack_groupchats_handler, c, data, len)) {
+    if (!bin_pack_obj(pack_groupchats_handler, m->log, c, data, len)) {
         LOGGER_FATAL(m->log, "failed to pack group chats into buffer of length %u", len);
         return data;
     }
@@ -3219,9 +3234,13 @@ static State_Load_Status groups_load(Messenger *m, const uint8_t *data, uint32_t
 
         if (group_number < 0) {
             LOGGER_WARNING(m->log, "Failed to load group %u", i);
+            // Can't recover trivially. We may need to skip over some data here.
+            break;
         }
     }
 
+    LOGGER_DEBUG(m->log, "Successfully loaded %u groups", gc_count_groups(m->group_handler));
+
     bin_unpack_free(bu);
 
     return STATE_LOAD_STATUS_CONTINUE;
@@ -3617,7 +3636,9 @@ Messenger *new_messenger(Mono_Time *mono_time, const Memory *mem, const Random *
     m->onion = new_onion(m->log, m->mem, m->mono_time, m->rng, m->dht);
     m->onion_a = new_onion_announce(m->log, m->mem, m->rng, m->mono_time, m->dht);
     m->onion_c = new_onion_client(m->log, m->mem, m->rng, m->mono_time, m->net_crypto);
-    m->fr_c = new_friend_connections(m->log, m->mono_time, m->ns, m->onion_c, options->local_discovery_enabled);
+    if (m->onion_c != nullptr) {
+        m->fr_c = new_friend_connections(m->log, m->mono_time, m->ns, m->onion_c, options->local_discovery_enabled);
+    }
 
     if ((options->dht_announcements_enabled && (m->forwarding == nullptr || m->announce == nullptr)) ||
             m->onion == nullptr || m->onion_a == nullptr || m->onion_c == nullptr || m->fr_c == nullptr) {

external/toxcore/c-toxcore/toxcore/attributes.h

@@ -18,7 +18,7 @@
 #define GNU_PRINTF(f, a)
 #endif
 
-#if defined(__GNUC__) && defined(_DEBUG) && !defined(__OPTIMIZE__)
+#if defined(__GNUC__) && defined(_DEBUG)
 #define non_null(...) __attribute__((__nonnull__(__VA_ARGS__)))
 #else
 #define non_null(...)

external/toxcore/c-toxcore/toxcore/bin_pack.c

@@ -62,21 +62,47 @@ static void bin_pack_init(Bin_Pack *bp, uint8_t *buf, uint32_t buf_size)
     cmp_init(&bp->ctx, bp, null_reader, null_skipper, buf_writer);
 }
 
-bool bin_pack_obj(bin_pack_cb *callback, const void *obj, uint8_t *buf, uint32_t buf_size)
+uint32_t bin_pack_obj_size(bin_pack_cb *callback, const Logger *logger, const void *obj)
-{
-    Bin_Pack bp;
-    bin_pack_init(&bp, buf, buf_size);
-    return callback(&bp, obj);
-}
-
-uint32_t bin_pack_obj_size(bin_pack_cb *callback, const void *obj)
 {
     Bin_Pack bp;
     bin_pack_init(&bp, nullptr, 0);
-    callback(&bp, obj);
+    if (!callback(&bp, logger, obj)) {
+        return UINT32_MAX;
+    }
     return bp.bytes_pos;
 }
 
+bool bin_pack_obj(bin_pack_cb *callback, const Logger *logger, const void *obj, uint8_t *buf, uint32_t buf_size)
+{
+    Bin_Pack bp;
+    bin_pack_init(&bp, buf, buf_size);
+    return callback(&bp, logger, obj);
+}
+
+uint32_t bin_pack_obj_array_size(bin_pack_array_cb *callback, const Logger *logger, const void *arr, uint32_t count)
+{
+    Bin_Pack bp;
+    bin_pack_init(&bp, nullptr, 0);
+    for (uint32_t i = 0; i < count; ++i) {
+        if (!callback(&bp, logger, arr, i)) {
+            return UINT32_MAX;
+        }
+    }
+    return bp.bytes_pos;
+}
+
+bool bin_pack_obj_array(bin_pack_array_cb *callback, const Logger *logger, const void *arr, uint32_t count, uint8_t *buf, uint32_t buf_size)
+{
+    Bin_Pack bp;
+    bin_pack_init(&bp, buf, buf_size);
+    for (uint32_t i = 0; i < count; ++i) {
+        if (!callback(&bp, logger, arr, i)) {
+            return false;
+        }
+    }
+    return true;
+}
+
 Bin_Pack *bin_pack_new(uint8_t *buf, uint32_t buf_size)
 {
     Bin_Pack *bp = (Bin_Pack *)calloc(1, sizeof(Bin_Pack));

external/toxcore/c-toxcore/toxcore/bin_pack.h

@@ -8,6 +8,7 @@
 #include <stdint.h>
 
 #include "attributes.h"
+#include "logger.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -23,18 +24,29 @@ typedef struct Bin_Pack Bin_Pack;
  * This function would typically cast the `void *` to the actual object pointer type and then call
  * more appropriately typed packing functions.
  */
-typedef bool bin_pack_cb(Bin_Pack *bp, const void *obj);
+typedef bool bin_pack_cb(Bin_Pack *bp, const Logger *logger, const void *obj);
+
+/** @brief Function used to pack an array of objects.
+ *
+ * This function would typically cast the `void *` to the actual object pointer type and then call
+ * more appropriately typed packing functions.
+ *
+ * @param arr is the object array as void pointer.
+ * @param index is the index in the object array that is currently being packed.
+ */
+typedef bool bin_pack_array_cb(Bin_Pack *bp, const Logger *logger, const void *arr, uint32_t index);
 
 /** @brief Determine the serialised size of an object.
  *
  * @param callback The function called on the created packer and packed object.
+ * @param logger Optional logger object to pass to the callback.
  * @param obj The object to be packed, passed as `obj` to the callback.
  *
- * @return The packed size of the passed object according to the callback. UINT32_MAX in case of
+ * @return The packed size of the passed object according to the callback.
- *   errors such as buffer overflow.
+ * @retval UINT32_MAX in case of errors such as buffer overflow.
  */
-non_null(1) nullable(2)
+non_null(1) nullable(2, 3)
-uint32_t bin_pack_obj_size(bin_pack_cb *callback, const void *obj);
+uint32_t bin_pack_obj_size(bin_pack_cb *callback, const Logger *logger, const void *obj);
 
 /** @brief Pack an object into a buffer of a given size.
  *
@@ -45,14 +57,57 @@ uint32_t bin_pack_obj_size(bin_pack_cb *callback, const void *obj);
  * overflows `uint32_t`, this function returns `false`.
  *
  * @param callback The function called on the created packer and packed object.
+ * @param logger Optional logger object to pass to the callback.
  * @param obj The object to be packed, passed as `obj` to the callback.
  * @param buf A byte array large enough to hold the serialised representation of `obj`.
  * @param buf_size The size of the byte array. Can be `UINT32_MAX` to disable bounds checking.
  *
  * @retval false if an error occurred (e.g. buffer overflow).
  */
+non_null(1, 4) nullable(2, 3)
+bool bin_pack_obj(bin_pack_cb *callback, const Logger *logger, const void *obj, uint8_t *buf, uint32_t buf_size);
+
+/** @brief Determine the serialised size of an object array.
+ *
+ * Calls the callback `count` times with increasing `index` argument from 0 to
+ * `count`. This function is here just so we don't need to write the same
+ * trivial loop many times and so we don't need an extra struct just to contain
+ * an array with size so it can be passed to `bin_pack_obj_size`.
+ *
+ * @param callback The function called on the created packer and each object to
+ *   be packed.
+ * @param logger Optional logger object to pass to the callback.
+ * @param arr The object array to be packed, passed as `arr` to the callback.
+ * @param count The number of elements in the object array.
+ *
+ * @return The packed size of the passed object array according to the callback.
+ * @retval UINT32_MAX in case of errors such as buffer overflow.
+ */
 non_null(1, 3) nullable(2)
-bool bin_pack_obj(bin_pack_cb *callback, const void *obj, uint8_t *buf, uint32_t buf_size);
+uint32_t bin_pack_obj_array_size(bin_pack_array_cb *callback, const Logger *logger, const void *arr, uint32_t count);
+
+/** @brief Pack an object array into a buffer of a given size.
+ *
+ * Calls the callback `count` times with increasing `index` argument from 0 to
+ * `count`. This function is here just so we don't need to write the same
+ * trivial loop many times and so we don't need an extra struct just to contain
+ * an array with size so it can be passed to `bin_pack_obj`.
+ *
+ * Similar to `bin_pack_obj` but for arrays. Does not write the array length, so
+ * if you need that, write it manually using `bin_pack_array`.
+ *
+ * @param callback The function called on the created packer and packed object
+ *   array.
+ * @param logger Optional logger object to pass to the callback.
+ * @param arr The object array to be packed, passed as `arr` to the callback.
+ * @param count The number of elements in the object array.
+ * @param buf A byte array large enough to hold the serialised representation of `arr`.
+ * @param buf_size The size of the byte array. Can be `UINT32_MAX` to disable bounds checking.
+ *
+ * @retval false if an error occurred (e.g. buffer overflow).
+ */
+non_null(1, 3, 5) nullable(2)
+bool bin_pack_obj_array(bin_pack_array_cb *callback, const Logger *logger, const void *arr, uint32_t count, uint8_t *buf, uint32_t buf_size);
 
 /** @brief Allocate a new packer object.
  *

external/toxcore/c-toxcore/toxcore/bin_unpack.c

@@ -73,10 +73,14 @@ bool bin_unpack_array(Bin_Unpack *bu, uint32_t *size)
     return cmp_read_array(&bu->ctx, size) && *size <= bu->bytes_size;
 }
 
-bool bin_unpack_array_fixed(Bin_Unpack *bu, uint32_t required_size)
+bool bin_unpack_array_fixed(Bin_Unpack *bu, uint32_t required_size, uint32_t *actual_size)
 {
-    uint32_t size;
+    uint32_t size = 0;
-    return cmp_read_array(&bu->ctx, &size) && size == required_size;
+    const bool success = cmp_read_array(&bu->ctx, &size) && size == required_size;
+    if (actual_size != nullptr) {
+        *actual_size = size;
+    }
+    return success;
 }
 
 bool bin_unpack_bool(Bin_Unpack *bu, bool *val)
@@ -128,6 +132,18 @@ bool bin_unpack_bin(Bin_Unpack *bu, uint8_t **data_ptr, uint32_t *data_length_pt
     return true;
 }
 
+bool bin_unpack_bin_max(Bin_Unpack *bu, uint8_t *data, uint16_t *data_length_ptr, uint16_t max_data_length)
+{
+    uint32_t bin_size;
+    if (!bin_unpack_bin_size(bu, &bin_size) || bin_size > max_data_length) {
+        return false;
+    }
+
+    *data_length_ptr = bin_size;
+
+    return bin_unpack_bin_b(bu, data, bin_size);
+}
+
 bool bin_unpack_bin_fixed(Bin_Unpack *bu, uint8_t *data, uint32_t data_length)
 {
     uint32_t bin_size;

external/toxcore/c-toxcore/toxcore/bin_unpack.h

@@ -45,10 +45,14 @@ void bin_unpack_free(Bin_Unpack *bu);
 non_null() bool bin_unpack_array(Bin_Unpack *bu, uint32_t *size);
 
 /** @brief Start unpacking a fixed size MessagePack array.
+ *
+ * Fails if the array size is not the required size. If `actual_size` is passed a non-null
+ * pointer, the array size is written there.
  *
  * @retval false if the packed array size is not exactly the required size.
  */
-non_null() bool bin_unpack_array_fixed(Bin_Unpack *bu, uint32_t required_size);
+non_null(1) nullable(3)
+bool bin_unpack_array_fixed(Bin_Unpack *bu, uint32_t required_size, uint32_t *actual_size);
 
 /** @brief Unpack a MessagePack bool. */
 non_null() bool bin_unpack_bool(Bin_Unpack *bu, bool *val);
@@ -71,10 +75,16 @@ non_null() bool bin_unpack_nil(Bin_Unpack *bu);
  * large allocation unless the input array was already that large.
  */
 non_null() bool bin_unpack_bin(Bin_Unpack *bu, uint8_t **data_ptr, uint32_t *data_length_ptr);
+/** @brief Unpack a variable size MessagePack bin into a fixed size byte array.
+ *
+ * Stores unpacked data into `data` with its length stored in `data_length_ptr`. This function does
+ * not allocate memory and requires that `max_data_length` is less than or equal to `sizeof(arr)`
+ * when `arr` is passed as `data` pointer.
+ */
+non_null() bool bin_unpack_bin_max(Bin_Unpack *bu, uint8_t *data, uint16_t *data_length_ptr, uint16_t max_data_length);
 /** @brief Unpack a MessagePack bin of a fixed length into a pre-allocated byte array.
  *
- * Unlike the function above, this function does not allocate any memory, but requires the size to
+ * Similar to the function above, but doesn't output the data length.
- * be known up front.
  */
 non_null() bool bin_unpack_bin_fixed(Bin_Unpack *bu, uint8_t *data, uint32_t data_length);
 

external/toxcore/c-toxcore/toxcore/crypto_core_test.cc

@@ -12,11 +12,30 @@ namespace {
 
 using HmacKey = std::array<uint8_t, CRYPTO_HMAC_KEY_SIZE>;
 using Hmac = std::array<uint8_t, CRYPTO_HMAC_SIZE>;
+using PublicKey = std::array<uint8_t, CRYPTO_PUBLIC_KEY_SIZE>;
+using SecretKey = std::array<uint8_t, CRYPTO_SECRET_KEY_SIZE>;
 using ExtPublicKey = std::array<uint8_t, EXT_PUBLIC_KEY_SIZE>;
 using ExtSecretKey = std::array<uint8_t, EXT_SECRET_KEY_SIZE>;
 using Signature = std::array<uint8_t, CRYPTO_SIGNATURE_SIZE>;
 using Nonce = std::array<uint8_t, CRYPTO_NONCE_SIZE>;
 
+TEST(CryptoCore, EncryptLargeData)
+{
+    const Random *rng = system_random();
+    ASSERT_NE(rng, nullptr);
+
+    Nonce nonce{};
+    PublicKey pk;
+    SecretKey sk;
+    crypto_new_keypair(rng, pk.data(), sk.data());
+
+    // 100 MiB of data (all zeroes, doesn't matter what's inside).
+    std::vector<uint8_t> plain(100 * 1024 * 1024);
+    std::vector<uint8_t> encrypted(plain.size() + CRYPTO_MAC_SIZE);
+
+    encrypt_data(pk.data(), sk.data(), nonce.data(), plain.data(), plain.size(), encrypted.data());
+}
+
 TEST(CryptoCore, IncrementNonce)
 {
     Nonce nonce{};
@@ -60,7 +79,8 @@ TEST(CryptoCore, Signatures)
 
     EXPECT_TRUE(create_extended_keypair(pk.data(), sk.data()));
 
-    std::vector<uint8_t> message;
+    std::vector<uint8_t> message{0};
+    message.clear();
 
     // Try a few different sizes, including empty 0 length message.
     for (uint8_t i = 0; i < 100; ++i) {
@@ -82,7 +102,8 @@ TEST(CryptoCore, Hmac)
     HmacKey sk;
     new_hmac_key(rng, sk.data());
 
-    std::vector<uint8_t> message;
+    std::vector<uint8_t> message{0};
+    message.clear();
 
     // Try a few different sizes, including empty 0 length message.
     for (uint8_t i = 0; i < 100; ++i) {

external/toxcore/c-toxcore/toxcore/events/conference_invite.c

@@ -120,7 +120,7 @@ static bool tox_event_conference_invite_unpack(
     Tox_Event_Conference_Invite *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/conference_message.c

@@ -135,7 +135,7 @@ static bool tox_event_conference_message_unpack(
     Tox_Event_Conference_Message *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 4)) {
+    if (!bin_unpack_array_fixed(bu, 4, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/conference_peer_name.c

@@ -120,7 +120,7 @@ static bool tox_event_conference_peer_name_unpack(
     Tox_Event_Conference_Peer_Name *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/conference_title.c

@@ -119,7 +119,7 @@ static bool tox_event_conference_title_unpack(
     Tox_Event_Conference_Title *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/file_chunk_request.c

@@ -112,7 +112,7 @@ static bool tox_event_file_chunk_request_unpack(
     Tox_Event_File_Chunk_Request *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 4)) {
+    if (!bin_unpack_array_fixed(bu, 4, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/file_recv.c

@@ -149,7 +149,7 @@ static bool tox_event_file_recv_unpack(
     Tox_Event_File_Recv *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 5)) {
+    if (!bin_unpack_array_fixed(bu, 5, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/file_recv_chunk.c

@@ -134,7 +134,7 @@ static bool tox_event_file_recv_chunk_unpack(
     Tox_Event_File_Recv_Chunk *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 4)) {
+    if (!bin_unpack_array_fixed(bu, 4, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/file_recv_control.c

@@ -99,7 +99,7 @@ static bool tox_event_file_recv_control_unpack(
     Tox_Event_File_Recv_Control *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_connection_status.c

@@ -86,7 +86,7 @@ static bool tox_event_friend_connection_status_unpack(
     Tox_Event_Friend_Connection_Status *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_lossless_packet.c

@@ -105,7 +105,7 @@ static bool tox_event_friend_lossless_packet_unpack(
     Tox_Event_Friend_Lossless_Packet *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_lossy_packet.c

@@ -104,7 +104,7 @@ static bool tox_event_friend_lossy_packet_unpack(
     Tox_Event_Friend_Lossy_Packet *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_message.c

@@ -119,7 +119,7 @@ static bool tox_event_friend_message_unpack(
     Tox_Event_Friend_Message *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_name.c

@@ -104,7 +104,7 @@ static bool tox_event_friend_name_unpack(
     Tox_Event_Friend_Name *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_read_receipt.c

@@ -83,7 +83,7 @@ static bool tox_event_friend_read_receipt_unpack(
     Tox_Event_Friend_Read_Receipt *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_request.c

@@ -105,7 +105,7 @@ static bool tox_event_friend_request_unpack(
     Tox_Event_Friend_Request *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_status.c

@@ -84,7 +84,7 @@ static bool tox_event_friend_status_unpack(
     Tox_Event_Friend_Status *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_status_message.c

@@ -106,7 +106,7 @@ static bool tox_event_friend_status_message_unpack(
     Tox_Event_Friend_Status_Message *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/friend_typing.c

@@ -82,7 +82,7 @@ static bool tox_event_friend_typing_unpack(
     Tox_Event_Friend_Typing *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_custom_packet.c

@@ -120,7 +120,7 @@ static bool tox_event_group_custom_packet_unpack(
     Tox_Event_Group_Custom_Packet *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_custom_private_packet.c

@@ -120,7 +120,7 @@ static bool tox_event_group_custom_private_packet_unpack(
     Tox_Event_Group_Custom_Private_Packet *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_invite.c

@@ -142,7 +142,7 @@ static bool tox_event_group_invite_unpack(
     Tox_Event_Group_Invite *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 3)) {
+    if (!bin_unpack_array_fixed(bu, 3, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_join_fail.c

@@ -84,7 +84,7 @@ static bool tox_event_group_join_fail_unpack(
     Tox_Event_Group_Join_Fail *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_message.c

@@ -150,7 +150,7 @@ static bool tox_event_group_message_unpack(
     Tox_Event_Group_Message *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 5)) {
+    if (!bin_unpack_array_fixed(bu, 5, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_moderation.c

@@ -114,7 +114,7 @@ static bool tox_event_group_moderation_unpack(
     Tox_Event_Group_Moderation *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 4)) {
+    if (!bin_unpack_array_fixed(bu, 4, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_password.c

@@ -105,7 +105,7 @@ static bool tox_event_group_password_unpack(
     Tox_Event_Group_Password *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_peer_exit.c

@@ -172,7 +172,7 @@ static bool tox_event_group_peer_exit_unpack(
     Tox_Event_Group_Peer_Exit *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 5)) {
+    if (!bin_unpack_array_fixed(bu, 5, nullptr)) {
         return false;
     }
 

external/toxcore/c-toxcore/toxcore/events/group_peer_join.c

@@ -84,7 +84,7 @@ static bool tox_event_group_peer_join_unpack(
     Tox_Event_Group_Peer_Join *event, Bin_Unpack *bu)
 {
     assert(event != nullptr);
-    if (!bin_unpack_array_fixed(bu, 2)) {
+    if (!bin_unpack_array_fixed(bu, 2, nullptr)) {
         return false;
     }