Merge commit '227425b90e9a671118026689dd30967e127a1090' as 'external/toxcore/c-toxcore'

external/toxcore/c-toxcore/other/analysis/check_includes

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+from typing import Tuple
+
+ALLOWLIST: Tuple[str, ...] = (
+    # system headers
+    "pthread.h",
+    "stdarg.h",
+    "stdbool.h",
+    "stddef.h",
+    "stdint.h",
+    "time.h",  # time_t used in Messenger.h TODO(iphydf): maybe don't?
+    # toxav stuff, maybe not worth abstracting away
+    "opus.h",
+    "vpx/vp8cx.h",
+    "vpx/vp8dx.h",
+    "vpx/vpx_decoder.h",
+    "vpx/vpx_encoder.h",
+    "vpx/vpx_image.h",
+)
+
+out = (subprocess.run(
+    [
+        "grep", "-R", "^#include <.*>", "other", "toxav", "toxcore",
+        "toxencryptsave"
+    ],
+    check=True,
+    capture_output=True,
+).stdout.decode("utf-8").rstrip())
+
+errors = 0
+for line in out.split("\n"):
+    # other/fun can do what it wants.
+    if "/fun/" in line:
+        continue
+    filename, include = line.split(":", 1)
+    # We only check headers.
+    if not filename.endswith(".h"):
+        continue
+    # ccompat.h can include some things we don't really want elsewhere.
+    allowlist = ALLOWLIST
+    if filename == "toxcore/ccompat.h":
+        allowlist += ("assert.h", "alloca.h", "malloc.h", "stdlib.h")
+    header = include[include.index("<") + 1:include.index(">")]
+    if header not in allowlist:
+        source = filename[:-2] + ".c"
+        print(
+            f"{filename}: includes system header <{header}>, which is not allowed in .h files"
+        )
+        print(
+            " " * len(filename) +
+            f"  consider including it in {source} and exporting an abstraction, instead"
+        )
+        errors += 1
+
+if errors:
+    sys.exit(1)

external/toxcore/c-toxcore/other/analysis/check_logger_levels

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Make sure that logger levels in toxcore/logger.h, CMakeLists.txt and
+# configure.ac stay in sync.
+
+set -ex
+
+TMP_DIR="$(mktemp -d)"
+
+# ^\s+LOGGER_LEVEL_(\w+),?$
+sed -n 's/^\s\+LOGGER_LEVEL_\(\w\+\),\?$/\1/p' toxcore/logger.h > "${TMP_DIR}/logger.h"
+# ^.*\$\{MIN_LOGGER_LEVEL\}" STREQUAL "(\w+)".*$
+sed -n 's/^.*\${MIN_LOGGER_LEVEL}\" STREQUAL \"\(\w\+\)\".*$/\1/p' CMakeLists.txt > "${TMP_DIR}/CMakeLists.txt"
+# ^.*LOGGER_LEVEL_(\w+).*$
+sed -n 's/^.*LOGGER_LEVEL_\(\w\+\).*$/\1/p' configure.ac > "${TMP_DIR}/configure.ac"
+
+diff -u "${TMP_DIR}/CMakeLists.txt" "${TMP_DIR}/logger.h"
+diff -u "${TMP_DIR}/configure.ac" "${TMP_DIR}/logger.h"

external/toxcore/c-toxcore/other/analysis/gen-file.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+CPPFLAGS="-DMIN_LOGGER_LEVEL=LOGGER_LEVEL_TRACE"
+CPPFLAGS+=("-DCMP_NO_FLOAT=1")
+CPPFLAGS+=("-isystem" "/usr/include/opus")
+CPPFLAGS+=("-Iauto_tests")
+CPPFLAGS+=("-Iother")
+CPPFLAGS+=("-Iother/bootstrap_daemon/src")
+CPPFLAGS+=("-Iother/fun")
+CPPFLAGS+=("-Itesting")
+CPPFLAGS+=("-Itesting/fuzzing")
+CPPFLAGS+=("-Itoxcore")
+CPPFLAGS+=("-Itoxcore/events")
+CPPFLAGS+=("-Itoxav")
+CPPFLAGS+=("-Itoxencryptsave")
+CPPFLAGS+=("-Ithird_party/cmp")
+
+LDFLAGS=("-lopus" "-lsodium" "-lvpx" "-lpthread" "-lconfig" "-lgtest")
+LDFLAGS+=("-fuse-ld=gold")
+LDFLAGS+=("-Wl,--detect-odr-violations")
+LDFLAGS+=("-Wl,--warn-common")
+LDFLAGS+=("-Wl,--warn-execstack")
+LDFLAGS+=("-Wl,-z,noexecstack")
+LDFLAGS+=("-Wl,-z,now")
+
+put() {
+  if [ "$SKIP_LINES" = "" ]; then
+    echo "#line 1 \"$1\"" >>amalgamation.cc
+  fi
+  cat "$1" >>amalgamation.cc
+}
+
+putmain() {
+  NS=$(echo "${1//[^a-zA-Z0-9_]/_}" | sed -e 's/^__*//')
+  echo "namespace $NS {" >>amalgamation.cc
+  if [ "$SKIP_LINES" = "" ]; then
+    echo "#line 1 \"$1\"" >>amalgamation.cc
+  fi
+  sed -e 's/^int main(/static &/' "$1" >>amalgamation.cc
+  echo "} //  namespace $NS" >>amalgamation.cc
+}
+
+callmain() {
+  NS=$(echo "${1//[^a-zA-Z0-9_]/_}" | sed -e 's/^__*//')
+  echo "  call($NS::main, argc, argv);" >>amalgamation.cc
+}
+
+: >amalgamation.cc
+
+# Include all C and C++ code
+FIND_QUERY="find . '-(' -name '*.c' -or -name '*.cc' '-)'"
+# Excludes
+FIND_QUERY="$FIND_QUERY -and -not -wholename './_build/*'"
+FIND_QUERY="$FIND_QUERY -and -not -wholename './other/docker/*'"
+FIND_QUERY="$FIND_QUERY -and -not -wholename './super_donators/*'"
+FIND_QUERY="$FIND_QUERY -and -not -name amalgamation.cc"
+FIND_QUERY="$FIND_QUERY -and -not -name av_test.c"
+FIND_QUERY="$FIND_QUERY -and -not -name cracker.c"
+FIND_QUERY="$FIND_QUERY -and -not -name version_test.c"
+FIND_QUERY="$FIND_QUERY -and -not -name '*_fuzz_test.cc'"
+FIND_QUERY="$FIND_QUERY -and -not -wholename './testing/fuzzing/*'"
+FIND_QUERY="$FIND_QUERY -and -not -wholename './third_party/cmp/examples/*'"
+FIND_QUERY="$FIND_QUERY -and -not -wholename './third_party/cmp/test/*'"
+
+if [ "$SKIP_GTEST" == 1 ]; then
+  FIND_QUERY="$FIND_QUERY -and -not -name '*_test.cc'"
+fi
+
+readarray -t FILES <<<"$(eval "$FIND_QUERY")"
+
+(for i in "${FILES[@]}"; do
+  grep -o '#include <[^>]*>' "$i" |
+    grep -E -v '<win|<ws|<iphlp|<libc|<mach/|<crypto_|<randombytes|<u.h>|<sys/filio|<stropts.h>|<linux'
+done) | sort -u >>amalgamation.cc
+
+put auto_tests/check_compat.h
+
+echo 'namespace {' >>amalgamation.cc
+for i in "${FILES[@]}"; do
+  if ! grep -q '^int main(' "$i"; then
+    put "$i"
+  fi
+done
+
+for i in "${FILES[@]}"; do
+  if grep -q '^int main(' "$i"; then
+    putmain "$i"
+  fi
+done
+
+echo "static void call(int m(), int argc, char **argv) { m(); }" >>amalgamation.cc
+echo "static void call(int m(int, char **), int argc, char **argv) { m(argc, argv); }" >>amalgamation.cc
+echo "static void call(int m(int, const char *const *), int argc, char **argv) { m(argc, argv); }" >>amalgamation.cc
+echo '}  // namespace' >>amalgamation.cc
+
+echo "int main(int argc, char **argv) {" >>amalgamation.cc
+for i in "${FILES[@]}"; do
+  if grep -q '^int main(' "$i"; then
+    callmain "$i"
+  fi
+done
+echo "  return 0;" >>amalgamation.cc
+echo "}" >>amalgamation.cc

external/toxcore/c-toxcore/other/analysis/run-clang

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+. other/analysis/gen-file.sh
+
+set -e
+
+run() {
+  echo "Running Clang compiler in variant '$*'"
+  clang++ -o /dev/null amalgamation.cc \
+    "${CPPFLAGS[@]}" \
+    "${LDFLAGS[@]}" \
+    "$@" \
+    -std=c++11 \
+    -Werror \
+    -Weverything \
+    -Wno-alloca \
+    -Wno-c++98-compat-pedantic \
+    -Wno-c99-extensions \
+    -Wno-conversion \
+    -Wno-covered-switch-default \
+    -Wno-disabled-macro-expansion \
+    -Wno-documentation-deprecated-sync \
+    -Wno-documentation-unknown-command \
+    -Wno-global-constructors \
+    -Wno-missing-braces \
+    -Wno-missing-field-initializers \
+    -Wno-missing-noreturn \
+    -Wno-old-style-cast \
+    -Wno-padded \
+    -Wno-sign-compare \
+    -Wno-tautological-pointer-compare \
+    -Wno-unreachable-code-return \
+    -Wno-unused-parameter \
+    -Wno-used-but-marked-unused \
+    -Wno-source-uses-openmp
+}
+
+. other/analysis/variants.sh

external/toxcore/c-toxcore/other/analysis/run-clang-analyze

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+. other/analysis/gen-file.sh
+
+set -e
+
+run() {
+  echo "Running Clang static analyzer in variant '$*'"
+  clang++ --analyze amalgamation.cc \
+    "${CPPFLAGS[@]}" \
+    "$@" \
+    -std=c++11
+}
+
+. other/analysis/variants.sh

external/toxcore/c-toxcore/other/analysis/run-clang-tidy

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+CHECKS="*"
+
+# __attribute__((nonnull)) causes this warning on defensive null checks.
+CHECKS="$CHECKS,-clang-diagnostic-pointer-bool-conversion"
+CHECKS="$CHECKS,-clang-diagnostic-tautological-pointer-compare"
+
+# TODO(iphydf): We might want some of these. For the ones we don't want, add a
+# comment explaining why not.
+CHECKS="$CHECKS,-altera-unroll-loops"
+CHECKS="$CHECKS,-android-cloexec-accept"
+CHECKS="$CHECKS,-android-cloexec-fopen"
+CHECKS="$CHECKS,-bugprone-not-null-terminated-result"
+CHECKS="$CHECKS,-bugprone-reserved-identifier"
+CHECKS="$CHECKS,-bugprone-sizeof-expression"
+CHECKS="$CHECKS,-cert-dcl37-c"
+CHECKS="$CHECKS,-cert-dcl51-cpp"
+CHECKS="$CHECKS,-clang-analyzer-optin.performance.Padding"
+CHECKS="$CHECKS,-cppcoreguidelines-avoid-magic-numbers"
+CHECKS="$CHECKS,-cppcoreguidelines-init-variables"
+CHECKS="$CHECKS,-hicpp-multiway-paths-covered"
+CHECKS="$CHECKS,-hicpp-signed-bitwise"
+CHECKS="$CHECKS,-llvm-else-after-return"
+CHECKS="$CHECKS,-llvmlibc-restrict-system-libc-headers"
+CHECKS="$CHECKS,-misc-redundant-expression"
+CHECKS="$CHECKS,-misc-unused-parameters"
+CHECKS="$CHECKS,-readability-else-after-return"
+CHECKS="$CHECKS,-readability-function-cognitive-complexity"
+CHECKS="$CHECKS,-readability-inconsistent-declaration-parameter-name"
+CHECKS="$CHECKS,-readability-magic-numbers"
+CHECKS="$CHECKS,-readability-redundant-control-flow"
+
+# TODO(iphydf): Maybe fix these?
+CHECKS="$CHECKS,-altera-id-dependent-backward-branch"
+CHECKS="$CHECKS,-altera-struct-pack-align"
+CHECKS="$CHECKS,-bugprone-branch-clone"
+CHECKS="$CHECKS,-bugprone-easily-swappable-parameters"
+CHECKS="$CHECKS,-bugprone-implicit-widening-of-multiplication-result"
+CHECKS="$CHECKS,-bugprone-integer-division"
+CHECKS="$CHECKS,-bugprone-narrowing-conversions"
+CHECKS="$CHECKS,-clang-analyzer-core.NonNullParamChecker"
+CHECKS="$CHECKS,-clang-analyzer-core.NullDereference"
+CHECKS="$CHECKS,-clang-analyzer-optin.portability.UnixAPI"
+CHECKS="$CHECKS,-clang-analyzer-unix.Malloc"
+CHECKS="$CHECKS,-clang-analyzer-valist.Uninitialized"
+CHECKS="$CHECKS,-concurrency-mt-unsafe"
+CHECKS="$CHECKS,-cppcoreguidelines-avoid-non-const-global-variables"
+CHECKS="$CHECKS,-cppcoreguidelines-narrowing-conversions"
+CHECKS="$CHECKS,-google-readability-casting"
+CHECKS="$CHECKS,-misc-no-recursion"
+
+ERRORS="*"
+
+# TODO(iphydf): Fix these.
+ERRORS="$ERRORS,-bugprone-macro-parentheses"
+ERRORS="$ERRORS,-bugprone-posix-return"
+ERRORS="$ERRORS,-bugprone-signed-char-misuse"
+ERRORS="$ERRORS,-cert-err34-c"
+ERRORS="$ERRORS,-cert-str34-c"
+ERRORS="$ERRORS,-hicpp-uppercase-literal-suffix"
+ERRORS="$ERRORS,-readability-suspicious-call-argument"
+ERRORS="$ERRORS,-readability-uppercase-literal-suffix"
+
+set -eux
+
+run() {
+  echo "Running clang-tidy in variant '$*'"
+  EXTRA_ARGS=("$@")
+  for i in "${!EXTRA_ARGS[@]}"; do
+    EXTRA_ARGS[$i]="--extra-arg=${EXTRA_ARGS[$i]}"
+  done
+  clang-tidy-12 \
+    -p=_build \
+    --extra-arg=-DMIN_LOGGER_LEVEL=LOGGER_LEVEL_TRACE \
+    "${EXTRA_ARGS[@]}" \
+    --checks="$CHECKS" \
+    --warnings-as-errors="$ERRORS" \
+    --use-color \
+    other/bootstrap_daemon/src/*.c \
+    other/*.c \
+    toxav/*.c \
+    toxcore/*.c \
+    toxencryptsave/*.c
+}
+
+. other/analysis/variants.sh

external/toxcore/c-toxcore/other/analysis/run-cppcheck

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+SKIP_GTEST=1
+
+. other/analysis/gen-file.sh
+
+set -e
+
+CPPCHECK=("--enable=all")
+CPPCHECK+=("--inconclusive")
+CPPCHECK+=("--error-exitcode=1")
+# Used for VLA.
+CPPCHECK+=("--suppress=allocaCalled")
+# False positives in switch statements.
+CPPCHECK+=("--suppress=knownConditionTrueFalse")
+# Cppcheck does not need standard library headers to get proper results.
+CPPCHECK+=("--suppress=missingIncludeSystem")
+# TODO(iphydf): Maybe fix?
+CPPCHECK+=("--suppress=signConversion")
+# TODO(iphydf): Fixed in the toxav refactor PR.
+CPPCHECK+=("--suppress=redundantAssignment")
+# We have some redundant nullptr checks in assertions
+CPPCHECK+=("--suppress=nullPointerRedundantCheck")
+# Triggers a false warning in group.c
+CPPCHECK+=("--suppress=AssignmentAddressToInteger")
+# TODO(sudden6): This triggers a false positive, check again later to enable it
+CPPCHECK+=("--suppress=arrayIndexOutOfBoundsCond")
+
+# We're a library. This only works on whole programs.
+CPPCHECK_C=("--suppress=unusedFunction")
+
+# False positive in auto_tests.
+CPPCHECK_CXX+=("--suppress=shadowArgument")
+CPPCHECK_CXX+=("--suppress=shadowFunction")
+# False positive for callback functions
+CPPCHECK_CXX+=("--suppress=constParameter")
+# Used in Messenger.c for a static_assert(...)
+CPPCHECK_CXX+=("--suppress=sizeofFunctionCall")
+
+run() {
+  echo "Running cppcheck in variant '$*'"
+  cppcheck "${CPPCHECK[@]}" "${CPPCHECK_C[@]}" tox*/*.[ch] tox*/*/*.[ch] "${CPPFLAGS[@]}" "$@"
+  cppcheck "${CPPCHECK[@]}" "${CPPCHECK_CXX[@]}" amalgamation.cc "${CPPFLAGS[@]}" "$@"
+}
+
+. other/analysis/variants.sh

external/toxcore/c-toxcore/other/analysis/run-cpplint

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+FILTER="-whitespace"
+FILTER="$FILTER,-build/include_subdir"
+FILTER="$FILTER,-readability/casting"
+FILTER="$FILTER,-runtime/arrays"
+FILTER="$FILTER,-runtime/printf"
+FILTER="$FILTER,-runtime/int"
+FILTER="$FILTER,-build/header_guard"
+
+cpplint --filter="$FILTER" toxav/*.[ch] toxcore/*.[ch] toxencryptsave/*.[ch]

external/toxcore/c-toxcore/other/analysis/run-gcc

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+. other/analysis/gen-file.sh
+
+set -e
+
+run() {
+  echo "Running GCC in variant '$*'"
+  # TODO(iphydf): Get rid of all VLAs, then enable -fstack-protector -Wstack-protector
+  g++ -O3 -o /dev/null amalgamation.cc \
+    "${CPPFLAGS[@]}" \
+    "${LDFLAGS[@]}" \
+    "$@" \
+    -std=c++11 \
+    -fdiagnostics-color=always \
+    -Wall \
+    -Wextra \
+    -Werror \
+    -Wno-error=type-limits \
+    -Wno-aggressive-loop-optimizations \
+    -Wno-float-conversion \
+    -Wno-format-signedness \
+    -Wno-missing-field-initializers \
+    -Wno-nonnull-compare \
+    -Wno-padded \
+    -Wno-sign-compare \
+    -Wno-sign-conversion \
+    -Wno-switch-default \
+    -Wno-unused-parameter \
+    -Wstrict-aliasing=0 \
+    -Wstrict-overflow=1 \
+    \
+    -Wmissing-declarations \
+    -Wbool-compare \
+    -Wcast-align \
+    -Wcast-qual \
+    -Wchar-subscripts \
+    -Wdouble-promotion \
+    -Wduplicated-cond \
+    -Wempty-body \
+    -Wenum-compare \
+    -Wfloat-equal \
+    -Wformat=2 \
+    -Wframe-address \
+    -Wframe-larger-than=9000 \
+    -Wignored-attributes \
+    -Wignored-qualifiers \
+    -Winit-self \
+    -Winline \
+    -Wlarger-than=530000 \
+    -Wmaybe-uninitialized \
+    -Wmemset-transposed-args \
+    -Wmisleading-indentation \
+    -Wnonnull \
+    -Wnull-dereference \
+    -Wodr \
+    -Wredundant-decls \
+    -Wreturn-type \
+    -Wshadow \
+    -Wsuggest-attribute=format \
+    -Wundef \
+    -Wunsafe-loop-optimizations \
+    -Wunused-label \
+    -Wunused-local-typedefs \
+    -Wunused-value \
+    -Wunused-but-set-parameter \
+    -Wunused-but-set-variable \
+    -fopenmp
+}
+
+. other/analysis/variants.sh

external/toxcore/c-toxcore/other/analysis/run-infer

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# --bufferoverrun \
+# --pulse \
+
+read -r -d '' SCRIPT <<'EOF'
+infer \
+  --report-console-limit 100 \
+  --jobs 8 \
+  --biabduction \
+  --loop-hoisting \
+  --quandary \
+  --racerd \
+  --starvation \
+  --uninit \
+  -- clang++ -fsyntax-only \
+  $(pkg-config --cflags libconfig libsodium opus vpx) \
+  /work/other/bootstrap_daemon/src/*.c \
+  /work/other/bootstrap_node_packets.c \
+  /work/toxav/*.c \
+  /work/toxcore/*.c \
+  /work/toxencryptsave/*.c
+EOF
+
+docker run --rm -it -v "$PWD:/work" toxchat/infer bash -c "$SCRIPT"

external/toxcore/c-toxcore/other/analysis/variants.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+run