Merge commit '8eb4892b4976e82e020d0e30dcf8f0705b76bb4e'

2024-01-12 21:30:48 +01:00
parent 78b0e9a77f 8eb4892b49
commit 82fe4c3dd7
126 changed files with 1556 additions and 2484 deletions
--- a/external/toxcore/c-toxcore/other/docker/circleci/Dockerfile
+++ b/external/toxcore/c-toxcore/other/docker/circleci/Dockerfile
@ -7,6 +7,7 @@ RUN apt-get update && \
 clang \
 cmake \
 libconfig-dev \
+ libgmock-dev \
 libgtest-dev \
 libopus-dev \
 libsodium-dev \
--- a/external/toxcore/c-toxcore/other/docker/coverage/Dockerfile
+++ b/external/toxcore/c-toxcore/other/docker/coverage/Dockerfile
@ -1,53 +1,57 @@
 FROM toxchat/c-toxcore:sources AS src
 FROM ubuntu:20.04 AS build

+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 RUN apt-get update && \
 DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
 ca-certificates \
- clang \
- cmake \
 curl \
- gcc \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main" >> /etc/apt/sources.list \
+ && curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \
+ && apt-get update && \
+ DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
+ clang-17 \
+ cmake \
 git \
 golang-1.18 \
+ libclang-rt-17-dev \
 libconfig-dev \
+ libgmock-dev \
 libgtest-dev \
 libopus-dev \
 libsodium-dev \
+ libunwind-17-dev \
 libvpx-dev \
- llvm-dev \
+ lld-17 \
+ llvm-17-dev \
 make \
 ninja-build \
 pkg-config \
- python3-lxml \
- python3-pip \
- python3-pygments \
 && apt-get clean \
- && rm -rf /var/lib/apt/lists/* \
- && pip3 install --no-cache-dir gcovr
-# strip gtest so it doesn't end up in stack traces. This speeds up the
-# mallocfail run below by a lot.
-RUN ["strip", "-g",\
- "/usr/lib/x86_64-linux-gnu/libgtest.a",\
- "/usr/lib/x86_64-linux-gnu/libgtest_main.a"]
+ && rm -rf /var/lib/apt/lists/*
 RUN ["curl", "-s", "https://codecov.io/bash", "-o", "/usr/local/bin/codecov"]
 RUN ["chmod", "+x", "/usr/local/bin/codecov"]

-ENV CC=clang \
-    CXX=clang++ \
+ENV CC=clang-17 \
+    CXX=clang++-17 \
    PYTHONUNBUFFERED=1 \
    PATH=$PATH:/usr/lib/go-1.18/bin
-SHELL ["/bin/bash", "-c"]
+
+COPY --from=src /src/ /work/

 WORKDIR /work
-COPY --from=src /src/ /work/
+RUN git clone --depth=1 https://github.com/TokTok/toktok-fuzzer /work/testing/fuzzing/toktok-fuzzer
 RUN source .github/scripts/flags-coverage.sh \
 && go version \
 && (cd other/proxy && go get github.com/things-go/go-socks5 && go build proxy_server.go) \
 && cmake -B_build -H. -GNinja \
 -DCMAKE_C_FLAGS="$C_FLAGS" \
 -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
- -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
+ -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS -fuse-ld=lld" \
 -DCMAKE_UNITY_BUILD=ON \
 -DENABLE_SHARED=OFF \
 -DMIN_LOGGER_LEVEL=TRACE \
@ -56,6 +60,7 @@ RUN source .github/scripts/flags-coverage.sh \
 -DSTRICT_ABI=ON \
 -DAUTOTEST=ON \
 -DPROXY_TEST=ON \
+ -DBUILD_FUZZ_TESTS=ON \
 -DUSE_IPV6=OFF \
 -DTEST_TIMEOUT_SECONDS=40 \
 && cmake --build _build --parallel 8 --target install
@ -66,24 +71,15 @@ RUN /work/other/proxy/proxy_server \
    ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6)

 WORKDIR /work/mallocfail
-RUN ["git", "clone", "--depth=1", "https://github.com/ralight/mallocfail", "/work/mallocfail"]
-COPY other/docker/coverage/syscall_funcs.c src/
-RUN gcc -fPIC -shared -O2 -g3 -Wall -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lbacktrace \
+RUN ["git", "clone", "--depth=1", "https://github.com/TokTok/mallocfail", "/work/mallocfail"]
+RUN clang-17 -fuse-ld=lld -fPIC -shared -O2 -g3 -Wall -I/usr/lib/llvm-17/include -L/usr/lib/llvm-17/lib -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lunwind \
 && install mallocfail.so /usr/local/lib/mallocfail.so

 WORKDIR /work/_build
 COPY other/docker/coverage/run_mallocfail /usr/local/bin/
-RUN ["run_mallocfail", "--ctest=2", "--jobs=8"]
-RUN ["gcovr", \
- "--sort-percentage", \
- "--gcov-executable=llvm-cov gcov", \
- "--html-details=html/", \
- "--root=..", \
- "--exclude=CMakeFiles/", \
- "--exclude=_deps/", \
- "--exclude=(.+/)?auto_tests/", \
- "--exclude=.+_test.cc?$", \
- "--exclude=(.+/)?other/", \
- "--exclude=(.+/)?testing/"]
+RUN ["run_mallocfail", "--ctest=1", "--jobs=8"]
+RUN llvm-profdata-17 merge -sparse $(find . -name "*.profraw") -o toxcore.profdata
+RUN llvm-cov-17 show -format=text -instr-profile=toxcore.profdata -sources $(cmake --build . --target help | grep -o '[^:]*_test:' | grep -o '[^:]*' | xargs -n1 find . -type f -name | awk '{print "-object "$1}') > coverage.txt
+RUN llvm-cov-17 show -format=html -instr-profile=toxcore.profdata -sources $(cmake --build . --target help | grep -o '[^:]*_test:' | grep -o '[^:]*' | xargs -n1 find . -type f -name | awk '{print "-object "$1}') -output-dir=html

 WORKDIR /work
--- a/external/toxcore/c-toxcore/other/docker/coverage/Dockerfile.nginx
+++ b/external/toxcore/c-toxcore/other/docker/coverage/Dockerfile.nginx
@ -1,5 +1,4 @@
 # vim:ft=dockerfile
 FROM toxchat/c-toxcore:coverage AS build
 FROM nginx:alpine
-COPY --from=build /work/_build/html/coverage_details.html /usr/share/nginx/html/index.html
-COPY --from=build /work/_build/html/ /usr/share/nginx/html/
+COPY --from=build --chown=nginx:nginx /work/_build/html/ /usr/share/nginx/html/
--- a/external/toxcore/c-toxcore/other/docker/coverage/run
+++ b/external/toxcore/c-toxcore/other/docker/coverage/run
@ -6,4 +6,4 @@ read -a ci_env <<<"$(bash <(curl -s https://codecov.io/env))"

 docker build -t toxchat/c-toxcore:sources -f other/docker/sources/Dockerfile .
 docker build -t toxchat/c-toxcore:coverage -f other/docker/coverage/Dockerfile .
-docker run "${ci_env[@]}" -e CI=true --name toxcore-coverage --rm -t toxchat/c-toxcore:coverage /usr/local/bin/codecov -x "llvm-cov gcov"
+docker run "${ci_env[@]}" -e CI=true --name toxcore-coverage --rm -t toxchat/c-toxcore:coverage /usr/local/bin/codecov
--- a/external/toxcore/c-toxcore/other/docker/coverage/run_mallocfail
+++ b/external/toxcore/c-toxcore/other/docker/coverage/run_mallocfail
@ -27,11 +27,12 @@ from typing import NoReturn
 from typing import Optional
 from typing import Tuple

-_PRIMER = "./unit_util_test"
+_PRIMER = "auto_tests/auto_version_test"
 _MALLOCFAIL_SO = "/usr/local/lib/mallocfail.so"
 _HASHES = "mallocfail_hashes"
 _HASHES_PREV = "mallocfail_hashes.prev"
 _TIMEOUT = 3.0
+_BUILD_DIR = os.getcwd()

 _ENV = {
    "LD_PRELOAD": _MALLOCFAIL_SO,
@ -45,10 +46,19 @@ def run_mallocfail(tmpdir: str, timeout: float, exe: str, iteration: int,
    print(f"\x1b[1;33mmallocfail '{exe}' run #{iteration}\x1b[0m")
    hashes = os.path.join(tmpdir, _HASHES)
    hashes_prev = os.path.join(tmpdir, _HASHES_PREV)
+    profraw = os.path.join(_BUILD_DIR, "mallocfail.out", exe, "%p.profraw")
    if os.path.exists(hashes):
        shutil.copy(hashes, hashes_prev)
    try:
-        proc = subprocess.run([exe], timeout=timeout, env=_ENV, cwd=tmpdir)
+        proc = subprocess.run(
+            [exe],
+            timeout=timeout,
+            env={
+                "LLVM_PROFILE_FILE": profraw,
+                **_ENV,
+            },
+            cwd=tmpdir,
+        )
    except subprocess.TimeoutExpired:
        print(f"\x1b[1;34mProgram {exe} timed out\x1b[0m")
        return True
@ -65,13 +75,16 @@ def run_mallocfail(tmpdir: str, timeout: float, exe: str, iteration: int,
        # Process exited cleanly (success or failure).
        pass
    elif proc.returncode == -6:
-        # Assertion failed.
+        # abort(), we allow it.
+        pass
+    elif proc.returncode == 7:
+        # ck_assert failed, also fine for us.
        pass
    elif proc.returncode == -14:
        print(f"\x1b[0;34mProgram '{exe}' timed out\x1b[0m")
    else:
        print(
-            f"\x1b[1;32mProgram '{exe}' failed to handle OOM situation cleanly\x1b[0m"
+            f"\x1b[1;32mProgram '{exe}' failed to handle OOM situation cleanly (code {proc.returncode})\x1b[0m"
        )
        if not keep_going:
            raise Exception("Aborting test")
@ -96,8 +109,8 @@ def find_prog(name: str) -> Tuple[Optional[str], ...]:
            return path
        return None

-    return (attempt(f"./unit_{name}_test"),
-            attempt(f"auto_tests/auto_{name}_test"))
+    return (attempt(f"auto_tests/auto_{name}_test"),
+            )  # attempt(f"./unit_{name}_test"),


 def parse_flags(args: List[str]) -> Tuple[Dict[str, str], List[str]]:
@ -128,6 +141,9 @@ def isolated_mallocfail(timeout: int, exe: str) -> None:
        shutil.copy(exe, os.path.join(tmpdir, exe))
        shutil.copy(_HASHES, os.path.join(tmpdir, _HASHES))
        loop_mallocfail(tmpdir, timeout, exe)
+        profraw = os.path.join(tmpdir, "default.profraw")
+        if os.path.exists(profraw):
+            shutil.copy(profraw, exe + ".mallocfail.profraw")


 def main(args: List[str]) -> None:
@ -150,12 +166,12 @@ def main(args: List[str]) -> None:
    else:
        jobs = 1

-    # Start by running util_test, which allocates no memory of its own, just
+    # Start by running version_test, which allocates no memory of its own, just
    # to prime the mallocfail hashes so it doesn't make global initialisers
    # such as llvm_gcov_init fail.
    if os.path.exists(_PRIMER):
-        print(f"\x1b[1;33mPriming hashes with unit_util_test\x1b[0m")
-        loop_mallocfail(".", timeout, _PRIMER, keep_going=True)
+        print(f"\x1b[1;33mPriming hashes with {_PRIMER}\x1b[0m")
+        loop_mallocfail(os.getcwd(), timeout, _PRIMER, keep_going=True)

    print(f"\x1b[1;33m--------------------------------\x1b[0m")
    print(f"\x1b[1;33mStarting mallocfail for {len(exes)} programs:\x1b[0m")
--- a/external/toxcore/c-toxcore/other/docker/coverage/syscall_funcs.c
+++ b/external/toxcore/c-toxcore/other/docker/coverage/syscall_funcs.c
@ -1,149 +0,0 @@
-#define _GNU_SOURCE
-
-#include "mallocfail.h"
-
-#include <dlfcn.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-
-static int (*libc_ioctl)(int fd, unsigned long request, ...);
-static int (*libc_bind)(int sockfd, const struct sockaddr *addr,
-        socklen_t addrlen);
-static int (*libc_getsockopt)(int sockfd, int level, int optname,
-        void *optval, socklen_t *optlen);
-static int (*libc_setsockopt)(int sockfd, int level, int optname,
-        const void *optval, socklen_t optlen);
-static ssize_t (*libc_recv)(int sockfd, void *buf, size_t len, int flags);
-static ssize_t (*libc_recvfrom)(int sockfd, void *buf, size_t len, int flags,
-        struct sockaddr *src_addr, socklen_t *addrlen);
-static ssize_t (*libc_send)(int sockfd, const void *buf, size_t len, int flags);
-static ssize_t(*libc_sendto)(int sockfd, const void *buf, size_t len, int flags,
-        const struct sockaddr *dest_addr, socklen_t addrlen);
-static int (*libc_socket)(int domain, int type, int protocol);
-static int (*libc_listen)(int sockfd, int backlog);
-
-__attribute__((__constructor__))
-static void init(void)
-{
-    libc_ioctl = dlsym(RTLD_NEXT, "ioctl");
-    libc_bind = dlsym(RTLD_NEXT, "bind");
-    libc_getsockopt = dlsym(RTLD_NEXT, "getsockopt");
-    libc_setsockopt = dlsym(RTLD_NEXT, "setsockopt");
-    libc_recv = dlsym(RTLD_NEXT, "recv");
-    libc_recvfrom = dlsym(RTLD_NEXT, "recvfrom");
-    libc_send = dlsym(RTLD_NEXT, "send");
-    libc_sendto = dlsym(RTLD_NEXT, "sendto");
-    libc_socket = dlsym(RTLD_NEXT, "socket");
-    libc_listen = dlsym(RTLD_NEXT, "listen");
-}
-
-int ioctl(int fd, unsigned long request, ...)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-
-    va_list ap;
-    va_start(ap, request);
-    const int ret = libc_ioctl(fd, SIOCGIFCONF, va_arg(ap, void *));
-    va_end(ap);
-    return ret;
-}
-
-int bind(int sockfd, const struct sockaddr *addr,
-        socklen_t addrlen)
-{
-    // Unlike all others, if bind should fail once, it should fail always, because in toxcore we try
-    // many ports before giving up. If this only fails once, we'll never reach the code path where
-    // we give up.
-    static int should_fail = -1;
-    if (should_fail == -1) {
-        should_fail = should_malloc_fail();
-    }
-    if (should_fail) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_bind(sockfd, addr, addrlen);
-}
-
-int getsockopt(int sockfd, int level, int optname,
-        void *optval, socklen_t *optlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_getsockopt(sockfd, level, optname, optval, optlen);
-}
-
-int setsockopt(int sockfd, int level, int optname,
-        const void *optval, socklen_t optlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_setsockopt(sockfd, level, optname, optval, optlen);
-}
-
-ssize_t recv(int sockfd, void *buf, size_t len, int flags)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_recv(sockfd, buf, len, flags);
-}
-
-ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags,
-        struct sockaddr *src_addr, socklen_t *addrlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_recvfrom(sockfd, buf, len, flags, src_addr, addrlen);
-}
-
-ssize_t send(int sockfd, const void *buf, size_t len, int flags)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_send(sockfd, buf, len, flags);
-}
-
-ssize_t sendto(int sockfd, const void *buf, size_t len, int flags,
-        const struct sockaddr *dest_addr, socklen_t addrlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_sendto(sockfd, buf, len, flags, dest_addr, addrlen);
-}
-
-int socket(int domain, int type, int protocol)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_socket(domain, type, protocol);
-}
-
-int listen(int sockfd, int backlog)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_listen(sockfd, backlog);
-}
--- a/external/toxcore/c-toxcore/other/docker/doxygen/Dockerfile
+++ b/external/toxcore/c-toxcore/other/docker/doxygen/Dockerfile
@ -5,7 +5,7 @@ ENV LANG=en_US.UTF-8 \
    LC_CTYPE=en_US.UTF-8 \
    LC_ALL=en_US.UTF-8

-RUN apk add --no-cache doxygen git graphviz \
+RUN apk add --no-cache doxygen git graphviz texlive \
 && git clone --depth=1 https://github.com/jothepro/doxygen-awesome-css.git /work/doxygen-awesome-css
 WORKDIR /work
 COPY . /work/
--- a/external/toxcore/c-toxcore/other/docker/infer/Dockerfile
+++ b/external/toxcore/c-toxcore/other/docker/infer/Dockerfile
@ -0,0 +1,41 @@
+FROM toxchat/infer:latest
+
+COPY toxav/ /work/c-toxcore/toxav/
+COPY toxcore/ /work/c-toxcore/toxcore/
+COPY toxencryptsave/ /work/c-toxcore/toxencryptsave/
+COPY third_party/ /work/c-toxcore/third_party/
+RUN infer capture -- clang++ -fsyntax-only \
+  $(pkg-config --cflags libconfig libsodium opus vpx) \
+  /work/c-toxcore/toxav/*.c \
+  /work/c-toxcore/toxcore/*.c \
+  /work/c-toxcore/toxcore/*/*.c \
+  /work/c-toxcore/toxencryptsave/*.c
+RUN ["infer", "analyze",\
+ "--report-console-limit", "100",\
+ "--jobs", "8",\
+ "--no-bufferoverrun",\
+ "--no-datalog",\
+ "--print-active-checkers",\
+ "--loop-hoisting",\
+ "--quandary",\
+ "--racerd",\
+ "--starvation",\
+ "--uninit",\
+ "--disable-issue-type", "BUFFER_OVERRUN_L2",\
+ "--disable-issue-type", "PULSE_UNNECESSARY_COPY",\
+ "--enable-issue-type", "EXPENSIVE_EXECUTION_TIME",\
+ "--enable-issue-type", "INVARIANT_CALL",\
+ "--enable-issue-type", "PULSE_UNINITIALIZED_CONST",\
+ "--enable-issue-type", "SENSITIVE_DATA_FLOW",\
+ "--enable-issue-type", "UNTRUSTED_BUFFER_ACCESS",\
+ "--enable-issue-type", "UNTRUSTED_HEAP_ALLOCATION",\
+ "--disable-issue-type", "USE_AFTER_FREE_LATENT",\
+ "--disable-issue-type", "STACK_VARIABLE_ADDRESS_ESCAPE",\
+ "--disable-issue-type", "INVARIANT_CALL",\
+ "--fail-on-issue"]
+# In the above, the first 2 are disabled for extreme sensitivity and false
+# positives, the ones at the end are probably decent, but have some false
+# positives, so we can't fail-on-issue with them on.
+# INVARIANT_CALL is pretty fun, but currently wrong, because it can't see
+# through potential mutations via callbacks. Our code is bad and we should
+# feel bad, but until that's fixed, the invariant checker doesn't work.
--- a/external/toxcore/c-toxcore/other/docker/infer/run
+++ b/external/toxcore/c-toxcore/other/docker/infer/run
@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -eux
+BUILD=infer
+docker build -t "toxchat/c-toxcore:$BUILD" -f "other/docker/$BUILD/Dockerfile" .
--- a/external/toxcore/c-toxcore/other/docker/windows/Dockerfile
+++ b/external/toxcore/c-toxcore/other/docker/windows/Dockerfile
@ -2,7 +2,7 @@ FROM debian:bullseye-slim

 # Build-time environment variables
 ARG VERSION_MSGPACK=4.0.0 \
-    VERSION_SODIUM=1.0.18 \
+    VERSION_SODIUM=1.0.19 \
    VERSION_OPUS=1.3.1 \
    VERSION_VPX=1.11.0 \
 \
--- a/external/toxcore/c-toxcore/other/docker/windows/build_dependencies.sh
+++ b/external/toxcore/c-toxcore/other/docker/windows/build_dependencies.sh
@ -40,9 +40,9 @@ build() {

  echo
  echo "=== Building Sodium $VERSION_SODIUM $ARCH ==="
-  curl "${CURL_OPTIONS[@]}" -O "https://download.libsodium.org/libsodium/releases/libsodium-$VERSION_SODIUM.tar.gz"
+  curl "${CURL_OPTIONS[@]}" -O "https://github.com/jedisct1/libsodium/releases/download/$VERSION_SODIUM-RELEASE/libsodium-$VERSION_SODIUM.tar.gz"
  tar -xf "libsodium-$VERSION_SODIUM.tar.gz"
-  cd "libsodium-$VERSION_SODIUM"
+  cd "libsodium-stable"
  ./configure --host="$WINDOWS_TOOLCHAIN" --prefix="$PREFIX_DIR" --disable-shared --enable-static
  make
  make install